Imported Upstream version 2.2.0upstream/2.2.1upstream/2.2.0

1201 files changed, 112857 insertions, 32372 deletions

diff --git a/.gitignore b/.gitignore
index 2286d0a10..e32a58417 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,10 @@
 /trace/generated-tracers.dtrace
 /trace/generated-events.h
 /trace/generated-events.c
+/trace/generated-helpers-wrappers.h
+/trace/generated-helpers.h
+/trace/generated-helpers.c
+/trace/generated-tcg-tracers.h
 /trace/generated-ust-provider.h
 /trace/generated-ust.c
 /libcacard/trace/generated-tracers.c
diff --git a/.travis.yml b/.travis.yml
index 89c30aefa..ad66e5bca 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,7 +12,7 @@ notifications:
     on_failure: always
 env:
   global:
-    - TEST_CMD="make check"
+    - TEST_CMD=""
     - EXTRA_CONFIG=""
     # Development packages, EXTRA_PKGS saved for additional builds
     - CORE_PKGS="libusb-1.0-0-dev libiscsi-dev librados-dev libncurses5-dev"
@@ -20,31 +20,51 @@ env:
     - GUI_PKGS="libgtk-3-dev libvte-2.90-dev libsdl1.2-dev libpng12-dev libpixman-1-dev"
     - EXTRA_PKGS=""
   matrix:
+    # Group major targets together with their linux-user counterparts
     - TARGETS=alpha-softmmu,alpha-linux-user
-    - TARGETS=arm-softmmu,arm-linux-user
-    - TARGETS=aarch64-softmmu,aarch64-linux-user
-    - TARGETS=cris-softmmu
-    - TARGETS=i386-softmmu,x86_64-softmmu
-    - TARGETS=lm32-softmmu
-    - TARGETS=m68k-softmmu
-    - TARGETS=microblaze-softmmu,microblazeel-softmmu
+    - TARGETS=arm-softmmu,arm-linux-user,armeb-linux-user,aarch64-softmmu,aarch64-linux-user
+    - TARGETS=cris-softmmu,cris-linux-user
+    - TARGETS=i386-softmmu,i386-linux-user,x86_64-softmmu,x86_64-linux-user
+    - TARGETS=m68k-softmmu,m68k-linux-user
+    - TARGETS=microblaze-softmmu,microblazeel-softmmu,microblaze-linux-user,microblazeel-linux-user
     - TARGETS=mips-softmmu,mips64-softmmu,mips64el-softmmu,mipsel-softmmu
-    - TARGETS=moxie-softmmu
-    - TARGETS=or32-softmmu,
-    - TARGETS=ppc-softmmu,ppc64-softmmu,ppcemb-softmmu
-    - TARGETS=s390x-softmmu
-    - TARGETS=sh4-softmmu,sh4eb-softmmu
-    - TARGETS=sparc-softmmu,sparc64-softmmu
-    - TARGETS=unicore32-softmmu
-    - TARGETS=xtensa-softmmu,xtensaeb-softmmu
+    - TARGETS=mips-linux-user,mips64-linux-user,mips64el-linux-user,mipsel-linux-user,mipsn32-linux-user,mipsn32el-linux-user
+    - TARGETS=or32-softmmu,or32-linux-user
+    - TARGETS=ppc-softmmu,ppc64-softmmu,ppcemb-softmmu,ppc-linux-user,ppc64-linux-user,ppc64abi32-linux-user,ppc64le-linux-user
+    - TARGETS=s390x-softmmu,s390x-linux-user
+    - TARGETS=sh4-softmmu,sh4eb-softmmu,sh4-linux-user sh4eb-linux-user
+    - TARGETS=sparc-softmmu,sparc64-softmmu,sparc-linux-user,sparc32plus-linux-user,sparc64-linux-user
+    - TARGETS=unicore32-softmmu,unicore32-linux-user
+    # Group remaining softmmu only targets into one build
+    - TARGETS=lm32-softmmu,moxie-softmmu,tricore-softmmu,xtensa-softmmu,xtensaeb-softmmu
+git:
+  # we want to do this ourselves
+  submodules: false
 before_install:
+  - wget -O - http://people.linaro.org/~alex.bennee/qemu-submodule-git-seed.tar.xz | tar -xvJ
   - git submodule update --init --recursive
   - sudo apt-get update -qq
   - sudo apt-get install -qq ${CORE_PKGS} ${NET_PKGS} ${GUI_PKGS} ${EXTRA_PKGS}
-script: "./configure --target-list=${TARGETS} ${EXTRA_CONFIG} && make && ${TEST_CMD}"
+before_script:
+  - ./configure --target-list=${TARGETS} --enable-debug-tcg ${EXTRA_CONFIG}
+script:
+  - make -j2 && ${TEST_CMD}
 matrix:
   # We manually include a number of additional build for non-standard bits
   include:
+    # Make check target (we only do this once)
+    - env:
+        - TARGETS=alpha-softmmu,arm-softmmu,aarch64-softmmu,cris-softmmu,
+                  i386-softmmu,x86_64-softmmu,m68k-softmmu,microblaze-softmmu,
+                  microblazeel-softmmu,mips-softmmu,mips64-softmmu,
+                  mips64el-softmmu,mipsel-softmmu,or32-softmmu,ppc-softmmu,
+                  ppc64-softmmu,ppcemb-softmmu,s390x-softmmu,sh4-softmmu,
+                  sh4eb-softmmu,sparc-softmmu,sparc64-softmmu,
+                  unicore32-softmmu,unicore32-linux-user,
+                  lm32-softmmu,moxie-softmmu,tricore-softmmu,xtensa-softmmu,
+                  xtensaeb-softmmu
+          TEST_CMD="make check"
+      compiler: gcc
     # Debug related options
     - env: TARGETS=i386-softmmu,x86_64-softmmu
            EXTRA_CONFIG="--enable-debug"
@@ -73,7 +93,6 @@ matrix:
       compiler: gcc
     - env: TARGETS=i386-softmmu,x86_64-softmmu
            EXTRA_CONFIG="--enable-trace-backends=ftrace"
-           TEST_CMD=""
       compiler: gcc
     - env: TARGETS=i386-softmmu,x86_64-softmmu
           EXTRA_PKGS="liblttng-ust-dev liburcu-dev"
diff --git a/CODING_STYLE b/CODING_STYLE
index 4280945ff..d46cfa5f6 100644
--- a/CODING_STYLE
+++ b/CODING_STYLE
@@ -91,3 +91,17 @@ Mixed declarations (interleaving statements and declarations within blocks)
 are not allowed; declarations should be at the beginning of blocks.  In other
 words, the code should not generate warnings if using GCC's
 -Wdeclaration-after-statement option.
+
+6. Conditional statements
+
+When comparing a variable for (in)equality with a constant, list the
+constant on the right, as in:
+
+if (a == 1) {
+    /* Reads like: "If a equals 1" */
+    do_something();
+}
+
+Rationale: Yoda conditions (as in 'if (1 == a)') are awkward to read.
+Besides, good compilers already warn users when '==' is mis-typed as '=',
+even when the constant is on the right.
diff --git a/MAINTAINERS b/MAINTAINERS
index 906f25247..bcb69e80d 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -51,6 +51,7 @@ Descriptions of section entries:
 General Project Administration
 ------------------------------
 M: Anthony Liguori <aliguori@amazon.com>
+M: Peter Maydell <peter.maydell@linaro.org>
 
 Responsible Disclosure, Reporting Security Issues
 ------------------------------
@@ -61,11 +62,23 @@ L: secalert@redhat.com
 
 Guest CPU cores (TCG):
 ----------------------
+Overall
+L: qemu-devel@nongnu.org
+S: Odd fixes
+F: cpu-exec.c
+F: cputlb.c
+F: softmmu_template.h
+F: translate-all.c
+F: include/exec/cpu_ldst.h
+F: include/exec/cpu_ldst_template.h
+F: include/exec/helper*.h
+
 Alpha
 M: Richard Henderson <rth@twiddle.net>
 S: Maintained
 F: target-alpha/
 F: hw/alpha/
+F: tests/tcg/alpha/
 
 ARM
 M: Peter Maydell <peter.maydell@linaro.org>
@@ -79,6 +92,7 @@ M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
 S: Maintained
 F: target-cris/
 F: hw/cris/
+F: tests/tcg/cris/
 
 LM32
 M: Michael Walle <michael@walle.cc>
@@ -86,6 +100,7 @@ S: Maintained
 F: target-lm32/
 F: hw/lm32/
 F: hw/char/lm32_*
+F: tests/tcg/lm32/
 
 M68K
 S: Orphan
@@ -100,9 +115,11 @@ F: hw/microblaze/
 
 MIPS
 M: Aurelien Jarno <aurelien@aurel32.net>
-S: Odd Fixes
+M: Leon Alrae <leon.alrae@imgtec.com>
+S: Maintained
 F: target-mips/
 F: hw/mips/
+F: tests/tcg/mips/
 
 Moxie
 M: Anthony Green <green@moxielogic.com>
@@ -114,6 +131,7 @@ M: Jia Liu <proljc@gmail.com>
 S: Maintained
 F: target-openrisc/
 F: hw/openrisc/
+F: tests/tcg/openrisc/
 
 PowerPC
 M: Alexander Graf <agraf@suse.de>
@@ -149,7 +167,8 @@ F: target-unicore32/
 F: hw/unicore32/
 
 X86
-M: qemu-devel@nongnu.org
+M: Paolo Bonzini <pbonzini@redhat.com>
+M: Richard Henderson <rth@twiddle.net>
 S: Odd Fixes
 F: target-i386/
 F: hw/i386/
@@ -160,6 +179,13 @@ W: http://wiki.osll.spb.ru/doku.php?id=etc:users:jcmvbkbc:qemu-target-xtensa
 S: Maintained
 F: target-xtensa/
 F: hw/xtensa/
+F: tests/tcg/xtensa/
+
+TriCore
+M: Bastian Koppelmann <kbastian@mail.uni-paderborn.de>
+S: Maintained
+F: target-tricore/
+F: hw/tricore/
 
 Guest CPU Cores (KVM):
 ----------------------
@@ -192,9 +218,12 @@ M: Cornelia Huck <cornelia.huck@de.ibm.com>
 M: Alexander Graf <agraf@suse.de>
 S: Maintained
 F: target-s390x/kvm.c
-F: hw/intc/s390_flic.[hc]
+F: hw/intc/s390_flic.c
+F: hw/intc/s390_flic_kvm.c
+F: include/hw/s390x/s390_flic.h
 
 X86
+M: Paolo Bonzini <pbonzini@redhat.com>
 M: Marcelo Tosatti <mtosatti@redhat.com>
 L: kvm@vger.kernel.org
 S: Supported
@@ -260,7 +289,7 @@ F: include/hw/arm/digic.h
 F: hw/*/digic*
 
 Gumstix
-M: qemu-devel@nongnu.org
+L: qemu-devel@nongnu.org
 S: Orphan
 F: hw/arm/gumstix.c
 
@@ -276,7 +305,7 @@ S: Maintained
 F: hw/arm/integratorcp.c
 
 Mainstone
-M: qemu-devel@nongnu.org
+L: qemu-devel@nongnu.org
 S: Orphan
 F: hw/arm/mainstone.c
 
@@ -382,7 +411,7 @@ S: Maintained
 F: hw/mips/mips_malta.c
 
 Mipssim
-M: qemu-devel@nongnu.org
+L: qemu-devel@nongnu.org
 S: Orphan
 F: hw/mips/mips_mipssim.c
 
@@ -515,6 +544,8 @@ F: hw/s390x/s390-virtio-ccw.c
 F: hw/s390x/css.[hc]
 F: hw/s390x/sclp*.[hc]
 F: hw/s390x/ipl*.[hc]
+F: include/hw/s390x/
+F: pc-bios/s390-ccw/
 T: git git://github.com/cohuck/qemu virtio-ccw-upstr
 
 UniCore32 Machines
@@ -552,12 +583,13 @@ Xtensa Machines
 sim
 M: Max Filippov <jcmvbkbc@gmail.com>
 S: Maintained
-F: hw/xtensa/xtensa_sim.c
+F: hw/xtensa/sim.c
 
-Avnet LX60
+XTFPGA (LX60, LX200, ML605, KC705)
 M: Max Filippov <jcmvbkbc@gmail.com>
 S: Maintained
-F: hw/xtensa/xtensa_lx60.c
+F: hw/xtensa/xtfpga.c
+F: hw/net/opencores_eth.c
 
 Devices
 -------
@@ -614,7 +646,13 @@ USB
 M: Gerd Hoffmann <kraxel@redhat.com>
 S: Maintained
 F: hw/usb/*
-F: tests/usb-hcd-ehci-test.c
+F: tests/usb-*-test.c
+
+USB (serial adapter)
+M: Gerd Hoffmann <kraxel@redhat.com>
+M: Samuel Thibault <samuel.thibault@ens-lyon.org>
+S: Maintained
+F: hw/usb/dev-serial.c
 
 VFIO
 M: Alex Williamson <alex.williamson@redhat.com>
@@ -678,6 +716,12 @@ S: Maintained
 F: hw/*/xilinx_*
 F: include/hw/xilinx.h
 
+Vmware
+M: Dmitry Fleytman <dmitry@daynix.com>
+S: Maintained
+F: hw/net/vmxnet*
+F: hw/scsi/vmw_pvscsi*
+
 Subsystems
 ----------
 Audio
@@ -694,18 +738,30 @@ Block
 M: Kevin Wolf <kwolf@redhat.com>
 M: Stefan Hajnoczi <stefanha@redhat.com>
 S: Supported
+F: async.c
+F: aio-*.c
 F: block*
 F: block/
 F: hw/block/
 F: qemu-img*
 F: qemu-io*
+F: tests/image-fuzzer/
+F: tests/qemu-iotests/
 T: git git://repo.or.cz/qemu/kevin.git block
 T: git git://github.com/stefanha/qemu.git block
 
 Character Devices
 M: Anthony Liguori <aliguori@amazon.com>
+M: Paolo Bonzini <pbonzini@redhat.com>
 S: Maintained
 F: qemu-char.c
+F: backends/msmouse.c
+F: backends/testdev.c
+
+Character Devices (Braille)
+M: Samuel Thibault <samuel.thibault@ens-lyon.org>
+S: Maintained
+F: backends/baum.c
 
 CPU
 M: Andreas Färber <afaerber@suse.de>
@@ -727,7 +783,7 @@ S: Maintained
 F: device_tree.[ch]
 
 GDB stub
-M: qemu-devel@nongnu.org
+L: qemu-devel@nongnu.org
 S: Odd Fixes
 F: gdbstub*
 F: gdb-xml/
@@ -764,7 +820,11 @@ F: ui/cocoa.m
 
 Main loop
 M: Anthony Liguori <aliguori@amazon.com>
-S: Supported
+M: Paolo Bonzini <pbonzini@redhat.com>
+S: Maintained
+F: cpus.c
+F: main-loop.c
+F: qemu-timer.c
 F: vl.c
 
 Human Monitor (HMP)
@@ -803,6 +863,7 @@ M: Luiz Capitulino <lcapitulino@redhat.com>
 M: Michael Roth <mdroth@linux.vnet.ibm.com>
 S: Maintained
 F: qapi/
+F: tests/qapi-schema/
 T: git git://repo.or.cz/qemu/qmp-unstable.git queue/qmp
 
 QAPI Schema
@@ -813,6 +874,18 @@ S: Supported
 F: qapi-schema.json
 T: git git://repo.or.cz/qemu/qmp-unstable.git queue/qmp
 
+QObject
+M: Luiz Capitulino <lcapitulino@redhat.com>
+S: Maintained
+F: qobject/
+T: git git://repo.or.cz/qemu/qmp-unstable.git queue/qmp
+
+QEMU Guest Agent
+M: Michael Roth <mdroth@linux.vnet.ibm.com>
+S: Maintained
+F: qga/
+T: git git://github.com/mdroth/qemu.git qga
+
 QOM
 M: Anthony Liguori <aliguori@amazon.com>
 M: Andreas Färber <afaerber@suse.de>
@@ -853,6 +926,15 @@ M: Blue Swirl <blauwirbel@gmail.com>
 S: Odd Fixes
 F: scripts/checkpatch.pl
 
+Migration
+M: Juan Quintela <quintela@redhat.com>
+S: Maintained
+F: include/migration/
+F: migration*
+F: savevm.c
+F: arch_init.c
+F: vmstate.c
+
 Seccomp
 M: Eduardo Otubo <eduardo.otubo@profitbricks.com>
 S: Supported
@@ -861,6 +943,12 @@ F: include/sysemu/seccomp.h
 
 Usermode Emulation
 ------------------
+Overall
+M: Riku Voipio <riku.voipio@iki.fi>
+S: Maintained
+F: thunk.c
+F: user-exec.c
+
 BSD user
 M: Blue Swirl <blauwirbel@gmail.com>
 S: Maintained
@@ -874,7 +962,6 @@ F: linux-user/
 Tiny Code Generator (TCG)
 -------------------------
 Common code
-M: qemu-devel@nongnu.org
 M: Richard Henderson <rth@twiddle.net>
 S: Maintained
 F: tcg/
@@ -891,7 +978,7 @@ S: Maintained
 F: tcg/arm/
 
 i386 target
-M: qemu-devel@nongnu.org
+L: qemu-devel@nongnu.org
 S: Maintained
 F: tcg/i386/
 
@@ -968,7 +1055,7 @@ S: Supported
 F: block/rbd.c
 
 Sheepdog
-M: MORITA Kazutaka <morita.kazutaka@lab.ntt.co.jp>
+M: Hitoshi Mitake <mitake.hitoshi@lab.ntt.co.jp>
 M: Liu Yuan <namei.unix@gmail.com>
 L: sheepdog@lists.wpkg.org
 S: Supported
@@ -1000,3 +1087,14 @@ SSH
 M: Richard W.M. Jones <rjones@redhat.com>
 S: Supported
 F: block/ssh.c
+
+ARCHIPELAGO
+M: Chrysostomos Nanakos <cnanakos@grnet.gr>
+M: Chrysostomos Nanakos <chris@include.gr>
+S: Maintained
+F: block/archipelago.c
+
+Bootdevice
+M: Gonglei <arei.gonglei@huawei.com>
+S: Maintained
+F: bootdevice.c
diff --git a/Makefile b/Makefile
index d6b9dc1eb..f5052026d 100644
--- a/Makefile
+++ b/Makefile
@@ -57,6 +57,12 @@ GENERATED_HEADERS += trace/generated-tracers-dtrace.h
 endif
 GENERATED_SOURCES += trace/generated-tracers.c
 
+GENERATED_HEADERS += trace/generated-tcg-tracers.h
+
+GENERATED_HEADERS += trace/generated-helpers-wrappers.h
+GENERATED_HEADERS += trace/generated-helpers.h
+GENERATED_SOURCES += trace/generated-helpers.c
+
 ifeq ($(findstring ust,$(TRACE_BACKENDS)),ust)
 GENERATED_HEADERS += trace/generated-ust-provider.h
 GENERATED_SOURCES += trace/generated-ust.c
@@ -202,7 +208,7 @@ Makefile: $(version-obj-y) $(version-lobj-y)
 # Build libraries
 
 libqemustub.a: $(stub-obj-y)
-libqemuutil.a: $(util-obj-y) qapi-types.o qapi-visit.o qapi-event.o
+libqemuutil.a: $(util-obj-y)
 
 block-modules = $(foreach o,$(block-obj-m),"$(basename $(subst /,-,$o))",) NULL
 util/module.o-cflags = -D'CONFIG_BLOCK_MODULES=$(block-modules)'
@@ -412,6 +418,7 @@ endif
 	set -e; for x in $(KEYMAPS); do \
 		$(INSTALL_DATA) $(SRC_PATH)/pc-bios/keymaps/$$x "$(DESTDIR)$(qemu_datadir)/keymaps"; \
 	done
+	$(INSTALL_DATA) $(SRC_PATH)/trace-events "$(DESTDIR)$(qemu_datadir)/trace-events"
 	for d in $(TARGET_DIRS); do \
 	$(MAKE) $(SUBDIR_MAKEFLAGS) TARGET_DIR=$$d/ -C $$d $@ || exit 1 ; \
         done
diff --git a/Makefile.objs b/Makefile.objs
index 1f76cea56..18fd35cf1 100644
--- a/Makefile.objs
+++ b/Makefile.objs
@@ -1,7 +1,7 @@
 #######################################################################
 # Common libraries for tools and emulators
 stub-obj-y = stubs/
-util-obj-y = util/ qobject/ qapi/ trace/
+util-obj-y = util/ qobject/ qapi/ qapi-types.o qapi-visit.o qapi-event.o
 
 #######################################################################
 # block-obj-y is code used by both qemu system emulation and qemu-img
@@ -12,7 +12,6 @@ block-obj-y += main-loop.o iohandler.o qemu-timer.o
 block-obj-$(CONFIG_POSIX) += aio-posix.o
 block-obj-$(CONFIG_WIN32) += aio-win32.o
 block-obj-y += block/
-block-obj-y += qapi-types.o qapi-visit.o qapi-event.o
 block-obj-y += qemu-io-cmds.o
 
 block-obj-y += qemu-coroutine.o qemu-coroutine-lock.o qemu-coroutine-io.o
@@ -51,7 +50,7 @@ common-obj-$(CONFIG_LINUX) += fsdev/
 
 common-obj-y += migration.o migration-tcp.o
 common-obj-y += vmstate.o
-common-obj-y += qemu-file.o
+common-obj-y += qemu-file.o qemu-file-unix.o qemu-file-stdio.o
 common-obj-$(CONFIG_RDMA) += migration-rdma.o
 common-obj-y += qemu-char.o #aio.o
 common-obj-y += block-migration.o
@@ -63,6 +62,7 @@ common-obj-$(CONFIG_SPICE) += spice-qemu-char.o
 
 common-obj-y += audio/
 common-obj-y += hw/
+common-obj-y += accel.o
 
 common-obj-y += ui/
 common-obj-y += bt-host.o bt-vhci.o
@@ -88,11 +88,6 @@ common-obj-y += qmp-marshal.o
 common-obj-y += qmp.o hmp.o
 endif
 
-######################################################################
-# some qapi visitors are used by both system and user emulation:
-
-common-obj-y += qapi-visit.o qapi-types.o
-
 #######################################################################
 # Target-independent parts used in system and user emulation
 common-obj-y += qemu-log.o
@@ -107,9 +102,14 @@ version-obj-$(CONFIG_WIN32) += $(BUILD_DIR)/version.o
 version-lobj-$(CONFIG_WIN32) += $(BUILD_DIR)/version.lo
 
 ######################################################################
+# tracing
+util-obj-y +=  trace/
+target-obj-y += trace/
+
+######################################################################
 # guest agent
 
 # FIXME: a few definitions from qapi-types.o/qapi-visit.o are needed
 # by libqemuutil.a.  These should be moved to a separate .json schema.
-qga-obj-y = qga/ qapi-types.o qapi-visit.o
+qga-obj-y = qga/
 qga-vss-dll-obj-y = qga/
diff --git a/Makefile.target b/Makefile.target
index 137d0b051..e9ff1eed7 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -38,7 +38,7 @@ config-target.h: config-target.h-timestamp
 config-target.h-timestamp: config-target.mak
 
 ifdef CONFIG_TRACE_SYSTEMTAP
-stap: $(QEMU_PROG).stp-installed $(QEMU_PROG).stp
+stap: $(QEMU_PROG).stp-installed $(QEMU_PROG).stp $(QEMU_PROG)-simpletrace.stp
 
 ifdef CONFIG_USER_ONLY
 TARGET_TYPE=user
@@ -64,6 +64,13 @@ $(QEMU_PROG).stp: $(SRC_PATH)/trace-events
 		--target-type=$(TARGET_TYPE) \
 		< $< > $@,"  GEN   $(TARGET_DIR)$(QEMU_PROG).stp")
 
+$(QEMU_PROG)-simpletrace.stp: $(SRC_PATH)/trace-events
+	$(call quiet-command,$(TRACETOOL) \
+		--format=simpletrace-stap \
+		--backends=$(TRACE_BACKENDS) \
+		--probe-prefix=qemu.$(TARGET_TYPE).$(TARGET_NAME) \
+		< $< > $@,"  GEN   $(TARGET_DIR)$(QEMU_PROG)-simpletrace.stp")
+
 else
 stap:
 endif
@@ -120,7 +127,7 @@ endif #CONFIG_BSD_USER
 # System emulator target
 ifdef CONFIG_SOFTMMU
 obj-y += arch_init.o cpus.o monitor.o gdbstub.o balloon.o ioport.o numa.o
-obj-y += qtest.o
+obj-y += qtest.o bootdevice.o
 obj-y += hw/
 obj-$(CONFIG_FDT) += device_tree.o
 obj-$(CONFIG_KVM) += kvm-all.o
@@ -152,15 +159,20 @@ endif # CONFIG_SOFTMMU
 dummy := $(call unnest-vars,,obj-y)
 all-obj-y := $(obj-y)
 
+target-obj-y :=
 block-obj-y :=
 common-obj-y :=
 include $(SRC_PATH)/Makefile.objs
+dummy := $(call unnest-vars,,target-obj-y)
+target-obj-y-save := $(target-obj-y)
 dummy := $(call unnest-vars,.., \
                block-obj-y \
                block-obj-m \
                common-obj-y \
                common-obj-m)
+target-obj-y := $(target-obj-y-save)
 all-obj-y += $(common-obj-y)
+all-obj-y += $(target-obj-y)
 all-obj-$(CONFIG_SOFTMMU) += $(block-obj-y)
 
 # build either PROG or PROGW
@@ -191,6 +203,7 @@ endif
 ifdef CONFIG_TRACE_SYSTEMTAP
 	$(INSTALL_DIR) "$(DESTDIR)$(qemu_datadir)/../systemtap/tapset"
 	$(INSTALL_DATA) $(QEMU_PROG).stp-installed "$(DESTDIR)$(qemu_datadir)/../systemtap/tapset/$(QEMU_PROG).stp"
+	$(INSTALL_DATA) $(QEMU_PROG)-simpletrace.stp "$(DESTDIR)$(qemu_datadir)/../systemtap/tapset/$(QEMU_PROG)-simpletrace.stp"
 endif
 
 GENERATED_HEADERS += config-target.h
diff --git a/VERSION b/VERSION
index 7ec1d6db4..ccbccc3dc 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1.0
+2.2.0
diff --git a/accel.c b/accel.c
new file mode 100644
index 000000000..74e41daaa
--- /dev/null
+++ b/accel.c
@@ -0,0 +1,157 @@
+/*
+ * QEMU System Emulator, accelerator interfaces
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ * Copyright (c) 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "sysemu/accel.h"
+#include "hw/boards.h"
+#include "qemu-common.h"
+#include "sysemu/arch_init.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/kvm.h"
+#include "sysemu/qtest.h"
+#include "hw/xen/xen.h"
+#include "qom/object.h"
+#include "hw/boards.h"
+
+int tcg_tb_size;
+static bool tcg_allowed = true;
+
+static int tcg_init(MachineState *ms)
+{
+    tcg_exec_init(tcg_tb_size * 1024 * 1024);
+    return 0;
+}
+
+static const TypeInfo accel_type = {
+    .name = TYPE_ACCEL,
+    .parent = TYPE_OBJECT,
+    .class_size = sizeof(AccelClass),
+    .instance_size = sizeof(AccelState),
+};
+
+/* Lookup AccelClass from opt_name. Returns NULL if not found */
+static AccelClass *accel_find(const char *opt_name)
+{
+    char *class_name = g_strdup_printf(ACCEL_CLASS_NAME("%s"), opt_name);
+    AccelClass *ac = ACCEL_CLASS(object_class_by_name(class_name));
+    g_free(class_name);
+    return ac;
+}
+
+static int accel_init_machine(AccelClass *acc, MachineState *ms)
+{
+    ObjectClass *oc = OBJECT_CLASS(acc);
+    const char *cname = object_class_get_name(oc);
+    AccelState *accel = ACCEL(object_new(cname));
+    int ret;
+    ms->accelerator = accel;
+    *(acc->allowed) = true;
+    ret = acc->init_machine(ms);
+    if (ret < 0) {
+        ms->accelerator = NULL;
+        *(acc->allowed) = false;
+        object_unref(OBJECT(accel));
+    }
+    return ret;
+}
+
+int configure_accelerator(MachineState *ms)
+{
+    const char *p;
+    char buf[10];
+    int ret;
+    bool accel_initialised = false;
+    bool init_failed = false;
+    AccelClass *acc = NULL;
+
+    p = qemu_opt_get(qemu_get_machine_opts(), "accel");
+    if (p == NULL) {
+        /* Use the default "accelerator", tcg */
+        p = "tcg";
+    }
+
+    while (!accel_initialised && *p != '\0') {
+        if (*p == ':') {
+            p++;
+        }
+        p = get_opt_name(buf, sizeof(buf), p, ':');
+        acc = accel_find(buf);
+        if (!acc) {
+            fprintf(stderr, "\"%s\" accelerator not found.\n", buf);
+            continue;
+        }
+        if (acc->available && !acc->available()) {
+            printf("%s not supported for this target\n",
+                   acc->name);
+            continue;
+        }
+        ret = accel_init_machine(acc, ms);
+        if (ret < 0) {
+            init_failed = true;
+            fprintf(stderr, "failed to initialize %s: %s\n",
+                    acc->name,
+                    strerror(-ret));
+        } else {
+            accel_initialised = true;
+        }
+    }
+
+    if (!accel_initialised) {
+        if (!init_failed) {
+            fprintf(stderr, "No accelerator found!\n");
+        }
+        exit(1);
+    }
+
+    if (init_failed) {
+        fprintf(stderr, "Back to %s accelerator.\n", acc->name);
+    }
+
+    return !accel_initialised;
+}
+
+
+static void tcg_accel_class_init(ObjectClass *oc, void *data)
+{
+    AccelClass *ac = ACCEL_CLASS(oc);
+    ac->name = "tcg";
+    ac->init_machine = tcg_init;
+    ac->allowed = &tcg_allowed;
+}
+
+#define TYPE_TCG_ACCEL ACCEL_CLASS_NAME("tcg")
+
+static const TypeInfo tcg_accel_type = {
+    .name = TYPE_TCG_ACCEL,
+    .parent = TYPE_ACCEL,
+    .class_init = tcg_accel_class_init,
+};
+
+static void register_accel_types(void)
+{
+    type_register_static(&accel_type);
+    type_register_static(&tcg_accel_type);
+}
+
+type_init(register_accel_types);
diff --git a/aio-posix.c b/aio-posix.c
index 2eada2e04..d3ac06e23 100644
--- a/aio-posix.c
+++ b/aio-posix.c
@@ -100,6 +100,11 @@ void aio_set_event_notifier(AioContext *ctx,
                        (IOHandler *)io_read, NULL, notifier);
 }
 
+bool aio_prepare(AioContext *ctx)
+{
+    return false;
+}
+
 bool aio_pending(AioContext *ctx)
 {
     AioHandler *node;
@@ -119,12 +124,21 @@ bool aio_pending(AioContext *ctx)
     return false;
 }
 
-static bool aio_dispatch(AioContext *ctx)
+bool aio_dispatch(AioContext *ctx)
 {
     AioHandler *node;
     bool progress = false;
 
     /*
+     * If there are callbacks left that have been queued, we need to call them.
+     * Do not call select in this case, because it is possible that the caller
+     * does not need a complete flush (as is the case for aio_poll loops).
+     */
+    if (aio_bh_poll(ctx)) {
+        progress = true;
+    }
+
+    /*
      * We have to walk very carefully in case aio_set_fd_handler is
      * called while we're walking.
      */
@@ -184,22 +198,9 @@ bool aio_poll(AioContext *ctx, bool blocking)
 
     /* aio_notify can avoid the expensive event_notifier_set if
      * everything (file descriptors, bottom halves, timers) will
-     * be re-evaluated before the next blocking poll().  This happens
-     * in two cases:
-     *
-     * 1) when aio_poll is called with blocking == false
-     *
-     * 2) when we are called after poll().  If we are called before
-     *    poll(), bottom halves will not be re-evaluated and we need
-     *    aio_notify() if blocking == true.
-     *
-     * The first aio_dispatch() only does something when AioContext is
-     * running as a GSource, and in that case aio_poll is used only
-     * with blocking == false, so this optimization is already quite
-     * effective.  However, the code is ugly and should be restructured
-     * to have a single aio_dispatch() call.  To do this, we need to
-     * reorganize aio_poll into a prepare/poll/dispatch model like
-     * glib's.
+     * be re-evaluated before the next blocking poll().  This is
+     * already true when aio_poll is called with blocking == false;
+     * if blocking == true, it is only true after poll() returns.
      *
      * If we're in a nested event loop, ctx->dispatching might be true.
      * In that case we can restore it just before returning, but we
@@ -207,26 +208,6 @@ bool aio_poll(AioContext *ctx, bool blocking)
      */
     aio_set_dispatching(ctx, !blocking);
 
-    /*
-     * If there are callbacks left that have been queued, we need to call them.
-     * Do not call select in this case, because it is possible that the caller
-     * does not need a complete flush (as is the case for aio_poll loops).
-     */
-    if (aio_bh_poll(ctx)) {
-        blocking = false;
-        progress = true;
-    }
-
-    /* Re-evaluate condition (1) above.  */
-    aio_set_dispatching(ctx, !blocking);
-    if (aio_dispatch(ctx)) {
-        progress = true;
-    }
-
-    if (progress && !blocking) {
-        goto out;
-    }
-
     ctx->walking_handlers++;
 
     g_array_set_size(ctx->pollfds, 0);
@@ -249,7 +230,7 @@ bool aio_poll(AioContext *ctx, bool blocking)
     /* wait until next event */
     ret = qemu_poll_ns((GPollFD *)ctx->pollfds->data,
                          ctx->pollfds->len,
-                         blocking ? timerlistgroup_deadline_ns(&ctx->tlg) : 0);
+                         blocking ? aio_compute_timeout(ctx) : 0);
 
     /* if we have any readable fds, dispatch event */
     if (ret > 0) {
@@ -268,7 +249,6 @@ bool aio_poll(AioContext *ctx, bool blocking)
         progress = true;
     }
 
-out:
     aio_set_dispatching(ctx, was_dispatching);
     return progress;
 }
diff --git a/aio-win32.c b/aio-win32.c
index c12f61e97..d81313b00 100644
--- a/aio-win32.c
+++ b/aio-win32.c
@@ -22,12 +22,80 @@
 
 struct AioHandler {
     EventNotifier *e;
+    IOHandler *io_read;
+    IOHandler *io_write;
     EventNotifierHandler *io_notify;
     GPollFD pfd;
     int deleted;
+    void *opaque;
     QLIST_ENTRY(AioHandler) node;
 };
 
+void aio_set_fd_handler(AioContext *ctx,
+                        int fd,
+                        IOHandler *io_read,
+                        IOHandler *io_write,
+                        void *opaque)
+{
+    /* fd is a SOCKET in our case */
+    AioHandler *node;
+
+    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
+        if (node->pfd.fd == fd && !node->deleted) {
+            break;
+        }
+    }
+
+    /* Are we deleting the fd handler? */
+    if (!io_read && !io_write) {
+        if (node) {
+            /* If the lock is held, just mark the node as deleted */
+            if (ctx->walking_handlers) {
+                node->deleted = 1;
+                node->pfd.revents = 0;
+            } else {
+                /* Otherwise, delete it for real.  We can't just mark it as
+                 * deleted because deleted nodes are only cleaned up after
+                 * releasing the walking_handlers lock.
+                 */
+                QLIST_REMOVE(node, node);
+                g_free(node);
+            }
+        }
+    } else {
+        HANDLE event;
+
+        if (node == NULL) {
+            /* Alloc and insert if it's not already there */
+            node = g_malloc0(sizeof(AioHandler));
+            node->pfd.fd = fd;
+            QLIST_INSERT_HEAD(&ctx->aio_handlers, node, node);
+        }
+
+        node->pfd.events = 0;
+        if (node->io_read) {
+            node->pfd.events |= G_IO_IN;
+        }
+        if (node->io_write) {
+            node->pfd.events |= G_IO_OUT;
+        }
+
+        node->e = &ctx->notifier;
+
+        /* Update handler with latest information */
+        node->opaque = opaque;
+        node->io_read = io_read;
+        node->io_write = io_write;
+
+        event = event_notifier_get_handle(&ctx->notifier);
+        WSAEventSelect(node->pfd.fd, event,
+                       FD_READ | FD_ACCEPT | FD_CLOSE |
+                       FD_CONNECT | FD_WRITE | FD_OOB);
+    }
+
+    aio_notify(ctx);
+}
+
 void aio_set_event_notifier(AioContext *ctx,
                             EventNotifier *e,
                             EventNotifierHandler *io_notify)
@@ -76,55 +144,82 @@ void aio_set_event_notifier(AioContext *ctx,
     aio_notify(ctx);
 }
 
-bool aio_pending(AioContext *ctx)
+bool aio_prepare(AioContext *ctx)
 {
+    static struct timeval tv0;
     AioHandler *node;
+    bool have_select_revents = false;
+    fd_set rfds, wfds;
 
+    /* fill fd sets */
+    FD_ZERO(&rfds);
+    FD_ZERO(&wfds);
     QLIST_FOREACH(node, &ctx->aio_handlers, node) {
-        if (node->pfd.revents && node->io_notify) {
-            return true;
+        if (node->io_read) {
+            FD_SET ((SOCKET)node->pfd.fd, &rfds);
+        }
+        if (node->io_write) {
+            FD_SET ((SOCKET)node->pfd.fd, &wfds);
         }
     }
 
-    return false;
+    if (select(0, &rfds, &wfds, NULL, &tv0) > 0) {
+        QLIST_FOREACH(node, &ctx->aio_handlers, node) {
+            node->pfd.revents = 0;
+            if (FD_ISSET(node->pfd.fd, &rfds)) {
+                node->pfd.revents |= G_IO_IN;
+                have_select_revents = true;
+            }
+
+            if (FD_ISSET(node->pfd.fd, &wfds)) {
+                node->pfd.revents |= G_IO_OUT;
+                have_select_revents = true;
+            }
+        }
+    }
+
+    return have_select_revents;
 }
 
-bool aio_poll(AioContext *ctx, bool blocking)
+bool aio_pending(AioContext *ctx)
 {
     AioHandler *node;
-    HANDLE events[MAXIMUM_WAIT_OBJECTS + 1];
-    bool progress;
-    int count;
-    int timeout;
 
-    progress = false;
+    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
+        if (node->pfd.revents && node->io_notify) {
+            return true;
+        }
 
-    /*
-     * If there are callbacks left that have been queued, we need to call then.
-     * Do not call select in this case, because it is possible that the caller
-     * does not need a complete flush (as is the case for aio_poll loops).
-     */
-    if (aio_bh_poll(ctx)) {
-        blocking = false;
-        progress = true;
+        if ((node->pfd.revents & G_IO_IN) && node->io_read) {
+            return true;
+        }
+        if ((node->pfd.revents & G_IO_OUT) && node->io_write) {
+            return true;
+        }
     }
 
-    /* Run timers */
-    progress |= timerlistgroup_run_timers(&ctx->tlg);
+    return false;
+}
+
+static bool aio_dispatch_handlers(AioContext *ctx, HANDLE event)
+{
+    AioHandler *node;
+    bool progress = false;
 
     /*
-     * Then dispatch any pending callbacks from the GSource.
-     *
      * We have to walk very carefully in case aio_set_fd_handler is
      * called while we're walking.
      */
     node = QLIST_FIRST(&ctx->aio_handlers);
     while (node) {
         AioHandler *tmp;
+        int revents = node->pfd.revents;
 
         ctx->walking_handlers++;
 
-        if (node->pfd.revents && node->io_notify) {
+        if (!node->deleted &&
+            (revents || event_notifier_get_handle(node->e) == event) &&
+            node->io_notify) {
             node->pfd.revents = 0;
             node->io_notify(node->e);
 
@@ -134,6 +229,28 @@ bool aio_poll(AioContext *ctx, bool blocking)
             }
         }
 
+        if (!node->deleted &&
+            (node->io_read || node->io_write)) {
+            node->pfd.revents = 0;
+            if ((revents & G_IO_IN) && node->io_read) {
+                node->io_read(node->opaque);
+                progress = true;
+            }
+            if ((revents & G_IO_OUT) && node->io_write) {
+                node->io_write(node->opaque);
+                progress = true;
+            }
+
+            /* if the next select() will return an event, we have progressed */
+            if (event == event_notifier_get_handle(&ctx->notifier)) {
+                WSANETWORKEVENTS ev;
+                WSAEnumNetworkEvents(node->pfd.fd, event, &ev);
+                if (ev.lNetworkEvents) {
+                    progress = true;
+                }
+            }
+        }
+
         tmp = node;
         node = QLIST_NEXT(node, node);
 
@@ -145,10 +262,47 @@ bool aio_poll(AioContext *ctx, bool blocking)
         }
     }
 
-    if (progress && !blocking) {
-        return true;
+    return progress;
+}
+
+bool aio_dispatch(AioContext *ctx)
+{
+    bool progress;
+
+    progress = aio_bh_poll(ctx);
+    progress |= aio_dispatch_handlers(ctx, INVALID_HANDLE_VALUE);
+    progress |= timerlistgroup_run_timers(&ctx->tlg);
+    return progress;
+}
+
+bool aio_poll(AioContext *ctx, bool blocking)
+{
+    AioHandler *node;
+    HANDLE events[MAXIMUM_WAIT_OBJECTS + 1];
+    bool was_dispatching, progress, have_select_revents, first;
+    int count;
+    int timeout;
+
+    have_select_revents = aio_prepare(ctx);
+    if (have_select_revents) {
+        blocking = false;
     }
 
+    was_dispatching = ctx->dispatching;
+    progress = false;
+
+    /* aio_notify can avoid the expensive event_notifier_set if
+     * everything (file descriptors, bottom halves, timers) will
+     * be re-evaluated before the next blocking poll().  This is
+     * already true when aio_poll is called with blocking == false;
+     * if blocking == true, it is only true after poll() returns.
+     *
+     * If we're in a nested event loop, ctx->dispatching might be true.
+     * In that case we can restore it just before returning, but we
+     * have to clear it now.
+     */
+    aio_set_dispatching(ctx, !blocking);
+
     ctx->walking_handlers++;
 
     /* fill fd sets */
@@ -160,64 +314,40 @@ bool aio_poll(AioContext *ctx, bool blocking)
     }
 
     ctx->walking_handlers--;
+    first = true;
 
     /* wait until next event */
     while (count > 0) {
+        HANDLE event;
         int ret;
 
-        timeout = blocking ?
-            qemu_timeout_ns_to_ms(timerlistgroup_deadline_ns(&ctx->tlg)) : 0;
+        timeout = blocking
+            ? qemu_timeout_ns_to_ms(aio_compute_timeout(ctx)) : 0;
         ret = WaitForMultipleObjects(count, events, FALSE, timeout);
+        aio_set_dispatching(ctx, true);
+
+        if (first && aio_bh_poll(ctx)) {
+            progress = true;
+        }
+        first = false;
 
         /* if we have any signaled events, dispatch event */
-        if ((DWORD) (ret - WAIT_OBJECT_0) >= count) {
+        event = NULL;
+        if ((DWORD) (ret - WAIT_OBJECT_0) < count) {
+            event = events[ret - WAIT_OBJECT_0];
+            events[ret - WAIT_OBJECT_0] = events[--count];
+        } else if (!have_select_revents) {
             break;
         }
 
+        have_select_revents = false;
         blocking = false;
 
-        /* we have to walk very carefully in case
-         * aio_set_fd_handler is called while we're walking */
-        node = QLIST_FIRST(&ctx->aio_handlers);
-        while (node) {
-            AioHandler *tmp;
-
-            ctx->walking_handlers++;
-
-            if (!node->deleted &&
-                event_notifier_get_handle(node->e) == events[ret - WAIT_OBJECT_0] &&
-                node->io_notify) {
-                node->io_notify(node->e);
-
-                /* aio_notify() does not count as progress */
-                if (node->e != &ctx->notifier) {
-                    progress = true;
-                }
-            }
-
-            tmp = node;
-            node = QLIST_NEXT(node, node);
-
-            ctx->walking_handlers--;
-
-            if (!ctx->walking_handlers && tmp->deleted) {
-                QLIST_REMOVE(tmp, node);
-                g_free(tmp);
-            }
-        }
-
-        /* Try again, but only call each handler once.  */
-        events[ret - WAIT_OBJECT_0] = events[--count];
+        progress |= aio_dispatch_handlers(ctx, event);
     }
 
-    if (blocking) {
-        /* Run the timers a second time. We do this because otherwise aio_wait
-         * will not note progress - and will stop a drain early - if we have
-         * a timer that was not ready to run entering g_poll but is ready
-         * after g_poll. This will only do anything if a timer has expired.
-         */
-        progress |= timerlistgroup_run_timers(&ctx->tlg);
-    }
+    progress |= timerlistgroup_run_timers(&ctx->tlg);
 
+    aio_set_dispatching(ctx, was_dispatching);
     return progress;
 }
diff --git a/arch_init.c b/arch_init.c
index 8ddaf3519..7680d28be 100644
--- a/arch_init.c
+++ b/arch_init.c
@@ -104,6 +104,8 @@ int graphic_depth = 32;
 #define QEMU_ARCH QEMU_ARCH_XTENSA
 #elif defined(TARGET_UNICORE32)
 #define QEMU_ARCH QEMU_ARCH_UNICORE32
+#elif defined(TARGET_TRICORE)
+#define QEMU_ARCH QEMU_ARCH_TRICORE
 #endif
 
 const uint32_t arch_type = QEMU_ARCH;
@@ -484,15 +486,23 @@ static void migration_bitmap_sync_range(ram_addr_t start, ram_addr_t length)
 
 
 /* Needs iothread lock! */
+/* Fix me: there are too many global variables used in migration process. */
+static int64_t start_time;
+static int64_t bytes_xfer_prev;
+static int64_t num_dirty_pages_period;
+
+static void migration_bitmap_sync_init(void)
+{
+    start_time = 0;
+    bytes_xfer_prev = 0;
+    num_dirty_pages_period = 0;
+}
 
 static void migration_bitmap_sync(void)
 {
     RAMBlock *block;
     uint64_t num_dirty_pages_init = migration_dirty_pages;
     MigrationState *s = migrate_get_current();
-    static int64_t start_time;
-    static int64_t bytes_xfer_prev;
-    static int64_t num_dirty_pages_period;
     int64_t end_time;
     int64_t bytes_xfer_now;
     static uint64_t xbzrle_cache_miss_prev;
@@ -772,6 +782,7 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
     mig_throttle_on = false;
     dirty_rate_high_cnt = 0;
     bitmap_sync_count = 0;
+    migration_bitmap_sync_init();
 
     if (migrate_use_xbzrle()) {
         XBZRLE_cache_lock();
@@ -1004,7 +1015,7 @@ static inline void *host_from_stream_offset(QEMUFile *f,
     uint8_t len;
 
     if (flags & RAM_SAVE_FLAG_CONTINUE) {
-        if (!block) {
+        if (!block || block->length <= offset) {
             error_report("Ack, bad migration stream!");
             return NULL;
         }
@@ -1017,8 +1028,9 @@ static inline void *host_from_stream_offset(QEMUFile *f,
     id[len] = 0;
 
     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-        if (!strncmp(id, block->idstr, sizeof(id)))
+        if (!strncmp(id, block->idstr, sizeof(id)) && block->length > offset) {
             return memory_region_get_ram_ptr(block->mr) + offset;
+        }
     }
 
     error_report("Can't find block %s!", id);
@@ -1038,8 +1050,7 @@ void ram_handle_compressed(void *host, uint8_t ch, uint64_t size)
 
 static int ram_load(QEMUFile *f, void *opaque, int version_id)
 {
-    ram_addr_t addr;
-    int flags, ret = 0;
+    int flags = 0, ret = 0;
     static uint64_t seq_iter;
 
     seq_iter++;
@@ -1048,21 +1059,24 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
         ret = -EINVAL;
     }
 
-    while (!ret) {
-        addr = qemu_get_be64(f);
+    while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
+        ram_addr_t addr, total_ram_bytes;
+        void *host;
+        uint8_t ch;
 
+        addr = qemu_get_be64(f);
         flags = addr & ~TARGET_PAGE_MASK;
         addr &= TARGET_PAGE_MASK;
 
-        if (flags & RAM_SAVE_FLAG_MEM_SIZE) {
+        switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
+        case RAM_SAVE_FLAG_MEM_SIZE:
             /* Synchronize RAM block list */
-            char id[256];
-            ram_addr_t length;
-            ram_addr_t total_ram_bytes = addr;
-
-            while (total_ram_bytes) {
+            total_ram_bytes = addr;
+            while (!ret && total_ram_bytes) {
                 RAMBlock *block;
                 uint8_t len;
+                char id[256];
+                ram_addr_t length;
 
                 len = qemu_get_byte(f);
                 qemu_get_buffer(f, (uint8_t *)id, len);
@@ -1072,8 +1086,8 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
                 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
                     if (!strncmp(id, block->idstr, sizeof(id))) {
                         if (block->length != length) {
-                            error_report("Length mismatch: %s: " RAM_ADDR_FMT
-                                         " in != " RAM_ADDR_FMT, id, length,
+                            error_report("Length mismatch: %s: 0x" RAM_ADDR_FMT
+                                         " in != 0x" RAM_ADDR_FMT, id, length,
                                          block->length);
                             ret =  -EINVAL;
                         }
@@ -1086,16 +1100,11 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
                                  "accept migration", id);
                     ret = -EINVAL;
                 }
-                if (ret) {
-                    break;
-                }
 
                 total_ram_bytes -= length;
             }
-        } else if (flags & RAM_SAVE_FLAG_COMPRESS) {
-            void *host;
-            uint8_t ch;
-
+            break;
+        case RAM_SAVE_FLAG_COMPRESS:
             host = host_from_stream_offset(f, addr, flags);
             if (!host) {
                 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
@@ -1105,9 +1114,8 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
 
             ch = qemu_get_byte(f);
             ram_handle_compressed(host, ch, TARGET_PAGE_SIZE);
-        } else if (flags & RAM_SAVE_FLAG_PAGE) {
-            void *host;
-
+            break;
+        case RAM_SAVE_FLAG_PAGE:
             host = host_from_stream_offset(f, addr, flags);
             if (!host) {
                 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
@@ -1116,8 +1124,9 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
             }
 
             qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
-        } else if (flags & RAM_SAVE_FLAG_XBZRLE) {
-            void *host = host_from_stream_offset(f, addr, flags);
+            break;
+        case RAM_SAVE_FLAG_XBZRLE:
+            host = host_from_stream_offset(f, addr, flags);
             if (!host) {
                 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
                 ret = -EINVAL;
@@ -1130,17 +1139,22 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
                 ret = -EINVAL;
                 break;
             }
-        } else if (flags & RAM_SAVE_FLAG_HOOK) {
-            ram_control_load_hook(f, flags);
-        } else if (flags & RAM_SAVE_FLAG_EOS) {
-            /* normal exit */
             break;
-        } else {
-            error_report("Unknown migration flags: %#x", flags);
-            ret = -EINVAL;
+        case RAM_SAVE_FLAG_EOS:
+            /* normal exit */
             break;
+        default:
+            if (flags & RAM_SAVE_FLAG_HOOK) {
+                ram_control_load_hook(f, flags);
+            } else {
+                error_report("Unknown combination of migration flags: %#x",
+                             flags);
+                ret = -EINVAL;
+            }
+        }
+        if (!ret) {
+            ret = qemu_file_get_error(f);
         }
-        ret = qemu_file_get_error(f);
     }
 
     DPRINTF("Completed load of VM with exit code %d seq iteration "
@@ -1335,11 +1349,6 @@ void cpudef_init(void)
 #endif
 }
 
-int tcg_available(void)
-{
-    return 1;
-}
-
 int kvm_available(void)
 {
 #ifdef CONFIG_KVM
diff --git a/async.c b/async.c
index 34af0b25c..6e1b282aa 100644
--- a/async.c
+++ b/async.c
@@ -152,39 +152,48 @@ void qemu_bh_delete(QEMUBH *bh)
     bh->deleted = 1;
 }
 
-static gboolean
-aio_ctx_prepare(GSource *source, gint    *timeout)
+int64_t
+aio_compute_timeout(AioContext *ctx)
 {
-    AioContext *ctx = (AioContext *) source;
+    int64_t deadline;
+    int timeout = -1;
     QEMUBH *bh;
-    int deadline;
 
-    /* We assume there is no timeout already supplied */
-    *timeout = -1;
     for (bh = ctx->first_bh; bh; bh = bh->next) {
         if (!bh->deleted && bh->scheduled) {
             if (bh->idle) {
                 /* idle bottom halves will be polled at least
                  * every 10ms */
-                *timeout = 10;
+                timeout = 10000000;
             } else {
                 /* non-idle bottom halves will be executed
                  * immediately */
-                *timeout = 0;
-                return true;
+                return 0;
             }
         }
     }
 
-    deadline = qemu_timeout_ns_to_ms(timerlistgroup_deadline_ns(&ctx->tlg));
+    deadline = timerlistgroup_deadline_ns(&ctx->tlg);
     if (deadline == 0) {
-        *timeout = 0;
-        return true;
+        return 0;
     } else {
-        *timeout = qemu_soonest_timeout(*timeout, deadline);
+        return qemu_soonest_timeout(timeout, deadline);
+    }
+}
+
+static gboolean
+aio_ctx_prepare(GSource *source, gint    *timeout)
+{
+    AioContext *ctx = (AioContext *) source;
+
+    /* We assume there is no timeout already supplied */
+    *timeout = qemu_timeout_ns_to_ms(aio_compute_timeout(ctx));
+
+    if (aio_prepare(ctx)) {
+        *timeout = 0;
     }
 
-    return false;
+    return *timeout == 0;
 }
 
 static gboolean
@@ -209,7 +218,7 @@ aio_ctx_dispatch(GSource     *source,
     AioContext *ctx = (AioContext *) source;
 
     assert(callback == NULL);
-    aio_poll(ctx, false);
+    aio_dispatch(ctx);
     return true;
 }
 
@@ -280,18 +289,24 @@ static void aio_rfifolock_cb(void *opaque)
     aio_notify(opaque);
 }
 
-AioContext *aio_context_new(void)
+AioContext *aio_context_new(Error **errp)
 {
+    int ret;
     AioContext *ctx;
     ctx = (AioContext *) g_source_new(&aio_source_funcs, sizeof(AioContext));
+    ret = event_notifier_init(&ctx->notifier, false);
+    if (ret < 0) {
+        g_source_destroy(&ctx->source);
+        error_setg_errno(errp, -ret, "Failed to initialize event notifier");
+        return NULL;
+    }
+    aio_set_event_notifier(ctx, &ctx->notifier,
+                           (EventNotifierHandler *)
+                           event_notifier_test_and_clear);
     ctx->pollfds = g_array_new(FALSE, FALSE, sizeof(GPollFD));
     ctx->thread_pool = NULL;
     qemu_mutex_init(&ctx->bh_lock);
     rfifolock_init(&ctx->lock, aio_rfifolock_cb, ctx);
-    event_notifier_init(&ctx->notifier, false);
-    aio_set_event_notifier(ctx, &ctx->notifier, 
-                           (EventNotifierHandler *)
-                           event_notifier_test_and_clear);
     timerlistgroup_init(&ctx->tlg, aio_timerlist_notify, ctx);
 
     return ctx;
diff --git a/backends/Makefile.objs b/backends/Makefile.objs
index 506a46c33..31a3a894f 100644
--- a/backends/Makefile.objs
+++ b/backends/Makefile.objs
@@ -1,7 +1,7 @@
 common-obj-y += rng.o rng-egd.o
 common-obj-$(CONFIG_POSIX) += rng-random.o
 
-common-obj-y += msmouse.o
+common-obj-y += msmouse.o testdev.o
 common-obj-$(CONFIG_BRLAPI) += baum.o
 baum.o-cflags := $(SDL_CFLAGS)
 
diff --git a/backends/baum.c b/backends/baum.c
index 796512d38..a69aafff4 100644
--- a/backends/baum.c
+++ b/backends/baum.c
@@ -629,7 +629,7 @@ fail_handle:
 
 static void register_types(void)
 {
-    register_char_driver_qapi("braille", CHARDEV_BACKEND_KIND_BRAILLE, NULL);
+    register_char_driver("braille", CHARDEV_BACKEND_KIND_BRAILLE, NULL);
 }
 
 type_init(register_types);
diff --git a/backends/hostmem-ram.c b/backends/hostmem-ram.c
index d9a8290dc..a67a13452 100644
--- a/backends/hostmem-ram.c
+++ b/backends/hostmem-ram.c
@@ -27,7 +27,7 @@ ram_backend_memory_alloc(HostMemoryBackend *backend, Error **errp)
 
     path = object_get_canonical_path_component(OBJECT(backend));
     memory_region_init_ram(&backend->mr, OBJECT(backend), path,
-                           backend->size);
+                           backend->size, errp);
     g_free(path);
 }
 
diff --git a/backends/hostmem.c b/backends/hostmem.c
index ca10c51b5..99e8f99da 100644
--- a/backends/hostmem.c
+++ b/backends/hostmem.c
@@ -257,15 +257,6 @@ static void host_memory_backend_init(Object *obj)
                         host_memory_backend_set_policy, NULL, NULL, NULL);
 }
 
-static void host_memory_backend_finalize(Object *obj)
-{
-    HostMemoryBackend *backend = MEMORY_BACKEND(obj);
-
-    if (memory_region_size(&backend->mr)) {
-        memory_region_destroy(&backend->mr);
-    }
-}
-
 MemoryRegion *
 host_memory_backend_get_memory(HostMemoryBackend *backend, Error **errp)
 {
@@ -304,7 +295,7 @@ host_memory_backend_memory_complete(UserCreatable *uc, Error **errp)
         /* ensure policy won't be ignored in case memory is preallocated
          * before mbind(). note: MPOL_MF_STRICT is ignored on hugepages so
          * this doesn't catch hugepage case. */
-        unsigned flags = MPOL_MF_STRICT;
+        unsigned flags = MPOL_MF_STRICT | MPOL_MF_MOVE;
 
         /* check for invalid host-nodes and policies and give more verbose
          * error messages than mbind(). */
@@ -360,7 +351,6 @@ static const TypeInfo host_memory_backend_info = {
     .class_init = host_memory_backend_class_init,
     .instance_size = sizeof(HostMemoryBackend),
     .instance_init = host_memory_backend_init,
-    .instance_finalize = host_memory_backend_finalize,
     .interfaces = (InterfaceInfo[]) {
         { TYPE_USER_CREATABLE },
         { }
diff --git a/backends/msmouse.c b/backends/msmouse.c
index 650a5314d..0119110a4 100644
--- a/backends/msmouse.c
+++ b/backends/msmouse.c
@@ -79,7 +79,7 @@ CharDriverState *qemu_chr_open_msmouse(void)
 
 static void register_types(void)
 {
-    register_char_driver_qapi("msmouse", CHARDEV_BACKEND_KIND_MSMOUSE, NULL);
+    register_char_driver("msmouse", CHARDEV_BACKEND_KIND_MSMOUSE, NULL);
 }
 
 type_init(register_types);
diff --git a/backends/rng-egd.c b/backends/rng-egd.c
index 25bb3b453..2962795a8 100644
--- a/backends/rng-egd.c
+++ b/backends/rng-egd.c
@@ -169,6 +169,7 @@ static void rng_egd_set_chardev(Object *obj, const char *value, Error **errp)
     if (b->opened) {
         error_set(errp, QERR_PERMISSION_DENIED);
     } else {
+        g_free(s->chr_name);
         s->chr_name = g_strdup(value);
     }
 }
diff --git a/backends/testdev.c b/backends/testdev.c
new file mode 100644
index 000000000..eba396aeb
--- /dev/null
+++ b/backends/testdev.c
@@ -0,0 +1,131 @@
+/*
+ * QEMU Char Device for testsuite control
+ *
+ * Copyright (c) 2014 Red Hat, Inc.
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu-common.h"
+#include "sysemu/char.h"
+
+#define BUF_SIZE 32
+
+typedef struct {
+    CharDriverState *chr;
+    uint8_t in_buf[32];
+    int in_buf_used;
+} TestdevCharState;
+
+/* Try to interpret a whole incoming packet */
+static int testdev_eat_packet(TestdevCharState *testdev)
+{
+    const uint8_t *cur = testdev->in_buf;
+    int len = testdev->in_buf_used;
+    uint8_t c;
+    int arg;
+
+#define EAT(c) do { \
+    if (!len--) {   \
+        return 0;   \
+    }               \
+    c = *cur++;     \
+} while (0)
+
+    EAT(c);
+
+    while (isspace(c)) {
+        EAT(c);
+    }
+
+    arg = 0;
+    while (isdigit(c)) {
+        arg = arg * 10 + c - '0';
+        EAT(c);
+    }
+
+    while (isspace(c)) {
+        EAT(c);
+    }
+
+    switch (c) {
+    case 'q':
+        exit((arg << 1) | 1);
+        break;
+    default:
+        break;
+    }
+    return cur - testdev->in_buf;
+}
+
+/* The other end is writing some data.  Store it and try to interpret */
+static int testdev_write(CharDriverState *chr, const uint8_t *buf, int len)
+{
+    TestdevCharState *testdev = chr->opaque;
+    int tocopy, eaten, orig_len = len;
+
+    while (len) {
+        /* Complete our buffer as much as possible */
+        tocopy = MIN(len, BUF_SIZE - testdev->in_buf_used);
+
+        memcpy(testdev->in_buf + testdev->in_buf_used, buf, tocopy);
+        testdev->in_buf_used += tocopy;
+        buf += tocopy;
+        len -= tocopy;
+
+        /* Interpret it as much as possible */
+        while (testdev->in_buf_used > 0 &&
+               (eaten = testdev_eat_packet(testdev)) > 0) {
+            memmove(testdev->in_buf, testdev->in_buf + eaten,
+                    testdev->in_buf_used - eaten);
+            testdev->in_buf_used -= eaten;
+        }
+    }
+    return orig_len;
+}
+
+static void testdev_close(struct CharDriverState *chr)
+{
+    TestdevCharState *testdev = chr->opaque;
+
+    g_free(testdev);
+}
+
+CharDriverState *chr_testdev_init(void)
+{
+    TestdevCharState *testdev;
+    CharDriverState *chr;
+
+    testdev = g_malloc0(sizeof(TestdevCharState));
+    testdev->chr = chr = g_malloc0(sizeof(CharDriverState));
+
+    chr->opaque = testdev;
+    chr->chr_write = testdev_write;
+    chr->chr_close = testdev_close;
+
+    return chr;
+}
+
+static void register_types(void)
+{
+    register_char_driver("testdev", CHARDEV_BACKEND_KIND_TESTDEV, NULL);
+}
+
+type_init(register_types);
diff --git a/block-migration.c b/block-migration.c
index 73cdd07e8..08db01a36 100644
--- a/block-migration.c
+++ b/block-migration.c
@@ -14,7 +14,9 @@
  */
 
 #include "qemu-common.h"
-#include "block/block_int.h"
+#include "block/block.h"
+#include "qemu/error-report.h"
+#include "qemu/main-loop.h"
 #include "hw/hw.h"
 #include "qemu/queue.h"
 #include "qemu/timer.h"
@@ -70,7 +72,7 @@ typedef struct BlkMigBlock {
     int nr_sectors;
     struct iovec iov;
     QEMUIOVector qiov;
-    BlockDriverAIOCB *aiocb;
+    BlockAIOCB *aiocb;
 
     /* Protected by block migration lock.  */
     int ret;
@@ -130,9 +132,9 @@ static void blk_send(QEMUFile *f, BlkMigBlock * blk)
                      | flags);
 
     /* device name */
-    len = strlen(blk->bmds->bs->device_name);
+    len = strlen(bdrv_get_device_name(blk->bmds->bs));
     qemu_put_byte(f, len);
-    qemu_put_buffer(f, (uint8_t *)blk->bmds->bs->device_name, len);
+    qemu_put_buffer(f, (uint8_t *)bdrv_get_device_name(blk->bmds->bs), len);
 
     /* if a block is zero we need to flush here since the network
      * bandwidth is now a lot higher than the storage device bandwidth.
@@ -186,7 +188,7 @@ static int bmds_aio_inflight(BlkMigDevState *bmds, int64_t sector)
 {
     int64_t chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;
 
-    if ((sector << BDRV_SECTOR_BITS) < bdrv_getlength(bmds->bs)) {
+    if (sector < bdrv_nb_sectors(bmds->bs)) {
         return !!(bmds->aio_bitmap[chunk / (sizeof(unsigned long) * 8)] &
             (1UL << (chunk % (sizeof(unsigned long) * 8))));
     } else {
@@ -223,8 +225,7 @@ static void alloc_aio_bitmap(BlkMigDevState *bmds)
     BlockDriverState *bs = bmds->bs;
     int64_t bitmap_size;
 
-    bitmap_size = (bdrv_getlength(bs) >> BDRV_SECTOR_BITS) +
-            BDRV_SECTORS_PER_DIRTY_CHUNK * 8 - 1;
+    bitmap_size = bdrv_nb_sectors(bs) + BDRV_SECTORS_PER_DIRTY_CHUNK * 8 - 1;
     bitmap_size /= BDRV_SECTORS_PER_DIRTY_CHUNK * 8;
 
     bmds->aio_bitmap = g_malloc0(bitmap_size);
@@ -284,7 +285,7 @@ static int mig_save_device_bulk(QEMUFile *f, BlkMigDevState *bmds)
         nr_sectors = total_sectors - cur_sector;
     }
 
-    blk = g_malloc(sizeof(BlkMigBlock));
+    blk = g_new(BlkMigBlock, 1);
     blk->buf = g_malloc(BLOCK_SIZE);
     blk->bmds = bmds;
     blk->sector = cur_sector;
@@ -344,18 +345,31 @@ static void unset_dirty_tracking(void)
     }
 }
 
-static void init_blk_migration_it(void *opaque, BlockDriverState *bs)
+static void init_blk_migration(QEMUFile *f)
 {
+    BlockDriverState *bs;
     BlkMigDevState *bmds;
     int64_t sectors;
 
-    if (!bdrv_is_read_only(bs)) {
-        sectors = bdrv_getlength(bs) >> BDRV_SECTOR_BITS;
+    block_mig_state.submitted = 0;
+    block_mig_state.read_done = 0;
+    block_mig_state.transferred = 0;
+    block_mig_state.total_sector_sum = 0;
+    block_mig_state.prev_progress = -1;
+    block_mig_state.bulk_completed = 0;
+    block_mig_state.zero_blocks = migrate_zero_blocks();
+
+    for (bs = bdrv_next(NULL); bs; bs = bdrv_next(bs)) {
+        if (bdrv_is_read_only(bs)) {
+            continue;
+        }
+
+        sectors = bdrv_nb_sectors(bs);
         if (sectors <= 0) {
             return;
         }
 
-        bmds = g_malloc0(sizeof(BlkMigDevState));
+        bmds = g_new0(BlkMigDevState, 1);
         bmds->bs = bs;
         bmds->bulk_completed = 0;
         bmds->total_sectors = sectors;
@@ -370,28 +384,15 @@ static void init_blk_migration_it(void *opaque, BlockDriverState *bs)
 
         if (bmds->shared_base) {
             DPRINTF("Start migration for %s with shared base image\n",
-                    bs->device_name);
+                    bdrv_get_device_name(bs));
         } else {
-            DPRINTF("Start full migration for %s\n", bs->device_name);
+            DPRINTF("Start full migration for %s\n", bdrv_get_device_name(bs));
         }
 
         QSIMPLEQ_INSERT_TAIL(&block_mig_state.bmds_list, bmds, entry);
     }
 }
 
-static void init_blk_migration(QEMUFile *f)
-{
-    block_mig_state.submitted = 0;
-    block_mig_state.read_done = 0;
-    block_mig_state.transferred = 0;
-    block_mig_state.total_sector_sum = 0;
-    block_mig_state.prev_progress = -1;
-    block_mig_state.bulk_completed = 0;
-    block_mig_state.zero_blocks = migrate_zero_blocks();
-
-    bdrv_iterate(init_blk_migration_it, NULL);
-}
-
 /* Called with no lock taken.  */
 
 static int blk_mig_save_bulked_block(QEMUFile *f)
@@ -466,7 +467,7 @@ static int mig_save_device_dirty(QEMUFile *f, BlkMigDevState *bmds,
             } else {
                 nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
             }
-            blk = g_malloc(sizeof(BlkMigBlock));
+            blk = g_new(BlkMigBlock, 1);
             blk->buf = g_malloc(BLOCK_SIZE);
             blk->bmds = bmds;
             blk->sector = sector;
@@ -799,7 +800,7 @@ static int block_load(QEMUFile *f, void *opaque, int version_id)
 
             if (bs != bs_prev) {
                 bs_prev = bs;
-                total_sectors = bdrv_getlength(bs) >> BDRV_SECTOR_BITS;
+                total_sectors = bdrv_nb_sectors(bs);
                 if (total_sectors <= 0) {
                     error_report("Error getting length of block device %s",
                                  device_name);
diff --git a/block.c b/block.c
index 8cf519ba3..a612594db 100644
--- a/block.c
+++ b/block.c
@@ -28,6 +28,7 @@
 #include "block/blockjob.h"
 #include "qemu/module.h"
 #include "qapi/qmp/qjson.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/sysemu.h"
 #include "qemu/notify.h"
 #include "block/coroutine.h"
@@ -57,13 +58,12 @@ struct BdrvDirtyBitmap {
 
 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
 
-static void bdrv_dev_change_media_cb(BlockDriverState *bs, bool load);
-static BlockDriverAIOCB *bdrv_aio_readv_em(BlockDriverState *bs,
+static BlockAIOCB *bdrv_aio_readv_em(BlockDriverState *bs,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque);
-static BlockDriverAIOCB *bdrv_aio_writev_em(BlockDriverState *bs,
+        BlockCompletionFunc *cb, void *opaque);
+static BlockAIOCB *bdrv_aio_writev_em(BlockDriverState *bs,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque);
+        BlockCompletionFunc *cb, void *opaque);
 static int coroutine_fn bdrv_co_readv_em(BlockDriverState *bs,
                                          int64_t sector_num, int nb_sectors,
                                          QEMUIOVector *iov);
@@ -76,14 +76,14 @@ static int coroutine_fn bdrv_co_do_preadv(BlockDriverState *bs,
 static int coroutine_fn bdrv_co_do_pwritev(BlockDriverState *bs,
     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
     BdrvRequestFlags flags);
-static BlockDriverAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
-                                               int64_t sector_num,
-                                               QEMUIOVector *qiov,
-                                               int nb_sectors,
-                                               BdrvRequestFlags flags,
-                                               BlockDriverCompletionFunc *cb,
-                                               void *opaque,
-                                               bool is_write);
+static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
+                                         int64_t sector_num,
+                                         QEMUIOVector *qiov,
+                                         int nb_sectors,
+                                         BdrvRequestFlags flags,
+                                         BlockCompletionFunc *cb,
+                                         void *opaque,
+                                         bool is_write);
 static void coroutine_fn bdrv_co_do_rw(void *opaque);
 static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
     int64_t sector_num, int nb_sectors, BdrvRequestFlags flags);
@@ -332,29 +332,21 @@ void bdrv_register(BlockDriver *bdrv)
     QLIST_INSERT_HEAD(&bdrv_drivers, bdrv, list);
 }
 
-/* create a new block device (by default it is empty) */
-BlockDriverState *bdrv_new(const char *device_name, Error **errp)
+BlockDriverState *bdrv_new_root(void)
+{
+    BlockDriverState *bs = bdrv_new();
+
+    QTAILQ_INSERT_TAIL(&bdrv_states, bs, device_list);
+    return bs;
+}
+
+BlockDriverState *bdrv_new(void)
 {
     BlockDriverState *bs;
     int i;
 
-    if (bdrv_find(device_name)) {
-        error_setg(errp, "Device with id '%s' already exists",
-                   device_name);
-        return NULL;
-    }
-    if (bdrv_find_node(device_name)) {
-        error_setg(errp, "Device with node-name '%s' already exists",
-                   device_name);
-        return NULL;
-    }
-
-    bs = g_malloc0(sizeof(BlockDriverState));
+    bs = g_new0(BlockDriverState, 1);
     QLIST_INIT(&bs->dirty_bitmaps);
-    pstrcpy(bs->device_name, sizeof(bs->device_name), device_name);
-    if (device_name[0] != '\0') {
-        QTAILQ_INSERT_TAIL(&bdrv_states, bs, device_list);
-    }
     for (i = 0; i < BLOCK_OP_TYPE_MAX; i++) {
         QLIST_INIT(&bs->op_blockers[i]);
     }
@@ -527,6 +519,7 @@ void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
             return;
         }
         bs->bl.opt_transfer_length = bs->file->bl.opt_transfer_length;
+        bs->bl.max_transfer_length = bs->file->bl.max_transfer_length;
         bs->bl.opt_mem_alignment = bs->file->bl.opt_mem_alignment;
     } else {
         bs->bl.opt_mem_alignment = 512;
@@ -541,6 +534,9 @@ void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
         bs->bl.opt_transfer_length =
             MAX(bs->bl.opt_transfer_length,
                 bs->backing_hd->bl.opt_transfer_length);
+        bs->bl.max_transfer_length =
+            MIN_NON_ZERO(bs->bl.max_transfer_length,
+                         bs->backing_hd->bl.max_transfer_length);
         bs->bl.opt_mem_alignment =
             MAX(bs->bl.opt_mem_alignment,
                 bs->backing_hd->bl.opt_mem_alignment);
@@ -701,6 +697,7 @@ static int find_image_format(BlockDriverState *bs, const char *filename,
 
 /**
  * Set the current 'total_sectors' value
+ * Return 0 on success, -errno on error.
  */
 static int refresh_total_sectors(BlockDriverState *bs, int64_t hint)
 {
@@ -858,14 +855,14 @@ static void bdrv_assign_node_name(BlockDriverState *bs,
         return;
     }
 
-    /* empty string node name is invalid */
-    if (node_name[0] == '\0') {
-        error_setg(errp, "Empty node name");
+    /* Check for empty string or invalid characters */
+    if (!id_wellformed(node_name)) {
+        error_setg(errp, "Invalid node name");
         return;
     }
 
     /* takes care of avoiding namespaces collisions */
-    if (bdrv_find(node_name)) {
+    if (blk_by_name(node_name)) {
         error_setg(errp, "node-name=%s is conflicting with a device id",
                    node_name);
         return;
@@ -961,6 +958,7 @@ static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
     } else {
         bs->filename[0] = '\0';
     }
+    pstrcpy(bs->exact_filename, sizeof(bs->exact_filename), bs->filename);
 
     bs->drv = drv;
     bs->opaque = g_malloc0(drv->instance_size);
@@ -1148,7 +1146,7 @@ void bdrv_set_backing_hd(BlockDriverState *bs, BlockDriverState *backing_hd)
     } else if (backing_hd) {
         error_setg(&bs->backing_blocker,
                    "device is used as backing hd of '%s'",
-                   bs->device_name);
+                   bdrv_get_device_name(bs));
     }
 
     bs->backing_hd = backing_hd;
@@ -1213,7 +1211,7 @@ int bdrv_open_backing_file(BlockDriverState *bs, QDict *options, Error **errp)
         goto free_exit;
     }
 
-    backing_hd = bdrv_new("", errp);
+    backing_hd = bdrv_new();
 
     if (bs->backing_format[0] != '\0') {
         back_drv = bdrv_find_format(bs->backing_format);
@@ -1313,7 +1311,6 @@ int bdrv_append_temp_snapshot(BlockDriverState *bs, int flags, Error **errp)
         error_setg_errno(errp, -total_size, "Could not get image size");
         goto out;
     }
-    total_size &= BDRV_SECTOR_MASK;
 
     /* Create the temporary image */
     ret = get_tmp_filename(tmp_filename, PATH_MAX + 1);
@@ -1343,7 +1340,7 @@ int bdrv_append_temp_snapshot(BlockDriverState *bs, int flags, Error **errp)
     qdict_put(snapshot_options, "file.filename",
               qstring_from_str(tmp_filename));
 
-    bs_snapshot = bdrv_new("", &error_abort);
+    bs_snapshot = bdrv_new();
 
     ret = bdrv_open(&bs_snapshot, NULL, NULL, snapshot_options,
                     flags, bdrv_qcow2, &local_err);
@@ -1414,7 +1411,7 @@ int bdrv_open(BlockDriverState **pbs, const char *filename,
     if (*pbs) {
         bs = *pbs;
     } else {
-        bs = bdrv_new("", &error_abort);
+        bs = bdrv_new();
     }
 
     /* NULL means an empty set of options */
@@ -1503,6 +1500,8 @@ int bdrv_open(BlockDriverState **pbs, const char *filename,
         }
     }
 
+    bdrv_refresh_filename(bs);
+
     /* For snapshot=on, create a temporary qcow2 overlay. bs points to the
      * temporary snapshot afterwards. */
     if (snapshot_flags) {
@@ -1521,7 +1520,7 @@ int bdrv_open(BlockDriverState **pbs, const char *filename,
         } else {
             error_setg(errp, "Block format '%s' used by device '%s' doesn't "
                        "support the option '%s'", drv->format_name,
-                       bs->device_name, entry->key);
+                       bdrv_get_device_name(bs), entry->key);
         }
 
         ret = -EINVAL;
@@ -1529,7 +1528,9 @@ int bdrv_open(BlockDriverState **pbs, const char *filename,
     }
 
     if (!bdrv_key_required(bs)) {
-        bdrv_dev_change_media_cb(bs, true);
+        if (bs->blk) {
+            blk_dev_change_media_cb(bs->blk, true);
+        }
     } else if (!runstate_check(RUN_STATE_PRELAUNCH)
                && !runstate_check(RUN_STATE_INMIGRATE)
                && !runstate_check(RUN_STATE_PAUSED)) { /* HACK */
@@ -1728,7 +1729,7 @@ int bdrv_reopen_prepare(BDRVReopenState *reopen_state, BlockReopenQueue *queue,
     if (!(reopen_state->bs->open_flags & BDRV_O_ALLOW_RDWR) &&
         reopen_state->flags & BDRV_O_RDWR) {
         error_set(errp, QERR_DEVICE_IS_READ_ONLY,
-                  reopen_state->bs->device_name);
+                  bdrv_get_device_name(reopen_state->bs));
         goto error;
     }
 
@@ -1755,7 +1756,7 @@ int bdrv_reopen_prepare(BDRVReopenState *reopen_state, BlockReopenQueue *queue,
         /* It is currently mandatory to have a bdrv_reopen_prepare()
          * handler for each supported drv. */
         error_set(errp, QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-                  drv->format_name, reopen_state->bs->device_name,
+                  drv->format_name, bdrv_get_device_name(reopen_state->bs),
                  "reopening of file");
         ret = -1;
         goto error;
@@ -1814,6 +1815,8 @@ void bdrv_reopen_abort(BDRVReopenState *reopen_state)
 
 void bdrv_close(BlockDriverState *bs)
 {
+    BdrvAioNotifier *ban, *ban_next;
+
     if (bs->job) {
         block_job_cancel_sync(bs->job);
     }
@@ -1843,6 +1846,8 @@ void bdrv_close(BlockDriverState *bs)
         bs->zero_beyond_eof = false;
         QDECREF(bs->options);
         bs->options = NULL;
+        QDECREF(bs->full_open_options);
+        bs->full_open_options = NULL;
 
         if (bs->file != NULL) {
             bdrv_unref(bs->file);
@@ -1850,12 +1855,19 @@ void bdrv_close(BlockDriverState *bs)
         }
     }
 
-    bdrv_dev_change_media_cb(bs, false);
+    if (bs->blk) {
+        blk_dev_change_media_cb(bs->blk, false);
+    }
 
     /*throttling disk I/O limits*/
     if (bs->io_limits_enabled) {
         bdrv_io_limits_disable(bs);
     }
+
+    QLIST_FOREACH_SAFE(ban, &bs->aio_notifiers, list, ban_next) {
+        g_free(ban);
+    }
+    QLIST_INIT(&bs->aio_notifiers);
 }
 
 void bdrv_close_all(void)
@@ -1892,6 +1904,34 @@ static bool bdrv_requests_pending(BlockDriverState *bs)
     return false;
 }
 
+static bool bdrv_drain_one(BlockDriverState *bs)
+{
+    bool bs_busy;
+
+    bdrv_flush_io_queue(bs);
+    bdrv_start_throttled_reqs(bs);
+    bs_busy = bdrv_requests_pending(bs);
+    bs_busy |= aio_poll(bdrv_get_aio_context(bs), bs_busy);
+    return bs_busy;
+}
+
+/*
+ * Wait for pending requests to complete on a single BlockDriverState subtree
+ *
+ * See the warning in bdrv_drain_all().  This function can only be called if
+ * you are sure nothing can generate I/O because you have op blockers
+ * installed.
+ *
+ * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
+ * AioContext.
+ */
+void bdrv_drain(BlockDriverState *bs)
+{
+    while (bdrv_drain_one(bs)) {
+        /* Keep iterating */
+    }
+}
+
 /*
  * Wait for pending requests to complete across all BlockDriverStates
  *
@@ -1915,16 +1955,10 @@ void bdrv_drain_all(void)
 
         QTAILQ_FOREACH(bs, &bdrv_states, device_list) {
             AioContext *aio_context = bdrv_get_aio_context(bs);
-            bool bs_busy;
 
             aio_context_acquire(aio_context);
-            bdrv_flush_io_queue(bs);
-            bdrv_start_throttled_reqs(bs);
-            bs_busy = bdrv_requests_pending(bs);
-            bs_busy |= aio_poll(aio_context, bs_busy);
+            busy |= bdrv_drain_one(bs);
             aio_context_release(aio_context);
-
-            busy |= bs_busy;
         }
     }
 }
@@ -1934,10 +1968,17 @@ void bdrv_drain_all(void)
    Also, NULL terminate the device_name to prevent double remove */
 void bdrv_make_anon(BlockDriverState *bs)
 {
-    if (bs->device_name[0] != '\0') {
+    /*
+     * Take care to remove bs from bdrv_states only when it's actually
+     * in it.  Note that bs->device_list.tqe_prev is initially null,
+     * and gets set to non-null by QTAILQ_INSERT_TAIL().  Establish
+     * the useful invariant "bs in bdrv_states iff bs->tqe_prev" by
+     * resetting it to null on remove.
+     */
+    if (bs->device_list.tqe_prev) {
         QTAILQ_REMOVE(&bdrv_states, bs, device_list);
+        bs->device_list.tqe_prev = NULL;
     }
-    bs->device_name[0] = '\0';
     if (bs->node_name[0] != '\0') {
         QTAILQ_REMOVE(&graph_bdrv_states, bs, node_list);
     }
@@ -1957,9 +1998,6 @@ static void bdrv_move_feature_fields(BlockDriverState *bs_dest,
     /* move some fields that need to stay attached to the device */
 
     /* dev info */
-    bs_dest->dev_ops            = bs_src->dev_ops;
-    bs_dest->dev_opaque         = bs_src->dev_opaque;
-    bs_dest->dev                = bs_src->dev;
     bs_dest->guest_block_size   = bs_src->guest_block_size;
     bs_dest->copy_on_read       = bs_src->copy_on_read;
 
@@ -1991,9 +2029,9 @@ static void bdrv_move_feature_fields(BlockDriverState *bs_dest,
     bs_dest->job                = bs_src->job;
 
     /* keep the same entry in bdrv_states */
-    pstrcpy(bs_dest->device_name, sizeof(bs_dest->device_name),
-            bs_src->device_name);
     bs_dest->device_list = bs_src->device_list;
+    bs_dest->blk = bs_src->blk;
+
     memcpy(bs_dest->op_blockers, bs_src->op_blockers,
            sizeof(bs_dest->op_blockers));
 }
@@ -2006,7 +2044,7 @@ static void bdrv_move_feature_fields(BlockDriverState *bs_dest,
  * This will modify the BlockDriverState fields, and swap contents
  * between bs_new and bs_old. Both bs_new and bs_old are modified.
  *
- * bs_new is required to be anonymous.
+ * bs_new must not be attached to a BlockBackend.
  *
  * This function does not create any image files.
  */
@@ -2025,11 +2063,10 @@ void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old)
         QTAILQ_REMOVE(&graph_bdrv_states, bs_old, node_list);
     }
 
-    /* bs_new must be anonymous and shouldn't have anything fancy enabled */
-    assert(bs_new->device_name[0] == '\0');
+    /* bs_new must be unattached and shouldn't have anything fancy enabled */
+    assert(!bs_new->blk);
     assert(QLIST_EMPTY(&bs_new->dirty_bitmaps));
     assert(bs_new->job == NULL);
-    assert(bs_new->dev == NULL);
     assert(bs_new->io_limits_enabled == false);
     assert(!throttle_have_timer(&bs_new->throttle_state));
 
@@ -2042,11 +2079,10 @@ void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old)
     bdrv_move_feature_fields(bs_old, bs_new);
     bdrv_move_feature_fields(bs_new, &tmp);
 
-    /* bs_new shouldn't be in bdrv_states even after the swap!  */
-    assert(bs_new->device_name[0] == '\0');
+    /* bs_new must remain unattached */
+    assert(!bs_new->blk);
 
     /* Check a few fields that should remain attached to the device */
-    assert(bs_new->dev == NULL);
     assert(bs_new->job == NULL);
     assert(bs_new->io_limits_enabled == false);
     assert(!throttle_have_timer(&bs_new->throttle_state));
@@ -2070,7 +2106,7 @@ void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old)
  * This will modify the BlockDriverState fields, and swap contents
  * between bs_new and bs_top. Both bs_new and bs_top are modified.
  *
- * bs_new is required to be anonymous.
+ * bs_new must not be attached to a BlockBackend.
  *
  * This function does not create any image files.
  */
@@ -2085,7 +2121,6 @@ void bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top)
 
 static void bdrv_delete(BlockDriverState *bs)
 {
-    assert(!bs->dev);
     assert(!bs->job);
     assert(bdrv_op_blocker_is_empty(bs));
     assert(!bs->refcnt);
@@ -2099,101 +2134,6 @@ static void bdrv_delete(BlockDriverState *bs)
     g_free(bs);
 }
 
-int bdrv_attach_dev(BlockDriverState *bs, void *dev)
-/* TODO change to DeviceState *dev when all users are qdevified */
-{
-    if (bs->dev) {
-        return -EBUSY;
-    }
-    bs->dev = dev;
-    bdrv_iostatus_reset(bs);
-    return 0;
-}
-
-/* TODO qdevified devices don't use this, remove when devices are qdevified */
-void bdrv_attach_dev_nofail(BlockDriverState *bs, void *dev)
-{
-    if (bdrv_attach_dev(bs, dev) < 0) {
-        abort();
-    }
-}
-
-void bdrv_detach_dev(BlockDriverState *bs, void *dev)
-/* TODO change to DeviceState *dev when all users are qdevified */
-{
-    assert(bs->dev == dev);
-    bs->dev = NULL;
-    bs->dev_ops = NULL;
-    bs->dev_opaque = NULL;
-    bs->guest_block_size = 512;
-}
-
-/* TODO change to return DeviceState * when all users are qdevified */
-void *bdrv_get_attached_dev(BlockDriverState *bs)
-{
-    return bs->dev;
-}
-
-void bdrv_set_dev_ops(BlockDriverState *bs, const BlockDevOps *ops,
-                      void *opaque)
-{
-    bs->dev_ops = ops;
-    bs->dev_opaque = opaque;
-}
-
-static void bdrv_dev_change_media_cb(BlockDriverState *bs, bool load)
-{
-    if (bs->dev_ops && bs->dev_ops->change_media_cb) {
-        bool tray_was_closed = !bdrv_dev_is_tray_open(bs);
-        bs->dev_ops->change_media_cb(bs->dev_opaque, load);
-        if (tray_was_closed) {
-            /* tray open */
-            qapi_event_send_device_tray_moved(bdrv_get_device_name(bs),
-                                              true, &error_abort);
-        }
-        if (load) {
-            /* tray close */
-            qapi_event_send_device_tray_moved(bdrv_get_device_name(bs),
-                                              false, &error_abort);
-        }
-    }
-}
-
-bool bdrv_dev_has_removable_media(BlockDriverState *bs)
-{
-    return !bs->dev || (bs->dev_ops && bs->dev_ops->change_media_cb);
-}
-
-void bdrv_dev_eject_request(BlockDriverState *bs, bool force)
-{
-    if (bs->dev_ops && bs->dev_ops->eject_request_cb) {
-        bs->dev_ops->eject_request_cb(bs->dev_opaque, force);
-    }
-}
-
-bool bdrv_dev_is_tray_open(BlockDriverState *bs)
-{
-    if (bs->dev_ops && bs->dev_ops->is_tray_open) {
-        return bs->dev_ops->is_tray_open(bs->dev_opaque);
-    }
-    return false;
-}
-
-static void bdrv_dev_resize_cb(BlockDriverState *bs)
-{
-    if (bs->dev_ops && bs->dev_ops->resize_cb) {
-        bs->dev_ops->resize_cb(bs->dev_opaque);
-    }
-}
-
-bool bdrv_dev_is_medium_locked(BlockDriverState *bs)
-{
-    if (bs->dev_ops && bs->dev_ops->is_medium_locked) {
-        return bs->dev_ops->is_medium_locked(bs->dev_opaque);
-    }
-    return false;
-}
-
 /*
  * Run consistency checks on an image
  *
@@ -2203,6 +2143,9 @@ bool bdrv_dev_is_medium_locked(BlockDriverState *bs)
  */
 int bdrv_check(BlockDriverState *bs, BdrvCheckResult *res, BdrvCheckMode fix)
 {
+    if (bs->drv == NULL) {
+        return -ENOMEDIUM;
+    }
     if (bs->drv->bdrv_check == NULL) {
         return -ENOTSUP;
     }
@@ -2225,7 +2168,7 @@ int bdrv_commit(BlockDriverState *bs)
 
     if (!drv)
         return -ENOMEDIUM;
-    
+
     if (!bs->backing_hd) {
         return -ENOTSUP;
     }
@@ -2269,7 +2212,14 @@ int bdrv_commit(BlockDriverState *bs)
     }
 
     total_sectors = length >> BDRV_SECTOR_BITS;
-    buf = g_malloc(COMMIT_BUF_SECTORS * BDRV_SECTOR_SIZE);
+
+    /* qemu_try_blockalign() for bs will choose an alignment that works for
+     * bs->backing_hd as well, so no need to compare the alignment manually. */
+    buf = qemu_try_blockalign(bs, COMMIT_BUF_SECTORS * BDRV_SECTOR_SIZE);
+    if (buf == NULL) {
+        ret = -ENOMEM;
+        goto ro_cleanup;
+    }
 
     for (sector = 0; sector < total_sectors; sector += n) {
         ret = bdrv_is_allocated(bs, sector, COMMIT_BUF_SECTORS, &n);
@@ -2307,7 +2257,7 @@ int bdrv_commit(BlockDriverState *bs)
 
     ret = 0;
 ro_cleanup:
-    g_free(buf);
+    qemu_vfree(buf);
 
     if (ro) {
         /* ignoring error return here */
@@ -2612,7 +2562,7 @@ int bdrv_drop_intermediate(BlockDriverState *active, BlockDriverState *top,
      * into our deletion queue, until we hit the 'base'
      */
     while (intermediate) {
-        intermediate_state = g_malloc0(sizeof(BlkIntermediateStates));
+        intermediate_state = g_new0(BlkIntermediateStates, 1);
         intermediate_state->bs = intermediate;
         QSIMPLEQ_INSERT_TAIL(&states_to_delete, intermediate_state, entry);
 
@@ -2827,23 +2777,21 @@ int bdrv_write_zeroes(BlockDriverState *bs, int64_t sector_num,
  */
 int bdrv_make_zero(BlockDriverState *bs, BdrvRequestFlags flags)
 {
-    int64_t target_size;
-    int64_t ret, nb_sectors, sector_num = 0;
+    int64_t target_sectors, ret, nb_sectors, sector_num = 0;
     int n;
 
-    target_size = bdrv_getlength(bs);
-    if (target_size < 0) {
-        return target_size;
+    target_sectors = bdrv_nb_sectors(bs);
+    if (target_sectors < 0) {
+        return target_sectors;
     }
-    target_size /= BDRV_SECTOR_SIZE;
 
     for (;;) {
-        nb_sectors = target_size - sector_num;
+        nb_sectors = target_sectors - sector_num;
         if (nb_sectors <= 0) {
             return 0;
         }
-        if (nb_sectors > INT_MAX) {
-            nb_sectors = INT_MAX;
+        if (nb_sectors > INT_MAX / BDRV_SECTOR_SIZE) {
+            nb_sectors = INT_MAX / BDRV_SECTOR_SIZE;
         }
         ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n);
         if (ret < 0) {
@@ -2968,7 +2916,12 @@ static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs,
                                    cluster_sector_num, cluster_nb_sectors);
 
     iov.iov_len = cluster_nb_sectors * BDRV_SECTOR_SIZE;
-    iov.iov_base = bounce_buffer = qemu_blockalign(bs, iov.iov_len);
+    iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len);
+    if (bounce_buffer == NULL) {
+        ret = -ENOMEM;
+        goto err;
+    }
+
     qemu_iovec_init_external(&bounce_qiov, &iov, 1);
 
     ret = drv->bdrv_co_readv(bs, cluster_sector_num, cluster_nb_sectors,
@@ -3056,15 +3009,14 @@ static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs,
         ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
     } else {
         /* Read zeros after EOF of growable BDSes */
-        int64_t len, total_sectors, max_nb_sectors;
+        int64_t total_sectors, max_nb_sectors;
 
-        len = bdrv_getlength(bs);
-        if (len < 0) {
-            ret = len;
+        total_sectors = bdrv_nb_sectors(bs);
+        if (total_sectors < 0) {
+            ret = total_sectors;
             goto out;
         }
 
-        total_sectors = DIV_ROUND_UP(len, BDRV_SECTOR_SIZE);
         max_nb_sectors = ROUND_UP(MAX(0, total_sectors - sector_num),
                                   align >> BDRV_SECTOR_BITS);
         if (max_nb_sectors > 0) {
@@ -3253,7 +3205,11 @@ static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
             /* Fall back to bounce buffer if write zeroes is unsupported */
             iov.iov_len = num * BDRV_SECTOR_SIZE;
             if (iov.iov_base == NULL) {
-                iov.iov_base = qemu_blockalign(bs, num * BDRV_SECTOR_SIZE);
+                iov.iov_base = qemu_try_blockalign(bs, num * BDRV_SECTOR_SIZE);
+                if (iov.iov_base == NULL) {
+                    ret = -ENOMEM;
+                    goto fail;
+                }
                 memset(iov.iov_base, 0, num * BDRV_SECTOR_SIZE);
             }
             qemu_iovec_init_external(&qiov, &iov, 1);
@@ -3273,6 +3229,7 @@ static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
         nb_sectors -= num;
     }
 
+fail:
     qemu_vfree(iov.iov_base);
     return ret;
 }
@@ -3328,9 +3285,8 @@ static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs,
 
     bdrv_set_dirty(bs, sector_num, nb_sectors);
 
-    if (bs->wr_highest_sector < sector_num + nb_sectors - 1) {
-        bs->wr_highest_sector = sector_num + nb_sectors - 1;
-    }
+    block_acct_highest_sector(&bs->stats, sector_num, nb_sectors);
+
     if (bs->growable && ret >= 0) {
         bs->total_sectors = MAX(bs->total_sectors, sector_num + nb_sectors);
     }
@@ -3511,7 +3467,9 @@ int bdrv_truncate(BlockDriverState *bs, int64_t offset)
     ret = drv->bdrv_truncate(bs, offset);
     if (ret == 0) {
         ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
-        bdrv_dev_resize_cb(bs);
+        if (bs->blk) {
+            blk_dev_resize_cb(bs->blk);
+        }
     }
     return ret;
 }
@@ -3536,11 +3494,12 @@ int64_t bdrv_get_allocated_file_size(BlockDriverState *bs)
 }
 
 /**
- * Length of a file in bytes. Return < 0 if error or unknown.
+ * Return number of sectors on success, -errno on error.
  */
-int64_t bdrv_getlength(BlockDriverState *bs)
+int64_t bdrv_nb_sectors(BlockDriverState *bs)
 {
     BlockDriver *drv = bs->drv;
+
     if (!drv)
         return -ENOMEDIUM;
 
@@ -3550,19 +3509,26 @@ int64_t bdrv_getlength(BlockDriverState *bs)
             return ret;
         }
     }
-    return bs->total_sectors * BDRV_SECTOR_SIZE;
+    return bs->total_sectors;
+}
+
+/**
+ * Return length in bytes on success, -errno on error.
+ * The length is always a multiple of BDRV_SECTOR_SIZE.
+ */
+int64_t bdrv_getlength(BlockDriverState *bs)
+{
+    int64_t ret = bdrv_nb_sectors(bs);
+
+    return ret < 0 ? ret : ret * BDRV_SECTOR_SIZE;
 }
 
 /* return 0 as number of sectors if no device present or error */
 void bdrv_get_geometry(BlockDriverState *bs, uint64_t *nb_sectors_ptr)
 {
-    int64_t length;
-    length = bdrv_getlength(bs);
-    if (length < 0)
-        length = 0;
-    else
-        length = length >> BDRV_SECTOR_BITS;
-    *nb_sectors_ptr = length;
+    int64_t nb_sectors = bdrv_nb_sectors(bs);
+
+    *nb_sectors_ptr = nb_sectors < 0 ? 0 : nb_sectors;
 }
 
 void bdrv_set_on_error(BlockDriverState *bs, BlockdevOnError on_read_error,
@@ -3596,6 +3562,19 @@ BlockErrorAction bdrv_get_error_action(BlockDriverState *bs, bool is_read, int e
     }
 }
 
+static void send_qmp_error_event(BlockDriverState *bs,
+                                 BlockErrorAction action,
+                                 bool is_read, int error)
+{
+    IoOperationType optype;
+
+    optype = is_read ? IO_OPERATION_TYPE_READ : IO_OPERATION_TYPE_WRITE;
+    qapi_event_send_block_io_error(bdrv_get_device_name(bs), optype, action,
+                                   bdrv_iostatus_is_enabled(bs),
+                                   error == ENOSPC, strerror(error),
+                                   &error_abort);
+}
+
 /* This is done by device models because, while the block layer knows
  * about the error, it does not know whether an operation comes from
  * the device or the block layer (from a job, for example).
@@ -3621,16 +3600,10 @@ void bdrv_error_action(BlockDriverState *bs, BlockErrorAction action,
          * also ensures that the STOP/RESUME pair of events is emitted.
          */
         qemu_system_vmstop_request_prepare();
-        qapi_event_send_block_io_error(bdrv_get_device_name(bs),
-                                       is_read ? IO_OPERATION_TYPE_READ :
-                                       IO_OPERATION_TYPE_WRITE,
-                                       action, &error_abort);
+        send_qmp_error_event(bs, action, is_read, error);
         qemu_system_vmstop_request(RUN_STATE_IO_ERROR);
     } else {
-        qapi_event_send_block_io_error(bdrv_get_device_name(bs),
-                                       is_read ? IO_OPERATION_TYPE_READ :
-                                       IO_OPERATION_TYPE_WRITE,
-                                       action, &error_abort);
+        send_qmp_error_event(bs, action, is_read, error);
     }
 }
 
@@ -3697,8 +3670,10 @@ int bdrv_set_key(BlockDriverState *bs, const char *key)
         bs->valid_key = 0;
     } else if (!bs->valid_key) {
         bs->valid_key = 1;
-        /* call the change callback now, we skipped it on open */
-        bdrv_dev_change_media_cb(bs, true);
+        if (bs->blk) {
+            /* call the change callback now, we skipped it on open */
+            blk_dev_change_media_cb(bs->blk, true);
+        }
     }
     return ret;
 }
@@ -3708,11 +3683,17 @@ const char *bdrv_get_format_name(BlockDriverState *bs)
     return bs->drv ? bs->drv->format_name : NULL;
 }
 
+static int qsort_strcmp(const void *a, const void *b)
+{
+    return strcmp(a, b);
+}
+
 void bdrv_iterate_format(void (*it)(void *opaque, const char *name),
                          void *opaque)
 {
     BlockDriver *drv;
     int count = 0;
+    int i;
     const char **formats = NULL;
 
     QLIST_FOREACH(drv, &bdrv_drivers, list) {
@@ -3724,26 +3705,28 @@ void bdrv_iterate_format(void (*it)(void *opaque, const char *name),
             }
 
             if (!found) {
-                formats = g_realloc(formats, (count + 1) * sizeof(char *));
+                formats = g_renew(const char *, formats, count + 1);
                 formats[count++] = drv->format_name;
-                it(opaque, drv->format_name);
             }
         }
     }
+
+    qsort(formats, count, sizeof(formats[0]), qsort_strcmp);
+
+    for (i = 0; i < count; i++) {
+        it(opaque, formats[i]);
+    }
+
     g_free(formats);
 }
 
 /* This function is to find block backend bs */
+/* TODO convert callers to blk_by_name(), then remove */
 BlockDriverState *bdrv_find(const char *name)
 {
-    BlockDriverState *bs;
+    BlockBackend *blk = blk_by_name(name);
 
-    QTAILQ_FOREACH(bs, &bdrv_states, device_list) {
-        if (!strcmp(name, bs->device_name)) {
-            return bs;
-        }
-    }
-    return NULL;
+    return blk ? blk_bs(blk) : NULL;
 }
 
 /* This function is to find a node in the bs graph */
@@ -3782,13 +3765,14 @@ BlockDriverState *bdrv_lookup_bs(const char *device,
                                  const char *node_name,
                                  Error **errp)
 {
-    BlockDriverState *bs = NULL;
+    BlockBackend *blk;
+    BlockDriverState *bs;
 
     if (device) {
-        bs = bdrv_find(device);
+        blk = blk_by_name(device);
 
-        if (bs) {
-            return bs;
+        if (blk) {
+            return blk_bs(blk);
         }
     }
 
@@ -3825,18 +3809,10 @@ BlockDriverState *bdrv_next(BlockDriverState *bs)
     return QTAILQ_NEXT(bs, device_list);
 }
 
-void bdrv_iterate(void (*it)(void *opaque, BlockDriverState *bs), void *opaque)
-{
-    BlockDriverState *bs;
-
-    QTAILQ_FOREACH(bs, &bdrv_states, device_list) {
-        it(opaque, bs);
-    }
-}
-
-const char *bdrv_get_device_name(BlockDriverState *bs)
+/* TODO check what callers really want: bs->node_name or blk_name() */
+const char *bdrv_get_device_name(const BlockDriverState *bs)
 {
-    return bs->device_name;
+    return bs->blk ? blk_name(bs->blk) : "";
 }
 
 int bdrv_get_flags(BlockDriverState *bs)
@@ -3927,9 +3903,9 @@ typedef struct BdrvCoGetBlockStatusData {
 } BdrvCoGetBlockStatusData;
 
 /*
- * Returns true iff the specified sector is present in the disk image. Drivers
- * not implementing the functionality are assumed to not support backing files,
- * hence all their sectors are reported as allocated.
+ * Returns the allocation status of the specified sectors.
+ * Drivers not implementing the functionality are assumed to not support
+ * backing files, hence all their sectors are reported as allocated.
  *
  * If 'sector_num' is beyond the end of the disk image the return value is 0
  * and 'pnum' is set to 0.
@@ -3945,21 +3921,21 @@ static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
                                                      int64_t sector_num,
                                                      int nb_sectors, int *pnum)
 {
-    int64_t length;
+    int64_t total_sectors;
     int64_t n;
     int64_t ret, ret2;
 
-    length = bdrv_getlength(bs);
-    if (length < 0) {
-        return length;
+    total_sectors = bdrv_nb_sectors(bs);
+    if (total_sectors < 0) {
+        return total_sectors;
     }
 
-    if (sector_num >= (length >> BDRV_SECTOR_BITS)) {
+    if (sector_num >= total_sectors) {
         *pnum = 0;
         return 0;
     }
 
-    n = bs->total_sectors - sector_num;
+    n = total_sectors - sector_num;
     if (n < nb_sectors) {
         nb_sectors = n;
     }
@@ -3994,8 +3970,8 @@ static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
             ret |= BDRV_BLOCK_ZERO;
         } else if (bs->backing_hd) {
             BlockDriverState *bs2 = bs->backing_hd;
-            int64_t length2 = bdrv_getlength(bs2);
-            if (length2 >= 0 && sector_num >= (length2 >> BDRV_SECTOR_BITS)) {
+            int64_t nb_sectors2 = bdrv_nb_sectors(bs2);
+            if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) {
                 ret |= BDRV_BLOCK_ZERO;
             }
         }
@@ -4004,13 +3980,24 @@ static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
     if (bs->file &&
         (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
         (ret & BDRV_BLOCK_OFFSET_VALID)) {
+        int file_pnum;
+
         ret2 = bdrv_co_get_block_status(bs->file, ret >> BDRV_SECTOR_BITS,
-                                        *pnum, pnum);
+                                        *pnum, &file_pnum);
         if (ret2 >= 0) {
             /* Ignore errors.  This is just providing extra information, it
              * is useful but not necessary.
              */
-            ret |= (ret2 & BDRV_BLOCK_ZERO);
+            if (!file_pnum) {
+                /* !file_pnum indicates an offset at or beyond the EOF; it is
+                 * perfectly valid for the format block driver to point to such
+                 * offsets, so catch it and mark everything as zero */
+                ret |= BDRV_BLOCK_ZERO;
+            } else {
+                /* Limit request to the range reported by the protocol driver */
+                *pnum = file_pnum;
+                ret |= (ret2 & BDRV_BLOCK_ZERO);
+            }
         }
     }
 
@@ -4363,9 +4350,9 @@ int bdrv_get_backing_file_depth(BlockDriverState *bs)
 /**************************************************************/
 /* async I/Os */
 
-BlockDriverAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
-                                 QEMUIOVector *qiov, int nb_sectors,
-                                 BlockDriverCompletionFunc *cb, void *opaque)
+BlockAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
+                           QEMUIOVector *qiov, int nb_sectors,
+                           BlockCompletionFunc *cb, void *opaque)
 {
     trace_bdrv_aio_readv(bs, sector_num, nb_sectors, opaque);
 
@@ -4373,9 +4360,9 @@ BlockDriverAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
                                  cb, opaque, false);
 }
 
-BlockDriverAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num,
-                                  QEMUIOVector *qiov, int nb_sectors,
-                                  BlockDriverCompletionFunc *cb, void *opaque)
+BlockAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num,
+                            QEMUIOVector *qiov, int nb_sectors,
+                            BlockCompletionFunc *cb, void *opaque)
 {
     trace_bdrv_aio_writev(bs, sector_num, nb_sectors, opaque);
 
@@ -4383,9 +4370,9 @@ BlockDriverAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num,
                                  cb, opaque, true);
 }
 
-BlockDriverAIOCB *bdrv_aio_write_zeroes(BlockDriverState *bs,
+BlockAIOCB *bdrv_aio_write_zeroes(BlockDriverState *bs,
         int64_t sector_num, int nb_sectors, BdrvRequestFlags flags,
-        BlockDriverCompletionFunc *cb, void *opaque)
+        BlockCompletionFunc *cb, void *opaque)
 {
     trace_bdrv_aio_write_zeroes(bs, sector_num, nb_sectors, flags, opaque);
 
@@ -4400,7 +4387,7 @@ typedef struct MultiwriteCB {
     int num_requests;
     int num_callbacks;
     struct {
-        BlockDriverCompletionFunc *cb;
+        BlockCompletionFunc *cb;
         void *opaque;
         QEMUIOVector *free_qiov;
     } callbacks[];
@@ -4481,6 +4468,11 @@ static int multiwrite_merge(BlockDriverState *bs, BlockRequest *reqs,
             merge = 0;
         }
 
+        if (bs->bl.max_transfer_length && reqs[outidx].nb_sectors +
+            reqs[i].nb_sectors > bs->bl.max_transfer_length) {
+            merge = 0;
+        }
+
         if (merge) {
             size_t size;
             QEMUIOVector *qiov = g_malloc0(sizeof(*qiov));
@@ -4498,6 +4490,12 @@ static int multiwrite_merge(BlockDriverState *bs, BlockRequest *reqs,
             // Add the second request
             qemu_iovec_concat(qiov, reqs[i].qiov, 0, reqs[i].qiov->size);
 
+            // Add tail of first request, if necessary
+            if (qiov->size < reqs[outidx].qiov->size) {
+                qemu_iovec_concat(qiov, reqs[outidx].qiov, qiov->size,
+                                  reqs[outidx].qiov->size - qiov->size);
+            }
+
             reqs[outidx].nb_sectors = qiov->size >> 9;
             reqs[outidx].qiov = qiov;
 
@@ -4571,69 +4569,83 @@ int bdrv_aio_multiwrite(BlockDriverState *bs, BlockRequest *reqs, int num_reqs)
     return 0;
 }
 
-void bdrv_aio_cancel(BlockDriverAIOCB *acb)
+void bdrv_aio_cancel(BlockAIOCB *acb)
+{
+    qemu_aio_ref(acb);
+    bdrv_aio_cancel_async(acb);
+    while (acb->refcnt > 1) {
+        if (acb->aiocb_info->get_aio_context) {
+            aio_poll(acb->aiocb_info->get_aio_context(acb), true);
+        } else if (acb->bs) {
+            aio_poll(bdrv_get_aio_context(acb->bs), true);
+        } else {
+            abort();
+        }
+    }
+    qemu_aio_unref(acb);
+}
+
+/* Async version of aio cancel. The caller is not blocked if the acb implements
+ * cancel_async, otherwise we do nothing and let the request normally complete.
+ * In either case the completion callback must be called. */
+void bdrv_aio_cancel_async(BlockAIOCB *acb)
 {
-    acb->aiocb_info->cancel(acb);
+    if (acb->aiocb_info->cancel_async) {
+        acb->aiocb_info->cancel_async(acb);
+    }
 }
 
 /**************************************************************/
 /* async block device emulation */
 
-typedef struct BlockDriverAIOCBSync {
-    BlockDriverAIOCB common;
+typedef struct BlockAIOCBSync {
+    BlockAIOCB common;
     QEMUBH *bh;
     int ret;
     /* vector translation state */
     QEMUIOVector *qiov;
     uint8_t *bounce;
     int is_write;
-} BlockDriverAIOCBSync;
-
-static void bdrv_aio_cancel_em(BlockDriverAIOCB *blockacb)
-{
-    BlockDriverAIOCBSync *acb =
-        container_of(blockacb, BlockDriverAIOCBSync, common);
-    qemu_bh_delete(acb->bh);
-    acb->bh = NULL;
-    qemu_aio_release(acb);
-}
+} BlockAIOCBSync;
 
 static const AIOCBInfo bdrv_em_aiocb_info = {
-    .aiocb_size         = sizeof(BlockDriverAIOCBSync),
-    .cancel             = bdrv_aio_cancel_em,
+    .aiocb_size         = sizeof(BlockAIOCBSync),
 };
 
 static void bdrv_aio_bh_cb(void *opaque)
 {
-    BlockDriverAIOCBSync *acb = opaque;
+    BlockAIOCBSync *acb = opaque;
 
-    if (!acb->is_write)
+    if (!acb->is_write && acb->ret >= 0) {
         qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size);
+    }
     qemu_vfree(acb->bounce);
     acb->common.cb(acb->common.opaque, acb->ret);
     qemu_bh_delete(acb->bh);
     acb->bh = NULL;
-    qemu_aio_release(acb);
+    qemu_aio_unref(acb);
 }
 
-static BlockDriverAIOCB *bdrv_aio_rw_vector(BlockDriverState *bs,
-                                            int64_t sector_num,
-                                            QEMUIOVector *qiov,
-                                            int nb_sectors,
-                                            BlockDriverCompletionFunc *cb,
-                                            void *opaque,
-                                            int is_write)
+static BlockAIOCB *bdrv_aio_rw_vector(BlockDriverState *bs,
+                                      int64_t sector_num,
+                                      QEMUIOVector *qiov,
+                                      int nb_sectors,
+                                      BlockCompletionFunc *cb,
+                                      void *opaque,
+                                      int is_write)
 
 {
-    BlockDriverAIOCBSync *acb;
+    BlockAIOCBSync *acb;
 
     acb = qemu_aio_get(&bdrv_em_aiocb_info, bs, cb, opaque);
     acb->is_write = is_write;
     acb->qiov = qiov;
-    acb->bounce = qemu_blockalign(bs, qiov->size);
+    acb->bounce = qemu_try_blockalign(bs, qiov->size);
     acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_aio_bh_cb, acb);
 
-    if (is_write) {
+    if (acb->bounce == NULL) {
+        acb->ret = -ENOMEM;
+    } else if (is_write) {
         qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size);
         acb->ret = bs->drv->bdrv_write(bs, sector_num, acb->bounce, nb_sectors);
     } else {
@@ -4645,65 +4657,47 @@ static BlockDriverAIOCB *bdrv_aio_rw_vector(BlockDriverState *bs,
     return &acb->common;
 }
 
-static BlockDriverAIOCB *bdrv_aio_readv_em(BlockDriverState *bs,
+static BlockAIOCB *bdrv_aio_readv_em(BlockDriverState *bs,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque)
+        BlockCompletionFunc *cb, void *opaque)
 {
     return bdrv_aio_rw_vector(bs, sector_num, qiov, nb_sectors, cb, opaque, 0);
 }
 
-static BlockDriverAIOCB *bdrv_aio_writev_em(BlockDriverState *bs,
+static BlockAIOCB *bdrv_aio_writev_em(BlockDriverState *bs,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque)
+        BlockCompletionFunc *cb, void *opaque)
 {
     return bdrv_aio_rw_vector(bs, sector_num, qiov, nb_sectors, cb, opaque, 1);
 }
 
 
-typedef struct BlockDriverAIOCBCoroutine {
-    BlockDriverAIOCB common;
+typedef struct BlockAIOCBCoroutine {
+    BlockAIOCB common;
     BlockRequest req;
     bool is_write;
     bool *done;
     QEMUBH* bh;
-} BlockDriverAIOCBCoroutine;
-
-static void bdrv_aio_co_cancel_em(BlockDriverAIOCB *blockacb)
-{
-    AioContext *aio_context = bdrv_get_aio_context(blockacb->bs);
-    BlockDriverAIOCBCoroutine *acb =
-        container_of(blockacb, BlockDriverAIOCBCoroutine, common);
-    bool done = false;
-
-    acb->done = &done;
-    while (!done) {
-        aio_poll(aio_context, true);
-    }
-}
+} BlockAIOCBCoroutine;
 
 static const AIOCBInfo bdrv_em_co_aiocb_info = {
-    .aiocb_size         = sizeof(BlockDriverAIOCBCoroutine),
-    .cancel             = bdrv_aio_co_cancel_em,
+    .aiocb_size         = sizeof(BlockAIOCBCoroutine),
 };
 
 static void bdrv_co_em_bh(void *opaque)
 {
-    BlockDriverAIOCBCoroutine *acb = opaque;
+    BlockAIOCBCoroutine *acb = opaque;
 
     acb->common.cb(acb->common.opaque, acb->req.error);
 
-    if (acb->done) {
-        *acb->done = true;
-    }
-
     qemu_bh_delete(acb->bh);
-    qemu_aio_release(acb);
+    qemu_aio_unref(acb);
 }
 
 /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */
 static void coroutine_fn bdrv_co_do_rw(void *opaque)
 {
-    BlockDriverAIOCBCoroutine *acb = opaque;
+    BlockAIOCBCoroutine *acb = opaque;
     BlockDriverState *bs = acb->common.bs;
 
     if (!acb->is_write) {
@@ -4718,17 +4712,17 @@ static void coroutine_fn bdrv_co_do_rw(void *opaque)
     qemu_bh_schedule(acb->bh);
 }
 
-static BlockDriverAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
-                                               int64_t sector_num,
-                                               QEMUIOVector *qiov,
-                                               int nb_sectors,
-                                               BdrvRequestFlags flags,
-                                               BlockDriverCompletionFunc *cb,
-                                               void *opaque,
-                                               bool is_write)
+static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
+                                         int64_t sector_num,
+                                         QEMUIOVector *qiov,
+                                         int nb_sectors,
+                                         BdrvRequestFlags flags,
+                                         BlockCompletionFunc *cb,
+                                         void *opaque,
+                                         bool is_write)
 {
     Coroutine *co;
-    BlockDriverAIOCBCoroutine *acb;
+    BlockAIOCBCoroutine *acb;
 
     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
     acb->req.sector = sector_num;
@@ -4736,7 +4730,6 @@ static BlockDriverAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
     acb->req.qiov = qiov;
     acb->req.flags = flags;
     acb->is_write = is_write;
-    acb->done = NULL;
 
     co = qemu_coroutine_create(bdrv_co_do_rw);
     qemu_coroutine_enter(co, acb);
@@ -4746,7 +4739,7 @@ static BlockDriverAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
 
 static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque)
 {
-    BlockDriverAIOCBCoroutine *acb = opaque;
+    BlockAIOCBCoroutine *acb = opaque;
     BlockDriverState *bs = acb->common.bs;
 
     acb->req.error = bdrv_co_flush(bs);
@@ -4754,16 +4747,15 @@ static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque)
     qemu_bh_schedule(acb->bh);
 }
 
-BlockDriverAIOCB *bdrv_aio_flush(BlockDriverState *bs,
-        BlockDriverCompletionFunc *cb, void *opaque)
+BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs,
+        BlockCompletionFunc *cb, void *opaque)
 {
     trace_bdrv_aio_flush(bs, opaque);
 
     Coroutine *co;
-    BlockDriverAIOCBCoroutine *acb;
+    BlockAIOCBCoroutine *acb;
 
     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
-    acb->done = NULL;
 
     co = qemu_coroutine_create(bdrv_aio_flush_co_entry);
     qemu_coroutine_enter(co, acb);
@@ -4773,7 +4765,7 @@ BlockDriverAIOCB *bdrv_aio_flush(BlockDriverState *bs,
 
 static void coroutine_fn bdrv_aio_discard_co_entry(void *opaque)
 {
-    BlockDriverAIOCBCoroutine *acb = opaque;
+    BlockAIOCBCoroutine *acb = opaque;
     BlockDriverState *bs = acb->common.bs;
 
     acb->req.error = bdrv_co_discard(bs, acb->req.sector, acb->req.nb_sectors);
@@ -4781,19 +4773,18 @@ static void coroutine_fn bdrv_aio_discard_co_entry(void *opaque)
     qemu_bh_schedule(acb->bh);
 }
 
-BlockDriverAIOCB *bdrv_aio_discard(BlockDriverState *bs,
+BlockAIOCB *bdrv_aio_discard(BlockDriverState *bs,
         int64_t sector_num, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque)
+        BlockCompletionFunc *cb, void *opaque)
 {
     Coroutine *co;
-    BlockDriverAIOCBCoroutine *acb;
+    BlockAIOCBCoroutine *acb;
 
     trace_bdrv_aio_discard(bs, sector_num, nb_sectors, opaque);
 
     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
     acb->req.sector = sector_num;
     acb->req.nb_sectors = nb_sectors;
-    acb->done = NULL;
     co = qemu_coroutine_create(bdrv_aio_discard_co_entry);
     qemu_coroutine_enter(co, acb);
 
@@ -4812,22 +4803,32 @@ void bdrv_init_with_whitelist(void)
 }
 
 void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
-                   BlockDriverCompletionFunc *cb, void *opaque)
+                   BlockCompletionFunc *cb, void *opaque)
 {
-    BlockDriverAIOCB *acb;
+    BlockAIOCB *acb;
 
     acb = g_slice_alloc(aiocb_info->aiocb_size);
     acb->aiocb_info = aiocb_info;
     acb->bs = bs;
     acb->cb = cb;
     acb->opaque = opaque;
+    acb->refcnt = 1;
     return acb;
 }
 
-void qemu_aio_release(void *p)
+void qemu_aio_ref(void *p)
 {
-    BlockDriverAIOCB *acb = p;
-    g_slice_free1(acb->aiocb_info->aiocb_size, acb);
+    BlockAIOCB *acb = p;
+    acb->refcnt++;
+}
+
+void qemu_aio_unref(void *p)
+{
+    BlockAIOCB *acb = p;
+    assert(acb->refcnt > 0);
+    if (--acb->refcnt == 0) {
+        g_slice_free1(acb->aiocb_info->aiocb_size, acb);
+    }
 }
 
 /**************************************************************/
@@ -4853,7 +4854,7 @@ static int coroutine_fn bdrv_co_io_em(BlockDriverState *bs, int64_t sector_num,
     CoroutineIOCompletion co = {
         .coroutine = qemu_coroutine_self(),
     };
-    BlockDriverAIOCB *acb;
+    BlockAIOCB *acb;
 
     if (is_write) {
         acb = bs->drv->bdrv_aio_writev(bs, sector_num, iov, nb_sectors,
@@ -4919,7 +4920,7 @@ int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
     if (bs->drv->bdrv_co_flush_to_disk) {
         ret = bs->drv->bdrv_co_flush_to_disk(bs);
     } else if (bs->drv->bdrv_aio_flush) {
-        BlockDriverAIOCB *acb;
+        BlockAIOCB *acb;
         CoroutineIOCompletion co = {
             .coroutine = qemu_coroutine_self(),
         };
@@ -4965,6 +4966,11 @@ void bdrv_invalidate_cache(BlockDriverState *bs, Error **errp)
         return;
     }
 
+    if (!(bs->open_flags & BDRV_O_INCOMING)) {
+        return;
+    }
+    bs->open_flags &= ~BDRV_O_INCOMING;
+
     if (bs->drv->bdrv_invalidate_cache) {
         bs->drv->bdrv_invalidate_cache(bs, &local_err);
     } else if (bs->file) {
@@ -5000,19 +5006,6 @@ void bdrv_invalidate_cache_all(Error **errp)
     }
 }
 
-void bdrv_clear_incoming_migration_all(void)
-{
-    BlockDriverState *bs;
-
-    QTAILQ_FOREACH(bs, &bdrv_states, device_list) {
-        AioContext *aio_context = bdrv_get_aio_context(bs);
-
-        aio_context_acquire(aio_context);
-        bs->open_flags = bs->open_flags & ~(BDRV_O_INCOMING);
-        aio_context_release(aio_context);
-    }
-}
-
 int bdrv_flush(BlockDriverState *bs)
 {
     Coroutine *co;
@@ -5102,7 +5095,7 @@ int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
         if (bs->drv->bdrv_co_discard) {
             ret = bs->drv->bdrv_co_discard(bs, sector_num, num);
         } else {
-            BlockDriverAIOCB *acb;
+            BlockAIOCB *acb;
             CoroutineIOCompletion co = {
                 .coroutine = qemu_coroutine_self(),
             };
@@ -5189,13 +5182,15 @@ int bdrv_media_changed(BlockDriverState *bs)
 void bdrv_eject(BlockDriverState *bs, bool eject_flag)
 {
     BlockDriver *drv = bs->drv;
+    const char *device_name;
 
     if (drv && drv->bdrv_eject) {
         drv->bdrv_eject(bs, eject_flag);
     }
 
-    if (bs->device_name[0] != '\0') {
-        qapi_event_send_device_tray_moved(bdrv_get_device_name(bs),
+    device_name = bdrv_get_device_name(bs);
+    if (device_name[0] != '\0') {
+        qapi_event_send_device_tray_moved(device_name,
                                           eject_flag, &error_abort);
     }
 }
@@ -5226,9 +5221,9 @@ int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
     return -ENOTSUP;
 }
 
-BlockDriverAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
+BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
         unsigned long int req, void *buf,
-        BlockDriverCompletionFunc *cb, void *opaque)
+        BlockCompletionFunc *cb, void *opaque)
 {
     BlockDriver *drv = bs->drv;
 
@@ -5247,6 +5242,35 @@ void *qemu_blockalign(BlockDriverState *bs, size_t size)
     return qemu_memalign(bdrv_opt_mem_align(bs), size);
 }
 
+void *qemu_blockalign0(BlockDriverState *bs, size_t size)
+{
+    return memset(qemu_blockalign(bs, size), 0, size);
+}
+
+void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
+{
+    size_t align = bdrv_opt_mem_align(bs);
+
+    /* Ensure that NULL is never returned on success */
+    assert(align > 0);
+    if (size == 0) {
+        size = align;
+    }
+
+    return qemu_try_memalign(align, size);
+}
+
+void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
+{
+    void *mem = qemu_try_blockalign(bs, size);
+
+    if (mem) {
+        memset(mem, 0, size);
+    }
+
+    return mem;
+}
+
 /*
  * Check if all memory in this vector is sector aligned.
  */
@@ -5277,14 +5301,13 @@ BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState *bs, int granularity,
 
     granularity >>= BDRV_SECTOR_BITS;
     assert(granularity);
-    bitmap_size = bdrv_getlength(bs);
+    bitmap_size = bdrv_nb_sectors(bs);
     if (bitmap_size < 0) {
         error_setg_errno(errp, -bitmap_size, "could not get length of device");
         errno = -bitmap_size;
         return NULL;
     }
-    bitmap_size >>= BDRV_SECTOR_BITS;
-    bitmap = g_malloc0(sizeof(BdrvDirtyBitmap));
+    bitmap = g_new0(BdrvDirtyBitmap, 1);
     bitmap->bitmap = hbitmap_alloc(bitmap_size, ffs(granularity) - 1);
     QLIST_INSERT_HEAD(&bs->dirty_bitmaps, bitmap, list);
     return bitmap;
@@ -5310,8 +5333,8 @@ BlockDirtyInfoList *bdrv_query_dirty_bitmaps(BlockDriverState *bs)
     BlockDirtyInfoList **plist = &list;
 
     QLIST_FOREACH(bm, &bs->dirty_bitmaps, list) {
-        BlockDirtyInfo *info = g_malloc0(sizeof(BlockDirtyInfo));
-        BlockDirtyInfoList *entry = g_malloc0(sizeof(BlockDirtyInfoList));
+        BlockDirtyInfo *info = g_new0(BlockDirtyInfo, 1);
+        BlockDirtyInfoList *entry = g_new0(BlockDirtyInfoList, 1);
         info->count = bdrv_get_dirty_count(bs, bm);
         info->granularity =
             ((int64_t) BDRV_SECTOR_SIZE << hbitmap_granularity(bm->bitmap));
@@ -5371,6 +5394,9 @@ void bdrv_ref(BlockDriverState *bs)
  * deleted. */
 void bdrv_unref(BlockDriverState *bs)
 {
+    if (!bs) {
+        return;
+    }
     assert(bs->refcnt > 0);
     if (--bs->refcnt == 0) {
         bdrv_delete(bs);
@@ -5390,7 +5416,8 @@ bool bdrv_op_is_blocked(BlockDriverState *bs, BlockOpType op, Error **errp)
         blocker = QLIST_FIRST(&bs->op_blockers[op]);
         if (errp) {
             error_setg(errp, "Device '%s' is busy: %s",
-                       bs->device_name, error_get_pretty(blocker->reason));
+                       bdrv_get_device_name(bs),
+                       error_get_pretty(blocker->reason));
         }
         return true;
     }
@@ -5402,7 +5429,7 @@ void bdrv_op_block(BlockDriverState *bs, BlockOpType op, Error *reason)
     BdrvOpBlocker *blocker;
     assert((int) op >= 0 && op < BLOCK_OP_TYPE_MAX);
 
-    blocker = g_malloc0(sizeof(BdrvOpBlocker));
+    blocker = g_new0(BdrvOpBlocker, 1);
     blocker->reason = reason;
     QLIST_INSERT_HEAD(&bs->op_blockers[op], blocker, list);
 }
@@ -5487,27 +5514,6 @@ void bdrv_iostatus_set_err(BlockDriverState *bs, int error)
     }
 }
 
-void
-bdrv_acct_start(BlockDriverState *bs, BlockAcctCookie *cookie, int64_t bytes,
-        enum BlockAcctType type)
-{
-    assert(type < BDRV_MAX_IOTYPE);
-
-    cookie->bytes = bytes;
-    cookie->start_time_ns = get_clock();
-    cookie->type = type;
-}
-
-void
-bdrv_acct_done(BlockDriverState *bs, BlockAcctCookie *cookie)
-{
-    assert(cookie->type < BDRV_MAX_IOTYPE);
-
-    bs->nr_bytes[cookie->type] += cookie->bytes;
-    bs->nr_ops[cookie->type]++;
-    bs->total_time_ns[cookie->type] += get_clock() - cookie->start_time_ns;
-}
-
 void bdrv_img_create(const char *filename, const char *fmt,
                      const char *base_filename, const char *base_fmt,
                      char *options, uint64_t img_size, int flags,
@@ -5591,7 +5597,7 @@ void bdrv_img_create(const char *filename, const char *fmt,
     if (size == -1) {
         if (backing_file) {
             BlockDriverState *bs;
-            uint64_t size;
+            int64_t size;
             int back_flags;
 
             /* backing files always opened read-only */
@@ -5602,15 +5608,15 @@ void bdrv_img_create(const char *filename, const char *fmt,
             ret = bdrv_open(&bs, backing_file, NULL, NULL, back_flags,
                             backing_drv, &local_err);
             if (ret < 0) {
-                error_setg_errno(errp, -ret, "Could not open '%s': %s",
-                                 backing_file,
-                                 error_get_pretty(local_err));
-                error_free(local_err);
-                local_err = NULL;
                 goto out;
             }
-            bdrv_get_geometry(bs, &size);
-            size *= 512;
+            size = bdrv_getlength(bs);
+            if (size < 0) {
+                error_setg_errno(errp, -size, "Could not get size of '%s'",
+                                 backing_file);
+                bdrv_unref(bs);
+                goto out;
+            }
 
             qemu_opt_set_number(opts, BLOCK_OPT_SIZE, size);
 
@@ -5658,10 +5664,16 @@ AioContext *bdrv_get_aio_context(BlockDriverState *bs)
 
 void bdrv_detach_aio_context(BlockDriverState *bs)
 {
+    BdrvAioNotifier *baf;
+
     if (!bs->drv) {
         return;
     }
 
+    QLIST_FOREACH(baf, &bs->aio_notifiers, list) {
+        baf->detach_aio_context(baf->opaque);
+    }
+
     if (bs->io_limits_enabled) {
         throttle_detach_aio_context(&bs->throttle_state);
     }
@@ -5681,6 +5693,8 @@ void bdrv_detach_aio_context(BlockDriverState *bs)
 void bdrv_attach_aio_context(BlockDriverState *bs,
                              AioContext *new_context)
 {
+    BdrvAioNotifier *ban;
+
     if (!bs->drv) {
         return;
     }
@@ -5699,6 +5713,10 @@ void bdrv_attach_aio_context(BlockDriverState *bs,
     if (bs->io_limits_enabled) {
         throttle_attach_aio_context(&bs->throttle_state, new_context);
     }
+
+    QLIST_FOREACH(ban, &bs->aio_notifiers, list) {
+        ban->attached_aio_context(new_context, ban->opaque);
+    }
 }
 
 void bdrv_set_aio_context(BlockDriverState *bs, AioContext *new_context)
@@ -5715,18 +5733,56 @@ void bdrv_set_aio_context(BlockDriverState *bs, AioContext *new_context)
     aio_context_release(new_context);
 }
 
+void bdrv_add_aio_context_notifier(BlockDriverState *bs,
+        void (*attached_aio_context)(AioContext *new_context, void *opaque),
+        void (*detach_aio_context)(void *opaque), void *opaque)
+{
+    BdrvAioNotifier *ban = g_new(BdrvAioNotifier, 1);
+    *ban = (BdrvAioNotifier){
+        .attached_aio_context = attached_aio_context,
+        .detach_aio_context   = detach_aio_context,
+        .opaque               = opaque
+    };
+
+    QLIST_INSERT_HEAD(&bs->aio_notifiers, ban, list);
+}
+
+void bdrv_remove_aio_context_notifier(BlockDriverState *bs,
+                                      void (*attached_aio_context)(AioContext *,
+                                                                   void *),
+                                      void (*detach_aio_context)(void *),
+                                      void *opaque)
+{
+    BdrvAioNotifier *ban, *ban_next;
+
+    QLIST_FOREACH_SAFE(ban, &bs->aio_notifiers, list, ban_next) {
+        if (ban->attached_aio_context == attached_aio_context &&
+            ban->detach_aio_context   == detach_aio_context   &&
+            ban->opaque               == opaque)
+        {
+            QLIST_REMOVE(ban, list);
+            g_free(ban);
+
+            return;
+        }
+    }
+
+    abort();
+}
+
 void bdrv_add_before_write_notifier(BlockDriverState *bs,
                                     NotifierWithReturn *notifier)
 {
     notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
 }
 
-int bdrv_amend_options(BlockDriverState *bs, QemuOpts *opts)
+int bdrv_amend_options(BlockDriverState *bs, QemuOpts *opts,
+                       BlockDriverAmendStatusCB *status_cb)
 {
     if (!bs->drv->bdrv_amend_options) {
         return -ENOTSUP;
     }
-    return bs->drv->bdrv_amend_options(bs, opts);
+    return bs->drv->bdrv_amend_options(bs, opts, status_cb);
 }
 
 /* This function will be called by the bdrv_recurse_is_first_non_filter method
@@ -5789,13 +5845,19 @@ bool bdrv_is_first_non_filter(BlockDriverState *candidate)
 BlockDriverState *check_to_replace_node(const char *node_name, Error **errp)
 {
     BlockDriverState *to_replace_bs = bdrv_find_node(node_name);
+    AioContext *aio_context;
+
     if (!to_replace_bs) {
         error_setg(errp, "Node name '%s' not found", node_name);
         return NULL;
     }
 
+    aio_context = bdrv_get_aio_context(to_replace_bs);
+    aio_context_acquire(aio_context);
+
     if (bdrv_op_is_blocked(to_replace_bs, BLOCK_OP_TYPE_REPLACE, errp)) {
-        return NULL;
+        to_replace_bs = NULL;
+        goto out;
     }
 
     /* We don't want arbitrary node of the BDS chain to be replaced only the top
@@ -5805,9 +5867,12 @@ BlockDriverState *check_to_replace_node(const char *node_name, Error **errp)
      */
     if (!bdrv_is_first_non_filter(to_replace_bs)) {
         error_setg(errp, "Only top most non filter can be replaced");
-        return NULL;
+        to_replace_bs = NULL;
+        goto out;
     }
 
+out:
+    aio_context_release(aio_context);
     return to_replace_bs;
 }
 
@@ -5840,3 +5905,144 @@ void bdrv_flush_io_queue(BlockDriverState *bs)
         bdrv_flush_io_queue(bs->file);
     }
 }
+
+static bool append_open_options(QDict *d, BlockDriverState *bs)
+{
+    const QDictEntry *entry;
+    bool found_any = false;
+
+    for (entry = qdict_first(bs->options); entry;
+         entry = qdict_next(bs->options, entry))
+    {
+        /* Only take options for this level and exclude all non-driver-specific
+         * options */
+        if (!strchr(qdict_entry_key(entry), '.') &&
+            strcmp(qdict_entry_key(entry), "node-name"))
+        {
+            qobject_incref(qdict_entry_value(entry));
+            qdict_put_obj(d, qdict_entry_key(entry), qdict_entry_value(entry));
+            found_any = true;
+        }
+    }
+
+    return found_any;
+}
+
+/* Updates the following BDS fields:
+ *  - exact_filename: A filename which may be used for opening a block device
+ *                    which (mostly) equals the given BDS (even without any
+ *                    other options; so reading and writing must return the same
+ *                    results, but caching etc. may be different)
+ *  - full_open_options: Options which, when given when opening a block device
+ *                       (without a filename), result in a BDS (mostly)
+ *                       equalling the given one
+ *  - filename: If exact_filename is set, it is copied here. Otherwise,
+ *              full_open_options is converted to a JSON object, prefixed with
+ *              "json:" (for use through the JSON pseudo protocol) and put here.
+ */
+void bdrv_refresh_filename(BlockDriverState *bs)
+{
+    BlockDriver *drv = bs->drv;
+    QDict *opts;
+
+    if (!drv) {
+        return;
+    }
+
+    /* This BDS's file name will most probably depend on its file's name, so
+     * refresh that first */
+    if (bs->file) {
+        bdrv_refresh_filename(bs->file);
+    }
+
+    if (drv->bdrv_refresh_filename) {
+        /* Obsolete information is of no use here, so drop the old file name
+         * information before refreshing it */
+        bs->exact_filename[0] = '\0';
+        if (bs->full_open_options) {
+            QDECREF(bs->full_open_options);
+            bs->full_open_options = NULL;
+        }
+
+        drv->bdrv_refresh_filename(bs);
+    } else if (bs->file) {
+        /* Try to reconstruct valid information from the underlying file */
+        bool has_open_options;
+
+        bs->exact_filename[0] = '\0';
+        if (bs->full_open_options) {
+            QDECREF(bs->full_open_options);
+            bs->full_open_options = NULL;
+        }
+
+        opts = qdict_new();
+        has_open_options = append_open_options(opts, bs);
+
+        /* If no specific options have been given for this BDS, the filename of
+         * the underlying file should suffice for this one as well */
+        if (bs->file->exact_filename[0] && !has_open_options) {
+            strcpy(bs->exact_filename, bs->file->exact_filename);
+        }
+        /* Reconstructing the full options QDict is simple for most format block
+         * drivers, as long as the full options are known for the underlying
+         * file BDS. The full options QDict of that file BDS should somehow
+         * contain a representation of the filename, therefore the following
+         * suffices without querying the (exact_)filename of this BDS. */
+        if (bs->file->full_open_options) {
+            qdict_put_obj(opts, "driver",
+                          QOBJECT(qstring_from_str(drv->format_name)));
+            QINCREF(bs->file->full_open_options);
+            qdict_put_obj(opts, "file", QOBJECT(bs->file->full_open_options));
+
+            bs->full_open_options = opts;
+        } else {
+            QDECREF(opts);
+        }
+    } else if (!bs->full_open_options && qdict_size(bs->options)) {
+        /* There is no underlying file BDS (at least referenced by BDS.file),
+         * so the full options QDict should be equal to the options given
+         * specifically for this block device when it was opened (plus the
+         * driver specification).
+         * Because those options don't change, there is no need to update
+         * full_open_options when it's already set. */
+
+        opts = qdict_new();
+        append_open_options(opts, bs);
+        qdict_put_obj(opts, "driver",
+                      QOBJECT(qstring_from_str(drv->format_name)));
+
+        if (bs->exact_filename[0]) {
+            /* This may not work for all block protocol drivers (some may
+             * require this filename to be parsed), but we have to find some
+             * default solution here, so just include it. If some block driver
+             * does not support pure options without any filename at all or
+             * needs some special format of the options QDict, it needs to
+             * implement the driver-specific bdrv_refresh_filename() function.
+             */
+            qdict_put_obj(opts, "filename",
+                          QOBJECT(qstring_from_str(bs->exact_filename)));
+        }
+
+        bs->full_open_options = opts;
+    }
+
+    if (bs->exact_filename[0]) {
+        pstrcpy(bs->filename, sizeof(bs->filename), bs->exact_filename);
+    } else if (bs->full_open_options) {
+        QString *json = qobject_to_json(QOBJECT(bs->full_open_options));
+        snprintf(bs->filename, sizeof(bs->filename), "json:%s",
+                 qstring_get_str(json));
+        QDECREF(json);
+    }
+}
+
+/* This accessor function purpose is to allow the device models to access the
+ * BlockAcctStats structure embedded inside a BlockDriverState without being
+ * aware of the BlockDriverState structure layout.
+ * It will go away when the BlockAcctStats structure will be moved inside
+ * the device models.
+ */
+BlockAcctStats *bdrv_get_stats(BlockDriverState *bs)
+{
+    return &bs->stats;
+}
diff --git a/block/Makefile.objs b/block/Makefile.objs
index fd88c03ec..04b0e43eb 100644
--- a/block/Makefile.objs
+++ b/block/Makefile.objs
@@ -1,28 +1,28 @@
-block-obj-y += raw_bsd.o cow.o qcow.o vdi.o vmdk.o cloop.o dmg.o bochs.o vpc.o vvfat.o
+block-obj-y += raw_bsd.o qcow.o vdi.o vmdk.o cloop.o dmg.o bochs.o vpc.o vvfat.o
 block-obj-y += qcow2.o qcow2-refcount.o qcow2-cluster.o qcow2-snapshot.o qcow2-cache.o
 block-obj-y += qed.o qed-gencb.o qed-l2-cache.o qed-table.o qed-cluster.o
 block-obj-y += qed-check.o
 block-obj-$(CONFIG_VHDX) += vhdx.o vhdx-endian.o vhdx-log.o
 block-obj-$(CONFIG_QUORUM) += quorum.o
 block-obj-y += parallels.o blkdebug.o blkverify.o
-block-obj-y += snapshot.o qapi.o
+block-obj-y += block-backend.o snapshot.o qapi.o
 block-obj-$(CONFIG_WIN32) += raw-win32.o win32-aio.o
 block-obj-$(CONFIG_POSIX) += raw-posix.o
 block-obj-$(CONFIG_LINUX_AIO) += linux-aio.o
+block-obj-y += null.o mirror.o
 
-ifeq ($(CONFIG_POSIX),y)
 block-obj-y += nbd.o nbd-client.o sheepdog.o
 block-obj-$(CONFIG_LIBISCSI) += iscsi.o
 block-obj-$(CONFIG_LIBNFS) += nfs.o
 block-obj-$(CONFIG_CURL) += curl.o
 block-obj-$(CONFIG_RBD) += rbd.o
 block-obj-$(CONFIG_GLUSTERFS) += gluster.o
+block-obj-$(CONFIG_ARCHIPELAGO) += archipelago.o
 block-obj-$(CONFIG_LIBSSH2) += ssh.o
-endif
+block-obj-y += accounting.o
 
 common-obj-y += stream.o
 common-obj-y += commit.o
-common-obj-y += mirror.o
 common-obj-y += backup.o
 
 iscsi.o-cflags     := $(LIBISCSI_CFLAGS)
@@ -35,5 +35,6 @@ gluster.o-cflags   := $(GLUSTERFS_CFLAGS)
 gluster.o-libs     := $(GLUSTERFS_LIBS)
 ssh.o-cflags       := $(LIBSSH2_CFLAGS)
 ssh.o-libs         := $(LIBSSH2_LIBS)
+archipelago.o-libs := $(ARCHIPELAGO_LIBS)
 qcow.o-libs        := -lz
 linux-aio.o-libs   := -laio
diff --git a/block/accounting.c b/block/accounting.c
new file mode 100644
index 000000000..edbb1cc89
--- /dev/null
+++ b/block/accounting.c
@@ -0,0 +1,54 @@
+/*
+ * QEMU System Emulator block accounting
+ *
+ * Copyright (c) 2011 Christoph Hellwig
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "block/accounting.h"
+#include "block/block_int.h"
+
+void block_acct_start(BlockAcctStats *stats, BlockAcctCookie *cookie,
+                      int64_t bytes, enum BlockAcctType type)
+{
+    assert(type < BLOCK_MAX_IOTYPE);
+
+    cookie->bytes = bytes;
+    cookie->start_time_ns = get_clock();
+    cookie->type = type;
+}
+
+void block_acct_done(BlockAcctStats *stats, BlockAcctCookie *cookie)
+{
+    assert(cookie->type < BLOCK_MAX_IOTYPE);
+
+    stats->nr_bytes[cookie->type] += cookie->bytes;
+    stats->nr_ops[cookie->type]++;
+    stats->total_time_ns[cookie->type] += get_clock() - cookie->start_time_ns;
+}
+
+
+void block_acct_highest_sector(BlockAcctStats *stats, int64_t sector_num,
+                               unsigned int nb_sectors)
+{
+    if (stats->wr_highest_sector < sector_num + nb_sectors - 1) {
+        stats->wr_highest_sector = sector_num + nb_sectors - 1;
+    }
+}
diff --git a/block/archipelago.c b/block/archipelago.c
new file mode 100644
index 000000000..a8114b528
--- /dev/null
+++ b/block/archipelago.c
@@ -0,0 +1,1084 @@
+/*
+ * QEMU Block driver for Archipelago
+ *
+ * Copyright (C) 2014 Chrysostomos Nanakos <cnanakos@grnet.gr>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+/*
+ * VM Image on Archipelago volume is specified like this:
+ *
+ * file.driver=archipelago,file.volume=<volumename>
+ * [,file.mport=<mapperd_port>[,file.vport=<vlmcd_port>]
+ * [,file.segment=<segment_name>]]
+ *
+ * or
+ *
+ * file=archipelago:<volumename>[/mport=<mapperd_port>[:vport=<vlmcd_port>][:
+ * segment=<segment_name>]]
+ *
+ * 'archipelago' is the protocol.
+ *
+ * 'mport' is the port number on which mapperd is listening. This is optional
+ * and if not specified, QEMU will make Archipelago to use the default port.
+ *
+ * 'vport' is the port number on which vlmcd is listening. This is optional
+ * and if not specified, QEMU will make Archipelago to use the default port.
+ *
+ * 'segment' is the name of the shared memory segment Archipelago stack
+ * is using. This is optional and if not specified, QEMU will make Archipelago
+ * to use the default value, 'archipelago'.
+ *
+ * Examples:
+ *
+ * file.driver=archipelago,file.volume=my_vm_volume
+ * file.driver=archipelago,file.volume=my_vm_volume,file.mport=123
+ * file.driver=archipelago,file.volume=my_vm_volume,file.mport=123,
+ *  file.vport=1234
+ * file.driver=archipelago,file.volume=my_vm_volume,file.mport=123,
+ *  file.vport=1234,file.segment=my_segment
+ *
+ * or
+ *
+ * file=archipelago:my_vm_volume
+ * file=archipelago:my_vm_volume/mport=123
+ * file=archipelago:my_vm_volume/mport=123:vport=1234
+ * file=archipelago:my_vm_volume/mport=123:vport=1234:segment=my_segment
+ *
+ */
+
+#include "qemu-common.h"
+#include "block/block_int.h"
+#include "qemu/error-report.h"
+#include "qemu/thread.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qjson.h"
+#include "qemu/atomic.h"
+
+#include <inttypes.h>
+#include <xseg/xseg.h>
+#include <xseg/protocol.h>
+
+#define MAX_REQUEST_SIZE    524288
+
+#define ARCHIPELAGO_OPT_VOLUME      "volume"
+#define ARCHIPELAGO_OPT_SEGMENT     "segment"
+#define ARCHIPELAGO_OPT_MPORT       "mport"
+#define ARCHIPELAGO_OPT_VPORT       "vport"
+#define ARCHIPELAGO_DFL_MPORT       1001
+#define ARCHIPELAGO_DFL_VPORT       501
+
+#define archipelagolog(fmt, ...) \
+    do {                         \
+        fprintf(stderr, "archipelago\t%-24s: " fmt, __func__, ##__VA_ARGS__); \
+    } while (0)
+
+typedef enum {
+    ARCHIP_OP_READ,
+    ARCHIP_OP_WRITE,
+    ARCHIP_OP_FLUSH,
+    ARCHIP_OP_VOLINFO,
+    ARCHIP_OP_TRUNCATE,
+} ARCHIPCmd;
+
+typedef struct ArchipelagoAIOCB {
+    BlockAIOCB common;
+    QEMUBH *bh;
+    struct BDRVArchipelagoState *s;
+    QEMUIOVector *qiov;
+    ARCHIPCmd cmd;
+    int status;
+    int64_t size;
+    int64_t ret;
+} ArchipelagoAIOCB;
+
+typedef struct BDRVArchipelagoState {
+    ArchipelagoAIOCB *event_acb;
+    char *volname;
+    char *segment_name;
+    uint64_t size;
+    /* Archipelago specific */
+    struct xseg *xseg;
+    struct xseg_port *port;
+    xport srcport;
+    xport sport;
+    xport mportno;
+    xport vportno;
+    QemuMutex archip_mutex;
+    QemuCond archip_cond;
+    bool is_signaled;
+    /* Request handler specific */
+    QemuThread request_th;
+    QemuCond request_cond;
+    QemuMutex request_mutex;
+    bool th_is_signaled;
+    bool stopping;
+} BDRVArchipelagoState;
+
+typedef struct ArchipelagoSegmentedRequest {
+    size_t count;
+    size_t total;
+    int ref;
+    int failed;
+} ArchipelagoSegmentedRequest;
+
+typedef struct AIORequestData {
+    const char *volname;
+    off_t offset;
+    size_t size;
+    uint64_t bufidx;
+    int ret;
+    int op;
+    ArchipelagoAIOCB *aio_cb;
+    ArchipelagoSegmentedRequest *segreq;
+} AIORequestData;
+
+static void qemu_archipelago_complete_aio(void *opaque);
+
+static void init_local_signal(struct xseg *xseg, xport sport, xport srcport)
+{
+    if (xseg && (sport != srcport)) {
+        xseg_init_local_signal(xseg, srcport);
+        sport = srcport;
+    }
+}
+
+static void archipelago_finish_aiocb(AIORequestData *reqdata)
+{
+    if (reqdata->aio_cb->ret != reqdata->segreq->total) {
+        reqdata->aio_cb->ret = -EIO;
+    } else if (reqdata->aio_cb->ret == reqdata->segreq->total) {
+        reqdata->aio_cb->ret = 0;
+    }
+    reqdata->aio_cb->bh = aio_bh_new(
+                        bdrv_get_aio_context(reqdata->aio_cb->common.bs),
+                        qemu_archipelago_complete_aio, reqdata
+                        );
+    qemu_bh_schedule(reqdata->aio_cb->bh);
+}
+
+static int wait_reply(struct xseg *xseg, xport srcport, struct xseg_port *port,
+                      struct xseg_request *expected_req)
+{
+    struct xseg_request *req;
+    xseg_prepare_wait(xseg, srcport);
+    void *psd = xseg_get_signal_desc(xseg, port);
+    while (1) {
+        req = xseg_receive(xseg, srcport, X_NONBLOCK);
+        if (req) {
+            if (req != expected_req) {
+                archipelagolog("Unknown received request\n");
+                xseg_put_request(xseg, req, srcport);
+            } else if (!(req->state & XS_SERVED)) {
+                return -1;
+            } else {
+                break;
+            }
+        }
+        xseg_wait_signal(xseg, psd, 100000UL);
+    }
+    xseg_cancel_wait(xseg, srcport);
+    return 0;
+}
+
+static void xseg_request_handler(void *state)
+{
+    BDRVArchipelagoState *s = (BDRVArchipelagoState *) state;
+    void *psd = xseg_get_signal_desc(s->xseg, s->port);
+    qemu_mutex_lock(&s->request_mutex);
+
+    while (!s->stopping) {
+        struct xseg_request *req;
+        void *data;
+        xseg_prepare_wait(s->xseg, s->srcport);
+        req = xseg_receive(s->xseg, s->srcport, X_NONBLOCK);
+        if (req) {
+            AIORequestData *reqdata;
+            ArchipelagoSegmentedRequest *segreq;
+            xseg_get_req_data(s->xseg, req, (void **)&reqdata);
+
+            switch (reqdata->op) {
+            case ARCHIP_OP_READ:
+                data = xseg_get_data(s->xseg, req);
+                segreq = reqdata->segreq;
+                segreq->count += req->serviced;
+
+                qemu_iovec_from_buf(reqdata->aio_cb->qiov, reqdata->bufidx,
+                                    data,
+                                    req->serviced);
+
+                xseg_put_request(s->xseg, req, s->srcport);
+
+                if (atomic_fetch_dec(&segreq->ref) == 1) {
+                    if (!segreq->failed) {
+                        reqdata->aio_cb->ret = segreq->count;
+                        archipelago_finish_aiocb(reqdata);
+                        g_free(segreq);
+                    } else {
+                        g_free(segreq);
+                        g_free(reqdata);
+                    }
+                } else {
+                    g_free(reqdata);
+                }
+                break;
+            case ARCHIP_OP_WRITE:
+            case ARCHIP_OP_FLUSH:
+                segreq = reqdata->segreq;
+                segreq->count += req->serviced;
+                xseg_put_request(s->xseg, req, s->srcport);
+
+                if (atomic_fetch_dec(&segreq->ref) == 1) {
+                    if (!segreq->failed) {
+                        reqdata->aio_cb->ret = segreq->count;
+                        archipelago_finish_aiocb(reqdata);
+                        g_free(segreq);
+                    } else {
+                        g_free(segreq);
+                        g_free(reqdata);
+                    }
+                } else {
+                    g_free(reqdata);
+                }
+                break;
+            case ARCHIP_OP_VOLINFO:
+            case ARCHIP_OP_TRUNCATE:
+                s->is_signaled = true;
+                qemu_cond_signal(&s->archip_cond);
+                break;
+            }
+        } else {
+            xseg_wait_signal(s->xseg, psd, 100000UL);
+        }
+        xseg_cancel_wait(s->xseg, s->srcport);
+    }
+
+    s->th_is_signaled = true;
+    qemu_cond_signal(&s->request_cond);
+    qemu_mutex_unlock(&s->request_mutex);
+    qemu_thread_exit(NULL);
+}
+
+static int qemu_archipelago_xseg_init(BDRVArchipelagoState *s)
+{
+    if (xseg_initialize()) {
+        archipelagolog("Cannot initialize XSEG\n");
+        goto err_exit;
+    }
+
+    s->xseg = xseg_join("posix", s->segment_name,
+                        "posixfd", NULL);
+    if (!s->xseg) {
+        archipelagolog("Cannot join XSEG shared memory segment\n");
+        goto err_exit;
+    }
+    s->port = xseg_bind_dynport(s->xseg);
+    s->srcport = s->port->portno;
+    init_local_signal(s->xseg, s->sport, s->srcport);
+    return 0;
+
+err_exit:
+    return -1;
+}
+
+static int qemu_archipelago_init(BDRVArchipelagoState *s)
+{
+    int ret;
+
+    ret = qemu_archipelago_xseg_init(s);
+    if (ret < 0) {
+        error_report("Cannot initialize XSEG. Aborting...\n");
+        goto err_exit;
+    }
+
+    qemu_cond_init(&s->archip_cond);
+    qemu_mutex_init(&s->archip_mutex);
+    qemu_cond_init(&s->request_cond);
+    qemu_mutex_init(&s->request_mutex);
+    s->th_is_signaled = false;
+    qemu_thread_create(&s->request_th, "xseg_io_th",
+                       (void *) xseg_request_handler,
+                       (void *) s, QEMU_THREAD_JOINABLE);
+
+err_exit:
+    return ret;
+}
+
+static void qemu_archipelago_complete_aio(void *opaque)
+{
+    AIORequestData *reqdata = (AIORequestData *) opaque;
+    ArchipelagoAIOCB *aio_cb = (ArchipelagoAIOCB *) reqdata->aio_cb;
+
+    qemu_bh_delete(aio_cb->bh);
+    aio_cb->common.cb(aio_cb->common.opaque, aio_cb->ret);
+    aio_cb->status = 0;
+
+    qemu_aio_unref(aio_cb);
+    g_free(reqdata);
+}
+
+static void xseg_find_port(char *pstr, const char *needle, xport *aport)
+{
+    const char *a;
+    char *endptr = NULL;
+    unsigned long port;
+    if (strstart(pstr, needle, &a)) {
+        if (strlen(a) > 0) {
+            port = strtoul(a, &endptr, 10);
+            if (strlen(endptr)) {
+                *aport = -2;
+                return;
+            }
+            *aport = (xport) port;
+        }
+    }
+}
+
+static void xseg_find_segment(char *pstr, const char *needle,
+                              char **segment_name)
+{
+    const char *a;
+    if (strstart(pstr, needle, &a)) {
+        if (strlen(a) > 0) {
+            *segment_name = g_strdup(a);
+        }
+    }
+}
+
+static void parse_filename_opts(const char *filename, Error **errp,
+                                char **volume, char **segment_name,
+                                xport *mport, xport *vport)
+{
+    const char *start;
+    char *tokens[4], *ds;
+    int idx;
+    xport lmport = NoPort, lvport = NoPort;
+
+    strstart(filename, "archipelago:", &start);
+
+    ds = g_strdup(start);
+    tokens[0] = strtok(ds, "/");
+    tokens[1] = strtok(NULL, ":");
+    tokens[2] = strtok(NULL, ":");
+    tokens[3] = strtok(NULL, "\0");
+
+    if (!strlen(tokens[0])) {
+        error_setg(errp, "volume name must be specified first");
+        g_free(ds);
+        return;
+    }
+
+    for (idx = 1; idx < 4; idx++) {
+        if (tokens[idx] != NULL) {
+            if (strstart(tokens[idx], "mport=", NULL)) {
+                xseg_find_port(tokens[idx], "mport=", &lmport);
+            }
+            if (strstart(tokens[idx], "vport=", NULL)) {
+                xseg_find_port(tokens[idx], "vport=", &lvport);
+            }
+            if (strstart(tokens[idx], "segment=", NULL)) {
+                xseg_find_segment(tokens[idx], "segment=", segment_name);
+            }
+        }
+    }
+
+    if ((lmport == -2) || (lvport == -2)) {
+        error_setg(errp, "mport and/or vport must be set");
+        g_free(ds);
+        return;
+    }
+    *volume = g_strdup(tokens[0]);
+    *mport = lmport;
+    *vport = lvport;
+    g_free(ds);
+}
+
+static void archipelago_parse_filename(const char *filename, QDict *options,
+                                       Error **errp)
+{
+    const char *start;
+    char *volume = NULL, *segment_name = NULL;
+    xport mport = NoPort, vport = NoPort;
+
+    if (qdict_haskey(options, ARCHIPELAGO_OPT_VOLUME)
+            || qdict_haskey(options, ARCHIPELAGO_OPT_SEGMENT)
+            || qdict_haskey(options, ARCHIPELAGO_OPT_MPORT)
+            || qdict_haskey(options, ARCHIPELAGO_OPT_VPORT)) {
+        error_setg(errp, "volume/mport/vport/segment and a file name may not"
+                         " be specified at the same time");
+        return;
+    }
+
+    if (!strstart(filename, "archipelago:", &start)) {
+        error_setg(errp, "File name must start with 'archipelago:'");
+        return;
+    }
+
+    if (!strlen(start) || strstart(start, "/", NULL)) {
+        error_setg(errp, "volume name must be specified");
+        return;
+    }
+
+    parse_filename_opts(filename, errp, &volume, &segment_name, &mport, &vport);
+
+    if (volume) {
+        qdict_put(options, ARCHIPELAGO_OPT_VOLUME, qstring_from_str(volume));
+        g_free(volume);
+    }
+    if (segment_name) {
+        qdict_put(options, ARCHIPELAGO_OPT_SEGMENT,
+                  qstring_from_str(segment_name));
+        g_free(segment_name);
+    }
+    if (mport != NoPort) {
+        qdict_put(options, ARCHIPELAGO_OPT_MPORT, qint_from_int(mport));
+    }
+    if (vport != NoPort) {
+        qdict_put(options, ARCHIPELAGO_OPT_VPORT, qint_from_int(vport));
+    }
+}
+
+static QemuOptsList archipelago_runtime_opts = {
+    .name = "archipelago",
+    .head = QTAILQ_HEAD_INITIALIZER(archipelago_runtime_opts.head),
+    .desc = {
+        {
+            .name = ARCHIPELAGO_OPT_VOLUME,
+            .type = QEMU_OPT_STRING,
+            .help = "Name of the volume image",
+        },
+        {
+            .name = ARCHIPELAGO_OPT_SEGMENT,
+            .type = QEMU_OPT_STRING,
+            .help = "Name of the Archipelago shared memory segment",
+        },
+        {
+            .name = ARCHIPELAGO_OPT_MPORT,
+            .type = QEMU_OPT_NUMBER,
+            .help = "Archipelago mapperd port number"
+        },
+        {
+            .name = ARCHIPELAGO_OPT_VPORT,
+            .type = QEMU_OPT_NUMBER,
+            .help = "Archipelago vlmcd port number"
+
+        },
+        { /* end of list */ }
+    },
+};
+
+static int qemu_archipelago_open(BlockDriverState *bs,
+                                 QDict *options,
+                                 int bdrv_flags,
+                                 Error **errp)
+{
+    int ret = 0;
+    const char *volume, *segment_name;
+    QemuOpts *opts;
+    Error *local_err = NULL;
+    BDRVArchipelagoState *s = bs->opaque;
+
+    opts = qemu_opts_create(&archipelago_runtime_opts, NULL, 0, &error_abort);
+    qemu_opts_absorb_qdict(opts, options, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto err_exit;
+    }
+
+    s->mportno = qemu_opt_get_number(opts, ARCHIPELAGO_OPT_MPORT,
+                                     ARCHIPELAGO_DFL_MPORT);
+    s->vportno = qemu_opt_get_number(opts, ARCHIPELAGO_OPT_VPORT,
+                                     ARCHIPELAGO_DFL_VPORT);
+
+    segment_name = qemu_opt_get(opts, ARCHIPELAGO_OPT_SEGMENT);
+    if (segment_name == NULL) {
+        s->segment_name = g_strdup("archipelago");
+    } else {
+        s->segment_name = g_strdup(segment_name);
+    }
+
+    volume = qemu_opt_get(opts, ARCHIPELAGO_OPT_VOLUME);
+    if (volume == NULL) {
+        error_setg(errp, "archipelago block driver requires the 'volume'"
+                   " option");
+        ret = -EINVAL;
+        goto err_exit;
+    }
+    s->volname = g_strdup(volume);
+
+    /* Initialize XSEG, join shared memory segment */
+    ret = qemu_archipelago_init(s);
+    if (ret < 0) {
+        error_setg(errp, "cannot initialize XSEG and join shared "
+                   "memory segment");
+        goto err_exit;
+    }
+
+    qemu_opts_del(opts);
+    return 0;
+
+err_exit:
+    g_free(s->volname);
+    g_free(s->segment_name);
+    qemu_opts_del(opts);
+    return ret;
+}
+
+static void qemu_archipelago_close(BlockDriverState *bs)
+{
+    int r, targetlen;
+    char *target;
+    struct xseg_request *req;
+    BDRVArchipelagoState *s = bs->opaque;
+
+    s->stopping = true;
+
+    qemu_mutex_lock(&s->request_mutex);
+    while (!s->th_is_signaled) {
+        qemu_cond_wait(&s->request_cond,
+                       &s->request_mutex);
+    }
+    qemu_mutex_unlock(&s->request_mutex);
+    qemu_thread_join(&s->request_th);
+    qemu_cond_destroy(&s->request_cond);
+    qemu_mutex_destroy(&s->request_mutex);
+
+    qemu_cond_destroy(&s->archip_cond);
+    qemu_mutex_destroy(&s->archip_mutex);
+
+    targetlen = strlen(s->volname);
+    req = xseg_get_request(s->xseg, s->srcport, s->vportno, X_ALLOC);
+    if (!req) {
+        archipelagolog("Cannot get XSEG request\n");
+        goto err_exit;
+    }
+    r = xseg_prep_request(s->xseg, req, targetlen, 0);
+    if (r < 0) {
+        xseg_put_request(s->xseg, req, s->srcport);
+        archipelagolog("Cannot prepare XSEG close request\n");
+        goto err_exit;
+    }
+
+    target = xseg_get_target(s->xseg, req);
+    memcpy(target, s->volname, targetlen);
+    req->size = req->datalen;
+    req->offset = 0;
+    req->op = X_CLOSE;
+
+    xport p = xseg_submit(s->xseg, req, s->srcport, X_ALLOC);
+    if (p == NoPort) {
+        xseg_put_request(s->xseg, req, s->srcport);
+        archipelagolog("Cannot submit XSEG close request\n");
+        goto err_exit;
+    }
+
+    xseg_signal(s->xseg, p);
+    wait_reply(s->xseg, s->srcport, s->port, req);
+
+    xseg_put_request(s->xseg, req, s->srcport);
+
+err_exit:
+    g_free(s->volname);
+    g_free(s->segment_name);
+    xseg_quit_local_signal(s->xseg, s->srcport);
+    xseg_leave_dynport(s->xseg, s->port);
+    xseg_leave(s->xseg);
+}
+
+static int qemu_archipelago_create_volume(Error **errp, const char *volname,
+                                          char *segment_name,
+                                          uint64_t size, xport mportno,
+                                          xport vportno)
+{
+    int ret, targetlen;
+    struct xseg *xseg = NULL;
+    struct xseg_request *req;
+    struct xseg_request_clone *xclone;
+    struct xseg_port *port;
+    xport srcport = NoPort, sport = NoPort;
+    char *target;
+
+    /* Try default values if none has been set */
+    if (mportno == (xport) -1) {
+        mportno = ARCHIPELAGO_DFL_MPORT;
+    }
+
+    if (vportno == (xport) -1) {
+        vportno = ARCHIPELAGO_DFL_VPORT;
+    }
+
+    if (xseg_initialize()) {
+        error_setg(errp, "Cannot initialize XSEG");
+        return -1;
+    }
+
+    xseg = xseg_join("posix", segment_name,
+                     "posixfd", NULL);
+
+    if (!xseg) {
+        error_setg(errp, "Cannot join XSEG shared memory segment");
+        return -1;
+    }
+
+    port = xseg_bind_dynport(xseg);
+    srcport = port->portno;
+    init_local_signal(xseg, sport, srcport);
+
+    req = xseg_get_request(xseg, srcport, mportno, X_ALLOC);
+    if (!req) {
+        error_setg(errp, "Cannot get XSEG request");
+        return -1;
+    }
+
+    targetlen = strlen(volname);
+    ret = xseg_prep_request(xseg, req, targetlen,
+                            sizeof(struct xseg_request_clone));
+    if (ret < 0) {
+        error_setg(errp, "Cannot prepare XSEG request");
+        goto err_exit;
+    }
+
+    target = xseg_get_target(xseg, req);
+    if (!target) {
+        error_setg(errp, "Cannot get XSEG target.\n");
+        goto err_exit;
+    }
+    memcpy(target, volname, targetlen);
+    xclone = (struct xseg_request_clone *) xseg_get_data(xseg, req);
+    memset(xclone->target, 0 , XSEG_MAX_TARGETLEN);
+    xclone->targetlen = 0;
+    xclone->size = size;
+    req->offset = 0;
+    req->size = req->datalen;
+    req->op = X_CLONE;
+
+    xport p = xseg_submit(xseg, req, srcport, X_ALLOC);
+    if (p == NoPort) {
+        error_setg(errp, "Could not submit XSEG request");
+        goto err_exit;
+    }
+    xseg_signal(xseg, p);
+
+    ret = wait_reply(xseg, srcport, port, req);
+    if (ret < 0) {
+        error_setg(errp, "wait_reply() error.");
+    }
+
+    xseg_put_request(xseg, req, srcport);
+    xseg_quit_local_signal(xseg, srcport);
+    xseg_leave_dynport(xseg, port);
+    xseg_leave(xseg);
+    return ret;
+
+err_exit:
+    xseg_put_request(xseg, req, srcport);
+    xseg_quit_local_signal(xseg, srcport);
+    xseg_leave_dynport(xseg, port);
+    xseg_leave(xseg);
+    return -1;
+}
+
+static int qemu_archipelago_create(const char *filename,
+                                   QemuOpts *options,
+                                   Error **errp)
+{
+    int ret = 0;
+    uint64_t total_size = 0;
+    char *volname = NULL, *segment_name = NULL;
+    const char *start;
+    xport mport = NoPort, vport = NoPort;
+
+    if (!strstart(filename, "archipelago:", &start)) {
+        error_setg(errp, "File name must start with 'archipelago:'");
+        return -1;
+    }
+
+    if (!strlen(start) || strstart(start, "/", NULL)) {
+        error_setg(errp, "volume name must be specified");
+        return -1;
+    }
+
+    parse_filename_opts(filename, errp, &volname, &segment_name, &mport,
+                        &vport);
+    total_size = ROUND_UP(qemu_opt_get_size_del(options, BLOCK_OPT_SIZE, 0),
+                          BDRV_SECTOR_SIZE);
+
+    if (segment_name == NULL) {
+        segment_name = g_strdup("archipelago");
+    }
+
+    /* Create an Archipelago volume */
+    ret = qemu_archipelago_create_volume(errp, volname, segment_name,
+                                         total_size, mport,
+                                         vport);
+
+    g_free(volname);
+    g_free(segment_name);
+    return ret;
+}
+
+static const AIOCBInfo archipelago_aiocb_info = {
+    .aiocb_size = sizeof(ArchipelagoAIOCB),
+};
+
+static int archipelago_submit_request(BDRVArchipelagoState *s,
+                                        uint64_t bufidx,
+                                        size_t count,
+                                        off_t offset,
+                                        ArchipelagoAIOCB *aio_cb,
+                                        ArchipelagoSegmentedRequest *segreq,
+                                        int op)
+{
+    int ret, targetlen;
+    char *target;
+    void *data = NULL;
+    struct xseg_request *req;
+    AIORequestData *reqdata = g_new(AIORequestData, 1);
+
+    targetlen = strlen(s->volname);
+    req = xseg_get_request(s->xseg, s->srcport, s->vportno, X_ALLOC);
+    if (!req) {
+        archipelagolog("Cannot get XSEG request\n");
+        goto err_exit2;
+    }
+    ret = xseg_prep_request(s->xseg, req, targetlen, count);
+    if (ret < 0) {
+        archipelagolog("Cannot prepare XSEG request\n");
+        goto err_exit;
+    }
+    target = xseg_get_target(s->xseg, req);
+    if (!target) {
+        archipelagolog("Cannot get XSEG target\n");
+        goto err_exit;
+    }
+    memcpy(target, s->volname, targetlen);
+    req->size = count;
+    req->offset = offset;
+
+    switch (op) {
+    case ARCHIP_OP_READ:
+        req->op = X_READ;
+        break;
+    case ARCHIP_OP_WRITE:
+        req->op = X_WRITE;
+        break;
+    case ARCHIP_OP_FLUSH:
+        req->op = X_FLUSH;
+        break;
+    }
+    reqdata->volname = s->volname;
+    reqdata->offset = offset;
+    reqdata->size = count;
+    reqdata->bufidx = bufidx;
+    reqdata->aio_cb = aio_cb;
+    reqdata->segreq = segreq;
+    reqdata->op = op;
+
+    xseg_set_req_data(s->xseg, req, reqdata);
+    if (op == ARCHIP_OP_WRITE) {
+        data = xseg_get_data(s->xseg, req);
+        if (!data) {
+            archipelagolog("Cannot get XSEG data\n");
+            goto err_exit;
+        }
+        qemu_iovec_to_buf(aio_cb->qiov, bufidx, data, count);
+    }
+
+    xport p = xseg_submit(s->xseg, req, s->srcport, X_ALLOC);
+    if (p == NoPort) {
+        archipelagolog("Could not submit XSEG request\n");
+        goto err_exit;
+    }
+    xseg_signal(s->xseg, p);
+    return 0;
+
+err_exit:
+    g_free(reqdata);
+    xseg_put_request(s->xseg, req, s->srcport);
+    return -EIO;
+err_exit2:
+    g_free(reqdata);
+    return -EIO;
+}
+
+static int archipelago_aio_segmented_rw(BDRVArchipelagoState *s,
+                                        size_t count,
+                                        off_t offset,
+                                        ArchipelagoAIOCB *aio_cb,
+                                        int op)
+{
+    int ret, segments_nr;
+    size_t pos = 0;
+    ArchipelagoSegmentedRequest *segreq;
+
+    segreq = g_new0(ArchipelagoSegmentedRequest, 1);
+
+    if (op == ARCHIP_OP_FLUSH) {
+        segments_nr = 1;
+    } else {
+        segments_nr = (int)(count / MAX_REQUEST_SIZE) + \
+                      ((count % MAX_REQUEST_SIZE) ? 1 : 0);
+    }
+    segreq->total = count;
+    atomic_mb_set(&segreq->ref, segments_nr);
+
+    while (segments_nr > 1) {
+        ret = archipelago_submit_request(s, pos,
+                                            MAX_REQUEST_SIZE,
+                                            offset + pos,
+                                            aio_cb, segreq, op);
+
+        if (ret < 0) {
+            goto err_exit;
+        }
+        count -= MAX_REQUEST_SIZE;
+        pos += MAX_REQUEST_SIZE;
+        segments_nr--;
+    }
+    ret = archipelago_submit_request(s, pos, count, offset + pos,
+                                     aio_cb, segreq, op);
+
+    if (ret < 0) {
+        goto err_exit;
+    }
+    return 0;
+
+err_exit:
+    segreq->failed = 1;
+    if (atomic_fetch_sub(&segreq->ref, segments_nr) == segments_nr) {
+        g_free(segreq);
+    }
+    return ret;
+}
+
+static BlockAIOCB *qemu_archipelago_aio_rw(BlockDriverState *bs,
+                                           int64_t sector_num,
+                                           QEMUIOVector *qiov,
+                                           int nb_sectors,
+                                           BlockCompletionFunc *cb,
+                                           void *opaque,
+                                           int op)
+{
+    ArchipelagoAIOCB *aio_cb;
+    BDRVArchipelagoState *s = bs->opaque;
+    int64_t size, off;
+    int ret;
+
+    aio_cb = qemu_aio_get(&archipelago_aiocb_info, bs, cb, opaque);
+    aio_cb->cmd = op;
+    aio_cb->qiov = qiov;
+
+    aio_cb->ret = 0;
+    aio_cb->s = s;
+    aio_cb->status = -EINPROGRESS;
+
+    off = sector_num * BDRV_SECTOR_SIZE;
+    size = nb_sectors * BDRV_SECTOR_SIZE;
+    aio_cb->size = size;
+
+    ret = archipelago_aio_segmented_rw(s, size, off,
+                                       aio_cb, op);
+    if (ret < 0) {
+        goto err_exit;
+    }
+    return &aio_cb->common;
+
+err_exit:
+    error_report("qemu_archipelago_aio_rw(): I/O Error\n");
+    qemu_aio_unref(aio_cb);
+    return NULL;
+}
+
+static BlockAIOCB *qemu_archipelago_aio_readv(BlockDriverState *bs,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockCompletionFunc *cb, void *opaque)
+{
+    return qemu_archipelago_aio_rw(bs, sector_num, qiov, nb_sectors, cb,
+                                   opaque, ARCHIP_OP_READ);
+}
+
+static BlockAIOCB *qemu_archipelago_aio_writev(BlockDriverState *bs,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockCompletionFunc *cb, void *opaque)
+{
+    return qemu_archipelago_aio_rw(bs, sector_num, qiov, nb_sectors, cb,
+                                   opaque, ARCHIP_OP_WRITE);
+}
+
+static int64_t archipelago_volume_info(BDRVArchipelagoState *s)
+{
+    uint64_t size;
+    int ret, targetlen;
+    struct xseg_request *req;
+    struct xseg_reply_info *xinfo;
+    AIORequestData *reqdata = g_new(AIORequestData, 1);
+
+    const char *volname = s->volname;
+    targetlen = strlen(volname);
+    req = xseg_get_request(s->xseg, s->srcport, s->mportno, X_ALLOC);
+    if (!req) {
+        archipelagolog("Cannot get XSEG request\n");
+        goto err_exit2;
+    }
+    ret = xseg_prep_request(s->xseg, req, targetlen,
+                            sizeof(struct xseg_reply_info));
+    if (ret < 0) {
+        archipelagolog("Cannot prepare XSEG request\n");
+        goto err_exit;
+    }
+    char *target = xseg_get_target(s->xseg, req);
+    if (!target) {
+        archipelagolog("Cannot get XSEG target\n");
+        goto err_exit;
+    }
+    memcpy(target, volname, targetlen);
+    req->size = req->datalen;
+    req->offset = 0;
+    req->op = X_INFO;
+
+    reqdata->op = ARCHIP_OP_VOLINFO;
+    reqdata->volname = volname;
+    xseg_set_req_data(s->xseg, req, reqdata);
+
+    xport p = xseg_submit(s->xseg, req, s->srcport, X_ALLOC);
+    if (p == NoPort) {
+        archipelagolog("Cannot submit XSEG request\n");
+        goto err_exit;
+    }
+    xseg_signal(s->xseg, p);
+    qemu_mutex_lock(&s->archip_mutex);
+    while (!s->is_signaled) {
+        qemu_cond_wait(&s->archip_cond, &s->archip_mutex);
+    }
+    s->is_signaled = false;
+    qemu_mutex_unlock(&s->archip_mutex);
+
+    xinfo = (struct xseg_reply_info *) xseg_get_data(s->xseg, req);
+    size = xinfo->size;
+    xseg_put_request(s->xseg, req, s->srcport);
+    g_free(reqdata);
+    s->size = size;
+    return size;
+
+err_exit:
+    xseg_put_request(s->xseg, req, s->srcport);
+err_exit2:
+    g_free(reqdata);
+    return -EIO;
+}
+
+static int64_t qemu_archipelago_getlength(BlockDriverState *bs)
+{
+    int64_t ret;
+    BDRVArchipelagoState *s = bs->opaque;
+
+    ret = archipelago_volume_info(s);
+    return ret;
+}
+
+static int qemu_archipelago_truncate(BlockDriverState *bs, int64_t offset)
+{
+    int ret, targetlen;
+    struct xseg_request *req;
+    BDRVArchipelagoState *s = bs->opaque;
+    AIORequestData *reqdata = g_new(AIORequestData, 1);
+
+    const char *volname = s->volname;
+    targetlen = strlen(volname);
+    req = xseg_get_request(s->xseg, s->srcport, s->mportno, X_ALLOC);
+    if (!req) {
+        archipelagolog("Cannot get XSEG request\n");
+        goto err_exit2;
+    }
+
+    ret = xseg_prep_request(s->xseg, req, targetlen, 0);
+    if (ret < 0) {
+        archipelagolog("Cannot prepare XSEG request\n");
+        goto err_exit;
+    }
+    char *target = xseg_get_target(s->xseg, req);
+    if (!target) {
+        archipelagolog("Cannot get XSEG target\n");
+        goto err_exit;
+    }
+    memcpy(target, volname, targetlen);
+    req->offset = offset;
+    req->op = X_TRUNCATE;
+
+    reqdata->op = ARCHIP_OP_TRUNCATE;
+    reqdata->volname = volname;
+
+    xseg_set_req_data(s->xseg, req, reqdata);
+
+    xport p = xseg_submit(s->xseg, req, s->srcport, X_ALLOC);
+    if (p == NoPort) {
+        archipelagolog("Cannot submit XSEG request\n");
+        goto err_exit;
+    }
+
+    xseg_signal(s->xseg, p);
+    qemu_mutex_lock(&s->archip_mutex);
+    while (!s->is_signaled) {
+        qemu_cond_wait(&s->archip_cond, &s->archip_mutex);
+    }
+    s->is_signaled = false;
+    qemu_mutex_unlock(&s->archip_mutex);
+    xseg_put_request(s->xseg, req, s->srcport);
+    g_free(reqdata);
+    return 0;
+
+err_exit:
+    xseg_put_request(s->xseg, req, s->srcport);
+err_exit2:
+    g_free(reqdata);
+    return -EIO;
+}
+
+static QemuOptsList qemu_archipelago_create_opts = {
+    .name = "archipelago-create-opts",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_archipelago_create_opts.head),
+    .desc = {
+        {
+            .name = BLOCK_OPT_SIZE,
+            .type = QEMU_OPT_SIZE,
+            .help = "Virtual disk size"
+        },
+        { /* end of list */ }
+    }
+};
+
+static BlockAIOCB *qemu_archipelago_aio_flush(BlockDriverState *bs,
+        BlockCompletionFunc *cb, void *opaque)
+{
+    return qemu_archipelago_aio_rw(bs, 0, NULL, 0, cb, opaque,
+                                   ARCHIP_OP_FLUSH);
+}
+
+static BlockDriver bdrv_archipelago = {
+    .format_name         = "archipelago",
+    .protocol_name       = "archipelago",
+    .instance_size       = sizeof(BDRVArchipelagoState),
+    .bdrv_parse_filename = archipelago_parse_filename,
+    .bdrv_file_open      = qemu_archipelago_open,
+    .bdrv_close          = qemu_archipelago_close,
+    .bdrv_create         = qemu_archipelago_create,
+    .bdrv_getlength      = qemu_archipelago_getlength,
+    .bdrv_truncate       = qemu_archipelago_truncate,
+    .bdrv_aio_readv      = qemu_archipelago_aio_readv,
+    .bdrv_aio_writev     = qemu_archipelago_aio_writev,
+    .bdrv_aio_flush      = qemu_archipelago_aio_flush,
+    .bdrv_has_zero_init  = bdrv_has_zero_init_1,
+    .create_opts         = &qemu_archipelago_create_opts,
+};
+
+static void bdrv_archipelago_init(void)
+{
+    bdrv_register(&bdrv_archipelago);
+}
+
+block_init(bdrv_archipelago_init);
diff --git a/block/backup.c b/block/backup.c
index d0b02255c..792e65514 100644
--- a/block/backup.c
+++ b/block/backup.c
@@ -227,9 +227,25 @@ static BlockErrorAction backup_error_action(BackupBlockJob *job,
     }
 }
 
+typedef struct {
+    int ret;
+} BackupCompleteData;
+
+static void backup_complete(BlockJob *job, void *opaque)
+{
+    BackupBlockJob *s = container_of(job, BackupBlockJob, common);
+    BackupCompleteData *data = opaque;
+
+    bdrv_unref(s->target);
+
+    block_job_completed(job, data->ret);
+    g_free(data);
+}
+
 static void coroutine_fn backup_run(void *opaque)
 {
     BackupBlockJob *job = opaque;
+    BackupCompleteData *data;
     BlockDriverState *bs = job->common.bs;
     BlockDriverState *target = job->target;
     BlockdevOnError on_target_error = job->on_target_error;
@@ -344,16 +360,17 @@ static void coroutine_fn backup_run(void *opaque)
     hbitmap_free(job->bitmap);
 
     bdrv_iostatus_disable(target);
-    bdrv_unref(target);
 
-    block_job_completed(&job->common, ret);
+    data = g_malloc(sizeof(*data));
+    data->ret = ret;
+    block_job_defer_to_main_loop(&job->common, backup_complete, data);
 }
 
 void backup_start(BlockDriverState *bs, BlockDriverState *target,
                   int64_t speed, MirrorSyncMode sync_mode,
                   BlockdevOnError on_source_error,
                   BlockdevOnError on_target_error,
-                  BlockDriverCompletionFunc *cb, void *opaque,
+                  BlockCompletionFunc *cb, void *opaque,
                   Error **errp)
 {
     int64_t len;
diff --git a/block/blkdebug.c b/block/blkdebug.c
index f51407de3..862d93b59 100644
--- a/block/blkdebug.c
+++ b/block/blkdebug.c
@@ -26,6 +26,10 @@
 #include "qemu/config-file.h"
 #include "block/block_int.h"
 #include "qemu/module.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qstring.h"
 
 typedef struct BDRVBlkdebugState {
     int state;
@@ -37,7 +41,7 @@ typedef struct BDRVBlkdebugState {
 } BDRVBlkdebugState;
 
 typedef struct BlkdebugAIOCB {
-    BlockDriverAIOCB common;
+    BlockAIOCB common;
     QEMUBH *bh;
     int ret;
 } BlkdebugAIOCB;
@@ -48,11 +52,8 @@ typedef struct BlkdebugSuspendedReq {
     QLIST_ENTRY(BlkdebugSuspendedReq) next;
 } BlkdebugSuspendedReq;
 
-static void blkdebug_aio_cancel(BlockDriverAIOCB *blockacb);
-
 static const AIOCBInfo blkdebug_aiocb_info = {
-    .aiocb_size = sizeof(BlkdebugAIOCB),
-    .cancel     = blkdebug_aio_cancel,
+    .aiocb_size    = sizeof(BlkdebugAIOCB),
 };
 
 enum {
@@ -194,6 +195,8 @@ static const char *event_names[BLKDBG_EVENT_MAX] = {
     [BLKDBG_PWRITEV]                        = "pwritev",
     [BLKDBG_PWRITEV_ZERO]                   = "pwritev_zero",
     [BLKDBG_PWRITEV_DONE]                   = "pwritev_done",
+
+    [BLKDBG_EMPTY_IMAGE_PREPARE]            = "empty_image_prepare",
 };
 
 static int get_event_by_name(const char *name, BlkDebugEvent *event)
@@ -213,6 +216,7 @@ static int get_event_by_name(const char *name, BlkDebugEvent *event)
 struct add_rule_data {
     BDRVBlkdebugState *s;
     int action;
+    Error **errp;
 };
 
 static int add_rule(QemuOpts *opts, void *opaque)
@@ -225,7 +229,11 @@ static int add_rule(QemuOpts *opts, void *opaque)
 
     /* Find the right event for the rule */
     event_name = qemu_opt_get(opts, "event");
-    if (!event_name || get_event_by_name(event_name, &event) < 0) {
+    if (!event_name) {
+        error_setg(d->errp, "Missing event name for rule");
+        return -1;
+    } else if (get_event_by_name(event_name, &event) < 0) {
+        error_setg(d->errp, "Invalid event name \"%s\"", event_name);
         return -1;
     }
 
@@ -311,10 +319,21 @@ static int read_config(BDRVBlkdebugState *s, const char *filename,
 
     d.s = s;
     d.action = ACTION_INJECT_ERROR;
-    qemu_opts_foreach(&inject_error_opts, add_rule, &d, 0);
+    d.errp = &local_err;
+    qemu_opts_foreach(&inject_error_opts, add_rule, &d, 1);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto fail;
+    }
 
     d.action = ACTION_SET_STATE;
-    qemu_opts_foreach(&set_state_opts, add_rule, &d, 0);
+    qemu_opts_foreach(&set_state_opts, add_rule, &d, 1);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto fail;
+    }
 
     ret = 0;
 fail:
@@ -443,17 +462,11 @@ static void error_callback_bh(void *opaque)
     struct BlkdebugAIOCB *acb = opaque;
     qemu_bh_delete(acb->bh);
     acb->common.cb(acb->common.opaque, acb->ret);
-    qemu_aio_release(acb);
-}
-
-static void blkdebug_aio_cancel(BlockDriverAIOCB *blockacb)
-{
-    BlkdebugAIOCB *acb = container_of(blockacb, BlkdebugAIOCB, common);
-    qemu_aio_release(acb);
+    qemu_aio_unref(acb);
 }
 
-static BlockDriverAIOCB *inject_error(BlockDriverState *bs,
-    BlockDriverCompletionFunc *cb, void *opaque, BlkdebugRule *rule)
+static BlockAIOCB *inject_error(BlockDriverState *bs,
+    BlockCompletionFunc *cb, void *opaque, BlkdebugRule *rule)
 {
     BDRVBlkdebugState *s = bs->opaque;
     int error = rule->options.inject.error;
@@ -478,9 +491,9 @@ static BlockDriverAIOCB *inject_error(BlockDriverState *bs,
     return &acb->common;
 }
 
-static BlockDriverAIOCB *blkdebug_aio_readv(BlockDriverState *bs,
+static BlockAIOCB *blkdebug_aio_readv(BlockDriverState *bs,
     int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-    BlockDriverCompletionFunc *cb, void *opaque)
+    BlockCompletionFunc *cb, void *opaque)
 {
     BDRVBlkdebugState *s = bs->opaque;
     BlkdebugRule *rule = NULL;
@@ -500,9 +513,9 @@ static BlockDriverAIOCB *blkdebug_aio_readv(BlockDriverState *bs,
     return bdrv_aio_readv(bs->file, sector_num, qiov, nb_sectors, cb, opaque);
 }
 
-static BlockDriverAIOCB *blkdebug_aio_writev(BlockDriverState *bs,
+static BlockAIOCB *blkdebug_aio_writev(BlockDriverState *bs,
     int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-    BlockDriverCompletionFunc *cb, void *opaque)
+    BlockCompletionFunc *cb, void *opaque)
 {
     BDRVBlkdebugState *s = bs->opaque;
     BlkdebugRule *rule = NULL;
@@ -522,6 +535,25 @@ static BlockDriverAIOCB *blkdebug_aio_writev(BlockDriverState *bs,
     return bdrv_aio_writev(bs->file, sector_num, qiov, nb_sectors, cb, opaque);
 }
 
+static BlockAIOCB *blkdebug_aio_flush(BlockDriverState *bs,
+    BlockCompletionFunc *cb, void *opaque)
+{
+    BDRVBlkdebugState *s = bs->opaque;
+    BlkdebugRule *rule = NULL;
+
+    QSIMPLEQ_FOREACH(rule, &s->active_rules, active_next) {
+        if (rule->options.inject.sector == -1) {
+            break;
+        }
+    }
+
+    if (rule && rule->options.inject.error) {
+        return inject_error(bs, cb, opaque, rule);
+    }
+
+    return bdrv_aio_flush(bs->file, cb, opaque);
+}
+
 
 static void blkdebug_close(BlockDriverState *bs)
 {
@@ -687,6 +719,98 @@ static int64_t blkdebug_getlength(BlockDriverState *bs)
     return bdrv_getlength(bs->file);
 }
 
+static void blkdebug_refresh_filename(BlockDriverState *bs)
+{
+    BDRVBlkdebugState *s = bs->opaque;
+    struct BlkdebugRule *rule;
+    QDict *opts;
+    QList *inject_error_list = NULL, *set_state_list = NULL;
+    QList *suspend_list = NULL;
+    int event;
+
+    if (!bs->file->full_open_options) {
+        /* The config file cannot be recreated, so creating a plain filename
+         * is impossible */
+        return;
+    }
+
+    opts = qdict_new();
+    qdict_put_obj(opts, "driver", QOBJECT(qstring_from_str("blkdebug")));
+
+    QINCREF(bs->file->full_open_options);
+    qdict_put_obj(opts, "image", QOBJECT(bs->file->full_open_options));
+
+    for (event = 0; event < BLKDBG_EVENT_MAX; event++) {
+        QLIST_FOREACH(rule, &s->rules[event], next) {
+            if (rule->action == ACTION_INJECT_ERROR) {
+                QDict *inject_error = qdict_new();
+
+                qdict_put_obj(inject_error, "event", QOBJECT(qstring_from_str(
+                              BlkdebugEvent_lookup[rule->event])));
+                qdict_put_obj(inject_error, "state",
+                              QOBJECT(qint_from_int(rule->state)));
+                qdict_put_obj(inject_error, "errno", QOBJECT(qint_from_int(
+                              rule->options.inject.error)));
+                qdict_put_obj(inject_error, "sector", QOBJECT(qint_from_int(
+                              rule->options.inject.sector)));
+                qdict_put_obj(inject_error, "once", QOBJECT(qbool_from_int(
+                              rule->options.inject.once)));
+                qdict_put_obj(inject_error, "immediately",
+                              QOBJECT(qbool_from_int(
+                              rule->options.inject.immediately)));
+
+                if (!inject_error_list) {
+                    inject_error_list = qlist_new();
+                }
+
+                qlist_append_obj(inject_error_list, QOBJECT(inject_error));
+            } else if (rule->action == ACTION_SET_STATE) {
+                QDict *set_state = qdict_new();
+
+                qdict_put_obj(set_state, "event", QOBJECT(qstring_from_str(
+                              BlkdebugEvent_lookup[rule->event])));
+                qdict_put_obj(set_state, "state",
+                              QOBJECT(qint_from_int(rule->state)));
+                qdict_put_obj(set_state, "new_state", QOBJECT(qint_from_int(
+                              rule->options.set_state.new_state)));
+
+                if (!set_state_list) {
+                    set_state_list = qlist_new();
+                }
+
+                qlist_append_obj(set_state_list, QOBJECT(set_state));
+            } else if (rule->action == ACTION_SUSPEND) {
+                QDict *suspend = qdict_new();
+
+                qdict_put_obj(suspend, "event", QOBJECT(qstring_from_str(
+                              BlkdebugEvent_lookup[rule->event])));
+                qdict_put_obj(suspend, "state",
+                              QOBJECT(qint_from_int(rule->state)));
+                qdict_put_obj(suspend, "tag", QOBJECT(qstring_from_str(
+                              rule->options.suspend.tag)));
+
+                if (!suspend_list) {
+                    suspend_list = qlist_new();
+                }
+
+                qlist_append_obj(suspend_list, QOBJECT(suspend));
+            }
+        }
+    }
+
+    if (inject_error_list) {
+        qdict_put_obj(opts, "inject-error", QOBJECT(inject_error_list));
+    }
+    if (set_state_list) {
+        qdict_put_obj(opts, "set-state", QOBJECT(set_state_list));
+    }
+    if (suspend_list) {
+        qdict_put_obj(opts, "suspend", QOBJECT(suspend_list));
+    }
+
+    bs->full_open_options = opts;
+}
+
 static BlockDriver bdrv_blkdebug = {
     .format_name            = "blkdebug",
     .protocol_name          = "blkdebug",
@@ -696,9 +820,11 @@ static BlockDriver bdrv_blkdebug = {
     .bdrv_file_open         = blkdebug_open,
     .bdrv_close             = blkdebug_close,
     .bdrv_getlength         = blkdebug_getlength,
+    .bdrv_refresh_filename  = blkdebug_refresh_filename,
 
     .bdrv_aio_readv         = blkdebug_aio_readv,
     .bdrv_aio_writev        = blkdebug_aio_writev,
+    .bdrv_aio_flush         = blkdebug_aio_flush,
 
     .bdrv_debug_event           = blkdebug_debug_event,
     .bdrv_debug_breakpoint      = blkdebug_debug_breakpoint,
diff --git a/block/blkverify.c b/block/blkverify.c
index 621b78593..438dff8bc 100644
--- a/block/blkverify.c
+++ b/block/blkverify.c
@@ -10,6 +10,8 @@
 #include <stdarg.h>
 #include "qemu/sockets.h" /* for EINPROGRESS on Windows */
 #include "block/block_int.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qstring.h"
 
 typedef struct {
     BlockDriverState *test_file;
@@ -17,7 +19,7 @@ typedef struct {
 
 typedef struct BlkverifyAIOCB BlkverifyAIOCB;
 struct BlkverifyAIOCB {
-    BlockDriverAIOCB common;
+    BlockAIOCB common;
     QEMUBH *bh;
 
     /* Request metadata */
@@ -27,7 +29,6 @@ struct BlkverifyAIOCB {
 
     int ret;                    /* first completed request's result */
     unsigned int done;          /* completion counter */
-    bool *finished;             /* completion signal for cancel */
 
     QEMUIOVector *qiov;         /* user I/O vector */
     QEMUIOVector raw_qiov;      /* cloned I/O vector for raw file */
@@ -36,22 +37,8 @@ struct BlkverifyAIOCB {
     void (*verify)(BlkverifyAIOCB *acb);
 };
 
-static void blkverify_aio_cancel(BlockDriverAIOCB *blockacb)
-{
-    BlkverifyAIOCB *acb = (BlkverifyAIOCB *)blockacb;
-    AioContext *aio_context = bdrv_get_aio_context(blockacb->bs);
-    bool finished = false;
-
-    /* Wait until request completes, invokes its callback, and frees itself */
-    acb->finished = &finished;
-    while (!finished) {
-        aio_poll(aio_context, true);
-    }
-}
-
 static const AIOCBInfo blkverify_aiocb_info = {
     .aiocb_size         = sizeof(BlkverifyAIOCB),
-    .cancel             = blkverify_aio_cancel,
 };
 
 static void GCC_FMT_ATTR(2, 3) blkverify_err(BlkverifyAIOCB *acb,
@@ -156,6 +143,7 @@ static int blkverify_open(BlockDriverState *bs, QDict *options, int flags,
 
     ret = 0;
 fail:
+    qemu_opts_del(opts);
     return ret;
 }
 
@@ -177,7 +165,7 @@ static int64_t blkverify_getlength(BlockDriverState *bs)
 static BlkverifyAIOCB *blkverify_aio_get(BlockDriverState *bs, bool is_write,
                                          int64_t sector_num, QEMUIOVector *qiov,
                                          int nb_sectors,
-                                         BlockDriverCompletionFunc *cb,
+                                         BlockCompletionFunc *cb,
                                          void *opaque)
 {
     BlkverifyAIOCB *acb = qemu_aio_get(&blkverify_aiocb_info, bs, cb, opaque);
@@ -191,7 +179,6 @@ static BlkverifyAIOCB *blkverify_aio_get(BlockDriverState *bs, bool is_write,
     acb->qiov = qiov;
     acb->buf = NULL;
     acb->verify = NULL;
-    acb->finished = NULL;
     return acb;
 }
 
@@ -205,10 +192,7 @@ static void blkverify_aio_bh(void *opaque)
         qemu_vfree(acb->buf);
     }
     acb->common.cb(acb->common.opaque, acb->ret);
-    if (acb->finished) {
-        *acb->finished = true;
-    }
-    qemu_aio_release(acb);
+    qemu_aio_unref(acb);
 }
 
 static void blkverify_aio_cb(void *opaque, int ret)
@@ -245,9 +229,9 @@ static void blkverify_verify_readv(BlkverifyAIOCB *acb)
     }
 }
 
-static BlockDriverAIOCB *blkverify_aio_readv(BlockDriverState *bs,
+static BlockAIOCB *blkverify_aio_readv(BlockDriverState *bs,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque)
+        BlockCompletionFunc *cb, void *opaque)
 {
     BDRVBlkverifyState *s = bs->opaque;
     BlkverifyAIOCB *acb = blkverify_aio_get(bs, false, sector_num, qiov,
@@ -265,9 +249,9 @@ static BlockDriverAIOCB *blkverify_aio_readv(BlockDriverState *bs,
     return &acb->common;
 }
 
-static BlockDriverAIOCB *blkverify_aio_writev(BlockDriverState *bs,
+static BlockAIOCB *blkverify_aio_writev(BlockDriverState *bs,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque)
+        BlockCompletionFunc *cb, void *opaque)
 {
     BDRVBlkverifyState *s = bs->opaque;
     BlkverifyAIOCB *acb = blkverify_aio_get(bs, true, sector_num, qiov,
@@ -280,9 +264,9 @@ static BlockDriverAIOCB *blkverify_aio_writev(BlockDriverState *bs,
     return &acb->common;
 }
 
-static BlockDriverAIOCB *blkverify_aio_flush(BlockDriverState *bs,
-                                             BlockDriverCompletionFunc *cb,
-                                             void *opaque)
+static BlockAIOCB *blkverify_aio_flush(BlockDriverState *bs,
+                                       BlockCompletionFunc *cb,
+                                       void *opaque)
 {
     BDRVBlkverifyState *s = bs->opaque;
 
@@ -320,6 +304,32 @@ static void blkverify_attach_aio_context(BlockDriverState *bs,
     bdrv_attach_aio_context(s->test_file, new_context);
 }
 
+static void blkverify_refresh_filename(BlockDriverState *bs)
+{
+    BDRVBlkverifyState *s = bs->opaque;
+
+    /* bs->file has already been refreshed */
+    bdrv_refresh_filename(s->test_file);
+
+    if (bs->file->full_open_options && s->test_file->full_open_options) {
+        QDict *opts = qdict_new();
+        qdict_put_obj(opts, "driver", QOBJECT(qstring_from_str("blkverify")));
+
+        QINCREF(bs->file->full_open_options);
+        qdict_put_obj(opts, "raw", QOBJECT(bs->file->full_open_options));
+        QINCREF(s->test_file->full_open_options);
+        qdict_put_obj(opts, "test", QOBJECT(s->test_file->full_open_options));
+
+        bs->full_open_options = opts;
+    }
+
+    if (bs->file->exact_filename[0] && s->test_file->exact_filename[0]) {
+        snprintf(bs->exact_filename, sizeof(bs->exact_filename),
+                 "blkverify:%s:%s",
+                 bs->file->exact_filename, s->test_file->exact_filename);
+    }
+}
+
 static BlockDriver bdrv_blkverify = {
     .format_name                      = "blkverify",
     .protocol_name                    = "blkverify",
@@ -329,6 +339,7 @@ static BlockDriver bdrv_blkverify = {
     .bdrv_file_open                   = blkverify_open,
     .bdrv_close                       = blkverify_close,
     .bdrv_getlength                   = blkverify_getlength,
+    .bdrv_refresh_filename            = blkverify_refresh_filename,
 
     .bdrv_aio_readv                   = blkverify_aio_readv,
     .bdrv_aio_writev                  = blkverify_aio_writev,
diff --git a/block/block-backend.c b/block/block-backend.c
new file mode 100644
index 000000000..d0692b18e
--- /dev/null
+++ b/block/block-backend.c
@@ -0,0 +1,631 @@
+/*
+ * QEMU Block backends
+ *
+ * Copyright (C) 2014 Red Hat, Inc.
+ *
+ * Authors:
+ *  Markus Armbruster <armbru@redhat.com>,
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1
+ * or later.  See the COPYING.LIB file in the top-level directory.
+ */
+
+#include "sysemu/block-backend.h"
+#include "block/block_int.h"
+#include "sysemu/blockdev.h"
+#include "qapi-event.h"
+
+/* Number of coroutines to reserve per attached device model */
+#define COROUTINE_POOL_RESERVATION 64
+
+struct BlockBackend {
+    char *name;
+    int refcnt;
+    BlockDriverState *bs;
+    DriveInfo *legacy_dinfo;    /* null unless created by drive_new() */
+    QTAILQ_ENTRY(BlockBackend) link; /* for blk_backends */
+
+    void *dev;                  /* attached device model, if any */
+    /* TODO change to DeviceState when all users are qdevified */
+    const BlockDevOps *dev_ops;
+    void *dev_opaque;
+};
+
+static void drive_info_del(DriveInfo *dinfo);
+
+/* All the BlockBackends (except for hidden ones) */
+static QTAILQ_HEAD(, BlockBackend) blk_backends =
+    QTAILQ_HEAD_INITIALIZER(blk_backends);
+
+/*
+ * Create a new BlockBackend with @name, with a reference count of one.
+ * @name must not be null or empty.
+ * Fail if a BlockBackend with this name already exists.
+ * Store an error through @errp on failure, unless it's null.
+ * Return the new BlockBackend on success, null on failure.
+ */
+BlockBackend *blk_new(const char *name, Error **errp)
+{
+    BlockBackend *blk;
+
+    assert(name && name[0]);
+    if (!id_wellformed(name)) {
+        error_setg(errp, "Invalid device name");
+        return NULL;
+    }
+    if (blk_by_name(name)) {
+        error_setg(errp, "Device with id '%s' already exists", name);
+        return NULL;
+    }
+    if (bdrv_find_node(name)) {
+        error_setg(errp,
+                   "Device name '%s' conflicts with an existing node name",
+                   name);
+        return NULL;
+    }
+
+    blk = g_new0(BlockBackend, 1);
+    blk->name = g_strdup(name);
+    blk->refcnt = 1;
+    QTAILQ_INSERT_TAIL(&blk_backends, blk, link);
+    return blk;
+}
+
+/*
+ * Create a new BlockBackend with a new BlockDriverState attached.
+ * Otherwise just like blk_new(), which see.
+ */
+BlockBackend *blk_new_with_bs(const char *name, Error **errp)
+{
+    BlockBackend *blk;
+    BlockDriverState *bs;
+
+    blk = blk_new(name, errp);
+    if (!blk) {
+        return NULL;
+    }
+
+    bs = bdrv_new_root();
+    blk->bs = bs;
+    bs->blk = blk;
+    return blk;
+}
+
+static void blk_delete(BlockBackend *blk)
+{
+    assert(!blk->refcnt);
+    assert(!blk->dev);
+    if (blk->bs) {
+        assert(blk->bs->blk == blk);
+        blk->bs->blk = NULL;
+        bdrv_unref(blk->bs);
+        blk->bs = NULL;
+    }
+    /* Avoid double-remove after blk_hide_on_behalf_of_do_drive_del() */
+    if (blk->name[0]) {
+        QTAILQ_REMOVE(&blk_backends, blk, link);
+    }
+    g_free(blk->name);
+    drive_info_del(blk->legacy_dinfo);
+    g_free(blk);
+}
+
+static void drive_info_del(DriveInfo *dinfo)
+{
+    if (!dinfo) {
+        return;
+    }
+    qemu_opts_del(dinfo->opts);
+    g_free(dinfo->serial);
+    g_free(dinfo);
+}
+
+/*
+ * Increment @blk's reference count.
+ * @blk must not be null.
+ */
+void blk_ref(BlockBackend *blk)
+{
+    blk->refcnt++;
+}
+
+/*
+ * Decrement @blk's reference count.
+ * If this drops it to zero, destroy @blk.
+ * For convenience, do nothing if @blk is null.
+ */
+void blk_unref(BlockBackend *blk)
+{
+    if (blk) {
+        assert(blk->refcnt > 0);
+        if (!--blk->refcnt) {
+            blk_delete(blk);
+        }
+    }
+}
+
+/*
+ * Return the BlockBackend after @blk.
+ * If @blk is null, return the first one.
+ * Else, return @blk's next sibling, which may be null.
+ *
+ * To iterate over all BlockBackends, do
+ * for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
+ *     ...
+ * }
+ */
+BlockBackend *blk_next(BlockBackend *blk)
+{
+    return blk ? QTAILQ_NEXT(blk, link) : QTAILQ_FIRST(&blk_backends);
+}
+
+/*
+ * Return @blk's name, a non-null string.
+ * Wart: the name is empty iff @blk has been hidden with
+ * blk_hide_on_behalf_of_do_drive_del().
+ */
+const char *blk_name(BlockBackend *blk)
+{
+    return blk->name;
+}
+
+/*
+ * Return the BlockBackend with name @name if it exists, else null.
+ * @name must not be null.
+ */
+BlockBackend *blk_by_name(const char *name)
+{
+    BlockBackend *blk;
+
+    assert(name);
+    QTAILQ_FOREACH(blk, &blk_backends, link) {
+        if (!strcmp(name, blk->name)) {
+            return blk;
+        }
+    }
+    return NULL;
+}
+
+/*
+ * Return the BlockDriverState attached to @blk if any, else null.
+ */
+BlockDriverState *blk_bs(BlockBackend *blk)
+{
+    return blk->bs;
+}
+
+/*
+ * Return @blk's DriveInfo if any, else null.
+ */
+DriveInfo *blk_legacy_dinfo(BlockBackend *blk)
+{
+    return blk->legacy_dinfo;
+}
+
+/*
+ * Set @blk's DriveInfo to @dinfo, and return it.
+ * @blk must not have a DriveInfo set already.
+ * No other BlockBackend may have the same DriveInfo set.
+ */
+DriveInfo *blk_set_legacy_dinfo(BlockBackend *blk, DriveInfo *dinfo)
+{
+    assert(!blk->legacy_dinfo);
+    return blk->legacy_dinfo = dinfo;
+}
+
+/*
+ * Return the BlockBackend with DriveInfo @dinfo.
+ * It must exist.
+ */
+BlockBackend *blk_by_legacy_dinfo(DriveInfo *dinfo)
+{
+    BlockBackend *blk;
+
+    QTAILQ_FOREACH(blk, &blk_backends, link) {
+        if (blk->legacy_dinfo == dinfo) {
+            return blk;
+        }
+    }
+    abort();
+}
+
+/*
+ * Hide @blk.
+ * @blk must not have been hidden already.
+ * Make attached BlockDriverState, if any, anonymous.
+ * Once hidden, @blk is invisible to all functions that don't receive
+ * it as argument.  For example, blk_by_name() won't return it.
+ * Strictly for use by do_drive_del().
+ * TODO get rid of it!
+ */
+void blk_hide_on_behalf_of_do_drive_del(BlockBackend *blk)
+{
+    QTAILQ_REMOVE(&blk_backends, blk, link);
+    blk->name[0] = 0;
+    if (blk->bs) {
+        bdrv_make_anon(blk->bs);
+    }
+}
+
+/*
+ * Attach device model @dev to @blk.
+ * Return 0 on success, -EBUSY when a device model is attached already.
+ */
+int blk_attach_dev(BlockBackend *blk, void *dev)
+/* TODO change to DeviceState *dev when all users are qdevified */
+{
+    if (blk->dev) {
+        return -EBUSY;
+    }
+    blk_ref(blk);
+    blk->dev = dev;
+    bdrv_iostatus_reset(blk->bs);
+
+    /* We're expecting I/O from the device so bump up coroutine pool size */
+    qemu_coroutine_adjust_pool_size(COROUTINE_POOL_RESERVATION);
+    return 0;
+}
+
+/*
+ * Attach device model @dev to @blk.
+ * @blk must not have a device model attached already.
+ * TODO qdevified devices don't use this, remove when devices are qdevified
+ */
+void blk_attach_dev_nofail(BlockBackend *blk, void *dev)
+{
+    if (blk_attach_dev(blk, dev) < 0) {
+        abort();
+    }
+}
+
+/*
+ * Detach device model @dev from @blk.
+ * @dev must be currently attached to @blk.
+ */
+void blk_detach_dev(BlockBackend *blk, void *dev)
+/* TODO change to DeviceState *dev when all users are qdevified */
+{
+    assert(blk->dev == dev);
+    blk->dev = NULL;
+    blk->dev_ops = NULL;
+    blk->dev_opaque = NULL;
+    bdrv_set_guest_block_size(blk->bs, 512);
+    qemu_coroutine_adjust_pool_size(-COROUTINE_POOL_RESERVATION);
+    blk_unref(blk);
+}
+
+/*
+ * Return the device model attached to @blk if any, else null.
+ */
+void *blk_get_attached_dev(BlockBackend *blk)
+/* TODO change to return DeviceState * when all users are qdevified */
+{
+    return blk->dev;
+}
+
+/*
+ * Set @blk's device model callbacks to @ops.
+ * @opaque is the opaque argument to pass to the callbacks.
+ * This is for use by device models.
+ */
+void blk_set_dev_ops(BlockBackend *blk, const BlockDevOps *ops,
+                     void *opaque)
+{
+    blk->dev_ops = ops;
+    blk->dev_opaque = opaque;
+}
+
+/*
+ * Notify @blk's attached device model of media change.
+ * If @load is true, notify of media load.
+ * Else, notify of media eject.
+ * Also send DEVICE_TRAY_MOVED events as appropriate.
+ */
+void blk_dev_change_media_cb(BlockBackend *blk, bool load)
+{
+    if (blk->dev_ops && blk->dev_ops->change_media_cb) {
+        bool tray_was_closed = !blk_dev_is_tray_open(blk);
+
+        blk->dev_ops->change_media_cb(blk->dev_opaque, load);
+        if (tray_was_closed) {
+            /* tray open */
+            qapi_event_send_device_tray_moved(blk_name(blk),
+                                              true, &error_abort);
+        }
+        if (load) {
+            /* tray close */
+            qapi_event_send_device_tray_moved(blk_name(blk),
+                                              false, &error_abort);
+        }
+    }
+}
+
+/*
+ * Does @blk's attached device model have removable media?
+ * %true if no device model is attached.
+ */
+bool blk_dev_has_removable_media(BlockBackend *blk)
+{
+    return !blk->dev || (blk->dev_ops && blk->dev_ops->change_media_cb);
+}
+
+/*
+ * Notify @blk's attached device model of a media eject request.
+ * If @force is true, the medium is about to be yanked out forcefully.
+ */
+void blk_dev_eject_request(BlockBackend *blk, bool force)
+{
+    if (blk->dev_ops && blk->dev_ops->eject_request_cb) {
+        blk->dev_ops->eject_request_cb(blk->dev_opaque, force);
+    }
+}
+
+/*
+ * Does @blk's attached device model have a tray, and is it open?
+ */
+bool blk_dev_is_tray_open(BlockBackend *blk)
+{
+    if (blk->dev_ops && blk->dev_ops->is_tray_open) {
+        return blk->dev_ops->is_tray_open(blk->dev_opaque);
+    }
+    return false;
+}
+
+/*
+ * Does @blk's attached device model have the medium locked?
+ * %false if the device model has no such lock.
+ */
+bool blk_dev_is_medium_locked(BlockBackend *blk)
+{
+    if (blk->dev_ops && blk->dev_ops->is_medium_locked) {
+        return blk->dev_ops->is_medium_locked(blk->dev_opaque);
+    }
+    return false;
+}
+
+/*
+ * Notify @blk's attached device model of a backend size change.
+ */
+void blk_dev_resize_cb(BlockBackend *blk)
+{
+    if (blk->dev_ops && blk->dev_ops->resize_cb) {
+        blk->dev_ops->resize_cb(blk->dev_opaque);
+    }
+}
+
+void blk_iostatus_enable(BlockBackend *blk)
+{
+    bdrv_iostatus_enable(blk->bs);
+}
+
+int blk_read(BlockBackend *blk, int64_t sector_num, uint8_t *buf,
+             int nb_sectors)
+{
+    return bdrv_read(blk->bs, sector_num, buf, nb_sectors);
+}
+
+int blk_read_unthrottled(BlockBackend *blk, int64_t sector_num, uint8_t *buf,
+                         int nb_sectors)
+{
+    return bdrv_read_unthrottled(blk->bs, sector_num, buf, nb_sectors);
+}
+
+int blk_write(BlockBackend *blk, int64_t sector_num, const uint8_t *buf,
+              int nb_sectors)
+{
+    return bdrv_write(blk->bs, sector_num, buf, nb_sectors);
+}
+
+BlockAIOCB *blk_aio_write_zeroes(BlockBackend *blk, int64_t sector_num,
+                                 int nb_sectors, BdrvRequestFlags flags,
+                                 BlockCompletionFunc *cb, void *opaque)
+{
+    return bdrv_aio_write_zeroes(blk->bs, sector_num, nb_sectors, flags,
+                                 cb, opaque);
+}
+
+int blk_pread(BlockBackend *blk, int64_t offset, void *buf, int count)
+{
+    return bdrv_pread(blk->bs, offset, buf, count);
+}
+
+int blk_pwrite(BlockBackend *blk, int64_t offset, const void *buf, int count)
+{
+    return bdrv_pwrite(blk->bs, offset, buf, count);
+}
+
+int64_t blk_getlength(BlockBackend *blk)
+{
+    return bdrv_getlength(blk->bs);
+}
+
+void blk_get_geometry(BlockBackend *blk, uint64_t *nb_sectors_ptr)
+{
+    bdrv_get_geometry(blk->bs, nb_sectors_ptr);
+}
+
+BlockAIOCB *blk_aio_readv(BlockBackend *blk, int64_t sector_num,
+                          QEMUIOVector *iov, int nb_sectors,
+                          BlockCompletionFunc *cb, void *opaque)
+{
+    return bdrv_aio_readv(blk->bs, sector_num, iov, nb_sectors, cb, opaque);
+}
+
+BlockAIOCB *blk_aio_writev(BlockBackend *blk, int64_t sector_num,
+                           QEMUIOVector *iov, int nb_sectors,
+                           BlockCompletionFunc *cb, void *opaque)
+{
+    return bdrv_aio_writev(blk->bs, sector_num, iov, nb_sectors, cb, opaque);
+}
+
+BlockAIOCB *blk_aio_flush(BlockBackend *blk,
+                          BlockCompletionFunc *cb, void *opaque)
+{
+    return bdrv_aio_flush(blk->bs, cb, opaque);
+}
+
+BlockAIOCB *blk_aio_discard(BlockBackend *blk,
+                            int64_t sector_num, int nb_sectors,
+                            BlockCompletionFunc *cb, void *opaque)
+{
+    return bdrv_aio_discard(blk->bs, sector_num, nb_sectors, cb, opaque);
+}
+
+void blk_aio_cancel(BlockAIOCB *acb)
+{
+    bdrv_aio_cancel(acb);
+}
+
+void blk_aio_cancel_async(BlockAIOCB *acb)
+{
+    bdrv_aio_cancel_async(acb);
+}
+
+int blk_aio_multiwrite(BlockBackend *blk, BlockRequest *reqs, int num_reqs)
+{
+    return bdrv_aio_multiwrite(blk->bs, reqs, num_reqs);
+}
+
+int blk_ioctl(BlockBackend *blk, unsigned long int req, void *buf)
+{
+    return bdrv_ioctl(blk->bs, req, buf);
+}
+
+BlockAIOCB *blk_aio_ioctl(BlockBackend *blk, unsigned long int req, void *buf,
+                          BlockCompletionFunc *cb, void *opaque)
+{
+    return bdrv_aio_ioctl(blk->bs, req, buf, cb, opaque);
+}
+
+int blk_flush(BlockBackend *blk)
+{
+    return bdrv_flush(blk->bs);
+}
+
+int blk_flush_all(void)
+{
+    return bdrv_flush_all();
+}
+
+void blk_drain_all(void)
+{
+    bdrv_drain_all();
+}
+
+BlockdevOnError blk_get_on_error(BlockBackend *blk, bool is_read)
+{
+    return bdrv_get_on_error(blk->bs, is_read);
+}
+
+BlockErrorAction blk_get_error_action(BlockBackend *blk, bool is_read,
+                                      int error)
+{
+    return bdrv_get_error_action(blk->bs, is_read, error);
+}
+
+void blk_error_action(BlockBackend *blk, BlockErrorAction action,
+                      bool is_read, int error)
+{
+    bdrv_error_action(blk->bs, action, is_read, error);
+}
+
+int blk_is_read_only(BlockBackend *blk)
+{
+    return bdrv_is_read_only(blk->bs);
+}
+
+int blk_is_sg(BlockBackend *blk)
+{
+    return bdrv_is_sg(blk->bs);
+}
+
+int blk_enable_write_cache(BlockBackend *blk)
+{
+    return bdrv_enable_write_cache(blk->bs);
+}
+
+void blk_set_enable_write_cache(BlockBackend *blk, bool wce)
+{
+    bdrv_set_enable_write_cache(blk->bs, wce);
+}
+
+int blk_is_inserted(BlockBackend *blk)
+{
+    return bdrv_is_inserted(blk->bs);
+}
+
+void blk_lock_medium(BlockBackend *blk, bool locked)
+{
+    bdrv_lock_medium(blk->bs, locked);
+}
+
+void blk_eject(BlockBackend *blk, bool eject_flag)
+{
+    bdrv_eject(blk->bs, eject_flag);
+}
+
+int blk_get_flags(BlockBackend *blk)
+{
+    return bdrv_get_flags(blk->bs);
+}
+
+void blk_set_guest_block_size(BlockBackend *blk, int align)
+{
+    bdrv_set_guest_block_size(blk->bs, align);
+}
+
+void *blk_blockalign(BlockBackend *blk, size_t size)
+{
+    return qemu_blockalign(blk ? blk->bs : NULL, size);
+}
+
+bool blk_op_is_blocked(BlockBackend *blk, BlockOpType op, Error **errp)
+{
+    return bdrv_op_is_blocked(blk->bs, op, errp);
+}
+
+void blk_op_unblock(BlockBackend *blk, BlockOpType op, Error *reason)
+{
+    bdrv_op_unblock(blk->bs, op, reason);
+}
+
+void blk_op_block_all(BlockBackend *blk, Error *reason)
+{
+    bdrv_op_block_all(blk->bs, reason);
+}
+
+void blk_op_unblock_all(BlockBackend *blk, Error *reason)
+{
+    bdrv_op_unblock_all(blk->bs, reason);
+}
+
+AioContext *blk_get_aio_context(BlockBackend *blk)
+{
+    return bdrv_get_aio_context(blk->bs);
+}
+
+void blk_set_aio_context(BlockBackend *blk, AioContext *new_context)
+{
+    bdrv_set_aio_context(blk->bs, new_context);
+}
+
+void blk_io_plug(BlockBackend *blk)
+{
+    bdrv_io_plug(blk->bs);
+}
+
+void blk_io_unplug(BlockBackend *blk)
+{
+    bdrv_io_unplug(blk->bs);
+}
+
+BlockAcctStats *blk_get_stats(BlockBackend *blk)
+{
+    return bdrv_get_stats(blk->bs);
+}
+
+void *blk_aio_get(const AIOCBInfo *aiocb_info, BlockBackend *blk,
+                  BlockCompletionFunc *cb, void *opaque)
+{
+    return qemu_aio_get(aiocb_info, blk_bs(blk), cb, opaque);
+}
diff --git a/block/bochs.c b/block/bochs.c
index eba23df33..199ac2b9a 100644
--- a/block/bochs.c
+++ b/block/bochs.c
@@ -131,7 +131,11 @@ static int bochs_open(BlockDriverState *bs, QDict *options, int flags,
         return -EFBIG;
     }
 
-    s->catalog_bitmap = g_malloc(s->catalog_size * 4);
+    s->catalog_bitmap = g_try_new(uint32_t, s->catalog_size);
+    if (s->catalog_size && s->catalog_bitmap == NULL) {
+        error_setg(errp, "Could not allocate memory for catalog");
+        return -ENOMEM;
+    }
 
     ret = bdrv_pread(bs->file, le32_to_cpu(bochs.header), s->catalog_bitmap,
                      s->catalog_size * 4);
diff --git a/block/cloop.c b/block/cloop.c
index 845773792..f328be06f 100644
--- a/block/cloop.c
+++ b/block/cloop.c
@@ -116,7 +116,12 @@ static int cloop_open(BlockDriverState *bs, QDict *options, int flags,
                    "try increasing block size");
         return -EINVAL;
     }
-    s->offsets = g_malloc(offsets_size);
+
+    s->offsets = g_try_malloc(offsets_size);
+    if (s->offsets == NULL) {
+        error_setg(errp, "Could not allocate offsets table");
+        return -ENOMEM;
+    }
 
     ret = bdrv_pread(bs->file, 128 + 4 + 4, s->offsets, offsets_size);
     if (ret < 0) {
@@ -158,8 +163,20 @@ static int cloop_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     /* initialize zlib engine */
-    s->compressed_block = g_malloc(max_compressed_block_size + 1);
-    s->uncompressed_block = g_malloc(s->block_size);
+    s->compressed_block = g_try_malloc(max_compressed_block_size + 1);
+    if (s->compressed_block == NULL) {
+        error_setg(errp, "Could not allocate compressed_block");
+        ret = -ENOMEM;
+        goto fail;
+    }
+
+    s->uncompressed_block = g_try_malloc(s->block_size);
+    if (s->uncompressed_block == NULL) {
+        error_setg(errp, "Could not allocate uncompressed_block");
+        ret = -ENOMEM;
+        goto fail;
+    }
+
     if (inflateInit(&s->zstream) != Z_OK) {
         ret = -EINVAL;
         goto fail;
diff --git a/block/commit.c b/block/commit.c
index 91517d351..cfa2bbebc 100644
--- a/block/commit.c
+++ b/block/commit.c
@@ -60,17 +60,50 @@ static int coroutine_fn commit_populate(BlockDriverState *bs,
     return 0;
 }
 
-static void coroutine_fn commit_run(void *opaque)
+typedef struct {
+    int ret;
+} CommitCompleteData;
+
+static void commit_complete(BlockJob *job, void *opaque)
 {
-    CommitBlockJob *s = opaque;
+    CommitBlockJob *s = container_of(job, CommitBlockJob, common);
+    CommitCompleteData *data = opaque;
     BlockDriverState *active = s->active;
     BlockDriverState *top = s->top;
     BlockDriverState *base = s->base;
     BlockDriverState *overlay_bs;
+    int ret = data->ret;
+
+    if (!block_job_is_cancelled(&s->common) && ret == 0) {
+        /* success */
+        ret = bdrv_drop_intermediate(active, top, base, s->backing_file_str);
+    }
+
+    /* restore base open flags here if appropriate (e.g., change the base back
+     * to r/o). These reopens do not need to be atomic, since we won't abort
+     * even on failure here */
+    if (s->base_flags != bdrv_get_flags(base)) {
+        bdrv_reopen(base, s->base_flags, NULL);
+    }
+    overlay_bs = bdrv_find_overlay(active, top);
+    if (overlay_bs && s->orig_overlay_flags != bdrv_get_flags(overlay_bs)) {
+        bdrv_reopen(overlay_bs, s->orig_overlay_flags, NULL);
+    }
+    g_free(s->backing_file_str);
+    block_job_completed(&s->common, ret);
+    g_free(data);
+}
+
+static void coroutine_fn commit_run(void *opaque)
+{
+    CommitBlockJob *s = opaque;
+    CommitCompleteData *data;
+    BlockDriverState *top = s->top;
+    BlockDriverState *base = s->base;
     int64_t sector_num, end;
     int ret = 0;
     int n = 0;
-    void *buf;
+    void *buf = NULL;
     int bytes_written = 0;
     int64_t base_len;
 
@@ -78,18 +111,18 @@ static void coroutine_fn commit_run(void *opaque)
 
 
     if (s->common.len < 0) {
-        goto exit_restore_reopen;
+        goto out;
     }
 
     ret = base_len = bdrv_getlength(base);
     if (base_len < 0) {
-        goto exit_restore_reopen;
+        goto out;
     }
 
     if (base_len < s->common.len) {
         ret = bdrv_truncate(base, s->common.len);
         if (ret) {
-            goto exit_restore_reopen;
+            goto out;
         }
     }
 
@@ -128,7 +161,7 @@ wait:
             if (s->on_error == BLOCKDEV_ON_ERROR_STOP ||
                 s->on_error == BLOCKDEV_ON_ERROR_REPORT||
                 (s->on_error == BLOCKDEV_ON_ERROR_ENOSPC && ret == -ENOSPC)) {
-                goto exit_free_buf;
+                goto out;
             } else {
                 n = 0;
                 continue;
@@ -140,27 +173,12 @@ wait:
 
     ret = 0;
 
-    if (!block_job_is_cancelled(&s->common) && sector_num == end) {
-        /* success */
-        ret = bdrv_drop_intermediate(active, top, base, s->backing_file_str);
-    }
-
-exit_free_buf:
+out:
     qemu_vfree(buf);
 
-exit_restore_reopen:
-    /* restore base open flags here if appropriate (e.g., change the base back
-     * to r/o). These reopens do not need to be atomic, since we won't abort
-     * even on failure here */
-    if (s->base_flags != bdrv_get_flags(base)) {
-        bdrv_reopen(base, s->base_flags, NULL);
-    }
-    overlay_bs = bdrv_find_overlay(active, top);
-    if (overlay_bs && s->orig_overlay_flags != bdrv_get_flags(overlay_bs)) {
-        bdrv_reopen(overlay_bs, s->orig_overlay_flags, NULL);
-    }
-    g_free(s->backing_file_str);
-    block_job_completed(&s->common, ret);
+    data = g_malloc(sizeof(*data));
+    data->ret = ret;
+    block_job_defer_to_main_loop(&s->common, commit_complete, data);
 }
 
 static void commit_set_speed(BlockJob *job, int64_t speed, Error **errp)
@@ -182,7 +200,7 @@ static const BlockJobDriver commit_job_driver = {
 
 void commit_start(BlockDriverState *bs, BlockDriverState *base,
                   BlockDriverState *top, int64_t speed,
-                  BlockdevOnError on_error, BlockDriverCompletionFunc *cb,
+                  BlockdevOnError on_error, BlockCompletionFunc *cb,
                   void *opaque, const char *backing_file_str, Error **errp)
 {
     CommitBlockJob *s;
diff --git a/block/cow.c b/block/cow.c
deleted file mode 100644
index 6ee483327..000000000
--- a/block/cow.c
+++ /dev/null
@@ -1,432 +0,0 @@
-/*
- * Block driver for the COW format
- *
- * Copyright (c) 2004 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#include "qemu-common.h"
-#include "block/block_int.h"
-#include "qemu/module.h"
-
-/**************************************************************/
-/* COW block driver using file system holes */
-
-/* user mode linux compatible COW file */
-#define COW_MAGIC 0x4f4f4f4d  /* MOOO */
-#define COW_VERSION 2
-
-struct cow_header_v2 {
-    uint32_t magic;
-    uint32_t version;
-    char backing_file[1024];
-    int32_t mtime;
-    uint64_t size;
-    uint32_t sectorsize;
-};
-
-typedef struct BDRVCowState {
-    CoMutex lock;
-    int64_t cow_sectors_offset;
-} BDRVCowState;
-
-static int cow_probe(const uint8_t *buf, int buf_size, const char *filename)
-{
-    const struct cow_header_v2 *cow_header = (const void *)buf;
-
-    if (buf_size >= sizeof(struct cow_header_v2) &&
-        be32_to_cpu(cow_header->magic) == COW_MAGIC &&
-        be32_to_cpu(cow_header->version) == COW_VERSION)
-        return 100;
-    else
-        return 0;
-}
-
-static int cow_open(BlockDriverState *bs, QDict *options, int flags,
-                    Error **errp)
-{
-    BDRVCowState *s = bs->opaque;
-    struct cow_header_v2 cow_header;
-    int bitmap_size;
-    int64_t size;
-    int ret;
-
-    /* see if it is a cow image */
-    ret = bdrv_pread(bs->file, 0, &cow_header, sizeof(cow_header));
-    if (ret < 0) {
-        goto fail;
-    }
-
-    if (be32_to_cpu(cow_header.magic) != COW_MAGIC) {
-        error_setg(errp, "Image not in COW format");
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    if (be32_to_cpu(cow_header.version) != COW_VERSION) {
-        char version[64];
-        snprintf(version, sizeof(version),
-               "COW version %" PRIu32, cow_header.version);
-        error_set(errp, QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
-            bs->device_name, "cow", version);
-        ret = -ENOTSUP;
-        goto fail;
-    }
-
-    /* cow image found */
-    size = be64_to_cpu(cow_header.size);
-    bs->total_sectors = size / 512;
-
-    pstrcpy(bs->backing_file, sizeof(bs->backing_file),
-            cow_header.backing_file);
-
-    bitmap_size = ((bs->total_sectors + 7) >> 3) + sizeof(cow_header);
-    s->cow_sectors_offset = (bitmap_size + 511) & ~511;
-    qemu_co_mutex_init(&s->lock);
-    return 0;
- fail:
-    return ret;
-}
-
-static inline void cow_set_bits(uint8_t *bitmap, int start, int64_t nb_sectors)
-{
-    int64_t bitnum = start, last = start + nb_sectors;
-    while (bitnum < last) {
-        if ((bitnum & 7) == 0 && bitnum + 8 <= last) {
-            bitmap[bitnum / 8] = 0xFF;
-            bitnum += 8;
-            continue;
-        }
-        bitmap[bitnum/8] |= (1 << (bitnum % 8));
-        bitnum++;
-    }
-}
-
-#define BITS_PER_BITMAP_SECTOR (512 * 8)
-
-/* Cannot use bitmap.c on big-endian machines.  */
-static int cow_test_bit(int64_t bitnum, const uint8_t *bitmap)
-{
-    return (bitmap[bitnum / 8] & (1 << (bitnum & 7))) != 0;
-}
-
-static int cow_find_streak(const uint8_t *bitmap, int value, int start, int nb_sectors)
-{
-    int streak_value = value ? 0xFF : 0;
-    int last = MIN(start + nb_sectors, BITS_PER_BITMAP_SECTOR);
-    int bitnum = start;
-    while (bitnum < last) {
-        if ((bitnum & 7) == 0 && bitmap[bitnum / 8] == streak_value) {
-            bitnum += 8;
-            continue;
-        }
-        if (cow_test_bit(bitnum, bitmap) == value) {
-            bitnum++;
-            continue;
-        }
-        break;
-    }
-    return MIN(bitnum, last) - start;
-}
-
-/* Return true if first block has been changed (ie. current version is
- * in COW file).  Set the number of continuous blocks for which that
- * is true. */
-static int coroutine_fn cow_co_is_allocated(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, int *num_same)
-{
-    int64_t bitnum = sector_num + sizeof(struct cow_header_v2) * 8;
-    uint64_t offset = (bitnum / 8) & -BDRV_SECTOR_SIZE;
-    bool first = true;
-    int changed = 0, same = 0;
-
-    do {
-        int ret;
-        uint8_t bitmap[BDRV_SECTOR_SIZE];
-
-        bitnum &= BITS_PER_BITMAP_SECTOR - 1;
-        int sector_bits = MIN(nb_sectors, BITS_PER_BITMAP_SECTOR - bitnum);
-
-        ret = bdrv_pread(bs->file, offset, &bitmap, sizeof(bitmap));
-        if (ret < 0) {
-            return ret;
-        }
-
-        if (first) {
-            changed = cow_test_bit(bitnum, bitmap);
-            first = false;
-        }
-
-        same += cow_find_streak(bitmap, changed, bitnum, nb_sectors);
-
-        bitnum += sector_bits;
-        nb_sectors -= sector_bits;
-        offset += BDRV_SECTOR_SIZE;
-    } while (nb_sectors);
-
-    *num_same = same;
-    return changed;
-}
-
-static int64_t coroutine_fn cow_co_get_block_status(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, int *num_same)
-{
-    BDRVCowState *s = bs->opaque;
-    int ret = cow_co_is_allocated(bs, sector_num, nb_sectors, num_same);
-    int64_t offset = s->cow_sectors_offset + (sector_num << BDRV_SECTOR_BITS);
-    if (ret < 0) {
-        return ret;
-    }
-    return (ret ? BDRV_BLOCK_DATA : 0) | offset | BDRV_BLOCK_OFFSET_VALID;
-}
-
-static int cow_update_bitmap(BlockDriverState *bs, int64_t sector_num,
-        int nb_sectors)
-{
-    int64_t bitnum = sector_num + sizeof(struct cow_header_v2) * 8;
-    uint64_t offset = (bitnum / 8) & -BDRV_SECTOR_SIZE;
-    bool first = true;
-    int sector_bits;
-
-    for ( ; nb_sectors;
-            bitnum += sector_bits,
-            nb_sectors -= sector_bits,
-            offset += BDRV_SECTOR_SIZE) {
-        int ret, set;
-        uint8_t bitmap[BDRV_SECTOR_SIZE];
-
-        bitnum &= BITS_PER_BITMAP_SECTOR - 1;
-        sector_bits = MIN(nb_sectors, BITS_PER_BITMAP_SECTOR - bitnum);
-
-        ret = bdrv_pread(bs->file, offset, &bitmap, sizeof(bitmap));
-        if (ret < 0) {
-            return ret;
-        }
-
-        /* Skip over any already set bits */
-        set = cow_find_streak(bitmap, 1, bitnum, sector_bits);
-        bitnum += set;
-        sector_bits -= set;
-        nb_sectors -= set;
-        if (!sector_bits) {
-            continue;
-        }
-
-        if (first) {
-            ret = bdrv_flush(bs->file);
-            if (ret < 0) {
-                return ret;
-            }
-            first = false;
-        }
-
-        cow_set_bits(bitmap, bitnum, sector_bits);
-
-        ret = bdrv_pwrite(bs->file, offset, &bitmap, sizeof(bitmap));
-        if (ret < 0) {
-            return ret;
-        }
-    }
-
-    return 0;
-}
-
-static int coroutine_fn cow_read(BlockDriverState *bs, int64_t sector_num,
-                                 uint8_t *buf, int nb_sectors)
-{
-    BDRVCowState *s = bs->opaque;
-    int ret, n;
-
-    while (nb_sectors > 0) {
-        ret = cow_co_is_allocated(bs, sector_num, nb_sectors, &n);
-        if (ret < 0) {
-            return ret;
-        }
-        if (ret) {
-            ret = bdrv_pread(bs->file,
-                        s->cow_sectors_offset + sector_num * 512,
-                        buf, n * 512);
-            if (ret < 0) {
-                return ret;
-            }
-        } else {
-            if (bs->backing_hd) {
-                /* read from the base image */
-                ret = bdrv_read(bs->backing_hd, sector_num, buf, n);
-                if (ret < 0) {
-                    return ret;
-                }
-            } else {
-                memset(buf, 0, n * 512);
-            }
-        }
-        nb_sectors -= n;
-        sector_num += n;
-        buf += n * 512;
-    }
-    return 0;
-}
-
-static coroutine_fn int cow_co_read(BlockDriverState *bs, int64_t sector_num,
-                                    uint8_t *buf, int nb_sectors)
-{
-    int ret;
-    BDRVCowState *s = bs->opaque;
-    qemu_co_mutex_lock(&s->lock);
-    ret = cow_read(bs, sector_num, buf, nb_sectors);
-    qemu_co_mutex_unlock(&s->lock);
-    return ret;
-}
-
-static int cow_write(BlockDriverState *bs, int64_t sector_num,
-                     const uint8_t *buf, int nb_sectors)
-{
-    BDRVCowState *s = bs->opaque;
-    int ret;
-
-    ret = bdrv_pwrite(bs->file, s->cow_sectors_offset + sector_num * 512,
-                      buf, nb_sectors * 512);
-    if (ret < 0) {
-        return ret;
-    }
-
-    return cow_update_bitmap(bs, sector_num, nb_sectors);
-}
-
-static coroutine_fn int cow_co_write(BlockDriverState *bs, int64_t sector_num,
-                                     const uint8_t *buf, int nb_sectors)
-{
-    int ret;
-    BDRVCowState *s = bs->opaque;
-    qemu_co_mutex_lock(&s->lock);
-    ret = cow_write(bs, sector_num, buf, nb_sectors);
-    qemu_co_mutex_unlock(&s->lock);
-    return ret;
-}
-
-static void cow_close(BlockDriverState *bs)
-{
-}
-
-static int cow_create(const char *filename, QemuOpts *opts, Error **errp)
-{
-    struct cow_header_v2 cow_header;
-    struct stat st;
-    int64_t image_sectors = 0;
-    char *image_filename = NULL;
-    Error *local_err = NULL;
-    int ret;
-    BlockDriverState *cow_bs = NULL;
-
-    /* Read out options */
-    image_sectors = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0) / 512;
-    image_filename = qemu_opt_get_del(opts, BLOCK_OPT_BACKING_FILE);
-
-    ret = bdrv_create_file(filename, opts, &local_err);
-    if (ret < 0) {
-        error_propagate(errp, local_err);
-        goto exit;
-    }
-
-    ret = bdrv_open(&cow_bs, filename, NULL, NULL,
-                    BDRV_O_RDWR | BDRV_O_PROTOCOL, NULL, &local_err);
-    if (ret < 0) {
-        error_propagate(errp, local_err);
-        goto exit;
-    }
-
-    memset(&cow_header, 0, sizeof(cow_header));
-    cow_header.magic = cpu_to_be32(COW_MAGIC);
-    cow_header.version = cpu_to_be32(COW_VERSION);
-    if (image_filename) {
-        /* Note: if no file, we put a dummy mtime */
-        cow_header.mtime = cpu_to_be32(0);
-
-        if (stat(image_filename, &st) != 0) {
-            goto mtime_fail;
-        }
-        cow_header.mtime = cpu_to_be32(st.st_mtime);
-    mtime_fail:
-        pstrcpy(cow_header.backing_file, sizeof(cow_header.backing_file),
-                image_filename);
-    }
-    cow_header.sectorsize = cpu_to_be32(512);
-    cow_header.size = cpu_to_be64(image_sectors * 512);
-    ret = bdrv_pwrite(cow_bs, 0, &cow_header, sizeof(cow_header));
-    if (ret < 0) {
-        goto exit;
-    }
-
-    /* resize to include at least all the bitmap */
-    ret = bdrv_truncate(cow_bs,
-        sizeof(cow_header) + ((image_sectors + 7) >> 3));
-    if (ret < 0) {
-        goto exit;
-    }
-
-exit:
-    g_free(image_filename);
-    if (cow_bs) {
-        bdrv_unref(cow_bs);
-    }
-    return ret;
-}
-
-static QemuOptsList cow_create_opts = {
-    .name = "cow-create-opts",
-    .head = QTAILQ_HEAD_INITIALIZER(cow_create_opts.head),
-    .desc = {
-        {
-            .name = BLOCK_OPT_SIZE,
-            .type = QEMU_OPT_SIZE,
-            .help = "Virtual disk size"
-        },
-        {
-            .name = BLOCK_OPT_BACKING_FILE,
-            .type = QEMU_OPT_STRING,
-            .help = "File name of a base image"
-        },
-        { /* end of list */ }
-    }
-};
-
-static BlockDriver bdrv_cow = {
-    .format_name    = "cow",
-    .instance_size  = sizeof(BDRVCowState),
-
-    .bdrv_probe     = cow_probe,
-    .bdrv_open      = cow_open,
-    .bdrv_close     = cow_close,
-    .bdrv_create    = cow_create,
-    .bdrv_has_zero_init     = bdrv_has_zero_init_1,
-    .supports_backing       = true,
-
-    .bdrv_read              = cow_co_read,
-    .bdrv_write             = cow_co_write,
-    .bdrv_co_get_block_status   = cow_co_get_block_status,
-
-    .create_opts    = &cow_create_opts,
-};
-
-static void bdrv_cow_init(void)
-{
-    bdrv_register(&bdrv_cow);
-}
-
-block_init(bdrv_cow_init);
diff --git a/block/curl.c b/block/curl.c
index 79ff2f1e4..bbee3ca17 100644
--- a/block/curl.c
+++ b/block/curl.c
@@ -26,7 +26,7 @@
 #include "qapi/qmp/qbool.h"
 #include <curl/curl.h>
 
-// #define DEBUG
+// #define DEBUG_CURL
 // #define DEBUG_VERBOSE
 
 #ifdef DEBUG_CURL
@@ -63,6 +63,8 @@ static CURLMcode __curl_multi_socket_action(CURLM *multi_handle,
 #define CURL_NUM_ACB    8
 #define SECTOR_SIZE     512
 #define READ_AHEAD_DEFAULT (256 * 1024)
+#define CURL_TIMEOUT_DEFAULT 5
+#define CURL_TIMEOUT_MAX 10000
 
 #define FIND_RET_NONE   0
 #define FIND_RET_OK     1
@@ -71,11 +73,13 @@ static CURLMcode __curl_multi_socket_action(CURLM *multi_handle,
 #define CURL_BLOCK_OPT_URL       "url"
 #define CURL_BLOCK_OPT_READAHEAD "readahead"
 #define CURL_BLOCK_OPT_SSLVERIFY "sslverify"
+#define CURL_BLOCK_OPT_TIMEOUT "timeout"
+#define CURL_BLOCK_OPT_COOKIE    "cookie"
 
 struct BDRVCURLState;
 
 typedef struct CURLAIOCB {
-    BlockDriverAIOCB common;
+    BlockAIOCB common;
     QEMUBH *bh;
     QEMUIOVector *qiov;
 
@@ -109,6 +113,8 @@ typedef struct BDRVCURLState {
     char *url;
     size_t readahead_size;
     bool sslverify;
+    uint64_t timeout;
+    char *cookie;
     bool accept_range;
     AioContext *aio_context;
 } BDRVCURLState;
@@ -207,7 +213,7 @@ static size_t curl_read_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
             qemu_iovec_from_buf(acb->qiov, 0, s->orig_buf + acb->start,
                                 acb->end - acb->start);
             acb->common.cb(acb->common.opaque, 0);
-            qemu_aio_release(acb);
+            qemu_aio_unref(acb);
             s->acb[i] = NULL;
         }
     }
@@ -299,7 +305,7 @@ static void curl_multi_check_completion(BDRVCURLState *s)
                     }
 
                     acb->common.cb(acb->common.opaque, -EIO);
-                    qemu_aio_release(acb);
+                    qemu_aio_unref(acb);
                     state->acb[i] = NULL;
                 }
             }
@@ -352,7 +358,7 @@ static void curl_multi_timeout_do(void *arg)
 #endif
 }
 
-static CURLState *curl_init_state(BDRVCURLState *s)
+static CURLState *curl_init_state(BlockDriverState *bs, BDRVCURLState *s)
 {
     CURLState *state = NULL;
     int i, j;
@@ -370,7 +376,7 @@ static CURLState *curl_init_state(BDRVCURLState *s)
             break;
         }
         if (!state) {
-            aio_poll(state->s->aio_context, true);
+            aio_poll(bdrv_get_aio_context(bs), true);
         }
     } while(!state);
 
@@ -382,7 +388,10 @@ static CURLState *curl_init_state(BDRVCURLState *s)
         curl_easy_setopt(state->curl, CURLOPT_URL, s->url);
         curl_easy_setopt(state->curl, CURLOPT_SSL_VERIFYPEER,
                          (long) s->sslverify);
-        curl_easy_setopt(state->curl, CURLOPT_TIMEOUT, 5);
+        if (s->cookie) {
+            curl_easy_setopt(state->curl, CURLOPT_COOKIE, s->cookie);
+        }
+        curl_easy_setopt(state->curl, CURLOPT_TIMEOUT, (long)s->timeout);
         curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION,
                          (void *)curl_read_cb);
         curl_easy_setopt(state->curl, CURLOPT_WRITEDATA, (void *)state);
@@ -489,6 +498,16 @@ static QemuOptsList runtime_opts = {
             .type = QEMU_OPT_BOOL,
             .help = "Verify SSL certificate"
         },
+        {
+            .name = CURL_BLOCK_OPT_TIMEOUT,
+            .type = QEMU_OPT_NUMBER,
+            .help = "Curl timeout"
+        },
+        {
+            .name = CURL_BLOCK_OPT_COOKIE,
+            .type = QEMU_OPT_STRING,
+            .help = "Pass the cookie or list of cookies with each request"
+        },
         { /* end of list */ }
     },
 };
@@ -501,6 +520,7 @@ static int curl_open(BlockDriverState *bs, QDict *options, int flags,
     QemuOpts *opts;
     Error *local_err = NULL;
     const char *file;
+    const char *cookie;
     double d;
 
     static int inited = 0;
@@ -525,8 +545,18 @@ static int curl_open(BlockDriverState *bs, QDict *options, int flags,
         goto out_noclean;
     }
 
+    s->timeout = qemu_opt_get_number(opts, CURL_BLOCK_OPT_TIMEOUT,
+                                     CURL_TIMEOUT_DEFAULT);
+    if (s->timeout > CURL_TIMEOUT_MAX) {
+        error_setg(errp, "timeout parameter is too large or negative");
+        goto out_noclean;
+    }
+
     s->sslverify = qemu_opt_get_bool(opts, CURL_BLOCK_OPT_SSLVERIFY, true);
 
+    cookie = qemu_opt_get(opts, CURL_BLOCK_OPT_COOKIE);
+    s->cookie = g_strdup(cookie);
+
     file = qemu_opt_get(opts, CURL_BLOCK_OPT_URL);
     if (file == NULL) {
         error_setg(errp, "curl block driver requires an 'url' option");
@@ -541,7 +571,7 @@ static int curl_open(BlockDriverState *bs, QDict *options, int flags,
     DPRINTF("CURL: Opening %s\n", file);
     s->aio_context = bdrv_get_aio_context(bs);
     s->url = g_strdup(file);
-    state = curl_init_state(s);
+    state = curl_init_state(bs, s);
     if (!state)
         goto out_noclean;
 
@@ -582,19 +612,14 @@ out:
     curl_easy_cleanup(state->curl);
     state->curl = NULL;
 out_noclean:
+    g_free(s->cookie);
     g_free(s->url);
     qemu_opts_del(opts);
     return -EINVAL;
 }
 
-static void curl_aio_cancel(BlockDriverAIOCB *blockacb)
-{
-    // Do we have to implement canceling? Seems to work without...
-}
-
 static const AIOCBInfo curl_aiocb_info = {
     .aiocb_size         = sizeof(CURLAIOCB),
-    .cancel             = curl_aio_cancel,
 };
 
 
@@ -616,7 +641,7 @@ static void curl_readv_bh_cb(void *p)
     // we can just call the callback and be done.
     switch (curl_find_buf(s, start, acb->nb_sectors * SECTOR_SIZE, acb)) {
         case FIND_RET_OK:
-            qemu_aio_release(acb);
+            qemu_aio_unref(acb);
             // fall through
         case FIND_RET_WAIT:
             return;
@@ -625,10 +650,10 @@ static void curl_readv_bh_cb(void *p)
     }
 
     // No cache found, so let's start a new request
-    state = curl_init_state(s);
+    state = curl_init_state(acb->common.bs, s);
     if (!state) {
         acb->common.cb(acb->common.opaque, -EIO);
-        qemu_aio_release(acb);
+        qemu_aio_unref(acb);
         return;
     }
 
@@ -640,7 +665,13 @@ static void curl_readv_bh_cb(void *p)
     state->buf_start = start;
     state->buf_len = acb->end + s->readahead_size;
     end = MIN(start + state->buf_len, s->len) - 1;
-    state->orig_buf = g_malloc(state->buf_len);
+    state->orig_buf = g_try_malloc(state->buf_len);
+    if (state->buf_len && state->orig_buf == NULL) {
+        curl_clean_state(state);
+        acb->common.cb(acb->common.opaque, -ENOMEM);
+        qemu_aio_unref(acb);
+        return;
+    }
     state->acb[0] = acb;
 
     snprintf(state->range, 127, "%zd-%zd", start, end);
@@ -654,9 +685,9 @@ static void curl_readv_bh_cb(void *p)
     curl_multi_socket_action(s->multi, CURL_SOCKET_TIMEOUT, 0, &running);
 }
 
-static BlockDriverAIOCB *curl_aio_readv(BlockDriverState *bs,
+static BlockAIOCB *curl_aio_readv(BlockDriverState *bs,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque)
+        BlockCompletionFunc *cb, void *opaque)
 {
     CURLAIOCB *acb;
 
@@ -678,6 +709,7 @@ static void curl_close(BlockDriverState *bs)
     DPRINTF("CURL: Close\n");
     curl_detach_aio_context(bs);
 
+    g_free(s->cookie);
     g_free(s->url);
 }
 
diff --git a/block/dmg.c b/block/dmg.c
index 1e153cd76..e455886d2 100644
--- a/block/dmg.c
+++ b/block/dmg.c
@@ -284,8 +284,15 @@ static int dmg_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     /* initialize zlib engine */
-    s->compressed_chunk = g_malloc(max_compressed_size + 1);
-    s->uncompressed_chunk = g_malloc(512 * max_sectors_per_chunk);
+    s->compressed_chunk = qemu_try_blockalign(bs->file,
+                                              max_compressed_size + 1);
+    s->uncompressed_chunk = qemu_try_blockalign(bs->file,
+                                                512 * max_sectors_per_chunk);
+    if (s->compressed_chunk == NULL || s->uncompressed_chunk == NULL) {
+        ret = -ENOMEM;
+        goto fail;
+    }
+
     if (inflateInit(&s->zstream) != Z_OK) {
         ret = -EINVAL;
         goto fail;
@@ -302,8 +309,8 @@ fail:
     g_free(s->lengths);
     g_free(s->sectors);
     g_free(s->sectorcounts);
-    g_free(s->compressed_chunk);
-    g_free(s->uncompressed_chunk);
+    qemu_vfree(s->compressed_chunk);
+    qemu_vfree(s->uncompressed_chunk);
     return ret;
 }
 
@@ -426,8 +433,8 @@ static void dmg_close(BlockDriverState *bs)
     g_free(s->lengths);
     g_free(s->sectors);
     g_free(s->sectorcounts);
-    g_free(s->compressed_chunk);
-    g_free(s->uncompressed_chunk);
+    qemu_vfree(s->compressed_chunk);
+    qemu_vfree(s->uncompressed_chunk);
 
     inflateEnd(&s->zstream);
 }
diff --git a/block/gluster.c b/block/gluster.c
index 9274dead7..1eb3a8c39 100644
--- a/block/gluster.c
+++ b/block/gluster.c
@@ -291,7 +291,7 @@ static int qemu_gluster_open(BlockDriverState *bs,  QDict *options,
     BDRVGlusterState *s = bs->opaque;
     int open_flags = 0;
     int ret = 0;
-    GlusterConf *gconf = g_malloc0(sizeof(GlusterConf));
+    GlusterConf *gconf = g_new0(GlusterConf, 1);
     QemuOpts *opts;
     Error *local_err = NULL;
     const char *filename;
@@ -351,12 +351,12 @@ static int qemu_gluster_reopen_prepare(BDRVReopenState *state,
     assert(state != NULL);
     assert(state->bs != NULL);
 
-    state->opaque = g_malloc0(sizeof(BDRVGlusterReopenState));
+    state->opaque = g_new0(BDRVGlusterReopenState, 1);
     reop_s = state->opaque;
 
     qemu_gluster_parse_flags(state->flags, &open_flags);
 
-    gconf = g_malloc0(sizeof(GlusterConf));
+    gconf = g_new0(GlusterConf, 1);
 
     reop_s->glfs = qemu_gluster_init(gconf, state->bs->filename, errp);
     if (reop_s->glfs == NULL) {
@@ -486,7 +486,7 @@ static int qemu_gluster_create(const char *filename,
     int prealloc = 0;
     int64_t total_size = 0;
     char *tmp = NULL;
-    GlusterConf *gconf = g_malloc0(sizeof(GlusterConf));
+    GlusterConf *gconf = g_new0(GlusterConf, 1);
 
     glfs = qemu_gluster_init(gconf, filename, errp);
     if (!glfs) {
@@ -494,8 +494,8 @@ static int qemu_gluster_create(const char *filename,
         goto out;
     }
 
-    total_size =
-        qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0) / BDRV_SECTOR_SIZE;
+    total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                          BDRV_SECTOR_SIZE);
 
     tmp = qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC);
     if (!tmp || !strcmp(tmp, "off")) {
@@ -516,9 +516,8 @@ static int qemu_gluster_create(const char *filename,
     if (!fd) {
         ret = -errno;
     } else {
-        if (!glfs_ftruncate(fd, total_size * BDRV_SECTOR_SIZE)) {
-            if (prealloc && qemu_gluster_zerofill(fd, 0,
-                    total_size * BDRV_SECTOR_SIZE)) {
+        if (!glfs_ftruncate(fd, total_size)) {
+            if (prealloc && qemu_gluster_zerofill(fd, 0, total_size)) {
                 ret = -errno;
             }
         } else {
diff --git a/block/iscsi.c b/block/iscsi.c
index a7bb6970a..ed375fc30 100644
--- a/block/iscsi.c
+++ b/block/iscsi.c
@@ -34,7 +34,6 @@
 #include "qemu/bitops.h"
 #include "qemu/bitmap.h"
 #include "block/block_int.h"
-#include "trace.h"
 #include "block/scsi.h"
 #include "qemu/iov.h"
 #include "sysemu/sysemu.h"
@@ -81,14 +80,13 @@ typedef struct IscsiTask {
 } IscsiTask;
 
 typedef struct IscsiAIOCB {
-    BlockDriverAIOCB common;
+    BlockAIOCB common;
     QEMUIOVector *qiov;
     QEMUBH *bh;
     IscsiLun *iscsilun;
     struct scsi_task *task;
     uint8_t *buf;
     int status;
-    int canceled;
     int64_t sector_num;
     int nb_sectors;
 #ifdef __linux__
@@ -120,16 +118,14 @@ iscsi_bh_cb(void *p)
     g_free(acb->buf);
     acb->buf = NULL;
 
-    if (acb->canceled == 0) {
-        acb->common.cb(acb->common.opaque, acb->status);
-    }
+    acb->common.cb(acb->common.opaque, acb->status);
 
     if (acb->task != NULL) {
         scsi_free_scsi_task(acb->task);
         acb->task = NULL;
     }
 
-    qemu_aio_release(acb);
+    qemu_aio_unref(acb);
 }
 
 static void
@@ -231,7 +227,7 @@ iscsi_abort_task_cb(struct iscsi_context *iscsi, int status, void *command_data,
 }
 
 static void
-iscsi_aio_cancel(BlockDriverAIOCB *blockacb)
+iscsi_aio_cancel(BlockAIOCB *blockacb)
 {
     IscsiAIOCB *acb = (IscsiAIOCB *)blockacb;
     IscsiLun *iscsilun = acb->iscsilun;
@@ -240,20 +236,15 @@ iscsi_aio_cancel(BlockDriverAIOCB *blockacb)
         return;
     }
 
-    acb->canceled = 1;
-
     /* send a task mgmt call to the target to cancel the task on the target */
     iscsi_task_mgmt_abort_task_async(iscsilun->iscsi, acb->task,
                                      iscsi_abort_task_cb, acb);
 
-    while (acb->status == -EINPROGRESS) {
-        aio_poll(iscsilun->aio_context, true);
-    }
 }
 
 static const AIOCBInfo iscsi_aiocb_info = {
     .aiocb_size         = sizeof(IscsiAIOCB),
-    .cancel             = iscsi_aio_cancel,
+    .cancel_async       = iscsi_aio_cancel,
 };
 
 
@@ -325,6 +316,13 @@ static bool is_request_lun_aligned(int64_t sector_num, int nb_sectors,
     return 1;
 }
 
+static unsigned long *iscsi_allocationmap_init(IscsiLun *iscsilun)
+{
+    return bitmap_try_new(DIV_ROUND_UP(sector_lun2qemu(iscsilun->num_blocks,
+                                                       iscsilun),
+                                       iscsilun->cluster_sectors));
+}
+
 static void iscsi_allocationmap_set(IscsiLun *iscsilun, int64_t sector_num,
                                     int nb_sectors)
 {
@@ -364,6 +362,12 @@ static int coroutine_fn iscsi_co_writev(BlockDriverState *bs,
         return -EINVAL;
     }
 
+    if (bs->bl.max_transfer_length && nb_sectors > bs->bl.max_transfer_length) {
+        error_report("iSCSI Error: Write of %d sectors exceeds max_xfer_len "
+                     "of %d sectors", nb_sectors, bs->bl.max_transfer_length);
+        return -EINVAL;
+    }
+
     lba = sector_qemu2lun(sector_num, iscsilun);
     num_sectors = sector_qemu2lun(nb_sectors, iscsilun);
     iscsi_co_init_iscsitask(iscsilun, &iTask);
@@ -531,6 +535,12 @@ static int coroutine_fn iscsi_co_readv(BlockDriverState *bs,
         return -EINVAL;
     }
 
+    if (bs->bl.max_transfer_length && nb_sectors > bs->bl.max_transfer_length) {
+        error_report("iSCSI Error: Read of %d sectors exceeds max_xfer_len "
+                     "of %d sectors", nb_sectors, bs->bl.max_transfer_length);
+        return -EINVAL;
+    }
+
     if (iscsilun->lbprz && nb_sectors >= ISCSI_CHECKALLOC_THRES &&
         !iscsi_allocationmap_is_allocated(iscsilun, sector_num, nb_sectors)) {
         int64_t ret;
@@ -638,10 +648,6 @@ iscsi_aio_ioctl_cb(struct iscsi_context *iscsi, int status,
     g_free(acb->buf);
     acb->buf = NULL;
 
-    if (acb->canceled != 0) {
-        return;
-    }
-
     acb->status = 0;
     if (status < 0) {
         error_report("Failed to ioctl(SG_IO) to iSCSI lun. %s",
@@ -669,9 +675,9 @@ iscsi_aio_ioctl_cb(struct iscsi_context *iscsi, int status,
     iscsi_schedule_bh(acb);
 }
 
-static BlockDriverAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
+static BlockAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
         unsigned long int req, void *buf,
-        BlockDriverCompletionFunc *cb, void *opaque)
+        BlockCompletionFunc *cb, void *opaque)
 {
     IscsiLun *iscsilun = bs->opaque;
     struct iscsi_context *iscsi = iscsilun->iscsi;
@@ -683,7 +689,6 @@ static BlockDriverAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
     acb = qemu_aio_get(&iscsi_aiocb_info, bs, cb, opaque);
 
     acb->iscsilun = iscsilun;
-    acb->canceled    = 0;
     acb->bh          = NULL;
     acb->status      = -EINPROGRESS;
     acb->buf         = NULL;
@@ -693,7 +698,7 @@ static BlockDriverAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
     if (acb->task == NULL) {
         error_report("iSCSI: Failed to allocate task for scsi command. %s",
                      iscsi_get_error(iscsi));
-        qemu_aio_release(acb);
+        qemu_aio_unref(acb);
         return NULL;
     }
     memset(acb->task, 0, sizeof(struct scsi_task));
@@ -731,7 +736,7 @@ static BlockDriverAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
                                  (data.size > 0) ? &data : NULL,
                                  acb) != 0) {
         scsi_free_scsi_task(acb->task);
-        qemu_aio_release(acb);
+        qemu_aio_unref(acb);
         return NULL;
     }
 
@@ -893,7 +898,10 @@ coroutine_fn iscsi_co_write_zeroes(BlockDriverState *bs, int64_t sector_num,
     nb_blocks = sector_qemu2lun(nb_sectors, iscsilun);
 
     if (iscsilun->zeroblock == NULL) {
-        iscsilun->zeroblock = g_malloc0(iscsilun->block_size);
+        iscsilun->zeroblock = g_try_malloc0(iscsilun->block_size);
+        if (iscsilun->zeroblock == NULL) {
+            return -ENOMEM;
+        }
     }
 
     iscsi_co_init_iscsitask(iscsilun, &iTask);
@@ -1223,6 +1231,40 @@ static void iscsi_attach_aio_context(BlockDriverState *bs,
               qemu_clock_get_ms(QEMU_CLOCK_REALTIME) + NOP_INTERVAL);
 }
 
+static bool iscsi_is_write_protected(IscsiLun *iscsilun)
+{
+    struct scsi_task *task;
+    struct scsi_mode_sense *ms = NULL;
+    bool wrprotected = false;
+
+    task = iscsi_modesense6_sync(iscsilun->iscsi, iscsilun->lun,
+                                 1, SCSI_MODESENSE_PC_CURRENT,
+                                 0x3F, 0, 255);
+    if (task == NULL) {
+        error_report("iSCSI: Failed to send MODE_SENSE(6) command: %s",
+                     iscsi_get_error(iscsilun->iscsi));
+        goto out;
+    }
+
+    if (task->status != SCSI_STATUS_GOOD) {
+        error_report("iSCSI: Failed MODE_SENSE(6), LUN assumed writable");
+        goto out;
+    }
+    ms = scsi_datain_unmarshall(task);
+    if (!ms) {
+        error_report("iSCSI: Failed to unmarshall MODE_SENSE(6) data: %s",
+                     iscsi_get_error(iscsilun->iscsi));
+        goto out;
+    }
+    wrprotected = ms->device_specific_parameter & 0x80;
+
+out:
+    if (task) {
+        scsi_free_scsi_task(task);
+    }
+    return wrprotected;
+}
+
 /*
  * We support iscsi url's on the form
  * iscsi://[<username>%<password>@]<host>[:<port>]/<targetname>/<lun>
@@ -1343,6 +1385,14 @@ static int iscsi_open(BlockDriverState *bs, QDict *options, int flags,
     scsi_free_scsi_task(task);
     task = NULL;
 
+    /* Check the write protect flag of the LUN if we want to write */
+    if (iscsilun->type == TYPE_DISK && (flags & BDRV_O_RDWR) &&
+        iscsi_is_write_protected(iscsilun)) {
+        error_setg(errp, "Cannot open a write protected LUN as read-write");
+        ret = -EACCES;
+        goto out;
+    }
+
     iscsi_readcapacity_sync(iscsilun, &local_err);
     if (local_err != NULL) {
         error_propagate(errp, local_err);
@@ -1413,9 +1463,10 @@ static int iscsi_open(BlockDriverState *bs, QDict *options, int flags,
         iscsilun->cluster_sectors = (iscsilun->bl.opt_unmap_gran *
                                      iscsilun->block_size) >> BDRV_SECTOR_BITS;
         if (iscsilun->lbprz && !(bs->open_flags & BDRV_O_NOCACHE)) {
-            iscsilun->allocationmap =
-                bitmap_new(DIV_ROUND_UP(bs->total_sectors,
-                                        iscsilun->cluster_sectors));
+            iscsilun->allocationmap = iscsi_allocationmap_init(iscsilun);
+            if (iscsilun->allocationmap == NULL) {
+                ret = -ENOMEM;
+            }
         }
     }
 
@@ -1450,31 +1501,44 @@ static void iscsi_close(BlockDriverState *bs)
     memset(iscsilun, 0, sizeof(IscsiLun));
 }
 
-static void iscsi_refresh_limits(BlockDriverState *bs, Error **errp)
+static int sector_limits_lun2qemu(int64_t sector, IscsiLun *iscsilun)
 {
-    IscsiLun *iscsilun = bs->opaque;
+    return MIN(sector_lun2qemu(sector, iscsilun), INT_MAX / 2 + 1);
+}
 
+static void iscsi_refresh_limits(BlockDriverState *bs, Error **errp)
+{
     /* We don't actually refresh here, but just return data queried in
      * iscsi_open(): iscsi targets don't change their limits. */
+
+    IscsiLun *iscsilun = bs->opaque;
+    uint32_t max_xfer_len = iscsilun->use_16_for_rw ? 0xffffffff : 0xffff;
+
+    if (iscsilun->bl.max_xfer_len) {
+        max_xfer_len = MIN(max_xfer_len, iscsilun->bl.max_xfer_len);
+    }
+
+    bs->bl.max_transfer_length = sector_limits_lun2qemu(max_xfer_len, iscsilun);
+
     if (iscsilun->lbp.lbpu) {
         if (iscsilun->bl.max_unmap < 0xffffffff) {
-            bs->bl.max_discard = sector_lun2qemu(iscsilun->bl.max_unmap,
-                                                 iscsilun);
+            bs->bl.max_discard =
+                sector_limits_lun2qemu(iscsilun->bl.max_unmap, iscsilun);
         }
-        bs->bl.discard_alignment = sector_lun2qemu(iscsilun->bl.opt_unmap_gran,
-                                                   iscsilun);
+        bs->bl.discard_alignment =
+            sector_limits_lun2qemu(iscsilun->bl.opt_unmap_gran, iscsilun);
     }
 
     if (iscsilun->bl.max_ws_len < 0xffffffff) {
-        bs->bl.max_write_zeroes = sector_lun2qemu(iscsilun->bl.max_ws_len,
-                                                  iscsilun);
+        bs->bl.max_write_zeroes =
+            sector_limits_lun2qemu(iscsilun->bl.max_ws_len, iscsilun);
     }
     if (iscsilun->lbp.lbpws) {
-        bs->bl.write_zeroes_alignment = sector_lun2qemu(iscsilun->bl.opt_unmap_gran,
-                                                        iscsilun);
+        bs->bl.write_zeroes_alignment =
+            sector_limits_lun2qemu(iscsilun->bl.opt_unmap_gran, iscsilun);
     }
-    bs->bl.opt_transfer_length = sector_lun2qemu(iscsilun->bl.opt_xfer_len,
-                                                 iscsilun);
+    bs->bl.opt_transfer_length =
+        sector_limits_lun2qemu(iscsilun->bl.opt_xfer_len, iscsilun);
 }
 
 /* Since iscsi_open() ignores bdrv_flags, there is nothing to do here in
@@ -1508,9 +1572,7 @@ static int iscsi_truncate(BlockDriverState *bs, int64_t offset)
 
     if (iscsilun->allocationmap != NULL) {
         g_free(iscsilun->allocationmap);
-        iscsilun->allocationmap =
-            bitmap_new(DIV_ROUND_UP(bs->total_sectors,
-                                    iscsilun->cluster_sectors));
+        iscsilun->allocationmap = iscsi_allocationmap_init(iscsilun);
     }
 
     return 0;
@@ -1524,12 +1586,12 @@ static int iscsi_create(const char *filename, QemuOpts *opts, Error **errp)
     IscsiLun *iscsilun = NULL;
     QDict *bs_options;
 
-    bs = bdrv_new("", &error_abort);
+    bs = bdrv_new();
 
     /* Read out options */
-    total_size =
-        qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0) / BDRV_SECTOR_SIZE;
-    bs->opaque = g_malloc0(sizeof(struct IscsiLun));
+    total_size = DIV_ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                              BDRV_SECTOR_SIZE);
+    bs->opaque = g_new0(struct IscsiLun, 1);
     iscsilun = bs->opaque;
 
     bs_options = qdict_new();
diff --git a/block/linux-aio.c b/block/linux-aio.c
index 7ac7e8c99..d92513b0f 100644
--- a/block/linux-aio.c
+++ b/block/linux-aio.c
@@ -28,7 +28,7 @@
 #define MAX_QUEUED_IO  128
 
 struct qemu_laiocb {
-    BlockDriverAIOCB common;
+    BlockAIOCB common;
     struct qemu_laio_state *ctx;
     struct iocb iocb;
     ssize_t ret;
@@ -51,6 +51,12 @@ struct qemu_laio_state {
 
     /* io queue for submit at batch */
     LaioQueue io_q;
+
+    /* I/O completion processing */
+    QEMUBH *completion_bh;
+    struct io_event events[MAX_EVENTS];
+    int event_idx;
+    int event_max;
 };
 
 static inline ssize_t io_event_ret(struct io_event *ev)
@@ -79,72 +85,89 @@ static void qemu_laio_process_completion(struct qemu_laio_state *s,
                 ret = -EINVAL;
             }
         }
-
-        laiocb->common.cb(laiocb->common.opaque, ret);
     }
+    laiocb->common.cb(laiocb->common.opaque, ret);
 
-    qemu_aio_release(laiocb);
+    qemu_aio_unref(laiocb);
 }
 
-static void qemu_laio_completion_cb(EventNotifier *e)
+/* The completion BH fetches completed I/O requests and invokes their
+ * callbacks.
+ *
+ * The function is somewhat tricky because it supports nested event loops, for
+ * example when a request callback invokes aio_poll().  In order to do this,
+ * the completion events array and index are kept in qemu_laio_state.  The BH
+ * reschedules itself as long as there are completions pending so it will
+ * either be called again in a nested event loop or will be called after all
+ * events have been completed.  When there are no events left to complete, the
+ * BH returns without rescheduling.
+ */
+static void qemu_laio_completion_bh(void *opaque)
 {
-    struct qemu_laio_state *s = container_of(e, struct qemu_laio_state, e);
-
-    while (event_notifier_test_and_clear(&s->e)) {
-        struct io_event events[MAX_EVENTS];
-        struct timespec ts = { 0 };
-        int nevents, i;
+    struct qemu_laio_state *s = opaque;
 
+    /* Fetch more completion events when empty */
+    if (s->event_idx == s->event_max) {
         do {
-            nevents = io_getevents(s->ctx, MAX_EVENTS, MAX_EVENTS, events, &ts);
-        } while (nevents == -EINTR);
+            struct timespec ts = { 0 };
+            s->event_max = io_getevents(s->ctx, MAX_EVENTS, MAX_EVENTS,
+                                        s->events, &ts);
+        } while (s->event_max == -EINTR);
+
+        s->event_idx = 0;
+        if (s->event_max <= 0) {
+            s->event_max = 0;
+            return; /* no more events */
+        }
+    }
 
-        for (i = 0; i < nevents; i++) {
-            struct iocb *iocb = events[i].obj;
-            struct qemu_laiocb *laiocb =
-                    container_of(iocb, struct qemu_laiocb, iocb);
+    /* Reschedule so nested event loops see currently pending completions */
+    qemu_bh_schedule(s->completion_bh);
 
-            laiocb->ret = io_event_ret(&events[i]);
-            qemu_laio_process_completion(s, laiocb);
-        }
+    /* Process completion events */
+    while (s->event_idx < s->event_max) {
+        struct iocb *iocb = s->events[s->event_idx].obj;
+        struct qemu_laiocb *laiocb =
+                container_of(iocb, struct qemu_laiocb, iocb);
+
+        laiocb->ret = io_event_ret(&s->events[s->event_idx]);
+        s->event_idx++;
+
+        qemu_laio_process_completion(s, laiocb);
+    }
+}
+
+static void qemu_laio_completion_cb(EventNotifier *e)
+{
+    struct qemu_laio_state *s = container_of(e, struct qemu_laio_state, e);
+
+    if (event_notifier_test_and_clear(&s->e)) {
+        qemu_bh_schedule(s->completion_bh);
     }
 }
 
-static void laio_cancel(BlockDriverAIOCB *blockacb)
+static void laio_cancel(BlockAIOCB *blockacb)
 {
     struct qemu_laiocb *laiocb = (struct qemu_laiocb *)blockacb;
     struct io_event event;
     int ret;
 
-    if (laiocb->ret != -EINPROGRESS)
+    if (laiocb->ret != -EINPROGRESS) {
         return;
-
-    /*
-     * Note that as of Linux 2.6.31 neither the block device code nor any
-     * filesystem implements cancellation of AIO request.
-     * Thus the polling loop below is the normal code path.
-     */
+    }
     ret = io_cancel(laiocb->ctx->ctx, &laiocb->iocb, &event);
-    if (ret == 0) {
-        laiocb->ret = -ECANCELED;
+    laiocb->ret = -ECANCELED;
+    if (ret != 0) {
+        /* iocb is not cancelled, cb will be called by the event loop later */
         return;
     }
 
-    /*
-     * We have to wait for the iocb to finish.
-     *
-     * The only way to get the iocb status update is by polling the io context.
-     * We might be able to do this slightly more optimal by removing the
-     * O_NONBLOCK flag.
-     */
-    while (laiocb->ret == -EINPROGRESS) {
-        qemu_laio_completion_cb(&laiocb->ctx->e);
-    }
+    laiocb->common.cb(laiocb->common.opaque, laiocb->ret);
 }
 
 static const AIOCBInfo laio_aiocb_info = {
     .aiocb_size         = sizeof(struct qemu_laiocb),
-    .cancel             = laio_cancel,
+    .cancel_async       = laio_cancel,
 };
 
 static void ioq_init(LaioQueue *io_q)
@@ -220,9 +243,9 @@ int laio_io_unplug(BlockDriverState *bs, void *aio_ctx, bool unplug)
     return ret;
 }
 
-BlockDriverAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
+BlockAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque, int type)
+        BlockCompletionFunc *cb, void *opaque, int type)
 {
     struct qemu_laio_state *s = aio_ctx;
     struct qemu_laiocb *laiocb;
@@ -263,7 +286,7 @@ BlockDriverAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
     return &laiocb->common;
 
 out_free_aiocb:
-    qemu_aio_release(laiocb);
+    qemu_aio_unref(laiocb);
     return NULL;
 }
 
@@ -272,12 +295,14 @@ void laio_detach_aio_context(void *s_, AioContext *old_context)
     struct qemu_laio_state *s = s_;
 
     aio_set_event_notifier(old_context, &s->e, NULL);
+    qemu_bh_delete(s->completion_bh);
 }
 
 void laio_attach_aio_context(void *s_, AioContext *new_context)
 {
     struct qemu_laio_state *s = s_;
 
+    s->completion_bh = aio_bh_new(new_context, qemu_laio_completion_bh, s);
     aio_set_event_notifier(new_context, &s->e, qemu_laio_completion_cb);
 }
 
diff --git a/block/mirror.c b/block/mirror.c
index c7a655fc5..2c6dd2a4c 100644
--- a/block/mirror.c
+++ b/block/mirror.c
@@ -45,6 +45,7 @@ typedef struct MirrorBlockJob {
     int64_t sector_num;
     int64_t granularity;
     size_t buf_size;
+    int64_t bdev_length;
     unsigned long *cow_bitmap;
     BdrvDirtyBitmap *dirty_bitmap;
     HBitmapIter hbi;
@@ -54,6 +55,7 @@ typedef struct MirrorBlockJob {
 
     unsigned long *in_flight_bitmap;
     int in_flight;
+    int sectors_in_flight;
     int ret;
 } MirrorBlockJob;
 
@@ -87,6 +89,7 @@ static void mirror_iteration_done(MirrorOp *op, int ret)
     trace_mirror_iteration_done(s, op->sector_num, op->nb_sectors, ret);
 
     s->in_flight--;
+    s->sectors_in_flight -= op->nb_sectors;
     iov = op->qiov.iov;
     for (i = 0; i < op->qiov.niov; i++) {
         MirrorBuffer *buf = (MirrorBuffer *) iov[i].iov_base;
@@ -98,8 +101,11 @@ static void mirror_iteration_done(MirrorOp *op, int ret)
     chunk_num = op->sector_num / sectors_per_chunk;
     nb_chunks = op->nb_sectors / sectors_per_chunk;
     bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks);
-    if (s->cow_bitmap && ret >= 0) {
-        bitmap_set(s->cow_bitmap, chunk_num, nb_chunks);
+    if (ret >= 0) {
+        if (s->cow_bitmap) {
+            bitmap_set(s->cow_bitmap, chunk_num, nb_chunks);
+        }
+        s->common.offset += (uint64_t)op->nb_sectors * BDRV_SECTOR_SIZE;
     }
 
     qemu_iovec_destroy(&op->qiov);
@@ -157,7 +163,7 @@ static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
     BlockDriverState *source = s->common.bs;
     int nb_sectors, sectors_per_chunk, nb_chunks;
     int64_t end, sector_num, next_chunk, next_sector, hbitmap_next_sector;
-    uint64_t delay_ns;
+    uint64_t delay_ns = 0;
     MirrorOp *op;
 
     s->sector_num = hbitmap_iter_next(&s->hbi);
@@ -172,7 +178,7 @@ static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
     hbitmap_next_sector = s->sector_num;
     sector_num = s->sector_num;
     sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
-    end = s->common.len >> BDRV_SECTOR_BITS;
+    end = s->bdev_length / BDRV_SECTOR_SIZE;
 
     /* Extend the QEMUIOVector to include all adjacent blocks that will
      * be copied in this operation.
@@ -247,8 +253,6 @@ static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
         next_chunk += added_chunks;
         if (!s->synced && s->common.speed) {
             delay_ns = ratelimit_calculate_delay(&s->limit, added_sectors);
-        } else {
-            delay_ns = 0;
         }
     } while (delay_ns == 0 && next_sector < end);
 
@@ -286,6 +290,7 @@ static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
 
     /* Copy the dirty cluster.  */
     s->in_flight++;
+    s->sectors_in_flight += nb_sectors;
     trace_mirror_one_iteration(s, sector_num, nb_sectors);
     bdrv_aio_readv(source, sector_num, &op->qiov, nb_sectors,
                    mirror_read_complete, op);
@@ -316,9 +321,56 @@ static void mirror_drain(MirrorBlockJob *s)
     }
 }
 
+typedef struct {
+    int ret;
+} MirrorExitData;
+
+static void mirror_exit(BlockJob *job, void *opaque)
+{
+    MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
+    MirrorExitData *data = opaque;
+    AioContext *replace_aio_context = NULL;
+
+    if (s->to_replace) {
+        replace_aio_context = bdrv_get_aio_context(s->to_replace);
+        aio_context_acquire(replace_aio_context);
+    }
+
+    if (s->should_complete && data->ret == 0) {
+        BlockDriverState *to_replace = s->common.bs;
+        if (s->to_replace) {
+            to_replace = s->to_replace;
+        }
+        if (bdrv_get_flags(s->target) != bdrv_get_flags(to_replace)) {
+            bdrv_reopen(s->target, bdrv_get_flags(to_replace), NULL);
+        }
+        bdrv_swap(s->target, to_replace);
+        if (s->common.driver->job_type == BLOCK_JOB_TYPE_COMMIT) {
+            /* drop the bs loop chain formed by the swap: break the loop then
+             * trigger the unref from the top one */
+            BlockDriverState *p = s->base->backing_hd;
+            bdrv_set_backing_hd(s->base, NULL);
+            bdrv_unref(p);
+        }
+    }
+    if (s->to_replace) {
+        bdrv_op_unblock_all(s->to_replace, s->replace_blocker);
+        error_free(s->replace_blocker);
+        bdrv_unref(s->to_replace);
+    }
+    if (replace_aio_context) {
+        aio_context_release(replace_aio_context);
+    }
+    g_free(s->replaces);
+    bdrv_unref(s->target);
+    block_job_completed(&s->common, data->ret);
+    g_free(data);
+}
+
 static void coroutine_fn mirror_run(void *opaque)
 {
     MirrorBlockJob *s = opaque;
+    MirrorExitData *data;
     BlockDriverState *bs = s->common.bs;
     int64_t sector_num, end, sectors_per_chunk, length;
     uint64_t last_pause_ns;
@@ -331,11 +383,11 @@ static void coroutine_fn mirror_run(void *opaque)
         goto immediate_exit;
     }
 
-    s->common.len = bdrv_getlength(bs);
-    if (s->common.len < 0) {
-        ret = s->common.len;
+    s->bdev_length = bdrv_getlength(bs);
+    if (s->bdev_length < 0) {
+        ret = s->bdev_length;
         goto immediate_exit;
-    } else if (s->common.len == 0) {
+    } else if (s->bdev_length == 0) {
         /* Report BLOCK_JOB_READY and wait for complete. */
         block_job_event_ready(&s->common);
         s->synced = true;
@@ -346,7 +398,7 @@ static void coroutine_fn mirror_run(void *opaque)
         goto immediate_exit;
     }
 
-    length = DIV_ROUND_UP(s->common.len, s->granularity);
+    length = DIV_ROUND_UP(s->bdev_length, s->granularity);
     s->in_flight_bitmap = bitmap_new(length);
 
     /* If we have no backing file yet in the destination, we cannot let
@@ -366,8 +418,13 @@ static void coroutine_fn mirror_run(void *opaque)
         }
     }
 
-    end = s->common.len >> BDRV_SECTOR_BITS;
-    s->buf = qemu_blockalign(bs, s->buf_size);
+    end = s->bdev_length / BDRV_SECTOR_SIZE;
+    s->buf = qemu_try_blockalign(bs, s->buf_size);
+    if (s->buf == NULL) {
+        ret = -ENOMEM;
+        goto immediate_exit;
+    }
+
     sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
     mirror_free_init(s);
 
@@ -406,6 +463,12 @@ static void coroutine_fn mirror_run(void *opaque)
         }
 
         cnt = bdrv_get_dirty_count(bs, s->dirty_bitmap);
+        /* s->common.offset contains the number of bytes already processed so
+         * far, cnt is the number of dirty sectors remaining and
+         * s->sectors_in_flight is the number of sectors currently being
+         * processed; together those are the current total operation length */
+        s->common.len = s->common.offset +
+                        (cnt + s->sectors_in_flight) * BDRV_SECTOR_SIZE;
 
         /* Note that even when no rate limit is applied we need to yield
          * periodically with no pending I/O so that qemu_aio_flush() returns.
@@ -442,7 +505,6 @@ static void coroutine_fn mirror_run(void *opaque)
                  * report completion.  This way, block-job-cancel will leave
                  * the target in a consistent state.
                  */
-                s->common.offset = end * BDRV_SECTOR_SIZE;
                 if (!s->synced) {
                     block_job_event_ready(&s->common);
                     s->synced = true;
@@ -464,15 +526,13 @@ static void coroutine_fn mirror_run(void *opaque)
              * mirror_populate runs.
              */
             trace_mirror_before_drain(s, cnt);
-            bdrv_drain_all();
+            bdrv_drain(bs);
             cnt = bdrv_get_dirty_count(bs, s->dirty_bitmap);
         }
 
         ret = 0;
         trace_mirror_before_sleep(s, cnt, s->synced, delay_ns);
         if (!s->synced) {
-            /* Publish progress */
-            s->common.offset = (end - cnt) * BDRV_SECTOR_SIZE;
             block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, delay_ns);
             if (block_job_is_cancelled(&s->common)) {
                 break;
@@ -507,31 +567,10 @@ immediate_exit:
     g_free(s->in_flight_bitmap);
     bdrv_release_dirty_bitmap(bs, s->dirty_bitmap);
     bdrv_iostatus_disable(s->target);
-    if (s->should_complete && ret == 0) {
-        BlockDriverState *to_replace = s->common.bs;
-        if (s->to_replace) {
-            to_replace = s->to_replace;
-        }
-        if (bdrv_get_flags(s->target) != bdrv_get_flags(to_replace)) {
-            bdrv_reopen(s->target, bdrv_get_flags(to_replace), NULL);
-        }
-        bdrv_swap(s->target, to_replace);
-        if (s->common.driver->job_type == BLOCK_JOB_TYPE_COMMIT) {
-            /* drop the bs loop chain formed by the swap: break the loop then
-             * trigger the unref from the top one */
-            BlockDriverState *p = s->base->backing_hd;
-            bdrv_set_backing_hd(s->base, NULL);
-            bdrv_unref(p);
-        }
-    }
-    if (s->to_replace) {
-        bdrv_op_unblock_all(s->to_replace, s->replace_blocker);
-        error_free(s->replace_blocker);
-        bdrv_unref(s->to_replace);
-    }
-    g_free(s->replaces);
-    bdrv_unref(s->target);
-    block_job_completed(&s->common, ret);
+
+    data = g_malloc(sizeof(*data));
+    data->ret = ret;
+    block_job_defer_to_main_loop(&s->common, mirror_exit, data);
 }
 
 static void mirror_set_speed(BlockJob *job, int64_t speed, Error **errp)
@@ -564,22 +603,30 @@ static void mirror_complete(BlockJob *job, Error **errp)
         return;
     }
     if (!s->synced) {
-        error_set(errp, QERR_BLOCK_JOB_NOT_READY, job->bs->device_name);
+        error_set(errp, QERR_BLOCK_JOB_NOT_READY,
+                  bdrv_get_device_name(job->bs));
         return;
     }
 
     /* check the target bs is not blocked and block all operations on it */
     if (s->replaces) {
+        AioContext *replace_aio_context;
+
         s->to_replace = check_to_replace_node(s->replaces, &local_err);
         if (!s->to_replace) {
             error_propagate(errp, local_err);
             return;
         }
 
+        replace_aio_context = bdrv_get_aio_context(s->to_replace);
+        aio_context_acquire(replace_aio_context);
+
         error_setg(&s->replace_blocker,
                    "block device is in use by block-job-complete");
         bdrv_op_block_all(s->to_replace, s->replace_blocker);
         bdrv_ref(s->to_replace);
+
+        aio_context_release(replace_aio_context);
     }
 
     s->should_complete = true;
@@ -609,7 +656,7 @@ static void mirror_start_job(BlockDriverState *bs, BlockDriverState *target,
                              int64_t buf_size,
                              BlockdevOnError on_source_error,
                              BlockdevOnError on_target_error,
-                             BlockDriverCompletionFunc *cb,
+                             BlockCompletionFunc *cb,
                              void *opaque, Error **errp,
                              const BlockJobDriver *driver,
                              bool is_none_mode, BlockDriverState *base)
@@ -669,7 +716,7 @@ void mirror_start(BlockDriverState *bs, BlockDriverState *target,
                   int64_t speed, int64_t granularity, int64_t buf_size,
                   MirrorSyncMode mode, BlockdevOnError on_source_error,
                   BlockdevOnError on_target_error,
-                  BlockDriverCompletionFunc *cb,
+                  BlockCompletionFunc *cb,
                   void *opaque, Error **errp)
 {
     bool is_none_mode;
@@ -686,7 +733,7 @@ void mirror_start(BlockDriverState *bs, BlockDriverState *target,
 void commit_active_start(BlockDriverState *bs, BlockDriverState *base,
                          int64_t speed,
                          BlockdevOnError on_error,
-                         BlockDriverCompletionFunc *cb,
+                         BlockCompletionFunc *cb,
                          void *opaque, Error **errp)
 {
     int64_t length, base_length;
diff --git a/block/nbd.c b/block/nbd.c
index 4eda0958d..04cc84507 100644
--- a/block/nbd.c
+++ b/block/nbd.c
@@ -31,8 +31,10 @@
 #include "block/block_int.h"
 #include "qemu/module.h"
 #include "qemu/sockets.h"
+#include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qjson.h"
 #include "qapi/qmp/qint.h"
+#include "qapi/qmp/qstring.h"
 
 #include <sys/types.h>
 #include <unistd.h>
@@ -338,6 +340,51 @@ static void nbd_attach_aio_context(BlockDriverState *bs,
     nbd_client_session_attach_aio_context(&s->client, new_context);
 }
 
+static void nbd_refresh_filename(BlockDriverState *bs)
+{
+    QDict *opts = qdict_new();
+    const char *path   = qdict_get_try_str(bs->options, "path");
+    const char *host   = qdict_get_try_str(bs->options, "host");
+    const char *port   = qdict_get_try_str(bs->options, "port");
+    const char *export = qdict_get_try_str(bs->options, "export");
+
+    qdict_put_obj(opts, "driver", QOBJECT(qstring_from_str("nbd")));
+
+    if (path && export) {
+        snprintf(bs->exact_filename, sizeof(bs->exact_filename),
+                 "nbd+unix:///%s?socket=%s", export, path);
+    } else if (path && !export) {
+        snprintf(bs->exact_filename, sizeof(bs->exact_filename),
+                 "nbd+unix://?socket=%s", path);
+    } else if (!path && export && port) {
+        snprintf(bs->exact_filename, sizeof(bs->exact_filename),
+                 "nbd://%s:%s/%s", host, port, export);
+    } else if (!path && export && !port) {
+        snprintf(bs->exact_filename, sizeof(bs->exact_filename),
+                 "nbd://%s/%s", host, export);
+    } else if (!path && !export && port) {
+        snprintf(bs->exact_filename, sizeof(bs->exact_filename),
+                 "nbd://%s:%s", host, port);
+    } else if (!path && !export && !port) {
+        snprintf(bs->exact_filename, sizeof(bs->exact_filename),
+                 "nbd://%s", host);
+    }
+
+    if (path) {
+        qdict_put_obj(opts, "path", QOBJECT(qstring_from_str(path)));
+    } else if (port) {
+        qdict_put_obj(opts, "host", QOBJECT(qstring_from_str(host)));
+        qdict_put_obj(opts, "port", QOBJECT(qstring_from_str(port)));
+    } else {
+        qdict_put_obj(opts, "host", QOBJECT(qstring_from_str(host)));
+    }
+    if (export) {
+        qdict_put_obj(opts, "export", QOBJECT(qstring_from_str(export)));
+    }
+
+    bs->full_open_options = opts;
+}
+
 static BlockDriver bdrv_nbd = {
     .format_name                = "nbd",
     .protocol_name              = "nbd",
@@ -352,6 +399,7 @@ static BlockDriver bdrv_nbd = {
     .bdrv_getlength             = nbd_getlength,
     .bdrv_detach_aio_context    = nbd_detach_aio_context,
     .bdrv_attach_aio_context    = nbd_attach_aio_context,
+    .bdrv_refresh_filename      = nbd_refresh_filename,
 };
 
 static BlockDriver bdrv_nbd_tcp = {
@@ -368,6 +416,7 @@ static BlockDriver bdrv_nbd_tcp = {
     .bdrv_getlength             = nbd_getlength,
     .bdrv_detach_aio_context    = nbd_detach_aio_context,
     .bdrv_attach_aio_context    = nbd_attach_aio_context,
+    .bdrv_refresh_filename      = nbd_refresh_filename,
 };
 
 static BlockDriver bdrv_nbd_unix = {
@@ -384,6 +433,7 @@ static BlockDriver bdrv_nbd_unix = {
     .bdrv_getlength             = nbd_getlength,
     .bdrv_detach_aio_context    = nbd_detach_aio_context,
     .bdrv_attach_aio_context    = nbd_attach_aio_context,
+    .bdrv_refresh_filename      = nbd_refresh_filename,
 };
 
 static void bdrv_nbd_init(void)
diff --git a/block/nfs.c b/block/nfs.c
index 8439e0d38..c76e368b9 100644
--- a/block/nfs.c
+++ b/block/nfs.c
@@ -172,7 +172,11 @@ static int coroutine_fn nfs_co_writev(BlockDriverState *bs,
 
     nfs_co_init_task(client, &task);
 
-    buf = g_malloc(nb_sectors * BDRV_SECTOR_SIZE);
+    buf = g_try_malloc(nb_sectors * BDRV_SECTOR_SIZE);
+    if (nb_sectors && buf == NULL) {
+        return -ENOMEM;
+    }
+
     qemu_iovec_to_buf(iov, 0, buf, nb_sectors * BDRV_SECTOR_SIZE);
 
     if (nfs_pwrite_async(client->context, client->fh,
@@ -389,28 +393,33 @@ static int nfs_file_open(BlockDriverState *bs, QDict *options, int flags,
     qemu_opts_absorb_qdict(opts, options, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
-        return -EINVAL;
+        ret = -EINVAL;
+        goto out;
     }
     ret = nfs_client_open(client, qemu_opt_get(opts, "filename"),
                           (flags & BDRV_O_RDWR) ? O_RDWR : O_RDONLY,
                           errp);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
     bs->total_sectors = ret;
-    return 0;
+    ret = 0;
+out:
+    qemu_opts_del(opts);
+    return ret;
 }
 
 static int nfs_file_create(const char *url, QemuOpts *opts, Error **errp)
 {
     int ret = 0;
     int64_t total_size = 0;
-    NFSClient *client = g_malloc0(sizeof(NFSClient));
+    NFSClient *client = g_new0(NFSClient, 1);
 
     client->aio_context = qemu_get_aio_context();
 
     /* Read out options */
-    total_size = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0);
+    total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                          BDRV_SECTOR_SIZE);
 
     ret = nfs_client_open(client, url, O_CREAT, errp);
     if (ret < 0) {
diff --git a/block/null.c b/block/null.c
new file mode 100644
index 000000000..ec2bd27a4
--- /dev/null
+++ b/block/null.c
@@ -0,0 +1,168 @@
+/*
+ * Null block driver
+ *
+ * Authors:
+ *  Fam Zheng <famz@redhat.com>
+ *
+ * Copyright (C) 2014 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "block/block_int.h"
+
+typedef struct {
+    int64_t length;
+} BDRVNullState;
+
+static QemuOptsList runtime_opts = {
+    .name = "null",
+    .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
+    .desc = {
+        {
+            .name = "filename",
+            .type = QEMU_OPT_STRING,
+            .help = "",
+        },
+        {
+            .name = BLOCK_OPT_SIZE,
+            .type = QEMU_OPT_SIZE,
+            .help = "size of the null block",
+        },
+        { /* end of list */ }
+    },
+};
+
+static int null_file_open(BlockDriverState *bs, QDict *options, int flags,
+                          Error **errp)
+{
+    QemuOpts *opts;
+    BDRVNullState *s = bs->opaque;
+
+    opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
+    qemu_opts_absorb_qdict(opts, options, &error_abort);
+    s->length =
+        qemu_opt_get_size(opts, BLOCK_OPT_SIZE, 1 << 30);
+    qemu_opts_del(opts);
+    return 0;
+}
+
+static void null_close(BlockDriverState *bs)
+{
+}
+
+static int64_t null_getlength(BlockDriverState *bs)
+{
+    BDRVNullState *s = bs->opaque;
+    return s->length;
+}
+
+static coroutine_fn int null_co_readv(BlockDriverState *bs,
+                                      int64_t sector_num, int nb_sectors,
+                                      QEMUIOVector *qiov)
+{
+    return 0;
+}
+
+static coroutine_fn int null_co_writev(BlockDriverState *bs,
+                                       int64_t sector_num, int nb_sectors,
+                                       QEMUIOVector *qiov)
+{
+    return 0;
+}
+
+static coroutine_fn int null_co_flush(BlockDriverState *bs)
+{
+    return 0;
+}
+
+typedef struct {
+    BlockAIOCB common;
+    QEMUBH *bh;
+} NullAIOCB;
+
+static const AIOCBInfo null_aiocb_info = {
+    .aiocb_size = sizeof(NullAIOCB),
+};
+
+static void null_bh_cb(void *opaque)
+{
+    NullAIOCB *acb = opaque;
+    acb->common.cb(acb->common.opaque, 0);
+    qemu_bh_delete(acb->bh);
+    qemu_aio_unref(acb);
+}
+
+static inline BlockAIOCB *null_aio_common(BlockDriverState *bs,
+                                          BlockCompletionFunc *cb,
+                                          void *opaque)
+{
+    NullAIOCB *acb;
+
+    acb = qemu_aio_get(&null_aiocb_info, bs, cb, opaque);
+    acb->bh = aio_bh_new(bdrv_get_aio_context(bs), null_bh_cb, acb);
+    qemu_bh_schedule(acb->bh);
+    return &acb->common;
+}
+
+static BlockAIOCB *null_aio_readv(BlockDriverState *bs,
+                                  int64_t sector_num, QEMUIOVector *qiov,
+                                  int nb_sectors,
+                                  BlockCompletionFunc *cb,
+                                  void *opaque)
+{
+    return null_aio_common(bs, cb, opaque);
+}
+
+static BlockAIOCB *null_aio_writev(BlockDriverState *bs,
+                                   int64_t sector_num, QEMUIOVector *qiov,
+                                   int nb_sectors,
+                                   BlockCompletionFunc *cb,
+                                   void *opaque)
+{
+    return null_aio_common(bs, cb, opaque);
+}
+
+static BlockAIOCB *null_aio_flush(BlockDriverState *bs,
+                                  BlockCompletionFunc *cb,
+                                  void *opaque)
+{
+    return null_aio_common(bs, cb, opaque);
+}
+
+static BlockDriver bdrv_null_co = {
+    .format_name            = "null-co",
+    .protocol_name          = "null-co",
+    .instance_size          = sizeof(BDRVNullState),
+
+    .bdrv_file_open         = null_file_open,
+    .bdrv_close             = null_close,
+    .bdrv_getlength         = null_getlength,
+
+    .bdrv_co_readv          = null_co_readv,
+    .bdrv_co_writev         = null_co_writev,
+    .bdrv_co_flush_to_disk  = null_co_flush,
+};
+
+static BlockDriver bdrv_null_aio = {
+    .format_name            = "null-aio",
+    .protocol_name          = "null-aio",
+    .instance_size          = sizeof(BDRVNullState),
+
+    .bdrv_file_open         = null_file_open,
+    .bdrv_close             = null_close,
+    .bdrv_getlength         = null_getlength,
+
+    .bdrv_aio_readv         = null_aio_readv,
+    .bdrv_aio_writev        = null_aio_writev,
+    .bdrv_aio_flush         = null_aio_flush,
+};
+
+static void bdrv_null_init(void)
+{
+    bdrv_register(&bdrv_null_co);
+    bdrv_register(&bdrv_null_aio);
+}
+
+block_init(bdrv_null_init);
diff --git a/block/parallels.c b/block/parallels.c
index 1a5bd350b..4f9cd8dd2 100644
--- a/block/parallels.c
+++ b/block/parallels.c
@@ -30,6 +30,7 @@
 /**************************************************************/
 
 #define HEADER_MAGIC "WithoutFreeSpace"
+#define HEADER_MAGIC2 "WithouFreSpacExt"
 #define HEADER_VERSION 2
 #define HEADER_SIZE 64
 
@@ -41,8 +42,10 @@ struct parallels_header {
     uint32_t cylinders;
     uint32_t tracks;
     uint32_t catalog_entries;
-    uint32_t nb_sectors;
-    char padding[24];
+    uint64_t nb_sectors;
+    uint32_t inuse;
+    uint32_t data_off;
+    char padding[12];
 } QEMU_PACKED;
 
 typedef struct BDRVParallelsState {
@@ -52,6 +55,8 @@ typedef struct BDRVParallelsState {
     unsigned int catalog_size;
 
     unsigned int tracks;
+
+    unsigned int off_multiplier;
 } BDRVParallelsState;
 
 static int parallels_probe(const uint8_t *buf, int buf_size, const char *filename)
@@ -59,11 +64,12 @@ static int parallels_probe(const uint8_t *buf, int buf_size, const char *filenam
     const struct parallels_header *ph = (const void *)buf;
 
     if (buf_size < HEADER_SIZE)
-	return 0;
+        return 0;
 
-    if (!memcmp(ph->magic, HEADER_MAGIC, 16) &&
-	(le32_to_cpu(ph->version) == HEADER_VERSION))
-	return 100;
+    if ((!memcmp(ph->magic, HEADER_MAGIC, 16) ||
+        !memcmp(ph->magic, HEADER_MAGIC2, 16)) &&
+        (le32_to_cpu(ph->version) == HEADER_VERSION))
+        return 100;
 
     return 0;
 }
@@ -83,14 +89,19 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags,
         goto fail;
     }
 
-    if (memcmp(ph.magic, HEADER_MAGIC, 16) ||
-        (le32_to_cpu(ph.version) != HEADER_VERSION)) {
-        error_setg(errp, "Image not in Parallels format");
-        ret = -EINVAL;
-        goto fail;
-    }
+    bs->total_sectors = le64_to_cpu(ph.nb_sectors);
 
-    bs->total_sectors = le32_to_cpu(ph.nb_sectors);
+    if (le32_to_cpu(ph.version) != HEADER_VERSION) {
+        goto fail_format;
+    }
+    if (!memcmp(ph.magic, HEADER_MAGIC, 16)) {
+        s->off_multiplier = 1;
+        bs->total_sectors = 0xffffffff & bs->total_sectors;
+    } else if (!memcmp(ph.magic, HEADER_MAGIC2, 16)) {
+        s->off_multiplier = le32_to_cpu(ph.tracks);
+    } else {
+        goto fail_format;
+    }
 
     s->tracks = le32_to_cpu(ph.tracks);
     if (s->tracks == 0) {
@@ -98,6 +109,11 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags,
         ret = -EINVAL;
         goto fail;
     }
+    if (s->tracks > INT32_MAX/513) {
+        error_setg(errp, "Invalid image: Too big cluster");
+        ret = -EFBIG;
+        goto fail;
+    }
 
     s->catalog_size = le32_to_cpu(ph.catalog_entries);
     if (s->catalog_size > INT_MAX / 4) {
@@ -105,7 +121,11 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags,
         ret = -EFBIG;
         goto fail;
     }
-    s->catalog_bitmap = g_malloc(s->catalog_size * 4);
+    s->catalog_bitmap = g_try_new(uint32_t, s->catalog_size);
+    if (s->catalog_size && s->catalog_bitmap == NULL) {
+        ret = -ENOMEM;
+        goto fail;
+    }
 
     ret = bdrv_pread(bs->file, 64, s->catalog_bitmap, s->catalog_size * 4);
     if (ret < 0) {
@@ -113,11 +133,14 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     for (i = 0; i < s->catalog_size; i++)
-	le32_to_cpus(&s->catalog_bitmap[i]);
+        le32_to_cpus(&s->catalog_bitmap[i]);
 
     qemu_co_mutex_init(&s->lock);
     return 0;
 
+fail_format:
+    error_setg(errp, "Image not in Parallels format");
+    ret = -EINVAL;
 fail:
     g_free(s->catalog_bitmap);
     return ret;
@@ -132,9 +155,10 @@ static int64_t seek_to_sector(BlockDriverState *bs, int64_t sector_num)
     offset = sector_num % s->tracks;
 
     /* not allocated */
-    if ((index > s->catalog_size) || (s->catalog_bitmap[index] == 0))
-	return -1;
-    return (uint64_t)(s->catalog_bitmap[index] + offset) * 512;
+    if ((index >= s->catalog_size) || (s->catalog_bitmap[index] == 0))
+        return -1;
+    return
+        ((uint64_t)s->catalog_bitmap[index] * s->off_multiplier + offset) * 512;
 }
 
 static int parallels_read(BlockDriverState *bs, int64_t sector_num,
diff --git a/block/qapi.c b/block/qapi.c
index f44f6b401..a87a34a8b 100644
--- a/block/qapi.c
+++ b/block/qapi.c
@@ -28,6 +28,7 @@
 #include "qapi-visit.h"
 #include "qapi/qmp-output-visitor.h"
 #include "qapi/qmp/types.h"
+#include "sysemu/block-backend.h"
 
 BlockDeviceInfo *bdrv_block_device_info(BlockDriverState *bs)
 {
@@ -165,19 +166,25 @@ void bdrv_query_image_info(BlockDriverState *bs,
                            ImageInfo **p_info,
                            Error **errp)
 {
-    uint64_t total_sectors;
+    int64_t size;
     const char *backing_filename;
     char backing_filename2[1024];
     BlockDriverInfo bdi;
     int ret;
     Error *err = NULL;
-    ImageInfo *info = g_new0(ImageInfo, 1);
+    ImageInfo *info;
 
-    bdrv_get_geometry(bs, &total_sectors);
+    size = bdrv_getlength(bs);
+    if (size < 0) {
+        error_setg_errno(errp, -size, "Can't get size of device '%s'",
+                         bdrv_get_device_name(bs));
+        return;
+    }
 
+    info = g_new0(ImageInfo, 1);
     info->filename        = g_strdup(bs->filename);
     info->format          = g_strdup(bdrv_get_format_name(bs));
-    info->virtual_size    = total_sectors * 512;
+    info->virtual_size    = size;
     info->actual_size     = bdrv_get_allocated_file_size(bs);
     info->has_actual_size = info->actual_size >= 0;
     if (bdrv_is_encrypted(bs)) {
@@ -236,22 +243,22 @@ void bdrv_query_image_info(BlockDriverState *bs,
 }
 
 /* @p_info will be set only on success. */
-void bdrv_query_info(BlockDriverState *bs,
-                     BlockInfo **p_info,
-                     Error **errp)
+static void bdrv_query_info(BlockBackend *blk, BlockInfo **p_info,
+                            Error **errp)
 {
     BlockInfo *info = g_malloc0(sizeof(*info));
+    BlockDriverState *bs = blk_bs(blk);
     BlockDriverState *bs0;
     ImageInfo **p_image_info;
     Error *local_err = NULL;
-    info->device = g_strdup(bs->device_name);
+    info->device = g_strdup(blk_name(blk));
     info->type = g_strdup("unknown");
-    info->locked = bdrv_dev_is_medium_locked(bs);
-    info->removable = bdrv_dev_has_removable_media(bs);
+    info->locked = blk_dev_is_medium_locked(blk);
+    info->removable = blk_dev_has_removable_media(blk);
 
-    if (bdrv_dev_has_removable_media(bs)) {
+    if (blk_dev_has_removable_media(blk)) {
         info->has_tray_open = true;
-        info->tray_open = bdrv_dev_is_tray_open(bs);
+        info->tray_open = blk_dev_is_tray_open(blk);
     }
 
     if (bdrv_iostatus_is_enabled(bs)) {
@@ -299,21 +306,22 @@ static BlockStats *bdrv_query_stats(const BlockDriverState *bs)
 
     s = g_malloc0(sizeof(*s));
 
-    if (bs->device_name[0]) {
+    if (bdrv_get_device_name(bs)[0]) {
         s->has_device = true;
-        s->device = g_strdup(bs->device_name);
+        s->device = g_strdup(bdrv_get_device_name(bs));
     }
 
     s->stats = g_malloc0(sizeof(*s->stats));
-    s->stats->rd_bytes = bs->nr_bytes[BDRV_ACCT_READ];
-    s->stats->wr_bytes = bs->nr_bytes[BDRV_ACCT_WRITE];
-    s->stats->rd_operations = bs->nr_ops[BDRV_ACCT_READ];
-    s->stats->wr_operations = bs->nr_ops[BDRV_ACCT_WRITE];
-    s->stats->wr_highest_offset = bs->wr_highest_sector * BDRV_SECTOR_SIZE;
-    s->stats->flush_operations = bs->nr_ops[BDRV_ACCT_FLUSH];
-    s->stats->wr_total_time_ns = bs->total_time_ns[BDRV_ACCT_WRITE];
-    s->stats->rd_total_time_ns = bs->total_time_ns[BDRV_ACCT_READ];
-    s->stats->flush_total_time_ns = bs->total_time_ns[BDRV_ACCT_FLUSH];
+    s->stats->rd_bytes = bs->stats.nr_bytes[BLOCK_ACCT_READ];
+    s->stats->wr_bytes = bs->stats.nr_bytes[BLOCK_ACCT_WRITE];
+    s->stats->rd_operations = bs->stats.nr_ops[BLOCK_ACCT_READ];
+    s->stats->wr_operations = bs->stats.nr_ops[BLOCK_ACCT_WRITE];
+    s->stats->wr_highest_offset =
+        bs->stats.wr_highest_sector * BDRV_SECTOR_SIZE;
+    s->stats->flush_operations = bs->stats.nr_ops[BLOCK_ACCT_FLUSH];
+    s->stats->wr_total_time_ns = bs->stats.total_time_ns[BLOCK_ACCT_WRITE];
+    s->stats->rd_total_time_ns = bs->stats.total_time_ns[BLOCK_ACCT_READ];
+    s->stats->flush_total_time_ns = bs->stats.total_time_ns[BLOCK_ACCT_FLUSH];
 
     if (bs->file) {
         s->has_parent = true;
@@ -331,12 +339,12 @@ static BlockStats *bdrv_query_stats(const BlockDriverState *bs)
 BlockInfoList *qmp_query_block(Error **errp)
 {
     BlockInfoList *head = NULL, **p_next = &head;
-    BlockDriverState *bs = NULL;
+    BlockBackend *blk;
     Error *local_err = NULL;
 
-     while ((bs = bdrv_next(bs))) {
+    for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
         BlockInfoList *info = g_malloc0(sizeof(*info));
-        bdrv_query_info(bs, &info->value, &local_err);
+        bdrv_query_info(blk, &info->value, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
             goto err;
diff --git a/block/qcow.c b/block/qcow.c
index a874056cf..ece22697a 100644
--- a/block/qcow.c
+++ b/block/qcow.c
@@ -124,7 +124,7 @@ static int qcow_open(BlockDriverState *bs, QDict *options, int flags,
         snprintf(version, sizeof(version), "QCOW version %" PRIu32,
                  header.version);
         error_set(errp, QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
-                  bs->device_name, "qcow", version);
+                  bdrv_get_device_name(bs), "qcow", version);
         ret = -ENOTSUP;
         goto fail;
     }
@@ -182,7 +182,12 @@ static int qcow_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     s->l1_table_offset = header.l1_table_offset;
-    s->l1_table = g_malloc(s->l1_size * sizeof(uint64_t));
+    s->l1_table = g_try_new(uint64_t, s->l1_size);
+    if (s->l1_table == NULL) {
+        error_setg(errp, "Could not allocate memory for L1 table");
+        ret = -ENOMEM;
+        goto fail;
+    }
 
     ret = bdrv_pread(bs->file, s->l1_table_offset, s->l1_table,
                s->l1_size * sizeof(uint64_t));
@@ -193,8 +198,16 @@ static int qcow_open(BlockDriverState *bs, QDict *options, int flags,
     for(i = 0;i < s->l1_size; i++) {
         be64_to_cpus(&s->l1_table[i]);
     }
-    /* alloc L2 cache */
-    s->l2_cache = g_malloc(s->l2_size * L2_CACHE_SIZE * sizeof(uint64_t));
+
+    /* alloc L2 cache (max. 64k * 16 * 8 = 8 MB) */
+    s->l2_cache =
+        qemu_try_blockalign(bs->file,
+                            s->l2_size * L2_CACHE_SIZE * sizeof(uint64_t));
+    if (s->l2_cache == NULL) {
+        error_setg(errp, "Could not allocate L2 table cache");
+        ret = -ENOMEM;
+        goto fail;
+    }
     s->cluster_cache = g_malloc(s->cluster_size);
     s->cluster_data = g_malloc(s->cluster_size);
     s->cluster_cache_offset = -1;
@@ -218,7 +231,7 @@ static int qcow_open(BlockDriverState *bs, QDict *options, int flags,
     /* Disable migration when qcow images are used */
     error_set(&s->migration_blocker,
               QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-              "qcow", bs->device_name, "live migration");
+              "qcow", bdrv_get_device_name(bs), "live migration");
     migrate_add_blocker(s->migration_blocker);
 
     qemu_co_mutex_init(&s->lock);
@@ -226,7 +239,7 @@ static int qcow_open(BlockDriverState *bs, QDict *options, int flags,
 
  fail:
     g_free(s->l1_table);
-    g_free(s->l2_cache);
+    qemu_vfree(s->l2_cache);
     g_free(s->cluster_cache);
     g_free(s->cluster_data);
     return ret;
@@ -517,7 +530,10 @@ static coroutine_fn int qcow_co_readv(BlockDriverState *bs, int64_t sector_num,
     void *orig_buf;
 
     if (qiov->niov > 1) {
-        buf = orig_buf = qemu_blockalign(bs, qiov->size);
+        buf = orig_buf = qemu_try_blockalign(bs, qiov->size);
+        if (buf == NULL) {
+            return -ENOMEM;
+        }
     } else {
         orig_buf = NULL;
         buf = (uint8_t *)qiov->iov->iov_base;
@@ -619,7 +635,10 @@ static coroutine_fn int qcow_co_writev(BlockDriverState *bs, int64_t sector_num,
     s->cluster_cache_offset = -1; /* disable compressed cache */
 
     if (qiov->niov > 1) {
-        buf = orig_buf = qemu_blockalign(bs, qiov->size);
+        buf = orig_buf = qemu_try_blockalign(bs, qiov->size);
+        if (buf == NULL) {
+            return -ENOMEM;
+        }
         qemu_iovec_to_buf(qiov, 0, buf, qiov->size);
     } else {
         orig_buf = NULL;
@@ -685,7 +704,7 @@ static void qcow_close(BlockDriverState *bs)
     BDRVQcowState *s = bs->opaque;
 
     g_free(s->l1_table);
-    g_free(s->l2_cache);
+    qemu_vfree(s->l2_cache);
     g_free(s->cluster_cache);
     g_free(s->cluster_data);
 
@@ -706,7 +725,8 @@ static int qcow_create(const char *filename, QemuOpts *opts, Error **errp)
     BlockDriverState *qcow_bs;
 
     /* Read out options */
-    total_size = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0) / 512;
+    total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                          BDRV_SECTOR_SIZE);
     backing_file = qemu_opt_get_del(opts, BLOCK_OPT_BACKING_FILE);
     if (qemu_opt_get_bool_del(opts, BLOCK_OPT_ENCRYPT, false)) {
         flags |= BLOCK_FLAG_ENCRYPT;
@@ -734,7 +754,7 @@ static int qcow_create(const char *filename, QemuOpts *opts, Error **errp)
     memset(&header, 0, sizeof(header));
     header.magic = cpu_to_be32(QCOW_MAGIC);
     header.version = cpu_to_be32(QCOW_VERSION);
-    header.size = cpu_to_be64(total_size * 512);
+    header.size = cpu_to_be64(total_size);
     header_size = sizeof(header);
     backing_filename_len = 0;
     if (backing_file) {
@@ -756,7 +776,7 @@ static int qcow_create(const char *filename, QemuOpts *opts, Error **errp)
     }
     header_size = (header_size + 7) & ~7;
     shift = header.cluster_bits + header.l2_bits;
-    l1_size = ((total_size * 512) + (1LL << shift) - 1) >> shift;
+    l1_size = (total_size + (1LL << shift) - 1) >> shift;
 
     header.l1_table_offset = cpu_to_be64(header_size);
     if (flags & BLOCK_FLAG_ENCRYPT) {
diff --git a/block/qcow2-cache.c b/block/qcow2-cache.c
index 8ecbb5bc0..904f6b1f4 100644
--- a/block/qcow2-cache.c
+++ b/block/qcow2-cache.c
@@ -48,15 +48,31 @@ Qcow2Cache *qcow2_cache_create(BlockDriverState *bs, int num_tables)
     Qcow2Cache *c;
     int i;
 
-    c = g_malloc0(sizeof(*c));
+    c = g_new0(Qcow2Cache, 1);
     c->size = num_tables;
-    c->entries = g_malloc0(sizeof(*c->entries) * num_tables);
+    c->entries = g_try_new0(Qcow2CachedTable, num_tables);
+    if (!c->entries) {
+        goto fail;
+    }
 
     for (i = 0; i < c->size; i++) {
-        c->entries[i].table = qemu_blockalign(bs, s->cluster_size);
+        c->entries[i].table = qemu_try_blockalign(bs->file, s->cluster_size);
+        if (c->entries[i].table == NULL) {
+            goto fail;
+        }
     }
 
     return c;
+
+fail:
+    if (c->entries) {
+        for (i = 0; i < c->size; i++) {
+            qemu_vfree(c->entries[i].table);
+        }
+    }
+    g_free(c->entries);
+    g_free(c);
+    return NULL;
 }
 
 int qcow2_cache_destroy(BlockDriverState* bs, Qcow2Cache *c)
diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c
index 4208dc08b..df0b2c9ce 100644
--- a/block/qcow2-cluster.c
+++ b/block/qcow2-cluster.c
@@ -72,14 +72,20 @@ int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
 #endif
 
     new_l1_size2 = sizeof(uint64_t) * new_l1_size;
-    new_l1_table = g_malloc0(align_offset(new_l1_size2, 512));
+    new_l1_table = qemu_try_blockalign(bs->file,
+                                       align_offset(new_l1_size2, 512));
+    if (new_l1_table == NULL) {
+        return -ENOMEM;
+    }
+    memset(new_l1_table, 0, align_offset(new_l1_size2, 512));
+
     memcpy(new_l1_table, s->l1_table, s->l1_size * sizeof(uint64_t));
 
     /* write new table (align to cluster) */
     BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_ALLOC_TABLE);
     new_l1_table_offset = qcow2_alloc_clusters(bs, new_l1_size2);
     if (new_l1_table_offset < 0) {
-        g_free(new_l1_table);
+        qemu_vfree(new_l1_table);
         return new_l1_table_offset;
     }
 
@@ -113,7 +119,7 @@ int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
     if (ret < 0) {
         goto fail;
     }
-    g_free(s->l1_table);
+    qemu_vfree(s->l1_table);
     old_l1_table_offset = s->l1_table_offset;
     s->l1_table_offset = new_l1_table_offset;
     s->l1_table = new_l1_table;
@@ -123,7 +129,7 @@ int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
                         QCOW2_DISCARD_OTHER);
     return 0;
  fail:
-    g_free(new_l1_table);
+    qemu_vfree(new_l1_table);
     qcow2_free_clusters(bs, new_l1_table_offset, new_l1_size2,
                         QCOW2_DISCARD_OTHER);
     return ret;
@@ -158,12 +164,14 @@ static int l2_load(BlockDriverState *bs, uint64_t l2_offset,
 int qcow2_write_l1_entry(BlockDriverState *bs, int l1_index)
 {
     BDRVQcowState *s = bs->opaque;
-    uint64_t buf[L1_ENTRIES_PER_SECTOR];
+    uint64_t buf[L1_ENTRIES_PER_SECTOR] = { 0 };
     int l1_start_index;
     int i, ret;
 
     l1_start_index = l1_index & ~(L1_ENTRIES_PER_SECTOR - 1);
-    for (i = 0; i < L1_ENTRIES_PER_SECTOR; i++) {
+    for (i = 0; i < L1_ENTRIES_PER_SECTOR && l1_start_index + i < s->l1_size;
+         i++)
+    {
         buf[i] = cpu_to_be64(s->l1_table[l1_start_index + i]);
     }
 
@@ -372,7 +380,10 @@ static int coroutine_fn copy_sectors(BlockDriverState *bs,
     }
 
     iov.iov_len = n * BDRV_SECTOR_SIZE;
-    iov.iov_base = qemu_blockalign(bs, iov.iov_len);
+    iov.iov_base = qemu_try_blockalign(bs, iov.iov_len);
+    if (iov.iov_base == NULL) {
+        return -ENOMEM;
+    }
 
     qemu_iovec_init_external(&qiov, &iov, 1);
 
@@ -477,6 +488,13 @@ int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
         goto out;
     }
 
+    if (offset_into_cluster(s, l2_offset)) {
+        qcow2_signal_corruption(bs, true, -1, -1, "L2 table offset %#" PRIx64
+                                " unaligned (L1 index: %#" PRIx64 ")",
+                                l2_offset, l1_index);
+        return -EIO;
+    }
+
     /* load the l2 table in memory */
 
     ret = l2_load(bs, l2_offset, &l2_table);
@@ -499,8 +517,11 @@ int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
         break;
     case QCOW2_CLUSTER_ZERO:
         if (s->qcow_version < 3) {
-            qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-            return -EIO;
+            qcow2_signal_corruption(bs, true, -1, -1, "Zero cluster entry found"
+                                    " in pre-v3 image (L2 offset: %#" PRIx64
+                                    ", L2 index: %#x)", l2_offset, l2_index);
+            ret = -EIO;
+            goto fail;
         }
         c = count_contiguous_clusters(nb_clusters, s->cluster_size,
                 &l2_table[l2_index], QCOW_OFLAG_ZERO);
@@ -516,6 +537,14 @@ int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
         c = count_contiguous_clusters(nb_clusters, s->cluster_size,
                 &l2_table[l2_index], QCOW_OFLAG_ZERO);
         *cluster_offset &= L2E_OFFSET_MASK;
+        if (offset_into_cluster(s, *cluster_offset)) {
+            qcow2_signal_corruption(bs, true, -1, -1, "Data cluster offset %#"
+                                    PRIx64 " unaligned (L2 offset: %#" PRIx64
+                                    ", L2 index: %#x)", *cluster_offset,
+                                    l2_offset, l2_index);
+            ret = -EIO;
+            goto fail;
+        }
         break;
     default:
         abort();
@@ -532,6 +561,10 @@ out:
     *num = nb_available - index_in_cluster;
 
     return ret;
+
+fail:
+    qcow2_cache_put(bs, s->l2_table_cache, (void **)&l2_table);
+    return ret;
 }
 
 /*
@@ -567,6 +600,12 @@ static int get_cluster_table(BlockDriverState *bs, uint64_t offset,
 
     assert(l1_index < s->l1_size);
     l2_offset = s->l1_table[l1_index] & L1E_OFFSET_MASK;
+    if (offset_into_cluster(s, l2_offset)) {
+        qcow2_signal_corruption(bs, true, -1, -1, "L2 table offset %#" PRIx64
+                                " unaligned (L1 index: %#" PRIx64 ")",
+                                l2_offset, l1_index);
+        return -EIO;
+    }
 
     /* seek the l2 table of the given l2 offset */
 
@@ -702,7 +741,11 @@ int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m)
     trace_qcow2_cluster_link_l2(qemu_coroutine_self(), m->nb_clusters);
     assert(m->nb_clusters > 0);
 
-    old_cluster = g_malloc(m->nb_clusters * sizeof(uint64_t));
+    old_cluster = g_try_new(uint64_t, m->nb_clusters);
+    if (old_cluster == NULL) {
+        ret = -ENOMEM;
+        goto err;
+    }
 
     /* copy content of unmodified sectors */
     ret = perform_cow(bs, m, &m->cow_start);
@@ -935,6 +978,15 @@ static int handle_copied(BlockDriverState *bs, uint64_t guest_offset,
         bool offset_matches =
             (cluster_offset & L2E_OFFSET_MASK) == *host_offset;
 
+        if (offset_into_cluster(s, cluster_offset & L2E_OFFSET_MASK)) {
+            qcow2_signal_corruption(bs, true, -1, -1, "Data cluster offset "
+                                    "%#llx unaligned (guest offset: %#" PRIx64
+                                    ")", cluster_offset & L2E_OFFSET_MASK,
+                                    guest_offset);
+            ret = -EIO;
+            goto out;
+        }
+
         if (*host_offset != 0 && !offset_matches) {
             *bytes = 0;
             ret = 0;
@@ -966,7 +1018,7 @@ out:
 
     /* Only return a host offset if we actually made progress. Otherwise we
      * would make requirements for handle_alloc() that it can't fulfill */
-    if (ret) {
+    if (ret > 0) {
         *host_offset = (cluster_offset & L2E_OFFSET_MASK)
                      + offset_into_cluster(s, guest_offset);
     }
@@ -1106,6 +1158,17 @@ static int handle_alloc(BlockDriverState *bs, uint64_t guest_offset,
         return 0;
     }
 
+    /* !*host_offset would overwrite the image header and is reserved for "no
+     * host offset preferred". If 0 was a valid host offset, it'd trigger the
+     * following overlap check; do that now to avoid having an invalid value in
+     * *host_offset. */
+    if (!alloc_cluster_offset) {
+        ret = qcow2_pre_write_overlap_check(bs, 0, alloc_cluster_offset,
+                                            nb_clusters * s->cluster_size);
+        assert(ret < 0);
+        goto fail;
+    }
+
     /*
      * Save info needed for meta data update.
      *
@@ -1351,7 +1414,7 @@ int qcow2_decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset)
  * clusters.
  */
 static int discard_single_l2(BlockDriverState *bs, uint64_t offset,
-    unsigned int nb_clusters, enum qcow2_discard_type type)
+    unsigned int nb_clusters, enum qcow2_discard_type type, bool full_discard)
 {
     BDRVQcowState *s = bs->opaque;
     uint64_t *l2_table;
@@ -1373,23 +1436,30 @@ static int discard_single_l2(BlockDriverState *bs, uint64_t offset,
         old_l2_entry = be64_to_cpu(l2_table[l2_index + i]);
 
         /*
-         * Make sure that a discarded area reads back as zeroes for v3 images
-         * (we cannot do it for v2 without actually writing a zero-filled
-         * buffer). We can skip the operation if the cluster is already marked
-         * as zero, or if it's unallocated and we don't have a backing file.
+         * If full_discard is false, make sure that a discarded area reads back
+         * as zeroes for v3 images (we cannot do it for v2 without actually
+         * writing a zero-filled buffer). We can skip the operation if the
+         * cluster is already marked as zero, or if it's unallocated and we
+         * don't have a backing file.
          *
          * TODO We might want to use bdrv_get_block_status(bs) here, but we're
          * holding s->lock, so that doesn't work today.
+         *
+         * If full_discard is true, the sector should not read back as zeroes,
+         * but rather fall through to the backing file.
          */
         switch (qcow2_get_cluster_type(old_l2_entry)) {
             case QCOW2_CLUSTER_UNALLOCATED:
-                if (!bs->backing_hd) {
+                if (full_discard || !bs->backing_hd) {
                     continue;
                 }
                 break;
 
             case QCOW2_CLUSTER_ZERO:
-                continue;
+                if (!full_discard) {
+                    continue;
+                }
+                break;
 
             case QCOW2_CLUSTER_NORMAL:
             case QCOW2_CLUSTER_COMPRESSED:
@@ -1401,7 +1471,7 @@ static int discard_single_l2(BlockDriverState *bs, uint64_t offset,
 
         /* First remove L2 entries */
         qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
-        if (s->qcow_version >= 3) {
+        if (!full_discard && s->qcow_version >= 3) {
             l2_table[l2_index + i] = cpu_to_be64(QCOW_OFLAG_ZERO);
         } else {
             l2_table[l2_index + i] = cpu_to_be64(0);
@@ -1420,7 +1490,7 @@ static int discard_single_l2(BlockDriverState *bs, uint64_t offset,
 }
 
 int qcow2_discard_clusters(BlockDriverState *bs, uint64_t offset,
-    int nb_sectors, enum qcow2_discard_type type)
+    int nb_sectors, enum qcow2_discard_type type, bool full_discard)
 {
     BDRVQcowState *s = bs->opaque;
     uint64_t end_offset;
@@ -1443,7 +1513,7 @@ int qcow2_discard_clusters(BlockDriverState *bs, uint64_t offset,
 
     /* Each L2 table is handled by its own loop iteration */
     while (nb_clusters > 0) {
-        ret = discard_single_l2(bs, offset, nb_clusters, type);
+        ret = discard_single_l2(bs, offset, nb_clusters, type, full_discard);
         if (ret < 0) {
             goto fail;
         }
@@ -1543,15 +1613,14 @@ fail:
  * Expands all zero clusters in a specific L1 table (or deallocates them, for
  * non-backed non-pre-allocated zero clusters).
  *
- * expanded_clusters is a bitmap where every bit corresponds to one cluster in
- * the image file; a bit gets set if the corresponding cluster has been used for
- * zero expansion (i.e., has been filled with zeroes and is referenced from an
- * L2 table). nb_clusters contains the total cluster count of the image file,
- * i.e., the number of bits in expanded_clusters.
+ * l1_entries and *visited_l1_entries are used to keep track of progress for
+ * status_cb(). l1_entries contains the total number of L1 entries and
+ * *visited_l1_entries counts all visited L1 entries.
  */
 static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
-                                      int l1_size, uint8_t **expanded_clusters,
-                                      uint64_t *nb_clusters)
+                                      int l1_size, int64_t *visited_l1_entries,
+                                      int64_t l1_entries,
+                                      BlockDriverAmendStatusCB *status_cb)
 {
     BDRVQcowState *s = bs->opaque;
     bool is_active_l1 = (l1_table == s->l1_table);
@@ -1562,15 +1631,23 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
     if (!is_active_l1) {
         /* inactive L2 tables require a buffer to be stored in when loading
          * them from disk */
-        l2_table = qemu_blockalign(bs, s->cluster_size);
+        l2_table = qemu_try_blockalign(bs->file, s->cluster_size);
+        if (l2_table == NULL) {
+            return -ENOMEM;
+        }
     }
 
     for (i = 0; i < l1_size; i++) {
         uint64_t l2_offset = l1_table[i] & L1E_OFFSET_MASK;
         bool l2_dirty = false;
+        int l2_refcount;
 
         if (!l2_offset) {
             /* unallocated */
+            (*visited_l1_entries)++;
+            if (status_cb) {
+                status_cb(bs, *visited_l1_entries, l1_entries);
+            }
             continue;
         }
 
@@ -1587,33 +1664,19 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
             goto fail;
         }
 
+        l2_refcount = qcow2_get_refcount(bs, l2_offset >> s->cluster_bits);
+        if (l2_refcount < 0) {
+            ret = l2_refcount;
+            goto fail;
+        }
+
         for (j = 0; j < s->l2_size; j++) {
             uint64_t l2_entry = be64_to_cpu(l2_table[j]);
-            int64_t offset = l2_entry & L2E_OFFSET_MASK, cluster_index;
+            int64_t offset = l2_entry & L2E_OFFSET_MASK;
             int cluster_type = qcow2_get_cluster_type(l2_entry);
             bool preallocated = offset != 0;
 
-            if (cluster_type == QCOW2_CLUSTER_NORMAL) {
-                cluster_index = offset >> s->cluster_bits;
-                assert((cluster_index >= 0) && (cluster_index < *nb_clusters));
-                if ((*expanded_clusters)[cluster_index / 8] &
-                    (1 << (cluster_index % 8))) {
-                    /* Probably a shared L2 table; this cluster was a zero
-                     * cluster which has been expanded, its refcount
-                     * therefore most likely requires an update. */
-                    ret = qcow2_update_cluster_refcount(bs, cluster_index, 1,
-                                                        QCOW2_DISCARD_NEVER);
-                    if (ret < 0) {
-                        goto fail;
-                    }
-                    /* Since we just increased the refcount, the COPIED flag may
-                     * no longer be set. */
-                    l2_table[j] = cpu_to_be64(l2_entry & ~QCOW_OFLAG_COPIED);
-                    l2_dirty = true;
-                }
-                continue;
-            }
-            else if (qcow2_get_cluster_type(l2_entry) != QCOW2_CLUSTER_ZERO) {
+            if (cluster_type != QCOW2_CLUSTER_ZERO) {
                 continue;
             }
 
@@ -1631,6 +1694,19 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
                     ret = offset;
                     goto fail;
                 }
+
+                if (l2_refcount > 1) {
+                    /* For shared L2 tables, set the refcount accordingly (it is
+                     * already 1 and needs to be l2_refcount) */
+                    ret = qcow2_update_cluster_refcount(bs,
+                            offset >> s->cluster_bits, l2_refcount - 1,
+                            QCOW2_DISCARD_OTHER);
+                    if (ret < 0) {
+                        qcow2_free_clusters(bs, offset, s->cluster_size,
+                                            QCOW2_DISCARD_OTHER);
+                        goto fail;
+                    }
+                }
             }
 
             ret = qcow2_pre_write_overlap_check(bs, 0, offset, s->cluster_size);
@@ -1652,29 +1728,12 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
                 goto fail;
             }
 
-            l2_table[j] = cpu_to_be64(offset | QCOW_OFLAG_COPIED);
-            l2_dirty = true;
-
-            cluster_index = offset >> s->cluster_bits;
-
-            if (cluster_index >= *nb_clusters) {
-                uint64_t old_bitmap_size = (*nb_clusters + 7) / 8;
-                uint64_t new_bitmap_size;
-                /* The offset may lie beyond the old end of the underlying image
-                 * file for growable files only */
-                assert(bs->file->growable);
-                *nb_clusters = size_to_clusters(s, bs->file->total_sectors *
-                                                BDRV_SECTOR_SIZE);
-                new_bitmap_size = (*nb_clusters + 7) / 8;
-                *expanded_clusters = g_realloc(*expanded_clusters,
-                                               new_bitmap_size);
-                /* clear the newly allocated space */
-                memset(&(*expanded_clusters)[old_bitmap_size], 0,
-                       new_bitmap_size - old_bitmap_size);
+            if (l2_refcount == 1) {
+                l2_table[j] = cpu_to_be64(offset | QCOW_OFLAG_COPIED);
+            } else {
+                l2_table[j] = cpu_to_be64(offset);
             }
-
-            assert((cluster_index >= 0) && (cluster_index < *nb_clusters));
-            (*expanded_clusters)[cluster_index / 8] |= 1 << (cluster_index % 8);
+            l2_dirty = true;
         }
 
         if (is_active_l1) {
@@ -1703,6 +1762,11 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
                 }
             }
         }
+
+        (*visited_l1_entries)++;
+        if (status_cb) {
+            status_cb(bs, *visited_l1_entries, l1_entries);
+        }
     }
 
     ret = 0;
@@ -1729,21 +1793,25 @@ fail:
  * allocation for pre-allocated ones). This is important for downgrading to a
  * qcow2 version which doesn't yet support metadata zero clusters.
  */
-int qcow2_expand_zero_clusters(BlockDriverState *bs)
+int qcow2_expand_zero_clusters(BlockDriverState *bs,
+                               BlockDriverAmendStatusCB *status_cb)
 {
     BDRVQcowState *s = bs->opaque;
     uint64_t *l1_table = NULL;
-    uint64_t nb_clusters;
-    uint8_t *expanded_clusters;
+    int64_t l1_entries = 0, visited_l1_entries = 0;
     int ret;
     int i, j;
 
-    nb_clusters = size_to_clusters(s, bs->file->total_sectors *
-                                   BDRV_SECTOR_SIZE);
-    expanded_clusters = g_malloc0((nb_clusters + 7) / 8);
+    if (status_cb) {
+        l1_entries = s->l1_size;
+        for (i = 0; i < s->nb_snapshots; i++) {
+            l1_entries += s->snapshots[i].l1_size;
+        }
+    }
 
     ret = expand_zero_clusters_in_l1(bs, s->l1_table, s->l1_size,
-                                     &expanded_clusters, &nb_clusters);
+                                     &visited_l1_entries, l1_entries,
+                                     status_cb);
     if (ret < 0) {
         goto fail;
     }
@@ -1777,7 +1845,8 @@ int qcow2_expand_zero_clusters(BlockDriverState *bs)
         }
 
         ret = expand_zero_clusters_in_l1(bs, l1_table, s->snapshots[i].l1_size,
-                                         &expanded_clusters, &nb_clusters);
+                                         &visited_l1_entries, l1_entries,
+                                         status_cb);
         if (ret < 0) {
             goto fail;
         }
@@ -1786,7 +1855,6 @@ int qcow2_expand_zero_clusters(BlockDriverState *bs)
     ret = 0;
 
 fail:
-    g_free(expanded_clusters);
     g_free(l1_table);
     return ret;
 }
diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c
index cc6cf743d..9afdb40b4 100644
--- a/block/qcow2-refcount.c
+++ b/block/qcow2-refcount.c
@@ -26,8 +26,6 @@
 #include "block/block_int.h"
 #include "block/qcow2.h"
 #include "qemu/range.h"
-#include "qapi/qmp/types.h"
-#include "qapi-event.h"
 
 static int64_t alloc_clusters_noref(BlockDriverState *bs, uint64_t size);
 static int QEMU_WARN_UNUSED_RESULT update_refcount(BlockDriverState *bs,
@@ -46,19 +44,25 @@ int qcow2_refcount_init(BlockDriverState *bs)
 
     assert(s->refcount_table_size <= INT_MAX / sizeof(uint64_t));
     refcount_table_size2 = s->refcount_table_size * sizeof(uint64_t);
-    s->refcount_table = g_malloc(refcount_table_size2);
+    s->refcount_table = g_try_malloc(refcount_table_size2);
+
     if (s->refcount_table_size > 0) {
+        if (s->refcount_table == NULL) {
+            ret = -ENOMEM;
+            goto fail;
+        }
         BLKDBG_EVENT(bs->file, BLKDBG_REFTABLE_LOAD);
         ret = bdrv_pread(bs->file, s->refcount_table_offset,
                          s->refcount_table, refcount_table_size2);
-        if (ret != refcount_table_size2)
+        if (ret < 0) {
             goto fail;
+        }
         for(i = 0; i < s->refcount_table_size; i++)
             be64_to_cpus(&s->refcount_table[i]);
     }
     return 0;
  fail:
-    return -ENOMEM;
+    return ret;
 }
 
 void qcow2_refcount_close(BlockDriverState *bs)
@@ -87,7 +91,7 @@ static int load_refcount_block(BlockDriverState *bs,
  * return value is the refcount of the cluster, negative values are -errno
  * and indicate an error.
  */
-static int get_refcount(BlockDriverState *bs, int64_t cluster_index)
+int qcow2_get_refcount(BlockDriverState *bs, int64_t cluster_index)
 {
     BDRVQcowState *s = bs->opaque;
     uint64_t refcount_table_index, block_index;
@@ -96,7 +100,7 @@ static int get_refcount(BlockDriverState *bs, int64_t cluster_index)
     uint16_t *refcount_block;
     uint16_t refcount;
 
-    refcount_table_index = cluster_index >> (s->cluster_bits - REFCOUNT_SHIFT);
+    refcount_table_index = cluster_index >> s->refcount_block_bits;
     if (refcount_table_index >= s->refcount_table_size)
         return 0;
     refcount_block_offset =
@@ -104,14 +108,20 @@ static int get_refcount(BlockDriverState *bs, int64_t cluster_index)
     if (!refcount_block_offset)
         return 0;
 
+    if (offset_into_cluster(s, refcount_block_offset)) {
+        qcow2_signal_corruption(bs, true, -1, -1, "Refblock offset %#" PRIx64
+                                " unaligned (reftable index: %#" PRIx64 ")",
+                                refcount_block_offset, refcount_table_index);
+        return -EIO;
+    }
+
     ret = qcow2_cache_get(bs, s->refcount_block_cache, refcount_block_offset,
         (void**) &refcount_block);
     if (ret < 0) {
         return ret;
     }
 
-    block_index = cluster_index &
-        ((1 << (s->cluster_bits - REFCOUNT_SHIFT)) - 1);
+    block_index = cluster_index & (s->refcount_block_size - 1);
     refcount = be16_to_cpu(refcount_block[block_index]);
 
     ret = qcow2_cache_put(bs, s->refcount_block_cache,
@@ -146,8 +156,8 @@ static unsigned int next_refcount_table_size(BDRVQcowState *s,
 static int in_same_refcount_block(BDRVQcowState *s, uint64_t offset_a,
     uint64_t offset_b)
 {
-    uint64_t block_a = offset_a >> (2 * s->cluster_bits - REFCOUNT_SHIFT);
-    uint64_t block_b = offset_b >> (2 * s->cluster_bits - REFCOUNT_SHIFT);
+    uint64_t block_a = offset_a >> (s->cluster_bits + s->refcount_block_bits);
+    uint64_t block_b = offset_b >> (s->cluster_bits + s->refcount_block_bits);
 
     return (block_a == block_b);
 }
@@ -168,7 +178,7 @@ static int alloc_refcount_block(BlockDriverState *bs,
     BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC);
 
     /* Find the refcount block for the given cluster */
-    refcount_table_index = cluster_index >> (s->cluster_bits - REFCOUNT_SHIFT);
+    refcount_table_index = cluster_index >> s->refcount_block_bits;
 
     if (refcount_table_index < s->refcount_table_size) {
 
@@ -177,6 +187,14 @@ static int alloc_refcount_block(BlockDriverState *bs,
 
         /* If it's already there, we're done */
         if (refcount_block_offset) {
+            if (offset_into_cluster(s, refcount_block_offset)) {
+                qcow2_signal_corruption(bs, true, -1, -1, "Refblock offset %#"
+                                        PRIx64 " unaligned (reftable index: "
+                                        "%#x)", refcount_block_offset,
+                                        refcount_table_index);
+                return -EIO;
+            }
+
              return load_refcount_block(bs, refcount_block_offset,
                  (void**) refcount_block);
         }
@@ -237,7 +255,7 @@ static int alloc_refcount_block(BlockDriverState *bs,
 
         /* The block describes itself, need to update the cache */
         int block_index = (new_block >> s->cluster_bits) &
-            ((1 << (s->cluster_bits - REFCOUNT_SHIFT)) - 1);
+            (s->refcount_block_size - 1);
         (*refcount_block)[block_index] = cpu_to_be16(1);
     } else {
         /* Described somewhere else. This can recurse at most twice before we
@@ -309,8 +327,7 @@ static int alloc_refcount_block(BlockDriverState *bs,
     BLKDBG_EVENT(bs->file, BLKDBG_REFTABLE_GROW);
 
     /* Calculate the number of refcount blocks needed so far */
-    uint64_t refcount_block_clusters = 1 << (s->cluster_bits - REFCOUNT_SHIFT);
-    uint64_t blocks_used = DIV_ROUND_UP(cluster_index, refcount_block_clusters);
+    uint64_t blocks_used = DIV_ROUND_UP(cluster_index, s->refcount_block_size);
 
     if (blocks_used > QCOW_MAX_REFTABLE_SIZE / sizeof(uint64_t)) {
         return -EFBIG;
@@ -324,14 +341,14 @@ static int alloc_refcount_block(BlockDriverState *bs,
         uint64_t table_clusters =
             size_to_clusters(s, table_size * sizeof(uint64_t));
         blocks_clusters = 1 +
-            ((table_clusters + refcount_block_clusters - 1)
-            / refcount_block_clusters);
+            ((table_clusters + s->refcount_block_size - 1)
+            / s->refcount_block_size);
         uint64_t meta_clusters = table_clusters + blocks_clusters;
 
         last_table_size = table_size;
         table_size = next_refcount_table_size(s, blocks_used +
-            ((meta_clusters + refcount_block_clusters - 1)
-            / refcount_block_clusters));
+            ((meta_clusters + s->refcount_block_size - 1)
+            / s->refcount_block_size));
 
     } while (last_table_size != table_size);
 
@@ -341,11 +358,17 @@ static int alloc_refcount_block(BlockDriverState *bs,
 #endif
 
     /* Create the new refcount table and blocks */
-    uint64_t meta_offset = (blocks_used * refcount_block_clusters) *
+    uint64_t meta_offset = (blocks_used * s->refcount_block_size) *
         s->cluster_size;
     uint64_t table_offset = meta_offset + blocks_clusters * s->cluster_size;
-    uint16_t *new_blocks = g_malloc0(blocks_clusters * s->cluster_size);
-    uint64_t *new_table = g_malloc0(table_size * sizeof(uint64_t));
+    uint64_t *new_table = g_try_new0(uint64_t, table_size);
+    uint16_t *new_blocks = g_try_malloc0(blocks_clusters * s->cluster_size);
+
+    assert(table_size > 0 && blocks_clusters > 0);
+    if (new_table == NULL || new_blocks == NULL) {
+        ret = -ENOMEM;
+        goto fail_table;
+    }
 
     /* Fill the new refcount table */
     memcpy(new_table, s->refcount_table,
@@ -369,6 +392,7 @@ static int alloc_refcount_block(BlockDriverState *bs,
     ret = bdrv_pwrite_sync(bs->file, meta_offset, new_blocks,
         blocks_clusters * s->cluster_size);
     g_free(new_blocks);
+    new_blocks = NULL;
     if (ret < 0) {
         goto fail_table;
     }
@@ -424,6 +448,7 @@ static int alloc_refcount_block(BlockDriverState *bs,
     return -EAGAIN;
 
 fail_table:
+    g_free(new_blocks);
     g_free(new_table);
 fail_block:
     if (*refcount_block != NULL) {
@@ -497,6 +522,7 @@ found:
         QTAILQ_REMOVE(&s->discards, p, next);
         d->offset = MIN(d->offset, p->offset);
         d->bytes += p->bytes;
+        g_free(p);
     }
 }
 
@@ -532,8 +558,7 @@ static int QEMU_WARN_UNUSED_RESULT update_refcount(BlockDriverState *bs,
     {
         int block_index, refcount;
         int64_t cluster_index = cluster_offset >> s->cluster_bits;
-        int64_t table_index =
-            cluster_index >> (s->cluster_bits - REFCOUNT_SHIFT);
+        int64_t table_index = cluster_index >> s->refcount_block_bits;
 
         /* Load the refcount block and allocate it if needed */
         if (table_index != old_table_index) {
@@ -555,8 +580,7 @@ static int QEMU_WARN_UNUSED_RESULT update_refcount(BlockDriverState *bs,
         qcow2_cache_entry_mark_dirty(s->refcount_block_cache, refcount_block);
 
         /* we can update the count and save it */
-        block_index = cluster_index &
-            ((1 << (s->cluster_bits - REFCOUNT_SHIFT)) - 1);
+        block_index = cluster_index & (s->refcount_block_size - 1);
 
         refcount = be16_to_cpu(refcount_block[block_index]);
         refcount += addend;
@@ -605,8 +629,7 @@ fail:
 }
 
 /*
- * Increases or decreases the refcount of a given cluster by one.
- * addend must be 1 or -1.
+ * Increases or decreases the refcount of a given cluster.
  *
  * If the return value is non-negative, it is the new refcount of the cluster.
  * If it is negative, it is -errno and indicates an error.
@@ -625,7 +648,7 @@ int qcow2_update_cluster_refcount(BlockDriverState *bs,
         return ret;
     }
 
-    return get_refcount(bs, cluster_index);
+    return qcow2_get_refcount(bs, cluster_index);
 }
 
 
@@ -646,7 +669,7 @@ static int64_t alloc_clusters_noref(BlockDriverState *bs, uint64_t size)
 retry:
     for(i = 0; i < nb_clusters; i++) {
         uint64_t next_cluster_index = s->free_cluster_index++;
-        refcount = get_refcount(bs, next_cluster_index);
+        refcount = qcow2_get_refcount(bs, next_cluster_index);
 
         if (refcount < 0) {
             return refcount;
@@ -710,7 +733,7 @@ int qcow2_alloc_clusters_at(BlockDriverState *bs, uint64_t offset,
         /* Check how many clusters there are free */
         cluster_index = offset >> s->cluster_bits;
         for(i = 0; i < nb_clusters; i++) {
-            refcount = get_refcount(bs, cluster_index++);
+            refcount = qcow2_get_refcount(bs, cluster_index++);
 
             if (refcount < 0) {
                 return refcount;
@@ -824,8 +847,14 @@ void qcow2_free_any_clusters(BlockDriverState *bs, uint64_t l2_entry,
     case QCOW2_CLUSTER_NORMAL:
     case QCOW2_CLUSTER_ZERO:
         if (l2_entry & L2E_OFFSET_MASK) {
-            qcow2_free_clusters(bs, l2_entry & L2E_OFFSET_MASK,
-                                nb_clusters << s->cluster_bits, type);
+            if (offset_into_cluster(s, l2_entry & L2E_OFFSET_MASK)) {
+                qcow2_signal_corruption(bs, false, -1, -1,
+                                        "Cannot free unaligned cluster %#llx",
+                                        l2_entry & L2E_OFFSET_MASK);
+            } else {
+                qcow2_free_clusters(bs, l2_entry & L2E_OFFSET_MASK,
+                                    nb_clusters << s->cluster_bits, type);
+            }
         }
         break;
     case QCOW2_CLUSTER_UNALLOCATED:
@@ -847,7 +876,8 @@ int qcow2_update_snapshot_refcount(BlockDriverState *bs,
     int64_t l1_table_offset, int l1_size, int addend)
 {
     BDRVQcowState *s = bs->opaque;
-    uint64_t *l1_table, *l2_table, l2_offset, offset, l1_size2, l1_allocated;
+    uint64_t *l1_table, *l2_table, l2_offset, offset, l1_size2;
+    bool l1_allocated = false;
     int64_t old_offset, old_l2_offset;
     int i, j, l1_modified = 0, nb_csectors, refcount;
     int ret;
@@ -862,8 +892,12 @@ int qcow2_update_snapshot_refcount(BlockDriverState *bs,
      * l1_table_offset when it is the current s->l1_table_offset! Be careful
      * when changing this! */
     if (l1_table_offset != s->l1_table_offset) {
-        l1_table = g_malloc0(align_offset(l1_size2, 512));
-        l1_allocated = 1;
+        l1_table = g_try_malloc0(align_offset(l1_size2, 512));
+        if (l1_size2 && l1_table == NULL) {
+            ret = -ENOMEM;
+            goto fail;
+        }
+        l1_allocated = true;
 
         ret = bdrv_pread(bs->file, l1_table_offset, l1_table, l1_size2);
         if (ret < 0) {
@@ -875,7 +909,7 @@ int qcow2_update_snapshot_refcount(BlockDriverState *bs,
     } else {
         assert(l1_size == s->l1_size);
         l1_table = s->l1_table;
-        l1_allocated = 0;
+        l1_allocated = false;
     }
 
     for(i = 0; i < l1_size; i++) {
@@ -884,6 +918,14 @@ int qcow2_update_snapshot_refcount(BlockDriverState *bs,
             old_l2_offset = l2_offset;
             l2_offset &= L1E_OFFSET_MASK;
 
+            if (offset_into_cluster(s, l2_offset)) {
+                qcow2_signal_corruption(bs, true, -1, -1, "L2 table offset %#"
+                                        PRIx64 " unaligned (L1 index: %#x)",
+                                        l2_offset, i);
+                ret = -EIO;
+                goto fail;
+            }
+
             ret = qcow2_cache_get(bs, s->l2_table_cache, l2_offset,
                 (void**) &l2_table);
             if (ret < 0) {
@@ -916,6 +958,17 @@ int qcow2_update_snapshot_refcount(BlockDriverState *bs,
 
                     case QCOW2_CLUSTER_NORMAL:
                     case QCOW2_CLUSTER_ZERO:
+                        if (offset_into_cluster(s, offset & L2E_OFFSET_MASK)) {
+                            qcow2_signal_corruption(bs, true, -1, -1, "Data "
+                                                    "cluster offset %#llx "
+                                                    "unaligned (L2 offset: %#"
+                                                    PRIx64 ", L2 index: %#x)",
+                                                    offset & L2E_OFFSET_MASK,
+                                                    l2_offset, j);
+                            ret = -EIO;
+                            goto fail;
+                        }
+
                         cluster_index = (offset & L2E_OFFSET_MASK) >> s->cluster_bits;
                         if (!cluster_index) {
                             /* unallocated */
@@ -927,7 +980,7 @@ int qcow2_update_snapshot_refcount(BlockDriverState *bs,
                                     cluster_index, addend,
                                     QCOW2_DISCARD_SNAPSHOT);
                         } else {
-                            refcount = get_refcount(bs, cluster_index);
+                            refcount = qcow2_get_refcount(bs, cluster_index);
                         }
 
                         if (refcount < 0) {
@@ -967,7 +1020,7 @@ int qcow2_update_snapshot_refcount(BlockDriverState *bs,
                 refcount = qcow2_update_cluster_refcount(bs, l2_offset >>
                         s->cluster_bits, addend, QCOW2_DISCARD_SNAPSHOT);
             } else {
-                refcount = get_refcount(bs, l2_offset >> s->cluster_bits);
+                refcount = qcow2_get_refcount(bs, l2_offset >> s->cluster_bits);
             }
             if (refcount < 0) {
                 ret = refcount;
@@ -1023,36 +1076,52 @@ fail:
  *
  * Modifies the number of errors in res.
  */
-static void inc_refcounts(BlockDriverState *bs,
-                          BdrvCheckResult *res,
-                          uint16_t *refcount_table,
-                          int refcount_table_size,
-                          int64_t offset, int64_t size)
+static int inc_refcounts(BlockDriverState *bs,
+                         BdrvCheckResult *res,
+                         uint16_t **refcount_table,
+                         int64_t *refcount_table_size,
+                         int64_t offset, int64_t size)
 {
     BDRVQcowState *s = bs->opaque;
     uint64_t start, last, cluster_offset, k;
 
-    if (size <= 0)
-        return;
+    if (size <= 0) {
+        return 0;
+    }
 
     start = start_of_cluster(s, offset);
     last = start_of_cluster(s, offset + size - 1);
     for(cluster_offset = start; cluster_offset <= last;
         cluster_offset += s->cluster_size) {
         k = cluster_offset >> s->cluster_bits;
-        if (k >= refcount_table_size) {
-            fprintf(stderr, "Warning: cluster offset=0x%" PRIx64 " is after "
-                "the end of the image file, can't properly check refcounts.\n",
-                cluster_offset);
-            res->check_errors++;
-        } else {
-            if (++refcount_table[k] == 0) {
-                fprintf(stderr, "ERROR: overflow cluster offset=0x%" PRIx64
-                    "\n", cluster_offset);
-                res->corruptions++;
+        if (k >= *refcount_table_size) {
+            int64_t old_refcount_table_size = *refcount_table_size;
+            uint16_t *new_refcount_table;
+
+            *refcount_table_size = k + 1;
+            new_refcount_table = g_try_realloc(*refcount_table,
+                                               *refcount_table_size *
+                                               sizeof(**refcount_table));
+            if (!new_refcount_table) {
+                *refcount_table_size = old_refcount_table_size;
+                res->check_errors++;
+                return -ENOMEM;
             }
+            *refcount_table = new_refcount_table;
+
+            memset(*refcount_table + old_refcount_table_size, 0,
+                   (*refcount_table_size - old_refcount_table_size) *
+                   sizeof(**refcount_table));
+        }
+
+        if (++(*refcount_table)[k] == 0) {
+            fprintf(stderr, "ERROR: overflow cluster offset=0x%" PRIx64
+                    "\n", cluster_offset);
+            res->corruptions++;
         }
     }
+
+    return 0;
 }
 
 /* Flags for check_refcounts_l1() and check_refcounts_l2() */
@@ -1069,20 +1138,24 @@ enum {
  * error occurred.
  */
 static int check_refcounts_l2(BlockDriverState *bs, BdrvCheckResult *res,
-    uint16_t *refcount_table, int refcount_table_size, int64_t l2_offset,
+    uint16_t **refcount_table, int64_t *refcount_table_size, int64_t l2_offset,
     int flags)
 {
     BDRVQcowState *s = bs->opaque;
     uint64_t *l2_table, l2_entry;
     uint64_t next_contiguous_offset = 0;
-    int i, l2_size, nb_csectors;
+    int i, l2_size, nb_csectors, ret;
 
     /* Read L2 table from disk */
     l2_size = s->l2_size * sizeof(uint64_t);
     l2_table = g_malloc(l2_size);
 
-    if (bdrv_pread(bs->file, l2_offset, l2_table, l2_size) != l2_size)
+    ret = bdrv_pread(bs->file, l2_offset, l2_table, l2_size);
+    if (ret < 0) {
+        fprintf(stderr, "ERROR: I/O error in check_refcounts_l2\n");
+        res->check_errors++;
         goto fail;
+    }
 
     /* Do the actual checks */
     for(i = 0; i < s->l2_size; i++) {
@@ -1103,8 +1176,11 @@ static int check_refcounts_l2(BlockDriverState *bs, BdrvCheckResult *res,
             nb_csectors = ((l2_entry >> s->csize_shift) &
                            s->csize_mask) + 1;
             l2_entry &= s->cluster_offset_mask;
-            inc_refcounts(bs, res, refcount_table, refcount_table_size,
-                l2_entry & ~511, nb_csectors * 512);
+            ret = inc_refcounts(bs, res, refcount_table, refcount_table_size,
+                                l2_entry & ~511, nb_csectors * 512);
+            if (ret < 0) {
+                goto fail;
+            }
 
             if (flags & CHECK_FRAG_INFO) {
                 res->bfi.allocated_clusters++;
@@ -1139,8 +1215,11 @@ static int check_refcounts_l2(BlockDriverState *bs, BdrvCheckResult *res,
             }
 
             /* Mark cluster as used */
-            inc_refcounts(bs, res, refcount_table,refcount_table_size,
-                offset, s->cluster_size);
+            ret = inc_refcounts(bs, res, refcount_table, refcount_table_size,
+                                offset, s->cluster_size);
+            if (ret < 0) {
+                goto fail;
+            }
 
             /* Correct offsets are cluster aligned */
             if (offset_into_cluster(s, offset)) {
@@ -1163,9 +1242,8 @@ static int check_refcounts_l2(BlockDriverState *bs, BdrvCheckResult *res,
     return 0;
 
 fail:
-    fprintf(stderr, "ERROR: I/O error in check_refcounts_l2\n");
     g_free(l2_table);
-    return -EIO;
+    return ret;
 }
 
 /*
@@ -1178,29 +1256,38 @@ fail:
  */
 static int check_refcounts_l1(BlockDriverState *bs,
                               BdrvCheckResult *res,
-                              uint16_t *refcount_table,
-                              int refcount_table_size,
+                              uint16_t **refcount_table,
+                              int64_t *refcount_table_size,
                               int64_t l1_table_offset, int l1_size,
                               int flags)
 {
     BDRVQcowState *s = bs->opaque;
-    uint64_t *l1_table, l2_offset, l1_size2;
+    uint64_t *l1_table = NULL, l2_offset, l1_size2;
     int i, ret;
 
     l1_size2 = l1_size * sizeof(uint64_t);
 
     /* Mark L1 table as used */
-    inc_refcounts(bs, res, refcount_table, refcount_table_size,
-        l1_table_offset, l1_size2);
+    ret = inc_refcounts(bs, res, refcount_table, refcount_table_size,
+                        l1_table_offset, l1_size2);
+    if (ret < 0) {
+        goto fail;
+    }
 
     /* Read L1 table entries from disk */
-    if (l1_size2 == 0) {
-        l1_table = NULL;
-    } else {
-        l1_table = g_malloc(l1_size2);
-        if (bdrv_pread(bs->file, l1_table_offset,
-                       l1_table, l1_size2) != l1_size2)
+    if (l1_size2 > 0) {
+        l1_table = g_try_malloc(l1_size2);
+        if (l1_table == NULL) {
+            ret = -ENOMEM;
+            res->check_errors++;
+            goto fail;
+        }
+        ret = bdrv_pread(bs->file, l1_table_offset, l1_table, l1_size2);
+        if (ret < 0) {
+            fprintf(stderr, "ERROR: I/O error in check_refcounts_l1\n");
+            res->check_errors++;
             goto fail;
+        }
         for(i = 0;i < l1_size; i++)
             be64_to_cpus(&l1_table[i]);
     }
@@ -1211,8 +1298,11 @@ static int check_refcounts_l1(BlockDriverState *bs,
         if (l2_offset) {
             /* Mark L2 table as used */
             l2_offset &= L1E_OFFSET_MASK;
-            inc_refcounts(bs, res, refcount_table, refcount_table_size,
-                l2_offset, s->cluster_size);
+            ret = inc_refcounts(bs, res, refcount_table, refcount_table_size,
+                                l2_offset, s->cluster_size);
+            if (ret < 0) {
+                goto fail;
+            }
 
             /* L2 tables are cluster aligned */
             if (offset_into_cluster(s, l2_offset)) {
@@ -1233,18 +1323,16 @@ static int check_refcounts_l1(BlockDriverState *bs,
     return 0;
 
 fail:
-    fprintf(stderr, "ERROR: I/O error in check_refcounts_l1\n");
-    res->check_errors++;
     g_free(l1_table);
-    return -EIO;
+    return ret;
 }
 
 /*
  * Checks the OFLAG_COPIED flag for all L1 and L2 entries.
  *
  * This function does not print an error message nor does it increment
- * check_errors if get_refcount fails (this is because such an error will have
- * been already detected and sufficiently signaled by the calling function
+ * check_errors if qcow2_get_refcount fails (this is because such an error will
+ * have been already detected and sufficiently signaled by the calling function
  * (qcow2_check_refcounts) by the time this function is called).
  */
 static int check_oflag_copied(BlockDriverState *bs, BdrvCheckResult *res,
@@ -1265,7 +1353,7 @@ static int check_oflag_copied(BlockDriverState *bs, BdrvCheckResult *res,
             continue;
         }
 
-        refcount = get_refcount(bs, l2_offset >> s->cluster_bits);
+        refcount = qcow2_get_refcount(bs, l2_offset >> s->cluster_bits);
         if (refcount < 0) {
             /* don't print message nor increment check_errors */
             continue;
@@ -1307,7 +1395,8 @@ static int check_oflag_copied(BlockDriverState *bs, BdrvCheckResult *res,
 
             if ((cluster_type == QCOW2_CLUSTER_NORMAL) ||
                 ((cluster_type == QCOW2_CLUSTER_ZERO) && (data_offset != 0))) {
-                refcount = get_refcount(bs, data_offset >> s->cluster_bits);
+                refcount = qcow2_get_refcount(bs,
+                                              data_offset >> s->cluster_bits);
                 if (refcount < 0) {
                     /* don't print message nor increment check_errors */
                     continue;
@@ -1359,245 +1448,191 @@ fail:
 }
 
 /*
- * Writes one sector of the refcount table to the disk
+ * Checks consistency of refblocks and accounts for each refblock in
+ * *refcount_table.
  */
-#define RT_ENTRIES_PER_SECTOR (512 / sizeof(uint64_t))
-static int write_reftable_entry(BlockDriverState *bs, int rt_index)
+static int check_refblocks(BlockDriverState *bs, BdrvCheckResult *res,
+                           BdrvCheckMode fix, bool *rebuild,
+                           uint16_t **refcount_table, int64_t *nb_clusters)
 {
     BDRVQcowState *s = bs->opaque;
-    uint64_t buf[RT_ENTRIES_PER_SECTOR];
-    int rt_start_index;
-    int i, ret;
-
-    rt_start_index = rt_index & ~(RT_ENTRIES_PER_SECTOR - 1);
-    for (i = 0; i < RT_ENTRIES_PER_SECTOR; i++) {
-        buf[i] = cpu_to_be64(s->refcount_table[rt_start_index + i]);
-    }
+    int64_t i, size;
+    int ret;
 
-    ret = qcow2_pre_write_overlap_check(bs, QCOW2_OL_REFCOUNT_TABLE,
-            s->refcount_table_offset + rt_start_index * sizeof(uint64_t),
-            sizeof(buf));
-    if (ret < 0) {
-        return ret;
-    }
+    for(i = 0; i < s->refcount_table_size; i++) {
+        uint64_t offset, cluster;
+        offset = s->refcount_table[i];
+        cluster = offset >> s->cluster_bits;
 
-    BLKDBG_EVENT(bs->file, BLKDBG_REFTABLE_UPDATE);
-    ret = bdrv_pwrite_sync(bs->file, s->refcount_table_offset +
-            rt_start_index * sizeof(uint64_t), buf, sizeof(buf));
-    if (ret < 0) {
-        return ret;
-    }
+        /* Refcount blocks are cluster aligned */
+        if (offset_into_cluster(s, offset)) {
+            fprintf(stderr, "ERROR refcount block %" PRId64 " is not "
+                "cluster aligned; refcount table entry corrupted\n", i);
+            res->corruptions++;
+            *rebuild = true;
+            continue;
+        }
 
-    return 0;
-}
+        if (cluster >= *nb_clusters) {
+            fprintf(stderr, "%s refcount block %" PRId64 " is outside image\n",
+                    fix & BDRV_FIX_ERRORS ? "Repairing" : "ERROR", i);
 
-/*
- * Allocates a new cluster for the given refcount block (represented by its
- * offset in the image file) and copies the current content there. This function
- * does _not_ decrement the reference count for the currently occupied cluster.
- *
- * This function prints an informative message to stderr on error (and returns
- * -errno); on success, the offset of the newly allocated cluster is returned.
- */
-static int64_t realloc_refcount_block(BlockDriverState *bs, int reftable_index,
-                                      uint64_t offset)
-{
-    BDRVQcowState *s = bs->opaque;
-    int64_t new_offset = 0;
-    void *refcount_block = NULL;
-    int ret;
+            if (fix & BDRV_FIX_ERRORS) {
+                int64_t old_nb_clusters = *nb_clusters;
+                uint16_t *new_refcount_table;
 
-    /* allocate new refcount block */
-    new_offset = qcow2_alloc_clusters(bs, s->cluster_size);
-    if (new_offset < 0) {
-        fprintf(stderr, "Could not allocate new cluster: %s\n",
-                strerror(-new_offset));
-        ret = new_offset;
-        goto done;
-    }
+                if (offset > INT64_MAX - s->cluster_size) {
+                    ret = -EINVAL;
+                    goto resize_fail;
+                }
 
-    /* fetch current refcount block content */
-    ret = qcow2_cache_get(bs, s->refcount_block_cache, offset, &refcount_block);
-    if (ret < 0) {
-        fprintf(stderr, "Could not fetch refcount block: %s\n", strerror(-ret));
-        goto fail_free_cluster;
-    }
+                ret = bdrv_truncate(bs->file, offset + s->cluster_size);
+                if (ret < 0) {
+                    goto resize_fail;
+                }
+                size = bdrv_getlength(bs->file);
+                if (size < 0) {
+                    ret = size;
+                    goto resize_fail;
+                }
 
-    /* new block has not yet been entered into refcount table, therefore it is
-     * no refcount block yet (regarding this check) */
-    ret = qcow2_pre_write_overlap_check(bs, 0, new_offset, s->cluster_size);
-    if (ret < 0) {
-        fprintf(stderr, "Could not write refcount block; metadata overlap "
-                "check failed: %s\n", strerror(-ret));
-        /* the image will be marked corrupt, so don't even attempt on freeing
-         * the cluster */
-        goto done;
-    }
+                *nb_clusters = size_to_clusters(s, size);
+                assert(*nb_clusters >= old_nb_clusters);
 
-    /* write to new block */
-    ret = bdrv_write(bs->file, new_offset / BDRV_SECTOR_SIZE, refcount_block,
-            s->cluster_sectors);
-    if (ret < 0) {
-        fprintf(stderr, "Could not write refcount block: %s\n", strerror(-ret));
-        goto fail_free_cluster;
-    }
+                new_refcount_table = g_try_realloc(*refcount_table,
+                                                   *nb_clusters *
+                                                   sizeof(**refcount_table));
+                if (!new_refcount_table) {
+                    *nb_clusters = old_nb_clusters;
+                    res->check_errors++;
+                    return -ENOMEM;
+                }
+                *refcount_table = new_refcount_table;
 
-    /* update refcount table */
-    assert(!offset_into_cluster(s, new_offset));
-    s->refcount_table[reftable_index] = new_offset;
-    ret = write_reftable_entry(bs, reftable_index);
-    if (ret < 0) {
-        fprintf(stderr, "Could not update refcount table: %s\n",
-                strerror(-ret));
-        goto fail_free_cluster;
-    }
+                memset(*refcount_table + old_nb_clusters, 0,
+                       (*nb_clusters - old_nb_clusters) *
+                       sizeof(**refcount_table));
 
-    goto done;
+                if (cluster >= *nb_clusters) {
+                    ret = -EINVAL;
+                    goto resize_fail;
+                }
 
-fail_free_cluster:
-    qcow2_free_clusters(bs, new_offset, s->cluster_size, QCOW2_DISCARD_OTHER);
+                res->corruptions_fixed++;
+                ret = inc_refcounts(bs, res, refcount_table, nb_clusters,
+                                    offset, s->cluster_size);
+                if (ret < 0) {
+                    return ret;
+                }
+                /* No need to check whether the refcount is now greater than 1:
+                 * This area was just allocated and zeroed, so it can only be
+                 * exactly 1 after inc_refcounts() */
+                continue;
 
-done:
-    if (refcount_block) {
-        /* This should never fail, as it would only do so if the given refcount
-         * block cannot be found in the cache. As this is impossible as long as
-         * there are no bugs, assert the success. */
-        int tmp = qcow2_cache_put(bs, s->refcount_block_cache, &refcount_block);
-        assert(tmp == 0);
-    }
+resize_fail:
+                res->corruptions++;
+                *rebuild = true;
+                fprintf(stderr, "ERROR could not resize image: %s\n",
+                        strerror(-ret));
+            } else {
+                res->corruptions++;
+            }
+            continue;
+        }
 
-    if (ret < 0) {
-        return ret;
+        if (offset != 0) {
+            ret = inc_refcounts(bs, res, refcount_table, nb_clusters,
+                                offset, s->cluster_size);
+            if (ret < 0) {
+                return ret;
+            }
+            if ((*refcount_table)[cluster] != 1) {
+                fprintf(stderr, "ERROR refcount block %" PRId64
+                        " refcount=%d\n", i, (*refcount_table)[cluster]);
+                res->corruptions++;
+                *rebuild = true;
+            }
+        }
     }
 
-    return new_offset;
+    return 0;
 }
 
 /*
- * Checks an image for refcount consistency.
- *
- * Returns 0 if no errors are found, the number of errors in case the image is
- * detected as corrupted, and -errno when an internal error occurred.
+ * Calculates an in-memory refcount table.
  */
-int qcow2_check_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
-                          BdrvCheckMode fix)
+static int calculate_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
+                               BdrvCheckMode fix, bool *rebuild,
+                               uint16_t **refcount_table, int64_t *nb_clusters)
 {
     BDRVQcowState *s = bs->opaque;
-    int64_t size, i, highest_cluster, nb_clusters;
-    int refcount1, refcount2;
+    int64_t i;
     QCowSnapshot *sn;
-    uint16_t *refcount_table;
     int ret;
 
-    size = bdrv_getlength(bs->file);
-    if (size < 0) {
-        res->check_errors++;
-        return size;
-    }
-
-    nb_clusters = size_to_clusters(s, size);
-    if (nb_clusters > INT_MAX) {
-        res->check_errors++;
-        return -EFBIG;
+    if (!*refcount_table) {
+        *refcount_table = g_try_new0(uint16_t, *nb_clusters);
+        if (*nb_clusters && *refcount_table == NULL) {
+            res->check_errors++;
+            return -ENOMEM;
+        }
     }
 
-    refcount_table = g_malloc0(nb_clusters * sizeof(uint16_t));
-
-    res->bfi.total_clusters =
-        size_to_clusters(s, bs->total_sectors * BDRV_SECTOR_SIZE);
-
     /* header */
-    inc_refcounts(bs, res, refcount_table, nb_clusters,
-        0, s->cluster_size);
+    ret = inc_refcounts(bs, res, refcount_table, nb_clusters,
+                        0, s->cluster_size);
+    if (ret < 0) {
+        return ret;
+    }
 
     /* current L1 table */
     ret = check_refcounts_l1(bs, res, refcount_table, nb_clusters,
                              s->l1_table_offset, s->l1_size, CHECK_FRAG_INFO);
     if (ret < 0) {
-        goto fail;
+        return ret;
     }
 
     /* snapshots */
-    for(i = 0; i < s->nb_snapshots; i++) {
+    for (i = 0; i < s->nb_snapshots; i++) {
         sn = s->snapshots + i;
         ret = check_refcounts_l1(bs, res, refcount_table, nb_clusters,
-            sn->l1_table_offset, sn->l1_size, 0);
+                                 sn->l1_table_offset, sn->l1_size, 0);
         if (ret < 0) {
-            goto fail;
+            return ret;
         }
     }
-    inc_refcounts(bs, res, refcount_table, nb_clusters,
-        s->snapshots_offset, s->snapshots_size);
+    ret = inc_refcounts(bs, res, refcount_table, nb_clusters,
+                        s->snapshots_offset, s->snapshots_size);
+    if (ret < 0) {
+        return ret;
+    }
 
     /* refcount data */
-    inc_refcounts(bs, res, refcount_table, nb_clusters,
-        s->refcount_table_offset,
-        s->refcount_table_size * sizeof(uint64_t));
-
-    for(i = 0; i < s->refcount_table_size; i++) {
-        uint64_t offset, cluster;
-        offset = s->refcount_table[i];
-        cluster = offset >> s->cluster_bits;
-
-        /* Refcount blocks are cluster aligned */
-        if (offset_into_cluster(s, offset)) {
-            fprintf(stderr, "ERROR refcount block %" PRId64 " is not "
-                "cluster aligned; refcount table entry corrupted\n", i);
-            res->corruptions++;
-            continue;
-        }
-
-        if (cluster >= nb_clusters) {
-            fprintf(stderr, "ERROR refcount block %" PRId64
-                    " is outside image\n", i);
-            res->corruptions++;
-            continue;
-        }
-
-        if (offset != 0) {
-            inc_refcounts(bs, res, refcount_table, nb_clusters,
-                offset, s->cluster_size);
-            if (refcount_table[cluster] != 1) {
-                fprintf(stderr, "%s refcount block %" PRId64
-                    " refcount=%d\n",
-                    fix & BDRV_FIX_ERRORS ? "Repairing" :
-                                            "ERROR",
-                    i, refcount_table[cluster]);
-
-                if (fix & BDRV_FIX_ERRORS) {
-                    int64_t new_offset;
-
-                    new_offset = realloc_refcount_block(bs, i, offset);
-                    if (new_offset < 0) {
-                        res->corruptions++;
-                        continue;
-                    }
+    ret = inc_refcounts(bs, res, refcount_table, nb_clusters,
+                        s->refcount_table_offset,
+                        s->refcount_table_size * sizeof(uint64_t));
+    if (ret < 0) {
+        return ret;
+    }
 
-                    /* update refcounts */
-                    if ((new_offset >> s->cluster_bits) >= nb_clusters) {
-                        /* increase refcount_table size if necessary */
-                        int old_nb_clusters = nb_clusters;
-                        nb_clusters = (new_offset >> s->cluster_bits) + 1;
-                        refcount_table = g_realloc(refcount_table,
-                                nb_clusters * sizeof(uint16_t));
-                        memset(&refcount_table[old_nb_clusters], 0, (nb_clusters
-                                - old_nb_clusters) * sizeof(uint16_t));
-                    }
-                    refcount_table[cluster]--;
-                    inc_refcounts(bs, res, refcount_table, nb_clusters,
-                            new_offset, s->cluster_size);
+    return check_refblocks(bs, res, fix, rebuild, refcount_table, nb_clusters);
+}
 
-                    res->corruptions_fixed++;
-                } else {
-                    res->corruptions++;
-                }
-            }
-        }
-    }
+/*
+ * Compares the actual reference count for each cluster in the image against the
+ * refcount as reported by the refcount structures on-disk.
+ */
+static void compare_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
+                              BdrvCheckMode fix, bool *rebuild,
+                              int64_t *highest_cluster,
+                              uint16_t *refcount_table, int64_t nb_clusters)
+{
+    BDRVQcowState *s = bs->opaque;
+    int64_t i;
+    int refcount1, refcount2, ret;
 
-    /* compare ref counts */
-    for (i = 0, highest_cluster = 0; i < nb_clusters; i++) {
-        refcount1 = get_refcount(bs, i);
+    for (i = 0, *highest_cluster = 0; i < nb_clusters; i++) {
+        refcount1 = qcow2_get_refcount(bs, i);
         if (refcount1 < 0) {
             fprintf(stderr, "Can't get refcount for cluster %" PRId64 ": %s\n",
                 i, strerror(-refcount1));
@@ -1608,14 +1643,15 @@ int qcow2_check_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
         refcount2 = refcount_table[i];
 
         if (refcount1 > 0 || refcount2 > 0) {
-            highest_cluster = i;
+            *highest_cluster = i;
         }
 
         if (refcount1 != refcount2) {
-
             /* Check if we're allowed to fix the mismatch */
             int *num_fixed = NULL;
-            if (refcount1 > refcount2 && (fix & BDRV_FIX_LEAKS)) {
+            if (refcount1 == 0) {
+                *rebuild = true;
+            } else if (refcount1 > refcount2 && (fix & BDRV_FIX_LEAKS)) {
                 num_fixed = &res->leaks_fixed;
             } else if (refcount1 < refcount2 && (fix & BDRV_FIX_ERRORS)) {
                 num_fixed = &res->corruptions_fixed;
@@ -1645,6 +1681,397 @@ int qcow2_check_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
             }
         }
     }
+}
+
+/*
+ * Allocates clusters using an in-memory refcount table (IMRT) in contrast to
+ * the on-disk refcount structures.
+ *
+ * On input, *first_free_cluster tells where to start looking, and need not
+ * actually be a free cluster; the returned offset will not be before that
+ * cluster.  On output, *first_free_cluster points to the first gap found, even
+ * if that gap was too small to be used as the returned offset.
+ *
+ * Note that *first_free_cluster is a cluster index whereas the return value is
+ * an offset.
+ */
+static int64_t alloc_clusters_imrt(BlockDriverState *bs,
+                                   int cluster_count,
+                                   uint16_t **refcount_table,
+                                   int64_t *imrt_nb_clusters,
+                                   int64_t *first_free_cluster)
+{
+    BDRVQcowState *s = bs->opaque;
+    int64_t cluster = *first_free_cluster, i;
+    bool first_gap = true;
+    int contiguous_free_clusters;
+
+    /* Starting at *first_free_cluster, find a range of at least cluster_count
+     * continuously free clusters */
+    for (contiguous_free_clusters = 0;
+         cluster < *imrt_nb_clusters &&
+         contiguous_free_clusters < cluster_count;
+         cluster++)
+    {
+        if (!(*refcount_table)[cluster]) {
+            contiguous_free_clusters++;
+            if (first_gap) {
+                /* If this is the first free cluster found, update
+                 * *first_free_cluster accordingly */
+                *first_free_cluster = cluster;
+                first_gap = false;
+            }
+        } else if (contiguous_free_clusters) {
+            contiguous_free_clusters = 0;
+        }
+    }
+
+    /* If contiguous_free_clusters is greater than zero, it contains the number
+     * of continuously free clusters until the current cluster; the first free
+     * cluster in the current "gap" is therefore
+     * cluster - contiguous_free_clusters */
+
+    /* If no such range could be found, grow the in-memory refcount table
+     * accordingly to append free clusters at the end of the image */
+    if (contiguous_free_clusters < cluster_count) {
+        int64_t old_imrt_nb_clusters = *imrt_nb_clusters;
+        uint16_t *new_refcount_table;
+
+        /* contiguous_free_clusters clusters are already empty at the image end;
+         * we need cluster_count clusters; therefore, we have to allocate
+         * cluster_count - contiguous_free_clusters new clusters at the end of
+         * the image (which is the current value of cluster; note that cluster
+         * may exceed old_imrt_nb_clusters if *first_free_cluster pointed beyond
+         * the image end) */
+        *imrt_nb_clusters = cluster + cluster_count - contiguous_free_clusters;
+        new_refcount_table = g_try_realloc(*refcount_table,
+                                           *imrt_nb_clusters *
+                                           sizeof(**refcount_table));
+        if (!new_refcount_table) {
+            *imrt_nb_clusters = old_imrt_nb_clusters;
+            return -ENOMEM;
+        }
+        *refcount_table = new_refcount_table;
+
+        memset(*refcount_table + old_imrt_nb_clusters, 0,
+               (*imrt_nb_clusters - old_imrt_nb_clusters) *
+               sizeof(**refcount_table));
+    }
+
+    /* Go back to the first free cluster */
+    cluster -= contiguous_free_clusters;
+    for (i = 0; i < cluster_count; i++) {
+        (*refcount_table)[cluster + i] = 1;
+    }
+
+    return cluster << s->cluster_bits;
+}
+
+/*
+ * Creates a new refcount structure based solely on the in-memory information
+ * given through *refcount_table. All necessary allocations will be reflected
+ * in that array.
+ *
+ * On success, the old refcount structure is leaked (it will be covered by the
+ * new refcount structure).
+ */
+static int rebuild_refcount_structure(BlockDriverState *bs,
+                                      BdrvCheckResult *res,
+                                      uint16_t **refcount_table,
+                                      int64_t *nb_clusters)
+{
+    BDRVQcowState *s = bs->opaque;
+    int64_t first_free_cluster = 0, reftable_offset = -1, cluster = 0;
+    int64_t refblock_offset, refblock_start, refblock_index;
+    uint32_t reftable_size = 0;
+    uint64_t *on_disk_reftable = NULL;
+    uint16_t *on_disk_refblock;
+    int i, ret = 0;
+    struct {
+        uint64_t reftable_offset;
+        uint32_t reftable_clusters;
+    } QEMU_PACKED reftable_offset_and_clusters;
+
+    qcow2_cache_empty(bs, s->refcount_block_cache);
+
+write_refblocks:
+    for (; cluster < *nb_clusters; cluster++) {
+        if (!(*refcount_table)[cluster]) {
+            continue;
+        }
+
+        refblock_index = cluster >> s->refcount_block_bits;
+        refblock_start = refblock_index << s->refcount_block_bits;
+
+        /* Don't allocate a cluster in a refblock already written to disk */
+        if (first_free_cluster < refblock_start) {
+            first_free_cluster = refblock_start;
+        }
+        refblock_offset = alloc_clusters_imrt(bs, 1, refcount_table,
+                                              nb_clusters, &first_free_cluster);
+        if (refblock_offset < 0) {
+            fprintf(stderr, "ERROR allocating refblock: %s\n",
+                    strerror(-refblock_offset));
+            res->check_errors++;
+            ret = refblock_offset;
+            goto fail;
+        }
+
+        if (reftable_size <= refblock_index) {
+            uint32_t old_reftable_size = reftable_size;
+            uint64_t *new_on_disk_reftable;
+
+            reftable_size = ROUND_UP((refblock_index + 1) * sizeof(uint64_t),
+                                     s->cluster_size) / sizeof(uint64_t);
+            new_on_disk_reftable = g_try_realloc(on_disk_reftable,
+                                                 reftable_size *
+                                                 sizeof(uint64_t));
+            if (!new_on_disk_reftable) {
+                res->check_errors++;
+                ret = -ENOMEM;
+                goto fail;
+            }
+            on_disk_reftable = new_on_disk_reftable;
+
+            memset(on_disk_reftable + old_reftable_size, 0,
+                   (reftable_size - old_reftable_size) * sizeof(uint64_t));
+
+            /* The offset we have for the reftable is now no longer valid;
+             * this will leak that range, but we can easily fix that by running
+             * a leak-fixing check after this rebuild operation */
+            reftable_offset = -1;
+        }
+        on_disk_reftable[refblock_index] = refblock_offset;
+
+        /* If this is apparently the last refblock (for now), try to squeeze the
+         * reftable in */
+        if (refblock_index == (*nb_clusters - 1) >> s->refcount_block_bits &&
+            reftable_offset < 0)
+        {
+            uint64_t reftable_clusters = size_to_clusters(s, reftable_size *
+                                                          sizeof(uint64_t));
+            reftable_offset = alloc_clusters_imrt(bs, reftable_clusters,
+                                                  refcount_table, nb_clusters,
+                                                  &first_free_cluster);
+            if (reftable_offset < 0) {
+                fprintf(stderr, "ERROR allocating reftable: %s\n",
+                        strerror(-reftable_offset));
+                res->check_errors++;
+                ret = reftable_offset;
+                goto fail;
+            }
+        }
+
+        ret = qcow2_pre_write_overlap_check(bs, 0, refblock_offset,
+                                            s->cluster_size);
+        if (ret < 0) {
+            fprintf(stderr, "ERROR writing refblock: %s\n", strerror(-ret));
+            goto fail;
+        }
+
+        on_disk_refblock = qemu_blockalign0(bs->file, s->cluster_size);
+        for (i = 0; i < s->refcount_block_size &&
+                    refblock_start + i < *nb_clusters; i++)
+        {
+            on_disk_refblock[i] =
+                cpu_to_be16((*refcount_table)[refblock_start + i]);
+        }
+
+        ret = bdrv_write(bs->file, refblock_offset / BDRV_SECTOR_SIZE,
+                         (void *)on_disk_refblock, s->cluster_sectors);
+        qemu_vfree(on_disk_refblock);
+        if (ret < 0) {
+            fprintf(stderr, "ERROR writing refblock: %s\n", strerror(-ret));
+            goto fail;
+        }
+
+        /* Go to the end of this refblock */
+        cluster = refblock_start + s->refcount_block_size - 1;
+    }
+
+    if (reftable_offset < 0) {
+        uint64_t post_refblock_start, reftable_clusters;
+
+        post_refblock_start = ROUND_UP(*nb_clusters, s->refcount_block_size);
+        reftable_clusters = size_to_clusters(s,
+                                             reftable_size * sizeof(uint64_t));
+        /* Not pretty but simple */
+        if (first_free_cluster < post_refblock_start) {
+            first_free_cluster = post_refblock_start;
+        }
+        reftable_offset = alloc_clusters_imrt(bs, reftable_clusters,
+                                              refcount_table, nb_clusters,
+                                              &first_free_cluster);
+        if (reftable_offset < 0) {
+            fprintf(stderr, "ERROR allocating reftable: %s\n",
+                    strerror(-reftable_offset));
+            res->check_errors++;
+            ret = reftable_offset;
+            goto fail;
+        }
+
+        goto write_refblocks;
+    }
+
+    assert(on_disk_reftable);
+
+    for (refblock_index = 0; refblock_index < reftable_size; refblock_index++) {
+        cpu_to_be64s(&on_disk_reftable[refblock_index]);
+    }
+
+    ret = qcow2_pre_write_overlap_check(bs, 0, reftable_offset,
+                                        reftable_size * sizeof(uint64_t));
+    if (ret < 0) {
+        fprintf(stderr, "ERROR writing reftable: %s\n", strerror(-ret));
+        goto fail;
+    }
+
+    assert(reftable_size < INT_MAX / sizeof(uint64_t));
+    ret = bdrv_pwrite(bs->file, reftable_offset, on_disk_reftable,
+                      reftable_size * sizeof(uint64_t));
+    if (ret < 0) {
+        fprintf(stderr, "ERROR writing reftable: %s\n", strerror(-ret));
+        goto fail;
+    }
+
+    /* Enter new reftable into the image header */
+    cpu_to_be64w(&reftable_offset_and_clusters.reftable_offset,
+                 reftable_offset);
+    cpu_to_be32w(&reftable_offset_and_clusters.reftable_clusters,
+                 size_to_clusters(s, reftable_size * sizeof(uint64_t)));
+    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader,
+                                              refcount_table_offset),
+                           &reftable_offset_and_clusters,
+                           sizeof(reftable_offset_and_clusters));
+    if (ret < 0) {
+        fprintf(stderr, "ERROR setting reftable: %s\n", strerror(-ret));
+        goto fail;
+    }
+
+    for (refblock_index = 0; refblock_index < reftable_size; refblock_index++) {
+        be64_to_cpus(&on_disk_reftable[refblock_index]);
+    }
+    s->refcount_table = on_disk_reftable;
+    s->refcount_table_offset = reftable_offset;
+    s->refcount_table_size = reftable_size;
+
+    return 0;
+
+fail:
+    g_free(on_disk_reftable);
+    return ret;
+}
+
+/*
+ * Checks an image for refcount consistency.
+ *
+ * Returns 0 if no errors are found, the number of errors in case the image is
+ * detected as corrupted, and -errno when an internal error occurred.
+ */
+int qcow2_check_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
+                          BdrvCheckMode fix)
+{
+    BDRVQcowState *s = bs->opaque;
+    BdrvCheckResult pre_compare_res;
+    int64_t size, highest_cluster, nb_clusters;
+    uint16_t *refcount_table = NULL;
+    bool rebuild = false;
+    int ret;
+
+    size = bdrv_getlength(bs->file);
+    if (size < 0) {
+        res->check_errors++;
+        return size;
+    }
+
+    nb_clusters = size_to_clusters(s, size);
+    if (nb_clusters > INT_MAX) {
+        res->check_errors++;
+        return -EFBIG;
+    }
+
+    res->bfi.total_clusters =
+        size_to_clusters(s, bs->total_sectors * BDRV_SECTOR_SIZE);
+
+    ret = calculate_refcounts(bs, res, fix, &rebuild, &refcount_table,
+                              &nb_clusters);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* In case we don't need to rebuild the refcount structure (but want to fix
+     * something), this function is immediately called again, in which case the
+     * result should be ignored */
+    pre_compare_res = *res;
+    compare_refcounts(bs, res, 0, &rebuild, &highest_cluster, refcount_table,
+                      nb_clusters);
+
+    if (rebuild && (fix & BDRV_FIX_ERRORS)) {
+        BdrvCheckResult old_res = *res;
+        int fresh_leaks = 0;
+
+        fprintf(stderr, "Rebuilding refcount structure\n");
+        ret = rebuild_refcount_structure(bs, res, &refcount_table,
+                                         &nb_clusters);
+        if (ret < 0) {
+            goto fail;
+        }
+
+        res->corruptions = 0;
+        res->leaks = 0;
+
+        /* Because the old reftable has been exchanged for a new one the
+         * references have to be recalculated */
+        rebuild = false;
+        memset(refcount_table, 0, nb_clusters * sizeof(uint16_t));
+        ret = calculate_refcounts(bs, res, 0, &rebuild, &refcount_table,
+                                  &nb_clusters);
+        if (ret < 0) {
+            goto fail;
+        }
+
+        if (fix & BDRV_FIX_LEAKS) {
+            /* The old refcount structures are now leaked, fix it; the result
+             * can be ignored, aside from leaks which were introduced by
+             * rebuild_refcount_structure() that could not be fixed */
+            BdrvCheckResult saved_res = *res;
+            *res = (BdrvCheckResult){ 0 };
+
+            compare_refcounts(bs, res, BDRV_FIX_LEAKS, &rebuild,
+                              &highest_cluster, refcount_table, nb_clusters);
+            if (rebuild) {
+                fprintf(stderr, "ERROR rebuilt refcount structure is still "
+                        "broken\n");
+            }
+
+            /* Any leaks accounted for here were introduced by
+             * rebuild_refcount_structure() because that function has created a
+             * new refcount structure from scratch */
+            fresh_leaks = res->leaks;
+            *res = saved_res;
+        }
+
+        if (res->corruptions < old_res.corruptions) {
+            res->corruptions_fixed += old_res.corruptions - res->corruptions;
+        }
+        if (res->leaks < old_res.leaks) {
+            res->leaks_fixed += old_res.leaks - res->leaks;
+        }
+        res->leaks += fresh_leaks;
+    } else if (fix) {
+        if (rebuild) {
+            fprintf(stderr, "ERROR need to rebuild refcount structures\n");
+            res->check_errors++;
+            ret = -EIO;
+            goto fail;
+        }
+
+        if (res->leaks || res->corruptions) {
+            *res = pre_compare_res;
+            compare_refcounts(bs, res, fix, &rebuild, &highest_cluster,
+                              refcount_table, nb_clusters);
+        }
+    }
 
     /* check OFLAG_COPIED */
     ret = check_oflag_copied(bs, res, fix);
@@ -1753,9 +2180,13 @@ int qcow2_check_metadata_overlap(BlockDriverState *bs, int ign, int64_t offset,
             uint64_t l1_ofs = s->snapshots[i].l1_table_offset;
             uint32_t l1_sz  = s->snapshots[i].l1_size;
             uint64_t l1_sz2 = l1_sz * sizeof(uint64_t);
-            uint64_t *l1 = g_malloc(l1_sz2);
+            uint64_t *l1 = g_try_malloc(l1_sz2);
             int ret;
 
+            if (l1_sz2 && l1 == NULL) {
+                return -ENOMEM;
+            }
+
             ret = bdrv_pread(bs->file, l1_ofs, l1, l1_sz2);
             if (ret < 0) {
                 g_free(l1);
@@ -1807,26 +2238,11 @@ int qcow2_pre_write_overlap_check(BlockDriverState *bs, int ign, int64_t offset,
         return ret;
     } else if (ret > 0) {
         int metadata_ol_bitnr = ffs(ret) - 1;
-        char *message;
-
         assert(metadata_ol_bitnr < QCOW2_OL_MAX_BITNR);
 
-        fprintf(stderr, "qcow2: Preventing invalid write on metadata (overlaps "
-                "with %s); image marked as corrupt.\n",
-                metadata_ol_names[metadata_ol_bitnr]);
-        message = g_strdup_printf("Prevented %s overwrite",
-                metadata_ol_names[metadata_ol_bitnr]);
-        qapi_event_send_block_image_corrupted(bdrv_get_device_name(bs),
-                                              message,
-                                              true,
-                                              offset,
-                                              true,
-                                              size,
-                                              &error_abort);
-        g_free(message);
-
-        qcow2_mark_corrupt(bs);
-        bs->drv = NULL; /* make BDS unusable */
+        qcow2_signal_corruption(bs, true, offset, size, "Preventing invalid "
+                                "write on metadata (overlaps with %s)",
+                                metadata_ol_names[metadata_ol_bitnr]);
         return -EIO;
     }
 
diff --git a/block/qcow2-snapshot.c b/block/qcow2-snapshot.c
index 0aa9defbe..5b3903cf2 100644
--- a/block/qcow2-snapshot.c
+++ b/block/qcow2-snapshot.c
@@ -58,7 +58,7 @@ int qcow2_read_snapshots(BlockDriverState *bs)
     }
 
     offset = s->snapshots_offset;
-    s->snapshots = g_malloc0(s->nb_snapshots * sizeof(QCowSnapshot));
+    s->snapshots = g_new0(QCowSnapshot, s->nb_snapshots);
 
     for(i = 0; i < s->nb_snapshots; i++) {
         /* Read statically sized part of the snapshot header */
@@ -381,7 +381,12 @@ int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
     sn->l1_table_offset = l1_table_offset;
     sn->l1_size = s->l1_size;
 
-    l1_table = g_malloc(s->l1_size * sizeof(uint64_t));
+    l1_table = g_try_new(uint64_t, s->l1_size);
+    if (s->l1_size && l1_table == NULL) {
+        ret = -ENOMEM;
+        goto fail;
+    }
+
     for(i = 0; i < s->l1_size; i++) {
         l1_table[i] = cpu_to_be64(s->l1_table[i]);
     }
@@ -412,7 +417,7 @@ int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
     }
 
     /* Append the new snapshot to the snapshot list */
-    new_snapshot_list = g_malloc((s->nb_snapshots + 1) * sizeof(QCowSnapshot));
+    new_snapshot_list = g_new(QCowSnapshot, s->nb_snapshots + 1);
     if (s->snapshots) {
         memcpy(new_snapshot_list, s->snapshots,
                s->nb_snapshots * sizeof(QCowSnapshot));
@@ -436,7 +441,7 @@ int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
     qcow2_discard_clusters(bs, qcow2_vm_state_offset(s),
                            align_offset(sn->vm_state_size, s->cluster_size)
                                 >> BDRV_SECTOR_BITS,
-                           QCOW2_DISCARD_NEVER);
+                           QCOW2_DISCARD_NEVER, false);
 
 #ifdef DEBUG_ALLOC
     {
@@ -499,7 +504,11 @@ int qcow2_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
      * Decrease the refcount referenced by the old one only when the L1
      * table is overwritten.
      */
-    sn_l1_table = g_malloc0(cur_l1_bytes);
+    sn_l1_table = g_try_malloc0(cur_l1_bytes);
+    if (cur_l1_bytes && sn_l1_table == NULL) {
+        ret = -ENOMEM;
+        goto fail;
+    }
 
     ret = bdrv_pread(bs->file, sn->l1_table_offset, sn_l1_table, sn_l1_bytes);
     if (ret < 0) {
@@ -652,7 +661,7 @@ int qcow2_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab)
         return s->nb_snapshots;
     }
 
-    sn_tab = g_malloc0(s->nb_snapshots * sizeof(QEMUSnapshotInfo));
+    sn_tab = g_new0(QEMUSnapshotInfo, s->nb_snapshots);
     for(i = 0; i < s->nb_snapshots; i++) {
         sn_info = sn_tab + i;
         sn = s->snapshots + i;
@@ -698,17 +707,21 @@ int qcow2_snapshot_load_tmp(BlockDriverState *bs,
         return -EFBIG;
     }
     new_l1_bytes = sn->l1_size * sizeof(uint64_t);
-    new_l1_table = g_malloc0(align_offset(new_l1_bytes, 512));
+    new_l1_table = qemu_try_blockalign(bs->file,
+                                       align_offset(new_l1_bytes, 512));
+    if (new_l1_table == NULL) {
+        return -ENOMEM;
+    }
 
     ret = bdrv_pread(bs->file, sn->l1_table_offset, new_l1_table, new_l1_bytes);
     if (ret < 0) {
         error_setg(errp, "Failed to read l1 table for snapshot");
-        g_free(new_l1_table);
+        qemu_vfree(new_l1_table);
         return ret;
     }
 
     /* Switch the L1 table */
-    g_free(s->l1_table);
+    qemu_vfree(s->l1_table);
 
     s->l1_size = sn->l1_size;
     s->l1_table_offset = sn->l1_table_offset;
diff --git a/block/qcow2.c b/block/qcow2.c
index 1e3ab6bd0..d12049451 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -30,6 +30,9 @@
 #include "qemu/error-report.h"
 #include "qapi/qmp/qerror.h"
 #include "qapi/qmp/qbool.h"
+#include "qapi/util.h"
+#include "qapi/qmp/types.h"
+#include "qapi-event.h"
 #include "trace.h"
 #include "qemu/option_int.h"
 
@@ -203,8 +206,8 @@ static void GCC_FMT_ATTR(3, 4) report_unsupported(BlockDriverState *bs,
     vsnprintf(msg, sizeof(msg), fmt, ap);
     va_end(ap);
 
-    error_set(errp, QERR_UNKNOWN_BLOCK_FORMAT_FEATURE, bs->device_name, "qcow2",
-              msg);
+    error_set(errp, QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
+              bdrv_get_device_name(bs), "qcow2", msg);
 }
 
 static void report_unsupported_feature(BlockDriverState *bs,
@@ -403,6 +406,12 @@ static QemuOptsList qcow2_runtime_opts = {
                     "templates (none, constant, cached, all)",
         },
         {
+            .name = QCOW2_OPT_OVERLAP_TEMPLATE,
+            .type = QEMU_OPT_STRING,
+            .help = "Selects which overlap checks to perform from a range of "
+                    "templates (none, constant, cached, all)",
+        },
+        {
             .name = QCOW2_OPT_OVERLAP_MAIN_HEADER,
             .type = QEMU_OPT_BOOL,
             .help = "Check for unintended writes into the main qcow2 header",
@@ -442,6 +451,22 @@ static QemuOptsList qcow2_runtime_opts = {
             .type = QEMU_OPT_BOOL,
             .help = "Check for unintended writes into an inactive L2 table",
         },
+        {
+            .name = QCOW2_OPT_CACHE_SIZE,
+            .type = QEMU_OPT_SIZE,
+            .help = "Maximum combined metadata (L2 tables and refcount blocks) "
+                    "cache size",
+        },
+        {
+            .name = QCOW2_OPT_L2_CACHE_SIZE,
+            .type = QEMU_OPT_SIZE,
+            .help = "Maximum L2 table cache size",
+        },
+        {
+            .name = QCOW2_OPT_REFCOUNT_CACHE_SIZE,
+            .type = QEMU_OPT_SIZE,
+            .help = "Maximum refcount block cache size",
+        },
         { /* end of list */ }
     },
 };
@@ -457,6 +482,61 @@ static const char *overlap_bool_option_names[QCOW2_OL_MAX_BITNR] = {
     [QCOW2_OL_INACTIVE_L2_BITNR]    = QCOW2_OPT_OVERLAP_INACTIVE_L2,
 };
 
+static void read_cache_sizes(QemuOpts *opts, uint64_t *l2_cache_size,
+                             uint64_t *refcount_cache_size, Error **errp)
+{
+    uint64_t combined_cache_size;
+    bool l2_cache_size_set, refcount_cache_size_set, combined_cache_size_set;
+
+    combined_cache_size_set = qemu_opt_get(opts, QCOW2_OPT_CACHE_SIZE);
+    l2_cache_size_set = qemu_opt_get(opts, QCOW2_OPT_L2_CACHE_SIZE);
+    refcount_cache_size_set = qemu_opt_get(opts, QCOW2_OPT_REFCOUNT_CACHE_SIZE);
+
+    combined_cache_size = qemu_opt_get_size(opts, QCOW2_OPT_CACHE_SIZE, 0);
+    *l2_cache_size = qemu_opt_get_size(opts, QCOW2_OPT_L2_CACHE_SIZE, 0);
+    *refcount_cache_size = qemu_opt_get_size(opts,
+                                             QCOW2_OPT_REFCOUNT_CACHE_SIZE, 0);
+
+    if (combined_cache_size_set) {
+        if (l2_cache_size_set && refcount_cache_size_set) {
+            error_setg(errp, QCOW2_OPT_CACHE_SIZE ", " QCOW2_OPT_L2_CACHE_SIZE
+                       " and " QCOW2_OPT_REFCOUNT_CACHE_SIZE " may not be set "
+                       "the same time");
+            return;
+        } else if (*l2_cache_size > combined_cache_size) {
+            error_setg(errp, QCOW2_OPT_L2_CACHE_SIZE " may not exceed "
+                       QCOW2_OPT_CACHE_SIZE);
+            return;
+        } else if (*refcount_cache_size > combined_cache_size) {
+            error_setg(errp, QCOW2_OPT_REFCOUNT_CACHE_SIZE " may not exceed "
+                       QCOW2_OPT_CACHE_SIZE);
+            return;
+        }
+
+        if (l2_cache_size_set) {
+            *refcount_cache_size = combined_cache_size - *l2_cache_size;
+        } else if (refcount_cache_size_set) {
+            *l2_cache_size = combined_cache_size - *refcount_cache_size;
+        } else {
+            *refcount_cache_size = combined_cache_size
+                                 / (DEFAULT_L2_REFCOUNT_SIZE_RATIO + 1);
+            *l2_cache_size = combined_cache_size - *refcount_cache_size;
+        }
+    } else {
+        if (!l2_cache_size_set && !refcount_cache_size_set) {
+            *l2_cache_size = DEFAULT_L2_CACHE_BYTE_SIZE;
+            *refcount_cache_size = *l2_cache_size
+                                 / DEFAULT_L2_REFCOUNT_SIZE_RATIO;
+        } else if (!l2_cache_size_set) {
+            *l2_cache_size = *refcount_cache_size
+                           * DEFAULT_L2_REFCOUNT_SIZE_RATIO;
+        } else if (!refcount_cache_size_set) {
+            *refcount_cache_size = *l2_cache_size
+                                 / DEFAULT_L2_REFCOUNT_SIZE_RATIO;
+        }
+    }
+}
+
 static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
                       Error **errp)
 {
@@ -464,12 +544,13 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
     unsigned int len, i;
     int ret = 0;
     QCowHeader header;
-    QemuOpts *opts;
+    QemuOpts *opts = NULL;
     Error *local_err = NULL;
     uint64_t ext_end;
     uint64_t l1_vm_state_index;
-    const char *opt_overlap_check;
+    const char *opt_overlap_check, *opt_overlap_check_template;
     int overlap_check_template = 0;
+    uint64_t l2_cache_size, refcount_cache_size;
 
     ret = bdrv_pread(bs->file, 0, &header, sizeof(header));
     if (ret < 0) {
@@ -617,6 +698,9 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
 
     s->l2_bits = s->cluster_bits - 3; /* L2 is always one cluster */
     s->l2_size = 1 << s->l2_bits;
+    /* 2^(s->refcount_order - 3) is the refcount width in bytes */
+    s->refcount_block_bits = s->cluster_bits - (s->refcount_order - 3);
+    s->refcount_block_size = 1 << s->refcount_block_bits;
     bs->total_sectors = header.size / 512;
     s->csize_shift = (62 - (s->cluster_bits - 8));
     s->csize_mask = (1 << (s->cluster_bits - 8)) - 1;
@@ -688,8 +772,13 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
 
 
     if (s->l1_size > 0) {
-        s->l1_table = g_malloc0(
+        s->l1_table = qemu_try_blockalign(bs->file,
             align_offset(s->l1_size * sizeof(uint64_t), 512));
+        if (s->l1_table == NULL) {
+            error_setg(errp, "Could not allocate L1 table");
+            ret = -ENOMEM;
+            goto fail;
+        }
         ret = bdrv_pread(bs->file, s->l1_table_offset, s->l1_table,
                          s->l1_size * sizeof(uint64_t));
         if (ret < 0) {
@@ -701,14 +790,61 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
         }
     }
 
+    /* get L2 table/refcount block cache size from command line options */
+    opts = qemu_opts_create(&qcow2_runtime_opts, NULL, 0, &error_abort);
+    qemu_opts_absorb_qdict(opts, options, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    read_cache_sizes(opts, &l2_cache_size, &refcount_cache_size, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    l2_cache_size /= s->cluster_size;
+    if (l2_cache_size < MIN_L2_CACHE_SIZE) {
+        l2_cache_size = MIN_L2_CACHE_SIZE;
+    }
+    if (l2_cache_size > INT_MAX) {
+        error_setg(errp, "L2 cache size too big");
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    refcount_cache_size /= s->cluster_size;
+    if (refcount_cache_size < MIN_REFCOUNT_CACHE_SIZE) {
+        refcount_cache_size = MIN_REFCOUNT_CACHE_SIZE;
+    }
+    if (refcount_cache_size > INT_MAX) {
+        error_setg(errp, "Refcount cache size too big");
+        ret = -EINVAL;
+        goto fail;
+    }
+
     /* alloc L2 table/refcount block cache */
-    s->l2_table_cache = qcow2_cache_create(bs, L2_CACHE_SIZE);
-    s->refcount_block_cache = qcow2_cache_create(bs, REFCOUNT_CACHE_SIZE);
+    s->l2_table_cache = qcow2_cache_create(bs, l2_cache_size);
+    s->refcount_block_cache = qcow2_cache_create(bs, refcount_cache_size);
+    if (s->l2_table_cache == NULL || s->refcount_block_cache == NULL) {
+        error_setg(errp, "Could not allocate metadata caches");
+        ret = -ENOMEM;
+        goto fail;
+    }
 
     s->cluster_cache = g_malloc(s->cluster_size);
     /* one more sector for decompressed data alignment */
-    s->cluster_data = qemu_blockalign(bs, QCOW_MAX_CRYPT_CLUSTERS * s->cluster_size
-                                  + 512);
+    s->cluster_data = qemu_try_blockalign(bs->file, QCOW_MAX_CRYPT_CLUSTERS
+                                                    * s->cluster_size + 512);
+    if (s->cluster_data == NULL) {
+        error_setg(errp, "Could not allocate temporary cluster buffer");
+        ret = -ENOMEM;
+        goto fail;
+    }
+
     s->cluster_cache_offset = -1;
     s->flags = flags;
 
@@ -774,7 +910,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
         (s->incompatible_features & QCOW2_INCOMPAT_DIRTY)) {
         BdrvCheckResult result = {0};
 
-        ret = qcow2_check(bs, &result, BDRV_FIX_ERRORS);
+        ret = qcow2_check(bs, &result, BDRV_FIX_ERRORS | BDRV_FIX_LEAKS);
         if (ret < 0) {
             error_setg_errno(errp, -ret, "Could not repair dirty image");
             goto fail;
@@ -782,14 +918,6 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     /* Enable lazy_refcounts according to image and command line options */
-    opts = qemu_opts_create(&qcow2_runtime_opts, NULL, 0, &error_abort);
-    qemu_opts_absorb_qdict(opts, options, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        ret = -EINVAL;
-        goto fail;
-    }
-
     s->use_lazy_refcounts = qemu_opt_get_bool(opts, QCOW2_OPT_LAZY_REFCOUNTS,
         (s->compatible_features & QCOW2_COMPAT_LAZY_REFCOUNTS));
 
@@ -803,7 +931,21 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
     s->discard_passthrough[QCOW2_DISCARD_OTHER] =
         qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_OTHER, false);
 
-    opt_overlap_check = qemu_opt_get(opts, "overlap-check") ?: "cached";
+    opt_overlap_check = qemu_opt_get(opts, QCOW2_OPT_OVERLAP);
+    opt_overlap_check_template = qemu_opt_get(opts, QCOW2_OPT_OVERLAP_TEMPLATE);
+    if (opt_overlap_check_template && opt_overlap_check &&
+        strcmp(opt_overlap_check_template, opt_overlap_check))
+    {
+        error_setg(errp, "Conflicting values for qcow2 options '"
+                   QCOW2_OPT_OVERLAP "' ('%s') and '" QCOW2_OPT_OVERLAP_TEMPLATE
+                   "' ('%s')", opt_overlap_check, opt_overlap_check_template);
+        ret = -EINVAL;
+        goto fail;
+    }
+    if (!opt_overlap_check) {
+        opt_overlap_check = opt_overlap_check_template ?: "cached";
+    }
+
     if (!strcmp(opt_overlap_check, "none")) {
         overlap_check_template = 0;
     } else if (!strcmp(opt_overlap_check, "constant")) {
@@ -816,7 +958,6 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
         error_setg(errp, "Unsupported value '%s' for qcow2 option "
                    "'overlap-check'. Allowed are either of the following: "
                    "none, constant, cached, all", opt_overlap_check);
-        qemu_opts_del(opts);
         ret = -EINVAL;
         goto fail;
     }
@@ -831,6 +972,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     qemu_opts_del(opts);
+    opts = NULL;
 
     if (s->use_lazy_refcounts && s->qcow_version < 3) {
         error_setg(errp, "Lazy refcounts require a qcow2 image with at least "
@@ -848,11 +990,12 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
     return ret;
 
  fail:
+    qemu_opts_del(opts);
     g_free(s->unknown_header_fields);
     cleanup_unknown_header_ext(bs);
     qcow2_free_snapshots(bs);
     qcow2_refcount_close(bs);
-    g_free(s->l1_table);
+    qemu_vfree(s->l1_table);
     /* else pre-write overlap checks in cache_destroy may crash */
     s->l1_table = NULL;
     if (s->l2_table_cache) {
@@ -1082,7 +1225,12 @@ static coroutine_fn int qcow2_co_readv(BlockDriverState *bs, int64_t sector_num,
                  */
                 if (!cluster_data) {
                     cluster_data =
-                        qemu_blockalign(bs, QCOW_MAX_CRYPT_CLUSTERS * s->cluster_size);
+                        qemu_try_blockalign(bs->file, QCOW_MAX_CRYPT_CLUSTERS
+                                                      * s->cluster_size);
+                    if (cluster_data == NULL) {
+                        ret = -ENOMEM;
+                        goto fail;
+                    }
                 }
 
                 assert(cur_nr_sectors <=
@@ -1182,8 +1330,13 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
 
         if (s->crypt_method) {
             if (!cluster_data) {
-                cluster_data = qemu_blockalign(bs, QCOW_MAX_CRYPT_CLUSTERS *
-                                                 s->cluster_size);
+                cluster_data = qemu_try_blockalign(bs->file,
+                                                   QCOW_MAX_CRYPT_CLUSTERS
+                                                   * s->cluster_size);
+                if (cluster_data == NULL) {
+                    ret = -ENOMEM;
+                    goto fail;
+                }
             }
 
             assert(hd_qiov.size <=
@@ -1270,7 +1423,7 @@ fail:
 static void qcow2_close(BlockDriverState *bs)
 {
     BDRVQcowState *s = bs->opaque;
-    g_free(s->l1_table);
+    qemu_vfree(s->l1_table);
     /* else pre-write overlap checks in cache_destroy may crash */
     s->l1_table = NULL;
 
@@ -1557,7 +1710,7 @@ static int preallocate(BlockDriverState *bs)
     int ret;
     QCowL2Meta *meta;
 
-    nb_sectors = bdrv_getlength(bs) >> BDRV_SECTOR_BITS;
+    nb_sectors = bdrv_nb_sectors(bs);
     offset = 0;
 
     while (nb_sectors) {
@@ -1612,7 +1765,7 @@ static int preallocate(BlockDriverState *bs)
 
 static int qcow2_create2(const char *filename, int64_t total_size,
                          const char *backing_file, const char *backing_format,
-                         int flags, size_t cluster_size, int prealloc,
+                         int flags, size_t cluster_size, PreallocMode prealloc,
                          QemuOpts *opts, int version,
                          Error **errp)
 {
@@ -1645,6 +1798,56 @@ static int qcow2_create2(const char *filename, int64_t total_size,
     Error *local_err = NULL;
     int ret;
 
+    if (prealloc == PREALLOC_MODE_FULL || prealloc == PREALLOC_MODE_FALLOC) {
+        int64_t meta_size = 0;
+        uint64_t nreftablee, nrefblocke, nl1e, nl2e;
+        int64_t aligned_total_size = align_offset(total_size, cluster_size);
+
+        /* header: 1 cluster */
+        meta_size += cluster_size;
+
+        /* total size of L2 tables */
+        nl2e = aligned_total_size / cluster_size;
+        nl2e = align_offset(nl2e, cluster_size / sizeof(uint64_t));
+        meta_size += nl2e * sizeof(uint64_t);
+
+        /* total size of L1 tables */
+        nl1e = nl2e * sizeof(uint64_t) / cluster_size;
+        nl1e = align_offset(nl1e, cluster_size / sizeof(uint64_t));
+        meta_size += nl1e * sizeof(uint64_t);
+
+        /* total size of refcount blocks
+         *
+         * note: every host cluster is reference-counted, including metadata
+         * (even refcount blocks are recursively included).
+         * Let:
+         *   a = total_size (this is the guest disk size)
+         *   m = meta size not including refcount blocks and refcount tables
+         *   c = cluster size
+         *   y1 = number of refcount blocks entries
+         *   y2 = meta size including everything
+         * then,
+         *   y1 = (y2 + a)/c
+         *   y2 = y1 * sizeof(u16) + y1 * sizeof(u16) * sizeof(u64) / c + m
+         * we can get y1:
+         *   y1 = (a + m) / (c - sizeof(u16) - sizeof(u16) * sizeof(u64) / c)
+         */
+        nrefblocke = (aligned_total_size + meta_size + cluster_size) /
+            (cluster_size - sizeof(uint16_t) -
+             1.0 * sizeof(uint16_t) * sizeof(uint64_t) / cluster_size);
+        nrefblocke = align_offset(nrefblocke, cluster_size / sizeof(uint16_t));
+        meta_size += nrefblocke * sizeof(uint16_t);
+
+        /* total size of refcount tables */
+        nreftablee = nrefblocke * sizeof(uint16_t) / cluster_size;
+        nreftablee = align_offset(nreftablee, cluster_size / sizeof(uint64_t));
+        meta_size += nreftablee * sizeof(uint64_t);
+
+        qemu_opt_set_number(opts, BLOCK_OPT_SIZE,
+                            aligned_total_size + meta_size);
+        qemu_opt_set(opts, BLOCK_OPT_PREALLOC, PreallocMode_lookup[prealloc]);
+    }
+
     ret = bdrv_create_file(filename, opts, &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
@@ -1671,7 +1874,7 @@ static int qcow2_create2(const char *filename, int64_t total_size,
         .l1_size                    = cpu_to_be32(0),
         .refcount_table_offset      = cpu_to_be64(cluster_size),
         .refcount_table_clusters    = cpu_to_be32(1),
-        .refcount_order             = cpu_to_be32(3 + REFCOUNT_SHIFT),
+        .refcount_order             = cpu_to_be32(4),
         .header_length              = cpu_to_be32(sizeof(*header)),
     };
 
@@ -1733,7 +1936,7 @@ static int qcow2_create2(const char *filename, int64_t total_size,
     }
 
     /* Okay, now that we have a valid image, let's give it the right size */
-    ret = bdrv_truncate(bs, total_size * BDRV_SECTOR_SIZE);
+    ret = bdrv_truncate(bs, total_size);
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not resize image");
         goto out;
@@ -1750,7 +1953,7 @@ static int qcow2_create2(const char *filename, int64_t total_size,
     }
 
     /* And if we're supposed to preallocate metadata, do that now */
-    if (prealloc) {
+    if (prealloc != PREALLOC_MODE_OFF) {
         BDRVQcowState *s = bs->opaque;
         qemu_co_mutex_lock(&s->lock);
         ret = preallocate(bs);
@@ -1786,16 +1989,17 @@ static int qcow2_create(const char *filename, QemuOpts *opts, Error **errp)
     char *backing_file = NULL;
     char *backing_fmt = NULL;
     char *buf = NULL;
-    uint64_t sectors = 0;
+    uint64_t size = 0;
     int flags = 0;
     size_t cluster_size = DEFAULT_CLUSTER_SIZE;
-    int prealloc = 0;
+    PreallocMode prealloc;
     int version = 3;
     Error *local_err = NULL;
     int ret;
 
     /* Read out options */
-    sectors = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0) / 512;
+    size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                    BDRV_SECTOR_SIZE);
     backing_file = qemu_opt_get_del(opts, BLOCK_OPT_BACKING_FILE);
     backing_fmt = qemu_opt_get_del(opts, BLOCK_OPT_BACKING_FMT);
     if (qemu_opt_get_bool_del(opts, BLOCK_OPT_ENCRYPT, false)) {
@@ -1804,12 +2008,11 @@ static int qcow2_create(const char *filename, QemuOpts *opts, Error **errp)
     cluster_size = qemu_opt_get_size_del(opts, BLOCK_OPT_CLUSTER_SIZE,
                                          DEFAULT_CLUSTER_SIZE);
     buf = qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC);
-    if (!buf || !strcmp(buf, "off")) {
-        prealloc = 0;
-    } else if (!strcmp(buf, "metadata")) {
-        prealloc = 1;
-    } else {
-        error_setg(errp, "Invalid preallocation mode: '%s'", buf);
+    prealloc = qapi_enum_parse(PreallocMode_lookup, buf,
+                               PREALLOC_MODE_MAX, PREALLOC_MODE_OFF,
+                               &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
         ret = -EINVAL;
         goto finish;
     }
@@ -1831,7 +2034,7 @@ static int qcow2_create(const char *filename, QemuOpts *opts, Error **errp)
         flags |= BLOCK_FLAG_LAZY_REFCOUNTS;
     }
 
-    if (backing_file && prealloc) {
+    if (backing_file && prealloc != PREALLOC_MODE_OFF) {
         error_setg(errp, "Backing file and preallocation cannot be used at "
                    "the same time");
         ret = -EINVAL;
@@ -1845,7 +2048,7 @@ static int qcow2_create(const char *filename, QemuOpts *opts, Error **errp)
         goto finish;
     }
 
-    ret = qcow2_create2(filename, sectors, backing_file, backing_fmt, flags,
+    ret = qcow2_create2(filename, size, backing_file, backing_fmt, flags,
                         cluster_size, prealloc, opts, version, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
@@ -1886,7 +2089,7 @@ static coroutine_fn int qcow2_co_discard(BlockDriverState *bs,
 
     qemu_co_mutex_lock(&s->lock);
     ret = qcow2_discard_clusters(bs, sector_num << BDRV_SECTOR_BITS,
-        nb_sectors, QCOW2_DISCARD_REQUEST);
+        nb_sectors, QCOW2_DISCARD_REQUEST, false);
     qemu_co_mutex_unlock(&s->lock);
     return ret;
 }
@@ -1947,7 +2150,6 @@ static int qcow2_write_compressed(BlockDriverState *bs, int64_t sector_num,
         /* align end of file to a sector boundary to ease reading with
            sector based I/Os */
         cluster_offset = bdrv_getlength(bs->file);
-        cluster_offset = (cluster_offset + 511) & ~511;
         bdrv_truncate(bs->file, cluster_offset);
         return 0;
     }
@@ -2028,6 +2230,195 @@ fail:
     return ret;
 }
 
+static int make_completely_empty(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+    int ret, l1_clusters;
+    int64_t offset;
+    uint64_t *new_reftable = NULL;
+    uint64_t rt_entry, l1_size2;
+    struct {
+        uint64_t l1_offset;
+        uint64_t reftable_offset;
+        uint32_t reftable_clusters;
+    } QEMU_PACKED l1_ofs_rt_ofs_cls;
+
+    ret = qcow2_cache_empty(bs, s->l2_table_cache);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    ret = qcow2_cache_empty(bs, s->refcount_block_cache);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* Refcounts will be broken utterly */
+    ret = qcow2_mark_dirty(bs);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    BLKDBG_EVENT(bs->file, BLKDBG_L1_UPDATE);
+
+    l1_clusters = DIV_ROUND_UP(s->l1_size, s->cluster_size / sizeof(uint64_t));
+    l1_size2 = (uint64_t)s->l1_size * sizeof(uint64_t);
+
+    /* After this call, neither the in-memory nor the on-disk refcount
+     * information accurately describe the actual references */
+
+    ret = bdrv_write_zeroes(bs->file, s->l1_table_offset / BDRV_SECTOR_SIZE,
+                            l1_clusters * s->cluster_sectors, 0);
+    if (ret < 0) {
+        goto fail_broken_refcounts;
+    }
+    memset(s->l1_table, 0, l1_size2);
+
+    BLKDBG_EVENT(bs->file, BLKDBG_EMPTY_IMAGE_PREPARE);
+
+    /* Overwrite enough clusters at the beginning of the sectors to place
+     * the refcount table, a refcount block and the L1 table in; this may
+     * overwrite parts of the existing refcount and L1 table, which is not
+     * an issue because the dirty flag is set, complete data loss is in fact
+     * desired and partial data loss is consequently fine as well */
+    ret = bdrv_write_zeroes(bs->file, s->cluster_size / BDRV_SECTOR_SIZE,
+                            (2 + l1_clusters) * s->cluster_size /
+                            BDRV_SECTOR_SIZE, 0);
+    /* This call (even if it failed overall) may have overwritten on-disk
+     * refcount structures; in that case, the in-memory refcount information
+     * will probably differ from the on-disk information which makes the BDS
+     * unusable */
+    if (ret < 0) {
+        goto fail_broken_refcounts;
+    }
+
+    BLKDBG_EVENT(bs->file, BLKDBG_L1_UPDATE);
+    BLKDBG_EVENT(bs->file, BLKDBG_REFTABLE_UPDATE);
+
+    /* "Create" an empty reftable (one cluster) directly after the image
+     * header and an empty L1 table three clusters after the image header;
+     * the cluster between those two will be used as the first refblock */
+    cpu_to_be64w(&l1_ofs_rt_ofs_cls.l1_offset, 3 * s->cluster_size);
+    cpu_to_be64w(&l1_ofs_rt_ofs_cls.reftable_offset, s->cluster_size);
+    cpu_to_be32w(&l1_ofs_rt_ofs_cls.reftable_clusters, 1);
+    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, l1_table_offset),
+                           &l1_ofs_rt_ofs_cls, sizeof(l1_ofs_rt_ofs_cls));
+    if (ret < 0) {
+        goto fail_broken_refcounts;
+    }
+
+    s->l1_table_offset = 3 * s->cluster_size;
+
+    new_reftable = g_try_new0(uint64_t, s->cluster_size / sizeof(uint64_t));
+    if (!new_reftable) {
+        ret = -ENOMEM;
+        goto fail_broken_refcounts;
+    }
+
+    s->refcount_table_offset = s->cluster_size;
+    s->refcount_table_size   = s->cluster_size / sizeof(uint64_t);
+
+    g_free(s->refcount_table);
+    s->refcount_table = new_reftable;
+    new_reftable = NULL;
+
+    /* Now the in-memory refcount information again corresponds to the on-disk
+     * information (reftable is empty and no refblocks (the refblock cache is
+     * empty)); however, this means some clusters (e.g. the image header) are
+     * referenced, but not refcounted, but the normal qcow2 code assumes that
+     * the in-memory information is always correct */
+
+    BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC);
+
+    /* Enter the first refblock into the reftable */
+    rt_entry = cpu_to_be64(2 * s->cluster_size);
+    ret = bdrv_pwrite_sync(bs->file, s->cluster_size,
+                           &rt_entry, sizeof(rt_entry));
+    if (ret < 0) {
+        goto fail_broken_refcounts;
+    }
+    s->refcount_table[0] = 2 * s->cluster_size;
+
+    s->free_cluster_index = 0;
+    assert(3 + l1_clusters <= s->refcount_block_size);
+    offset = qcow2_alloc_clusters(bs, 3 * s->cluster_size + l1_size2);
+    if (offset < 0) {
+        ret = offset;
+        goto fail_broken_refcounts;
+    } else if (offset > 0) {
+        error_report("First cluster in emptied image is in use");
+        abort();
+    }
+
+    /* Now finally the in-memory information corresponds to the on-disk
+     * structures and is correct */
+    ret = qcow2_mark_clean(bs);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    ret = bdrv_truncate(bs->file, (3 + l1_clusters) * s->cluster_size);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    return 0;
+
+fail_broken_refcounts:
+    /* The BDS is unusable at this point. If we wanted to make it usable, we
+     * would have to call qcow2_refcount_close(), qcow2_refcount_init(),
+     * qcow2_check_refcounts(), qcow2_refcount_close() and qcow2_refcount_init()
+     * again. However, because the functions which could have caused this error
+     * path to be taken are used by those functions as well, it's very likely
+     * that that sequence will fail as well. Therefore, just eject the BDS. */
+    bs->drv = NULL;
+
+fail:
+    g_free(new_reftable);
+    return ret;
+}
+
+static int qcow2_make_empty(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+    uint64_t start_sector;
+    int sector_step = INT_MAX / BDRV_SECTOR_SIZE;
+    int l1_clusters, ret = 0;
+
+    l1_clusters = DIV_ROUND_UP(s->l1_size, s->cluster_size / sizeof(uint64_t));
+
+    if (s->qcow_version >= 3 && !s->snapshots &&
+        3 + l1_clusters <= s->refcount_block_size) {
+        /* The following function only works for qcow2 v3 images (it requires
+         * the dirty flag) and only as long as there are no snapshots (because
+         * it completely empties the image). Furthermore, the L1 table and three
+         * additional clusters (image header, refcount table, one refcount
+         * block) have to fit inside one refcount block. */
+        return make_completely_empty(bs);
+    }
+
+    /* This fallback code simply discards every active cluster; this is slow,
+     * but works in all cases */
+    for (start_sector = 0; start_sector < bs->total_sectors;
+         start_sector += sector_step)
+    {
+        /* As this function is generally used after committing an external
+         * snapshot, QCOW2_DISCARD_SNAPSHOT seems appropriate. Also, the
+         * default action for this kind of discard is to pass the discard,
+         * which will ideally result in an actually smaller image file, as
+         * is probably desired. */
+        ret = qcow2_discard_clusters(bs, start_sector * BDRV_SECTOR_SIZE,
+                                     MIN(sector_step,
+                                         bs->total_sectors - start_sector),
+                                     QCOW2_DISCARD_SNAPSHOT, true);
+        if (ret < 0) {
+            break;
+        }
+    }
+
+    return ret;
+}
+
 static coroutine_fn int qcow2_co_flush_to_os(BlockDriverState *bs)
 {
     BDRVQcowState *s = bs->opaque;
@@ -2083,6 +2474,9 @@ static ImageInfoSpecific *qcow2_get_specific_info(BlockDriverState *bs)
             .lazy_refcounts     = s->compatible_features &
                                   QCOW2_COMPAT_LAZY_REFCOUNTS,
             .has_lazy_refcounts = true,
+            .corrupt            = s->incompatible_features &
+                                  QCOW2_INCOMPAT_CORRUPT,
+            .has_corrupt        = true,
         };
     }
 
@@ -2156,7 +2550,8 @@ static int qcow2_load_vmstate(BlockDriverState *bs, uint8_t *buf,
  * Downgrades an image's version. To achieve this, any incompatible features
  * have to be removed.
  */
-static int qcow2_downgrade(BlockDriverState *bs, int target_version)
+static int qcow2_downgrade(BlockDriverState *bs, int target_version,
+                           BlockDriverAmendStatusCB *status_cb)
 {
     BDRVQcowState *s = bs->opaque;
     int current_version = s->qcow_version;
@@ -2205,7 +2600,7 @@ static int qcow2_downgrade(BlockDriverState *bs, int target_version)
     /* clearing autoclear features is trivial */
     s->autoclear_features = 0;
 
-    ret = qcow2_expand_zero_clusters(bs);
+    ret = qcow2_expand_zero_clusters(bs, status_cb);
     if (ret < 0) {
         return ret;
     }
@@ -2219,7 +2614,8 @@ static int qcow2_downgrade(BlockDriverState *bs, int target_version)
     return 0;
 }
 
-static int qcow2_amend_options(BlockDriverState *bs, QemuOpts *opts)
+static int qcow2_amend_options(BlockDriverState *bs, QemuOpts *opts,
+                               BlockDriverAmendStatusCB *status_cb)
 {
     BDRVQcowState *s = bs->opaque;
     int old_version = s->qcow_version, new_version = old_version;
@@ -2297,7 +2693,7 @@ static int qcow2_amend_options(BlockDriverState *bs, QemuOpts *opts)
                 return ret;
             }
         } else {
-            ret = qcow2_downgrade(bs, new_version);
+            ret = qcow2_downgrade(bs, new_version, status_cb);
             if (ret < 0) {
                 return ret;
             }
@@ -2353,6 +2749,52 @@ static int qcow2_amend_options(BlockDriverState *bs, QemuOpts *opts)
     return 0;
 }
 
+/*
+ * If offset or size are negative, respectively, they will not be included in
+ * the BLOCK_IMAGE_CORRUPTED event emitted.
+ * fatal will be ignored for read-only BDS; corruptions found there will always
+ * be considered non-fatal.
+ */
+void qcow2_signal_corruption(BlockDriverState *bs, bool fatal, int64_t offset,
+                             int64_t size, const char *message_format, ...)
+{
+    BDRVQcowState *s = bs->opaque;
+    char *message;
+    va_list ap;
+
+    fatal = fatal && !bs->read_only;
+
+    if (s->signaled_corruption &&
+        (!fatal || (s->incompatible_features & QCOW2_INCOMPAT_CORRUPT)))
+    {
+        return;
+    }
+
+    va_start(ap, message_format);
+    message = g_strdup_vprintf(message_format, ap);
+    va_end(ap);
+
+    if (fatal) {
+        fprintf(stderr, "qcow2: Marking image as corrupt: %s; further "
+                "corruption events will be suppressed\n", message);
+    } else {
+        fprintf(stderr, "qcow2: Image is corrupt: %s; further non-fatal "
+                "corruption events will be suppressed\n", message);
+    }
+
+    qapi_event_send_block_image_corrupted(bdrv_get_device_name(bs), message,
+                                          offset >= 0, offset, size >= 0, size,
+                                          fatal, &error_abort);
+    g_free(message);
+
+    if (fatal) {
+        qcow2_mark_corrupt(bs);
+        bs->drv = NULL; /* make BDS unusable */
+    }
+
+    s->signaled_corruption = true;
+}
+
 static QemuOptsList qcow2_create_opts = {
     .name = "qcow2-create-opts",
     .head = QTAILQ_HEAD_INITIALIZER(qcow2_create_opts.head),
@@ -2392,7 +2834,8 @@ static QemuOptsList qcow2_create_opts = {
         {
             .name = BLOCK_OPT_PREALLOC,
             .type = QEMU_OPT_STRING,
-            .help = "Preallocation mode (allowed values: off, metadata)"
+            .help = "Preallocation mode (allowed values: off, metadata, "
+                    "falloc, full)"
         },
         {
             .name = BLOCK_OPT_LAZY_REFCOUNTS,
@@ -2424,6 +2867,7 @@ static BlockDriver bdrv_qcow2 = {
     .bdrv_co_discard        = qcow2_co_discard,
     .bdrv_truncate          = qcow2_truncate,
     .bdrv_write_compressed  = qcow2_write_compressed,
+    .bdrv_make_empty        = qcow2_make_empty,
 
     .bdrv_snapshot_create   = qcow2_snapshot_create,
     .bdrv_snapshot_goto     = qcow2_snapshot_goto,
diff --git a/block/qcow2.h b/block/qcow2.h
index b49424b85..6e39a1b63 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -59,15 +59,19 @@
 /* The cluster reads as all zeros */
 #define QCOW_OFLAG_ZERO (1ULL << 0)
 
-#define REFCOUNT_SHIFT 1 /* refcount size is 2 bytes */
-
 #define MIN_CLUSTER_BITS 9
 #define MAX_CLUSTER_BITS 21
 
-#define L2_CACHE_SIZE 16
+#define MIN_L2_CACHE_SIZE 1 /* cluster */
 
 /* Must be at least 4 to cover all cases of refcount table growth */
-#define REFCOUNT_CACHE_SIZE 4
+#define MIN_REFCOUNT_CACHE_SIZE 4 /* clusters */
+
+#define DEFAULT_L2_CACHE_BYTE_SIZE 1048576 /* bytes */
+
+/* The refblock cache needs only a fourth of the L2 cache size to cover as many
+ * clusters */
+#define DEFAULT_L2_REFCOUNT_SIZE_RATIO 4
 
 #define DEFAULT_CLUSTER_SIZE 65536
 
@@ -77,6 +81,7 @@
 #define QCOW2_OPT_DISCARD_SNAPSHOT "pass-discard-snapshot"
 #define QCOW2_OPT_DISCARD_OTHER "pass-discard-other"
 #define QCOW2_OPT_OVERLAP "overlap-check"
+#define QCOW2_OPT_OVERLAP_TEMPLATE "overlap-check.template"
 #define QCOW2_OPT_OVERLAP_MAIN_HEADER "overlap-check.main-header"
 #define QCOW2_OPT_OVERLAP_ACTIVE_L1 "overlap-check.active-l1"
 #define QCOW2_OPT_OVERLAP_ACTIVE_L2 "overlap-check.active-l2"
@@ -85,6 +90,9 @@
 #define QCOW2_OPT_OVERLAP_SNAPSHOT_TABLE "overlap-check.snapshot-table"
 #define QCOW2_OPT_OVERLAP_INACTIVE_L1 "overlap-check.inactive-l1"
 #define QCOW2_OPT_OVERLAP_INACTIVE_L2 "overlap-check.inactive-l2"
+#define QCOW2_OPT_CACHE_SIZE "cache-size"
+#define QCOW2_OPT_L2_CACHE_SIZE "l2-cache-size"
+#define QCOW2_OPT_REFCOUNT_CACHE_SIZE "refcount-cache-size"
 
 typedef struct QCowHeader {
     uint32_t magic;
@@ -213,6 +221,8 @@ typedef struct BDRVQcowState {
     int l2_size;
     int l1_size;
     int l1_vm_state_index;
+    int refcount_block_bits;
+    int refcount_block_size;
     int csize_shift;
     int csize_mask;
     uint64_t cluster_offset_mask;
@@ -252,6 +262,7 @@ typedef struct BDRVQcowState {
     bool discard_passthrough[QCOW2_DISCARD_MAX];
 
     int overlap_check; /* bitmask of Qcow2MetadataOverlap values */
+    bool signaled_corruption;
 
     uint64_t incompatible_features;
     uint64_t compatible_features;
@@ -468,10 +479,16 @@ int qcow2_mark_corrupt(BlockDriverState *bs);
 int qcow2_mark_consistent(BlockDriverState *bs);
 int qcow2_update_header(BlockDriverState *bs);
 
+void qcow2_signal_corruption(BlockDriverState *bs, bool fatal, int64_t offset,
+                             int64_t size, const char *message_format, ...)
+                             GCC_FMT_ATTR(5, 6);
+
 /* qcow2-refcount.c functions */
 int qcow2_refcount_init(BlockDriverState *bs);
 void qcow2_refcount_close(BlockDriverState *bs);
 
+int qcow2_get_refcount(BlockDriverState *bs, int64_t cluster_index);
+
 int qcow2_update_cluster_refcount(BlockDriverState *bs, int64_t cluster_index,
                                   int addend, enum qcow2_discard_type type);
 
@@ -519,10 +536,11 @@ uint64_t qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs,
 
 int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m);
 int qcow2_discard_clusters(BlockDriverState *bs, uint64_t offset,
-    int nb_sectors, enum qcow2_discard_type type);
+    int nb_sectors, enum qcow2_discard_type type, bool full_discard);
 int qcow2_zero_clusters(BlockDriverState *bs, uint64_t offset, int nb_sectors);
 
-int qcow2_expand_zero_clusters(BlockDriverState *bs);
+int qcow2_expand_zero_clusters(BlockDriverState *bs,
+                               BlockDriverAmendStatusCB *status_cb);
 
 /* qcow2-snapshot.c functions */
 int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info);
diff --git a/block/qed-check.c b/block/qed-check.c
index b473dcd61..36ecd290d 100644
--- a/block/qed-check.c
+++ b/block/qed-check.c
@@ -227,8 +227,10 @@ int qed_check(BDRVQEDState *s, BdrvCheckResult *result, bool fix)
     };
     int ret;
 
-    check.used_clusters = g_malloc0(((check.nclusters + 31) / 32) *
-                                       sizeof(check.used_clusters[0]));
+    check.used_clusters = g_try_new0(uint32_t, (check.nclusters + 31) / 32);
+    if (check.nclusters && check.used_clusters == NULL) {
+        return -ENOMEM;
+    }
 
     check.result->bfi.total_clusters =
         (s->header.image_size + s->header.cluster_size - 1) /
diff --git a/block/qed-gencb.c b/block/qed-gencb.c
index 7d7ac1ffc..b817a8bf5 100644
--- a/block/qed-gencb.c
+++ b/block/qed-gencb.c
@@ -13,7 +13,7 @@
 
 #include "qed.h"
 
-void *gencb_alloc(size_t len, BlockDriverCompletionFunc *cb, void *opaque)
+void *gencb_alloc(size_t len, BlockCompletionFunc *cb, void *opaque)
 {
     GenericCB *gencb = g_malloc(len);
     gencb->cb = cb;
@@ -24,7 +24,7 @@ void *gencb_alloc(size_t len, BlockDriverCompletionFunc *cb, void *opaque)
 void gencb_complete(void *opaque, int ret)
 {
     GenericCB *gencb = opaque;
-    BlockDriverCompletionFunc *cb = gencb->cb;
+    BlockCompletionFunc *cb = gencb->cb;
     void *user_opaque = gencb->opaque;
 
     g_free(gencb);
diff --git a/block/qed-table.c b/block/qed-table.c
index f61107a1c..513aa872c 100644
--- a/block/qed-table.c
+++ b/block/qed-table.c
@@ -49,7 +49,7 @@ out:
 }
 
 static void qed_read_table(BDRVQEDState *s, uint64_t offset, QEDTable *table,
-                           BlockDriverCompletionFunc *cb, void *opaque)
+                           BlockCompletionFunc *cb, void *opaque)
 {
     QEDReadTableCB *read_table_cb = gencb_alloc(sizeof(*read_table_cb),
                                                 cb, opaque);
@@ -119,7 +119,7 @@ out:
  */
 static void qed_write_table(BDRVQEDState *s, uint64_t offset, QEDTable *table,
                             unsigned int index, unsigned int n, bool flush,
-                            BlockDriverCompletionFunc *cb, void *opaque)
+                            BlockCompletionFunc *cb, void *opaque)
 {
     QEDWriteTableCB *write_table_cb;
     unsigned int sector_mask = BDRV_SECTOR_SIZE / sizeof(uint64_t) - 1;
@@ -180,7 +180,7 @@ int qed_read_l1_table_sync(BDRVQEDState *s)
 }
 
 void qed_write_l1_table(BDRVQEDState *s, unsigned int index, unsigned int n,
-                        BlockDriverCompletionFunc *cb, void *opaque)
+                        BlockCompletionFunc *cb, void *opaque)
 {
     BLKDBG_EVENT(s->bs->file, BLKDBG_L1_UPDATE);
     qed_write_table(s, s->header.l1_table_offset,
@@ -235,7 +235,7 @@ static void qed_read_l2_table_cb(void *opaque, int ret)
 }
 
 void qed_read_l2_table(BDRVQEDState *s, QEDRequest *request, uint64_t offset,
-                       BlockDriverCompletionFunc *cb, void *opaque)
+                       BlockCompletionFunc *cb, void *opaque)
 {
     QEDReadL2TableCB *read_l2_table_cb;
 
@@ -275,7 +275,7 @@ int qed_read_l2_table_sync(BDRVQEDState *s, QEDRequest *request, uint64_t offset
 
 void qed_write_l2_table(BDRVQEDState *s, QEDRequest *request,
                         unsigned int index, unsigned int n, bool flush,
-                        BlockDriverCompletionFunc *cb, void *opaque)
+                        BlockCompletionFunc *cb, void *opaque)
 {
     BLKDBG_EVENT(s->bs->file, BLKDBG_L2_UPDATE);
     qed_write_table(s, request->l2_table->offset,
diff --git a/block/qed.c b/block/qed.c
index 794483218..80f18d82e 100644
--- a/block/qed.c
+++ b/block/qed.c
@@ -18,22 +18,8 @@
 #include "qapi/qmp/qerror.h"
 #include "migration/migration.h"
 
-static void qed_aio_cancel(BlockDriverAIOCB *blockacb)
-{
-    QEDAIOCB *acb = (QEDAIOCB *)blockacb;
-    AioContext *aio_context = bdrv_get_aio_context(blockacb->bs);
-    bool finished = false;
-
-    /* Wait for the request to finish */
-    acb->finished = &finished;
-    while (!finished) {
-        aio_poll(aio_context, true);
-    }
-}
-
 static const AIOCBInfo qed_aiocb_info = {
     .aiocb_size         = sizeof(QEDAIOCB),
-    .cancel             = qed_aio_cancel,
 };
 
 static int bdrv_qed_probe(const uint8_t *buf, int buf_size,
@@ -144,7 +130,7 @@ static void qed_write_header_read_cb(void *opaque, int ret)
  * This function only updates known header fields in-place and does not affect
  * extra data after the QED header.
  */
-static void qed_write_header(BDRVQEDState *s, BlockDriverCompletionFunc cb,
+static void qed_write_header(BDRVQEDState *s, BlockCompletionFunc cb,
                              void *opaque)
 {
     /* We must write full sectors for O_DIRECT but cannot necessarily generate
@@ -422,7 +408,7 @@ static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags,
         snprintf(buf, sizeof(buf), "%" PRIx64,
             s->header.features & ~QED_FEATURE_MASK);
         error_set(errp, QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
-            bs->device_name, "QED", buf);
+            bdrv_get_device_name(bs), "QED", buf);
         return -ENOTSUP;
     }
     if (!qed_is_cluster_size_valid(s->header.cluster_size)) {
@@ -648,7 +634,8 @@ static int bdrv_qed_create(const char *filename, QemuOpts *opts, Error **errp)
     char *backing_fmt = NULL;
     int ret;
 
-    image_size = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0);
+    image_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                          BDRV_SECTOR_SIZE);
     backing_file = qemu_opt_get_del(opts, BLOCK_OPT_BACKING_FILE);
     backing_fmt = qemu_opt_get_del(opts, BLOCK_OPT_BACKING_FMT);
     cluster_size = qemu_opt_get_size_del(opts,
@@ -772,7 +759,7 @@ static BDRVQEDState *acb_to_s(QEDAIOCB *acb)
 static void qed_read_backing_file(BDRVQEDState *s, uint64_t pos,
                                   QEMUIOVector *qiov,
                                   QEMUIOVector **backing_qiov,
-                                  BlockDriverCompletionFunc *cb, void *opaque)
+                                  BlockCompletionFunc *cb, void *opaque)
 {
     uint64_t backing_length = 0;
     size_t size;
@@ -864,7 +851,7 @@ static void qed_copy_from_backing_file_write(void *opaque, int ret)
  */
 static void qed_copy_from_backing_file(BDRVQEDState *s, uint64_t pos,
                                        uint64_t len, uint64_t offset,
-                                       BlockDriverCompletionFunc *cb,
+                                       BlockCompletionFunc *cb,
                                        void *opaque)
 {
     CopyFromBackingFileCB *copy_cb;
@@ -915,21 +902,15 @@ static void qed_update_l2_table(BDRVQEDState *s, QEDTable *table, int index,
 static void qed_aio_complete_bh(void *opaque)
 {
     QEDAIOCB *acb = opaque;
-    BlockDriverCompletionFunc *cb = acb->common.cb;
+    BlockCompletionFunc *cb = acb->common.cb;
     void *user_opaque = acb->common.opaque;
     int ret = acb->bh_ret;
-    bool *finished = acb->finished;
 
     qemu_bh_delete(acb->bh);
-    qemu_aio_release(acb);
+    qemu_aio_unref(acb);
 
     /* Invoke callback */
     cb(user_opaque, ret);
-
-    /* Signal cancel completion */
-    if (finished) {
-        *finished = true;
-    }
 }
 
 static void qed_aio_complete(QEDAIOCB *acb, int ret)
@@ -1083,7 +1064,7 @@ static void qed_aio_write_main(void *opaque, int ret)
     BDRVQEDState *s = acb_to_s(acb);
     uint64_t offset = acb->cur_cluster +
                       qed_offset_into_cluster(s, acb->cur_pos);
-    BlockDriverCompletionFunc *next_fn;
+    BlockCompletionFunc *next_fn;
 
     trace_qed_aio_write_main(s, acb, ret, offset, acb->cur_qiov.size);
 
@@ -1183,7 +1164,7 @@ static void qed_aio_write_zero_cluster(void *opaque, int ret)
 static void qed_aio_write_alloc(QEDAIOCB *acb, size_t len)
 {
     BDRVQEDState *s = acb_to_s(acb);
-    BlockDriverCompletionFunc *cb;
+    BlockCompletionFunc *cb;
 
     /* Cancel timer when the first allocating request comes in */
     if (QSIMPLEQ_EMPTY(&s->allocating_write_reqs)) {
@@ -1240,7 +1221,11 @@ static void qed_aio_write_inplace(QEDAIOCB *acb, uint64_t offset, size_t len)
         struct iovec *iov = acb->qiov->iov;
 
         if (!iov->iov_base) {
-            iov->iov_base = qemu_blockalign(acb->common.bs, iov->iov_len);
+            iov->iov_base = qemu_try_blockalign(acb->common.bs, iov->iov_len);
+            if (iov->iov_base == NULL) {
+                qed_aio_complete(acb, -ENOMEM);
+                return;
+            }
             memset(iov->iov_base, 0, iov->iov_len);
         }
     }
@@ -1380,11 +1365,11 @@ static void qed_aio_next_io(void *opaque, int ret)
                       io_fn, acb);
 }
 
-static BlockDriverAIOCB *qed_aio_setup(BlockDriverState *bs,
-                                       int64_t sector_num,
-                                       QEMUIOVector *qiov, int nb_sectors,
-                                       BlockDriverCompletionFunc *cb,
-                                       void *opaque, int flags)
+static BlockAIOCB *qed_aio_setup(BlockDriverState *bs,
+                                 int64_t sector_num,
+                                 QEMUIOVector *qiov, int nb_sectors,
+                                 BlockCompletionFunc *cb,
+                                 void *opaque, int flags)
 {
     QEDAIOCB *acb = qemu_aio_get(&qed_aiocb_info, bs, cb, opaque);
 
@@ -1392,7 +1377,6 @@ static BlockDriverAIOCB *qed_aio_setup(BlockDriverState *bs,
                         opaque, flags);
 
     acb->flags = flags;
-    acb->finished = NULL;
     acb->qiov = qiov;
     acb->qiov_offset = 0;
     acb->cur_pos = (uint64_t)sector_num * BDRV_SECTOR_SIZE;
@@ -1406,20 +1390,20 @@ static BlockDriverAIOCB *qed_aio_setup(BlockDriverState *bs,
     return &acb->common;
 }
 
-static BlockDriverAIOCB *bdrv_qed_aio_readv(BlockDriverState *bs,
-                                            int64_t sector_num,
-                                            QEMUIOVector *qiov, int nb_sectors,
-                                            BlockDriverCompletionFunc *cb,
-                                            void *opaque)
+static BlockAIOCB *bdrv_qed_aio_readv(BlockDriverState *bs,
+                                      int64_t sector_num,
+                                      QEMUIOVector *qiov, int nb_sectors,
+                                      BlockCompletionFunc *cb,
+                                      void *opaque)
 {
     return qed_aio_setup(bs, sector_num, qiov, nb_sectors, cb, opaque, 0);
 }
 
-static BlockDriverAIOCB *bdrv_qed_aio_writev(BlockDriverState *bs,
-                                             int64_t sector_num,
-                                             QEMUIOVector *qiov, int nb_sectors,
-                                             BlockDriverCompletionFunc *cb,
-                                             void *opaque)
+static BlockAIOCB *bdrv_qed_aio_writev(BlockDriverState *bs,
+                                       int64_t sector_num,
+                                       QEMUIOVector *qiov, int nb_sectors,
+                                       BlockCompletionFunc *cb,
+                                       void *opaque)
 {
     return qed_aio_setup(bs, sector_num, qiov, nb_sectors, cb,
                          opaque, QED_AIOCB_WRITE);
@@ -1447,7 +1431,7 @@ static int coroutine_fn bdrv_qed_co_write_zeroes(BlockDriverState *bs,
                                                  int nb_sectors,
                                                  BdrvRequestFlags flags)
 {
-    BlockDriverAIOCB *blockacb;
+    BlockAIOCB *blockacb;
     BDRVQEDState *s = bs->opaque;
     QEDWriteZeroesCB cb = { .done = false };
     QEMUIOVector qiov;
diff --git a/block/qed.h b/block/qed.h
index 2b0e724e0..d3934a05c 100644
--- a/block/qed.h
+++ b/block/qed.h
@@ -128,7 +128,7 @@ enum {
 };
 
 typedef struct QEDAIOCB {
-    BlockDriverAIOCB common;
+    BlockAIOCB common;
     QEMUBH *bh;
     int bh_ret;                     /* final return status for completion bh */
     QSIMPLEQ_ENTRY(QEDAIOCB) next;  /* next request */
@@ -203,11 +203,11 @@ typedef void QEDFindClusterFunc(void *opaque, int ret, uint64_t offset, size_t l
  * Generic callback for chaining async callbacks
  */
 typedef struct {
-    BlockDriverCompletionFunc *cb;
+    BlockCompletionFunc *cb;
     void *opaque;
 } GenericCB;
 
-void *gencb_alloc(size_t len, BlockDriverCompletionFunc *cb, void *opaque);
+void *gencb_alloc(size_t len, BlockCompletionFunc *cb, void *opaque);
 void gencb_complete(void *opaque, int ret);
 
 /**
@@ -230,16 +230,16 @@ void qed_commit_l2_cache_entry(L2TableCache *l2_cache, CachedL2Table *l2_table);
  */
 int qed_read_l1_table_sync(BDRVQEDState *s);
 void qed_write_l1_table(BDRVQEDState *s, unsigned int index, unsigned int n,
-                        BlockDriverCompletionFunc *cb, void *opaque);
+                        BlockCompletionFunc *cb, void *opaque);
 int qed_write_l1_table_sync(BDRVQEDState *s, unsigned int index,
                             unsigned int n);
 int qed_read_l2_table_sync(BDRVQEDState *s, QEDRequest *request,
                            uint64_t offset);
 void qed_read_l2_table(BDRVQEDState *s, QEDRequest *request, uint64_t offset,
-                       BlockDriverCompletionFunc *cb, void *opaque);
+                       BlockCompletionFunc *cb, void *opaque);
 void qed_write_l2_table(BDRVQEDState *s, QEDRequest *request,
                         unsigned int index, unsigned int n, bool flush,
-                        BlockDriverCompletionFunc *cb, void *opaque);
+                        BlockCompletionFunc *cb, void *opaque);
 int qed_write_l2_table_sync(BDRVQEDState *s, QEDRequest *request,
                             unsigned int index, unsigned int n, bool flush);
 
diff --git a/block/quorum.c b/block/quorum.c
index d5ee9c005..437b12251 100644
--- a/block/quorum.c
+++ b/block/quorum.c
@@ -16,7 +16,12 @@
 #include <gnutls/gnutls.h>
 #include <gnutls/crypto.h>
 #include "block/block_int.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qint.h"
 #include "qapi/qmp/qjson.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qstring.h"
 #include "qapi-event.h"
 
 #define HASH_LENGTH 32
@@ -24,6 +29,7 @@
 #define QUORUM_OPT_VOTE_THRESHOLD "vote-threshold"
 #define QUORUM_OPT_BLKVERIFY      "blkverify"
 #define QUORUM_OPT_REWRITE        "rewrite-corrupted"
+#define QUORUM_OPT_READ_PATTERN   "read-pattern"
 
 /* This union holds a vote hash value */
 typedef union QuorumVoteValue {
@@ -74,6 +80,8 @@ typedef struct BDRVQuorumState {
     bool rewrite_corrupted;/* true if the driver must rewrite-on-read corrupted
                             * block if Quorum is reached.
                             */
+
+    QuorumReadPattern read_pattern;
 } BDRVQuorumState;
 
 typedef struct QuorumAIOCB QuorumAIOCB;
@@ -84,7 +92,7 @@ typedef struct QuorumAIOCB QuorumAIOCB;
  * $children_count QuorumChildRequest.
  */
 typedef struct QuorumChildRequest {
-    BlockDriverAIOCB *aiocb;
+    BlockAIOCB *aiocb;
     QEMUIOVector qiov;
     uint8_t *buf;
     int ret;
@@ -97,7 +105,7 @@ typedef struct QuorumChildRequest {
  * used to do operations on each children and track overall progress.
  */
 struct QuorumAIOCB {
-    BlockDriverAIOCB common;
+    BlockAIOCB common;
 
     /* Request metadata */
     uint64_t sector_num;
@@ -117,11 +125,12 @@ struct QuorumAIOCB {
 
     bool is_read;
     int vote_ret;
+    int child_iter;             /* which child to read in fifo pattern */
 };
 
 static bool quorum_vote(QuorumAIOCB *acb);
 
-static void quorum_aio_cancel(BlockDriverAIOCB *blockacb)
+static void quorum_aio_cancel(BlockAIOCB *blockacb)
 {
     QuorumAIOCB *acb = container_of(blockacb, QuorumAIOCB, common);
     BDRVQuorumState *s = acb->common.bs->opaque;
@@ -129,21 +138,19 @@ static void quorum_aio_cancel(BlockDriverAIOCB *blockacb)
 
     /* cancel all callbacks */
     for (i = 0; i < s->num_children; i++) {
-        bdrv_aio_cancel(acb->qcrs[i].aiocb);
+        if (acb->qcrs[i].aiocb) {
+            bdrv_aio_cancel_async(acb->qcrs[i].aiocb);
+        }
     }
-
-    g_free(acb->qcrs);
-    qemu_aio_release(acb);
 }
 
 static AIOCBInfo quorum_aiocb_info = {
     .aiocb_size         = sizeof(QuorumAIOCB),
-    .cancel             = quorum_aio_cancel,
+    .cancel_async       = quorum_aio_cancel,
 };
 
 static void quorum_aio_finalize(QuorumAIOCB *acb)
 {
-    BDRVQuorumState *s = acb->common.bs->opaque;
     int i, ret = 0;
 
     if (acb->vote_ret) {
@@ -153,14 +160,15 @@ static void quorum_aio_finalize(QuorumAIOCB *acb)
     acb->common.cb(acb->common.opaque, ret);
 
     if (acb->is_read) {
-        for (i = 0; i < s->num_children; i++) {
+        /* on the quorum case acb->child_iter == s->num_children - 1 */
+        for (i = 0; i <= acb->child_iter; i++) {
             qemu_vfree(acb->qcrs[i].buf);
             qemu_iovec_destroy(&acb->qcrs[i].qiov);
         }
     }
 
     g_free(acb->qcrs);
-    qemu_aio_release(acb);
+    qemu_aio_unref(acb);
 }
 
 static bool quorum_sha256_compare(QuorumVoteValue *a, QuorumVoteValue *b)
@@ -178,7 +186,7 @@ static QuorumAIOCB *quorum_aio_get(BDRVQuorumState *s,
                                    QEMUIOVector *qiov,
                                    uint64_t sector_num,
                                    int nb_sectors,
-                                   BlockDriverCompletionFunc *cb,
+                                   BlockCompletionFunc *cb,
                                    void *opaque)
 {
     QuorumAIOCB *acb = qemu_aio_get(&quorum_aiocb_info, bs, cb, opaque);
@@ -218,8 +226,8 @@ static void quorum_report_bad(QuorumAIOCB *acb, char *node_name, int ret)
 
 static void quorum_report_failure(QuorumAIOCB *acb)
 {
-    const char *reference = acb->common.bs->device_name[0] ?
-                            acb->common.bs->device_name :
+    const char *reference = bdrv_get_device_name(acb->common.bs)[0] ?
+                            bdrv_get_device_name(acb->common.bs) :
                             acb->common.bs->node_name;
 
     qapi_event_send_quorum_failure(reference, acb->sector_num,
@@ -256,6 +264,21 @@ static void quorum_rewrite_aio_cb(void *opaque, int ret)
     quorum_aio_finalize(acb);
 }
 
+static BlockAIOCB *read_fifo_child(QuorumAIOCB *acb);
+
+static void quorum_copy_qiov(QEMUIOVector *dest, QEMUIOVector *source)
+{
+    int i;
+    assert(dest->niov == source->niov);
+    assert(dest->size == source->size);
+    for (i = 0; i < source->niov; i++) {
+        assert(dest->iov[i].iov_len == source->iov[i].iov_len);
+        memcpy(dest->iov[i].iov_base,
+               source->iov[i].iov_base,
+               source->iov[i].iov_len);
+    }
+}
+
 static void quorum_aio_cb(void *opaque, int ret)
 {
     QuorumChildRequest *sacb = opaque;
@@ -263,6 +286,21 @@ static void quorum_aio_cb(void *opaque, int ret)
     BDRVQuorumState *s = acb->common.bs->opaque;
     bool rewrite = false;
 
+    if (acb->is_read && s->read_pattern == QUORUM_READ_PATTERN_FIFO) {
+        /* We try to read next child in FIFO order if we fail to read */
+        if (ret < 0 && ++acb->child_iter < s->num_children) {
+            read_fifo_child(acb);
+            return;
+        }
+
+        if (ret == 0) {
+            quorum_copy_qiov(acb->qiov, &acb->qcrs[acb->child_iter].qiov);
+        }
+        acb->vote_ret = ret;
+        quorum_aio_finalize(acb);
+        return;
+    }
+
     sacb->ret = ret;
     acb->count++;
     if (ret == 0) {
@@ -343,19 +381,6 @@ static bool quorum_rewrite_bad_versions(BDRVQuorumState *s, QuorumAIOCB *acb,
     return count;
 }
 
-static void quorum_copy_qiov(QEMUIOVector *dest, QEMUIOVector *source)
-{
-    int i;
-    assert(dest->niov == source->niov);
-    assert(dest->size == source->size);
-    for (i = 0; i < source->niov; i++) {
-        assert(dest->iov[i].iov_len == source->iov[i].iov_len);
-        memcpy(dest->iov[i].iov_base,
-               source->iov[i].iov_base,
-               source->iov[i].iov_len);
-    }
-}
-
 static void quorum_count_vote(QuorumVotes *votes,
                               QuorumVoteValue *value,
                               int index)
@@ -615,40 +640,68 @@ free_exit:
     return rewrite;
 }
 
-static BlockDriverAIOCB *quorum_aio_readv(BlockDriverState *bs,
-                                         int64_t sector_num,
-                                         QEMUIOVector *qiov,
-                                         int nb_sectors,
-                                         BlockDriverCompletionFunc *cb,
-                                         void *opaque)
+static BlockAIOCB *read_quorum_children(QuorumAIOCB *acb)
 {
-    BDRVQuorumState *s = bs->opaque;
-    QuorumAIOCB *acb = quorum_aio_get(s, bs, qiov, sector_num,
-                                      nb_sectors, cb, opaque);
+    BDRVQuorumState *s = acb->common.bs->opaque;
     int i;
 
-    acb->is_read = true;
-
     for (i = 0; i < s->num_children; i++) {
-        acb->qcrs[i].buf = qemu_blockalign(s->bs[i], qiov->size);
-        qemu_iovec_init(&acb->qcrs[i].qiov, qiov->niov);
-        qemu_iovec_clone(&acb->qcrs[i].qiov, qiov, acb->qcrs[i].buf);
+        acb->qcrs[i].buf = qemu_blockalign(s->bs[i], acb->qiov->size);
+        qemu_iovec_init(&acb->qcrs[i].qiov, acb->qiov->niov);
+        qemu_iovec_clone(&acb->qcrs[i].qiov, acb->qiov, acb->qcrs[i].buf);
     }
 
     for (i = 0; i < s->num_children; i++) {
-        bdrv_aio_readv(s->bs[i], sector_num, &acb->qcrs[i].qiov, nb_sectors,
-                       quorum_aio_cb, &acb->qcrs[i]);
+        bdrv_aio_readv(s->bs[i], acb->sector_num, &acb->qcrs[i].qiov,
+                       acb->nb_sectors, quorum_aio_cb, &acb->qcrs[i]);
     }
 
     return &acb->common;
 }
 
-static BlockDriverAIOCB *quorum_aio_writev(BlockDriverState *bs,
-                                          int64_t sector_num,
-                                          QEMUIOVector *qiov,
-                                          int nb_sectors,
-                                          BlockDriverCompletionFunc *cb,
-                                          void *opaque)
+static BlockAIOCB *read_fifo_child(QuorumAIOCB *acb)
+{
+    BDRVQuorumState *s = acb->common.bs->opaque;
+
+    acb->qcrs[acb->child_iter].buf = qemu_blockalign(s->bs[acb->child_iter],
+                                                     acb->qiov->size);
+    qemu_iovec_init(&acb->qcrs[acb->child_iter].qiov, acb->qiov->niov);
+    qemu_iovec_clone(&acb->qcrs[acb->child_iter].qiov, acb->qiov,
+                     acb->qcrs[acb->child_iter].buf);
+    bdrv_aio_readv(s->bs[acb->child_iter], acb->sector_num,
+                   &acb->qcrs[acb->child_iter].qiov, acb->nb_sectors,
+                   quorum_aio_cb, &acb->qcrs[acb->child_iter]);
+
+    return &acb->common;
+}
+
+static BlockAIOCB *quorum_aio_readv(BlockDriverState *bs,
+                                    int64_t sector_num,
+                                    QEMUIOVector *qiov,
+                                    int nb_sectors,
+                                    BlockCompletionFunc *cb,
+                                    void *opaque)
+{
+    BDRVQuorumState *s = bs->opaque;
+    QuorumAIOCB *acb = quorum_aio_get(s, bs, qiov, sector_num,
+                                      nb_sectors, cb, opaque);
+    acb->is_read = true;
+
+    if (s->read_pattern == QUORUM_READ_PATTERN_QUORUM) {
+        acb->child_iter = s->num_children - 1;
+        return read_quorum_children(acb);
+    }
+
+    acb->child_iter = 0;
+    return read_fifo_child(acb);
+}
+
+static BlockAIOCB *quorum_aio_writev(BlockDriverState *bs,
+                                     int64_t sector_num,
+                                     QEMUIOVector *qiov,
+                                     int nb_sectors,
+                                     BlockCompletionFunc *cb,
+                                     void *opaque)
 {
     BDRVQuorumState *s = bs->opaque;
     QuorumAIOCB *acb = quorum_aio_get(s, bs, qiov, sector_num, nb_sectors,
@@ -782,16 +835,39 @@ static QemuOptsList quorum_runtime_opts = {
             .type = QEMU_OPT_BOOL,
             .help = "Rewrite corrupted block on read quorum",
         },
+        {
+            .name = QUORUM_OPT_READ_PATTERN,
+            .type = QEMU_OPT_STRING,
+            .help = "Allowed pattern: quorum, fifo. Quorum is default",
+        },
         { /* end of list */ }
     },
 };
 
+static int parse_read_pattern(const char *opt)
+{
+    int i;
+
+    if (!opt) {
+        /* Set quorum as default */
+        return QUORUM_READ_PATTERN_QUORUM;
+    }
+
+    for (i = 0; i < QUORUM_READ_PATTERN_MAX; i++) {
+        if (!strcmp(opt, QuorumReadPattern_lookup[i])) {
+            return i;
+        }
+    }
+
+    return -EINVAL;
+}
+
 static int quorum_open(BlockDriverState *bs, QDict *options, int flags,
                        Error **errp)
 {
     BDRVQuorumState *s = bs->opaque;
     Error *local_err = NULL;
-    QemuOpts *opts;
+    QemuOpts *opts = NULL;
     bool *opened;
     QDict *sub = NULL;
     QList *list = NULL;
@@ -827,28 +903,37 @@ static int quorum_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     s->threshold = qemu_opt_get_number(opts, QUORUM_OPT_VOTE_THRESHOLD, 0);
-
-    /* and validate it against s->num_children */
-    ret = quorum_valid_threshold(s->threshold, s->num_children, &local_err);
+    ret = parse_read_pattern(qemu_opt_get(opts, QUORUM_OPT_READ_PATTERN));
     if (ret < 0) {
+        error_setg(&local_err, "Please set read-pattern as fifo or quorum");
         goto exit;
     }
+    s->read_pattern = ret;
 
-    /* is the driver in blkverify mode */
-    if (qemu_opt_get_bool(opts, QUORUM_OPT_BLKVERIFY, false) &&
-        s->num_children == 2 && s->threshold == 2) {
-        s->is_blkverify = true;
-    } else if (qemu_opt_get_bool(opts, QUORUM_OPT_BLKVERIFY, false)) {
-        fprintf(stderr, "blkverify mode is set by setting blkverify=on "
-                "and using two files with vote_threshold=2\n");
-    }
+    if (s->read_pattern == QUORUM_READ_PATTERN_QUORUM) {
+        /* and validate it against s->num_children */
+        ret = quorum_valid_threshold(s->threshold, s->num_children, &local_err);
+        if (ret < 0) {
+            goto exit;
+        }
 
-    s->rewrite_corrupted = qemu_opt_get_bool(opts, QUORUM_OPT_REWRITE, false);
-    if (s->rewrite_corrupted && s->is_blkverify) {
-        error_setg(&local_err,
-                   "rewrite-corrupted=on cannot be used with blkverify=on");
-        ret = -EINVAL;
-        goto exit;
+        /* is the driver in blkverify mode */
+        if (qemu_opt_get_bool(opts, QUORUM_OPT_BLKVERIFY, false) &&
+            s->num_children == 2 && s->threshold == 2) {
+            s->is_blkverify = true;
+        } else if (qemu_opt_get_bool(opts, QUORUM_OPT_BLKVERIFY, false)) {
+            fprintf(stderr, "blkverify mode is set by setting blkverify=on "
+                    "and using two files with vote_threshold=2\n");
+        }
+
+        s->rewrite_corrupted = qemu_opt_get_bool(opts, QUORUM_OPT_REWRITE,
+                                                 false);
+        if (s->rewrite_corrupted && s->is_blkverify) {
+            error_setg(&local_err,
+                       "rewrite-corrupted=on cannot be used with blkverify=on");
+            ret = -EINVAL;
+            goto exit;
+        }
     }
 
     /* allocate the children BlockDriverState array */
@@ -903,6 +988,7 @@ close_exit:
     g_free(s->bs);
     g_free(opened);
 exit:
+    qemu_opts_del(opts);
     /* propagate error */
     if (local_err) {
         error_propagate(errp, local_err);
@@ -945,6 +1031,39 @@ static void quorum_attach_aio_context(BlockDriverState *bs,
     }
 }
 
+static void quorum_refresh_filename(BlockDriverState *bs)
+{
+    BDRVQuorumState *s = bs->opaque;
+    QDict *opts;
+    QList *children;
+    int i;
+
+    for (i = 0; i < s->num_children; i++) {
+        bdrv_refresh_filename(s->bs[i]);
+        if (!s->bs[i]->full_open_options) {
+            return;
+        }
+    }
+
+    children = qlist_new();
+    for (i = 0; i < s->num_children; i++) {
+        QINCREF(s->bs[i]->full_open_options);
+        qlist_append_obj(children, QOBJECT(s->bs[i]->full_open_options));
+    }
+
+    opts = qdict_new();
+    qdict_put_obj(opts, "driver", QOBJECT(qstring_from_str("quorum")));
+    qdict_put_obj(opts, QUORUM_OPT_VOTE_THRESHOLD,
+                  QOBJECT(qint_from_int(s->threshold)));
+    qdict_put_obj(opts, QUORUM_OPT_BLKVERIFY,
+                  QOBJECT(qbool_from_int(s->is_blkverify)));
+    qdict_put_obj(opts, QUORUM_OPT_REWRITE,
+                  QOBJECT(qbool_from_int(s->rewrite_corrupted)));
+    qdict_put_obj(opts, "children", QOBJECT(children));
+
+    bs->full_open_options = opts;
+}
+
 static BlockDriver bdrv_quorum = {
     .format_name                        = "quorum",
     .protocol_name                      = "quorum",
@@ -953,6 +1072,7 @@ static BlockDriver bdrv_quorum = {
 
     .bdrv_file_open                     = quorum_open,
     .bdrv_close                         = quorum_close,
+    .bdrv_refresh_filename              = quorum_refresh_filename,
 
     .bdrv_co_flush_to_disk              = quorum_co_flush,
 
diff --git a/block/raw-aio.h b/block/raw-aio.h
index e18c97509..80681ce17 100644
--- a/block/raw-aio.h
+++ b/block/raw-aio.h
@@ -35,9 +35,9 @@
 #ifdef CONFIG_LINUX_AIO
 void *laio_init(void);
 void laio_cleanup(void *s);
-BlockDriverAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
+BlockAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque, int type);
+        BlockCompletionFunc *cb, void *opaque, int type);
 void laio_detach_aio_context(void *s, AioContext *old_context);
 void laio_attach_aio_context(void *s, AioContext *new_context);
 void laio_io_plug(BlockDriverState *bs, void *aio_ctx);
@@ -49,10 +49,10 @@ typedef struct QEMUWin32AIOState QEMUWin32AIOState;
 QEMUWin32AIOState *win32_aio_init(void);
 void win32_aio_cleanup(QEMUWin32AIOState *aio);
 int win32_aio_attach(QEMUWin32AIOState *aio, HANDLE hfile);
-BlockDriverAIOCB *win32_aio_submit(BlockDriverState *bs,
+BlockAIOCB *win32_aio_submit(BlockDriverState *bs,
         QEMUWin32AIOState *aio, HANDLE hfile,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque, int type);
+        BlockCompletionFunc *cb, void *opaque, int type);
 void win32_aio_detach_aio_context(QEMUWin32AIOState *aio,
                                   AioContext *old_context);
 void win32_aio_attach_aio_context(QEMUWin32AIOState *aio,
diff --git a/block/raw-posix.c b/block/raw-posix.c
index 8e9758e92..b1af77e47 100644
--- a/block/raw-posix.c
+++ b/block/raw-posix.c
@@ -30,6 +30,7 @@
 #include "block/thread-pool.h"
 #include "qemu/iov.h"
 #include "raw-aio.h"
+#include "qapi/util.h"
 
 #if defined(__APPLE__) && (__MACH__)
 #include <paths.h>
@@ -59,9 +60,6 @@
 #define FS_NOCOW_FL                     0x00800000 /* Do not cow file */
 #endif
 #endif
-#ifdef CONFIG_FIEMAP
-#include <linux/fiemap.h>
-#endif
 #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
 #include <linux/falloc.h>
 #endif
@@ -149,9 +147,7 @@ typedef struct BDRVRawState {
     bool has_discard:1;
     bool has_write_zeroes:1;
     bool discard_zeroes:1;
-#ifdef CONFIG_FIEMAP
-    bool skip_fiemap;
-#endif
+    bool needs_alignment;
 } BDRVRawState;
 
 typedef struct BDRVRawReopenState {
@@ -229,7 +225,7 @@ static void raw_probe_alignment(BlockDriverState *bs, int fd, Error **errp)
 
     /* For /dev/sg devices the alignment is not really used.
        With buffered I/O, we don't have any restrictions. */
-    if (bs->sg || !(s->open_flags & O_DIRECT)) {
+    if (bs->sg || !s->needs_alignment) {
         bs->request_alignment = 1;
         s->buf_align = 1;
         return;
@@ -445,6 +441,9 @@ static int raw_open_common(BlockDriverState *bs, QDict *options,
 
     s->has_discard = true;
     s->has_write_zeroes = true;
+    if ((bs->open_flags & BDRV_O_NOCACHE) != 0) {
+        s->needs_alignment = true;
+    }
 
     if (fstat(s->fd, &st) < 0) {
         error_setg_errno(errp, errno, "Could not stat file");
@@ -471,6 +470,17 @@ static int raw_open_common(BlockDriverState *bs, QDict *options,
         }
 #endif
     }
+#ifdef __FreeBSD__
+    if (S_ISCHR(st.st_mode)) {
+        /*
+         * The file is a char device (disk), which on FreeBSD isn't behind
+         * a pager, so force all requests to be aligned. This is needed
+         * so QEMU makes sure all IO operations on the device are aligned
+         * to sector size, or else FreeBSD will reject them with EINVAL.
+         */
+        s->needs_alignment = true;
+    }
+#endif
 
 #ifdef CONFIG_XFS
     if (platform_test_xfs_fd(s->fd)) {
@@ -517,7 +527,7 @@ static int raw_reopen_prepare(BDRVReopenState *state,
 
     s = state->bs->opaque;
 
-    state->opaque = g_malloc0(sizeof(BDRVRawReopenState));
+    state->opaque = g_new0(BDRVRawReopenState, 1);
     raw_s = state->opaque;
 
 #ifdef CONFIG_LINUX_AIO
@@ -747,6 +757,15 @@ static ssize_t handle_aiocb_rw_linear(RawPosixAIOData *aiocb, char *buf)
         }
         if (len == -1 && errno == EINTR) {
             continue;
+        } else if (len == -1 && errno == EINVAL &&
+                   (aiocb->bs->open_flags & BDRV_O_NOCACHE) &&
+                   !(aiocb->aio_type & QEMU_AIO_WRITE) &&
+                   offset > 0) {
+            /* O_DIRECT pread() may fail with EINVAL when offset is unaligned
+             * after a short read.  Assume that O_DIRECT short reads only occur
+             * at EOF.  Therefore this is a short read, not an I/O error.
+             */
+            break;
         } else if (len == -1) {
             offset = -errno;
             break;
@@ -798,7 +817,11 @@ static ssize_t handle_aiocb_rw(RawPosixAIOData *aiocb)
      * Ok, we have to do it the hard way, copy all segments into
      * a single aligned buffer.
      */
-    buf = qemu_blockalign(aiocb->bs, aiocb->aio_nbytes);
+    buf = qemu_try_blockalign(aiocb->bs, aiocb->aio_nbytes);
+    if (buf == NULL) {
+        return -ENOMEM;
+    }
+
     if (aiocb->aio_type & QEMU_AIO_WRITE) {
         char *p = buf;
         int i;
@@ -1027,9 +1050,9 @@ static int paio_submit_co(BlockDriverState *bs, int fd,
     return thread_pool_submit_co(pool, aio_worker, acb);
 }
 
-static BlockDriverAIOCB *paio_submit(BlockDriverState *bs, int fd,
+static BlockAIOCB *paio_submit(BlockDriverState *bs, int fd,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque, int type)
+        BlockCompletionFunc *cb, void *opaque, int type)
 {
     RawPosixAIOData *acb = g_slice_new(RawPosixAIOData);
     ThreadPool *pool;
@@ -1052,9 +1075,9 @@ static BlockDriverAIOCB *paio_submit(BlockDriverState *bs, int fd,
     return thread_pool_submit_aio(pool, aio_worker, acb, cb, opaque);
 }
 
-static BlockDriverAIOCB *raw_aio_submit(BlockDriverState *bs,
+static BlockAIOCB *raw_aio_submit(BlockDriverState *bs,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque, int type)
+        BlockCompletionFunc *cb, void *opaque, int type)
 {
     BDRVRawState *s = bs->opaque;
 
@@ -1062,11 +1085,12 @@ static BlockDriverAIOCB *raw_aio_submit(BlockDriverState *bs,
         return NULL;
 
     /*
-     * If O_DIRECT is used the buffer needs to be aligned on a sector
-     * boundary.  Check if this is the case or tell the low-level
-     * driver that it needs to copy the buffer.
+     * Check if the underlying device requires requests to be aligned,
+     * and if the request we are trying to submit is aligned or not.
+     * If this is the case tell the low-level driver that it needs
+     * to copy the buffer.
      */
-    if ((bs->open_flags & BDRV_O_NOCACHE)) {
+    if (s->needs_alignment) {
         if (!bdrv_qiov_is_aligned(bs, qiov)) {
             type |= QEMU_AIO_MISALIGNED;
 #ifdef CONFIG_LINUX_AIO
@@ -1111,24 +1135,24 @@ static void raw_aio_flush_io_queue(BlockDriverState *bs)
 #endif
 }
 
-static BlockDriverAIOCB *raw_aio_readv(BlockDriverState *bs,
+static BlockAIOCB *raw_aio_readv(BlockDriverState *bs,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque)
+        BlockCompletionFunc *cb, void *opaque)
 {
     return raw_aio_submit(bs, sector_num, qiov, nb_sectors,
                           cb, opaque, QEMU_AIO_READ);
 }
 
-static BlockDriverAIOCB *raw_aio_writev(BlockDriverState *bs,
+static BlockAIOCB *raw_aio_writev(BlockDriverState *bs,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque)
+        BlockCompletionFunc *cb, void *opaque)
 {
     return raw_aio_submit(bs, sector_num, qiov, nb_sectors,
                           cb, opaque, QEMU_AIO_WRITE);
 }
 
-static BlockDriverAIOCB *raw_aio_flush(BlockDriverState *bs,
-        BlockDriverCompletionFunc *cb, void *opaque)
+static BlockAIOCB *raw_aio_flush(BlockDriverState *bs,
+        BlockCompletionFunc *cb, void *opaque)
 {
     BDRVRawState *s = bs->opaque;
 
@@ -1352,128 +1376,199 @@ static int raw_create(const char *filename, QemuOpts *opts, Error **errp)
     int result = 0;
     int64_t total_size = 0;
     bool nocow = false;
+    PreallocMode prealloc;
+    char *buf = NULL;
+    Error *local_err = NULL;
 
     strstart(filename, "file:", &filename);
 
     /* Read out options */
-    total_size =
-        qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0) / BDRV_SECTOR_SIZE;
+    total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                          BDRV_SECTOR_SIZE);
     nocow = qemu_opt_get_bool(opts, BLOCK_OPT_NOCOW, false);
+    buf = qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC);
+    prealloc = qapi_enum_parse(PreallocMode_lookup, buf,
+                               PREALLOC_MODE_MAX, PREALLOC_MODE_OFF,
+                               &local_err);
+    g_free(buf);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        result = -EINVAL;
+        goto out;
+    }
 
     fd = qemu_open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY,
                    0644);
     if (fd < 0) {
         result = -errno;
         error_setg_errno(errp, -result, "Could not create file");
-    } else {
-        if (nocow) {
+        goto out;
+    }
+
+    if (nocow) {
 #ifdef __linux__
-            /* Set NOCOW flag to solve performance issue on fs like btrfs.
-             * This is an optimisation. The FS_IOC_SETFLAGS ioctl return value
-             * will be ignored since any failure of this operation should not
-             * block the left work.
-             */
-            int attr;
-            if (ioctl(fd, FS_IOC_GETFLAGS, &attr) == 0) {
-                attr |= FS_NOCOW_FL;
-                ioctl(fd, FS_IOC_SETFLAGS, &attr);
-            }
-#endif
+        /* Set NOCOW flag to solve performance issue on fs like btrfs.
+         * This is an optimisation. The FS_IOC_SETFLAGS ioctl return value
+         * will be ignored since any failure of this operation should not
+         * block the left work.
+         */
+        int attr;
+        if (ioctl(fd, FS_IOC_GETFLAGS, &attr) == 0) {
+            attr |= FS_NOCOW_FL;
+            ioctl(fd, FS_IOC_SETFLAGS, &attr);
         }
+#endif
+    }
 
-        if (ftruncate(fd, total_size * BDRV_SECTOR_SIZE) != 0) {
-            result = -errno;
-            error_setg_errno(errp, -result, "Could not resize file");
+    if (ftruncate(fd, total_size) != 0) {
+        result = -errno;
+        error_setg_errno(errp, -result, "Could not resize file");
+        goto out_close;
+    }
+
+    switch (prealloc) {
+#ifdef CONFIG_POSIX_FALLOCATE
+    case PREALLOC_MODE_FALLOC:
+        /* posix_fallocate() doesn't set errno. */
+        result = -posix_fallocate(fd, 0, total_size);
+        if (result != 0) {
+            error_setg_errno(errp, -result,
+                             "Could not preallocate data for the new file");
+        }
+        break;
+#endif
+    case PREALLOC_MODE_FULL:
+    {
+        int64_t num = 0, left = total_size;
+        buf = g_malloc0(65536);
+
+        while (left > 0) {
+            num = MIN(left, 65536);
+            result = write(fd, buf, num);
+            if (result < 0) {
+                result = -errno;
+                error_setg_errno(errp, -result,
+                                 "Could not write to the new file");
+                break;
+            }
+            left -= result;
         }
-        if (qemu_close(fd) != 0) {
-            result = -errno;
-            error_setg_errno(errp, -result, "Could not close the new file");
+        if (result >= 0) {
+            result = fsync(fd);
+            if (result < 0) {
+                result = -errno;
+                error_setg_errno(errp, -result,
+                                 "Could not flush new file to disk");
+            }
         }
+        g_free(buf);
+        break;
+    }
+    case PREALLOC_MODE_OFF:
+        break;
+    default:
+        result = -EINVAL;
+        error_setg(errp, "Unsupported preallocation mode: %s",
+                   PreallocMode_lookup[prealloc]);
+        break;
     }
+
+out_close:
+    if (qemu_close(fd) != 0 && result == 0) {
+        result = -errno;
+        error_setg_errno(errp, -result, "Could not close the new file");
+    }
+out:
     return result;
 }
 
-static int64_t try_fiemap(BlockDriverState *bs, off_t start, off_t *data,
-                          off_t *hole, int nb_sectors, int *pnum)
+/*
+ * Find allocation range in @bs around offset @start.
+ * May change underlying file descriptor's file offset.
+ * If @start is not in a hole, store @start in @data, and the
+ * beginning of the next hole in @hole, and return 0.
+ * If @start is in a non-trailing hole, store @start in @hole and the
+ * beginning of the next non-hole in @data, and return 0.
+ * If @start is in a trailing hole or beyond EOF, return -ENXIO.
+ * If we can't find out, return a negative errno other than -ENXIO.
+ */
+static int find_allocation(BlockDriverState *bs, off_t start,
+                           off_t *data, off_t *hole)
 {
-#ifdef CONFIG_FIEMAP
+#if defined SEEK_HOLE && defined SEEK_DATA
     BDRVRawState *s = bs->opaque;
-    int64_t ret = BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID | start;
-    struct {
-        struct fiemap fm;
-        struct fiemap_extent fe;
-    } f;
-
-    if (s->skip_fiemap) {
-        return -ENOTSUP;
-    }
+    off_t offs;
 
-    f.fm.fm_start = start;
-    f.fm.fm_length = (int64_t)nb_sectors * BDRV_SECTOR_SIZE;
-    f.fm.fm_flags = 0;
-    f.fm.fm_extent_count = 1;
-    f.fm.fm_reserved = 0;
-    if (ioctl(s->fd, FS_IOC_FIEMAP, &f) == -1) {
-        s->skip_fiemap = true;
-        return -errno;
+    /*
+     * SEEK_DATA cases:
+     * D1. offs == start: start is in data
+     * D2. offs > start: start is in a hole, next data at offs
+     * D3. offs < 0, errno = ENXIO: either start is in a trailing hole
+     *                              or start is beyond EOF
+     *     If the latter happens, the file has been truncated behind
+     *     our back since we opened it.  All bets are off then.
+     *     Treating like a trailing hole is simplest.
+     * D4. offs < 0, errno != ENXIO: we learned nothing
+     */
+    offs = lseek(s->fd, start, SEEK_DATA);
+    if (offs < 0) {
+        return -errno;          /* D3 or D4 */
     }
+    assert(offs >= start);
 
-    if (f.fm.fm_mapped_extents == 0) {
-        /* No extents found, data is beyond f.fm.fm_start + f.fm.fm_length.
-         * f.fm.fm_start + f.fm.fm_length must be clamped to the file size!
-         */
-        off_t length = lseek(s->fd, 0, SEEK_END);
-        *hole = f.fm.fm_start;
-        *data = MIN(f.fm.fm_start + f.fm.fm_length, length);
-    } else {
-        *data = f.fe.fe_logical;
-        *hole = f.fe.fe_logical + f.fe.fe_length;
-        if (f.fe.fe_flags & FIEMAP_EXTENT_UNWRITTEN) {
-            ret |= BDRV_BLOCK_ZERO;
-        }
+    if (offs > start) {
+        /* D2: in hole, next data at offs */
+        *hole = start;
+        *data = offs;
+        return 0;
     }
 
-    return ret;
-#else
-    return -ENOTSUP;
-#endif
-}
-
-static int64_t try_seek_hole(BlockDriverState *bs, off_t start, off_t *data,
-                             off_t *hole, int *pnum)
-{
-#if defined SEEK_HOLE && defined SEEK_DATA
-    BDRVRawState *s = bs->opaque;
-
-    *hole = lseek(s->fd, start, SEEK_HOLE);
-    if (*hole == -1) {
-        /* -ENXIO indicates that sector_num was past the end of the file.
-         * There is a virtual hole there.  */
-        assert(errno != -ENXIO);
+    /* D1: in data, end not yet known */
 
-        return -errno;
+    /*
+     * SEEK_HOLE cases:
+     * H1. offs == start: start is in a hole
+     *     If this happens here, a hole has been dug behind our back
+     *     since the previous lseek().
+     * H2. offs > start: either start is in data, next hole at offs,
+     *                   or start is in trailing hole, EOF at offs
+     *     Linux treats trailing holes like any other hole: offs ==
+     *     start.  Solaris seeks to EOF instead: offs > start (blech).
+     *     If that happens here, a hole has been dug behind our back
+     *     since the previous lseek().
+     * H3. offs < 0, errno = ENXIO: start is beyond EOF
+     *     If this happens, the file has been truncated behind our
+     *     back since we opened it.  Treat it like a trailing hole.
+     * H4. offs < 0, errno != ENXIO: we learned nothing
+     *     Pretend we know nothing at all, i.e. "forget" about D1.
+     */
+    offs = lseek(s->fd, start, SEEK_HOLE);
+    if (offs < 0) {
+        return -errno;          /* D1 and (H3 or H4) */
     }
+    assert(offs >= start);
 
-    if (*hole > start) {
+    if (offs > start) {
+        /*
+         * D1 and H2: either in data, next hole at offs, or it was in
+         * data but is now in a trailing hole.  In the latter case,
+         * all bets are off.  Treating it as if it there was data all
+         * the way to EOF is safe, so simply do that.
+         */
         *data = start;
-    } else {
-        /* On a hole.  We need another syscall to find its end.  */
-        *data = lseek(s->fd, start, SEEK_DATA);
-        if (*data == -1) {
-            *data = lseek(s->fd, 0, SEEK_END);
-        }
+        *hole = offs;
+        return 0;
     }
 
-    return BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID | start;
+    /* D1 and H1 */
+    return -EBUSY;
 #else
     return -ENOTSUP;
 #endif
 }
 
 /*
- * Returns true iff the specified sector is present in the disk image. Drivers
- * not implementing the functionality are assumed to not support backing files,
- * hence all their sectors are reported as allocated.
+ * Returns the allocation status of the specified sectors.
  *
  * If 'sector_num' is beyond the end of the disk image the return value is 0
  * and 'pnum' is set to 0.
@@ -1490,7 +1585,8 @@ static int64_t coroutine_fn raw_co_get_block_status(BlockDriverState *bs,
                                                     int nb_sectors, int *pnum)
 {
     off_t start, data = 0, hole = 0;
-    int64_t ret;
+    int64_t total_size;
+    int ret;
 
     ret = fd_open(bs);
     if (ret < 0) {
@@ -1498,34 +1594,41 @@ static int64_t coroutine_fn raw_co_get_block_status(BlockDriverState *bs,
     }
 
     start = sector_num * BDRV_SECTOR_SIZE;
-
-    ret = try_fiemap(bs, start, &data, &hole, nb_sectors, pnum);
-    if (ret < 0) {
-        ret = try_seek_hole(bs, start, &data, &hole, pnum);
-        if (ret < 0) {
-            /* Assume everything is allocated. */
-            data = 0;
-            hole = start + nb_sectors * BDRV_SECTOR_SIZE;
-            ret = BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID | start;
-        }
-    }
-
-    if (data <= start) {
+    total_size = bdrv_getlength(bs);
+    if (total_size < 0) {
+        return total_size;
+    } else if (start >= total_size) {
+        *pnum = 0;
+        return 0;
+    } else if (start + nb_sectors * BDRV_SECTOR_SIZE > total_size) {
+        nb_sectors = DIV_ROUND_UP(total_size - start, BDRV_SECTOR_SIZE);
+    }
+
+    ret = find_allocation(bs, start, &data, &hole);
+    if (ret == -ENXIO) {
+        /* Trailing hole */
+        *pnum = nb_sectors;
+        ret = BDRV_BLOCK_ZERO;
+    } else if (ret < 0) {
+        /* No info available, so pretend there are no holes */
+        *pnum = nb_sectors;
+        ret = BDRV_BLOCK_DATA;
+    } else if (data == start) {
         /* On a data extent, compute sectors to the end of the extent.  */
         *pnum = MIN(nb_sectors, (hole - start) / BDRV_SECTOR_SIZE);
+        ret = BDRV_BLOCK_DATA;
     } else {
         /* On a hole, compute sectors to the beginning of the next extent.  */
+        assert(hole == start);
         *pnum = MIN(nb_sectors, (data - start) / BDRV_SECTOR_SIZE);
-        ret &= ~BDRV_BLOCK_DATA;
-        ret |= BDRV_BLOCK_ZERO;
+        ret = BDRV_BLOCK_ZERO;
     }
-
-    return ret;
+    return ret | BDRV_BLOCK_OFFSET_VALID | start;
 }
 
-static coroutine_fn BlockDriverAIOCB *raw_aio_discard(BlockDriverState *bs,
+static coroutine_fn BlockAIOCB *raw_aio_discard(BlockDriverState *bs,
     int64_t sector_num, int nb_sectors,
-    BlockDriverCompletionFunc *cb, void *opaque)
+    BlockCompletionFunc *cb, void *opaque)
 {
     BDRVRawState *s = bs->opaque;
 
@@ -1572,6 +1675,11 @@ static QemuOptsList raw_create_opts = {
             .type = QEMU_OPT_BOOL,
             .help = "Turn off copy-on-write (valid only on btrfs)"
         },
+        {
+            .name = BLOCK_OPT_PREALLOC,
+            .type = QEMU_OPT_STRING,
+            .help = "Preallocation mode (allowed values: off, falloc, full)"
+        },
         { /* end of list */ }
     }
 };
@@ -1858,9 +1966,9 @@ static int hdev_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
     return ioctl(s->fd, req, buf);
 }
 
-static BlockDriverAIOCB *hdev_aio_ioctl(BlockDriverState *bs,
+static BlockAIOCB *hdev_aio_ioctl(BlockDriverState *bs,
         unsigned long int req, void *buf,
-        BlockDriverCompletionFunc *cb, void *opaque)
+        BlockCompletionFunc *cb, void *opaque)
 {
     BDRVRawState *s = bs->opaque;
     RawPosixAIOData *acb;
@@ -1899,9 +2007,9 @@ static int fd_open(BlockDriverState *bs)
 
 #endif /* !linux && !FreeBSD */
 
-static coroutine_fn BlockDriverAIOCB *hdev_aio_discard(BlockDriverState *bs,
+static coroutine_fn BlockAIOCB *hdev_aio_discard(BlockDriverState *bs,
     int64_t sector_num, int nb_sectors,
-    BlockDriverCompletionFunc *cb, void *opaque)
+    BlockCompletionFunc *cb, void *opaque)
 {
     BDRVRawState *s = bs->opaque;
 
@@ -1953,8 +2061,8 @@ static int hdev_create(const char *filename, QemuOpts *opts,
     (void)has_prefix;
 
     /* Read out options */
-    total_size =
-        qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0) / BDRV_SECTOR_SIZE;
+    total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                          BDRV_SECTOR_SIZE);
 
     fd = qemu_open(filename, O_WRONLY | O_BINARY);
     if (fd < 0) {
@@ -1970,7 +2078,7 @@ static int hdev_create(const char *filename, QemuOpts *opts,
         error_setg(errp,
                    "The given file is neither a block nor a character device");
         ret = -ENODEV;
-    } else if (lseek(fd, 0, SEEK_END) < total_size * BDRV_SECTOR_SIZE) {
+    } else if (lseek(fd, 0, SEEK_END) < total_size) {
         error_setg(errp, "Device is too small");
         ret = -ENOSPC;
     }
diff --git a/block/raw-win32.c b/block/raw-win32.c
index 902eab610..7b588815b 100644
--- a/block/raw-win32.c
+++ b/block/raw-win32.c
@@ -138,9 +138,9 @@ static int aio_worker(void *arg)
     return ret;
 }
 
-static BlockDriverAIOCB *paio_submit(BlockDriverState *bs, HANDLE hfile,
+static BlockAIOCB *paio_submit(BlockDriverState *bs, HANDLE hfile,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque, int type)
+        BlockCompletionFunc *cb, void *opaque, int type)
 {
     RawWin32AIOData *acb = g_slice_new(RawWin32AIOData);
     ThreadPool *pool;
@@ -369,9 +369,9 @@ fail:
     return ret;
 }
 
-static BlockDriverAIOCB *raw_aio_readv(BlockDriverState *bs,
+static BlockAIOCB *raw_aio_readv(BlockDriverState *bs,
                          int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-                         BlockDriverCompletionFunc *cb, void *opaque)
+                         BlockCompletionFunc *cb, void *opaque)
 {
     BDRVRawState *s = bs->opaque;
     if (s->aio) {
@@ -383,9 +383,9 @@ static BlockDriverAIOCB *raw_aio_readv(BlockDriverState *bs,
     }
 }
 
-static BlockDriverAIOCB *raw_aio_writev(BlockDriverState *bs,
+static BlockAIOCB *raw_aio_writev(BlockDriverState *bs,
                           int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-                          BlockDriverCompletionFunc *cb, void *opaque)
+                          BlockCompletionFunc *cb, void *opaque)
 {
     BDRVRawState *s = bs->opaque;
     if (s->aio) {
@@ -397,8 +397,8 @@ static BlockDriverAIOCB *raw_aio_writev(BlockDriverState *bs,
     }
 }
 
-static BlockDriverAIOCB *raw_aio_flush(BlockDriverState *bs,
-                         BlockDriverCompletionFunc *cb, void *opaque)
+static BlockAIOCB *raw_aio_flush(BlockDriverState *bs,
+                         BlockCompletionFunc *cb, void *opaque)
 {
     BDRVRawState *s = bs->opaque;
     return paio_submit(bs, s->hfile, 0, NULL, 0, cb, opaque, QEMU_AIO_FLUSH);
@@ -511,8 +511,8 @@ static int raw_create(const char *filename, QemuOpts *opts, Error **errp)
     strstart(filename, "file:", &filename);
 
     /* Read out options */
-    total_size =
-        qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0) / 512;
+    total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                          BDRV_SECTOR_SIZE);
 
     fd = qemu_open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY,
                    0644);
@@ -521,7 +521,7 @@ static int raw_create(const char *filename, QemuOpts *opts, Error **errp)
         return -EIO;
     }
     set_sparse(fd);
-    ftruncate(fd, total_size * 512);
+    ftruncate(fd, total_size);
     qemu_close(fd);
     return 0;
 }
diff --git a/block/raw_bsd.c b/block/raw_bsd.c
index f82f4c25d..401b967e8 100644
--- a/block/raw_bsd.c
+++ b/block/raw_bsd.c
@@ -129,10 +129,10 @@ static int raw_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
     return bdrv_ioctl(bs->file, req, buf);
 }
 
-static BlockDriverAIOCB *raw_aio_ioctl(BlockDriverState *bs,
-                                       unsigned long int req, void *buf,
-                                       BlockDriverCompletionFunc *cb,
-                                       void *opaque)
+static BlockAIOCB *raw_aio_ioctl(BlockDriverState *bs,
+                                 unsigned long int req, void *buf,
+                                 BlockCompletionFunc *cb,
+                                 void *opaque)
 {
     return bdrv_aio_ioctl(bs->file, req, buf, cb, opaque);
 }
diff --git a/block/rbd.c b/block/rbd.c
index 2b797d3e8..5b5a64a27 100644
--- a/block/rbd.c
+++ b/block/rbd.c
@@ -68,7 +68,7 @@ typedef enum {
 } RBDAIOCmd;
 
 typedef struct RBDAIOCB {
-    BlockDriverAIOCB common;
+    BlockAIOCB common;
     QEMUBH *bh;
     int64_t ret;
     QEMUIOVector *qiov;
@@ -77,7 +77,6 @@ typedef struct RBDAIOCB {
     int64_t sector_num;
     int error;
     struct BDRVRBDState *s;
-    int cancelled;
     int status;
 } RBDAIOCB;
 
@@ -314,7 +313,8 @@ static int qemu_rbd_create(const char *filename, QemuOpts *opts, Error **errp)
     }
 
     /* Read out options */
-    bytes = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0);
+    bytes = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                     BDRV_SECTOR_SIZE);
     objsize = qemu_opt_get_size_del(opts, BLOCK_OPT_CLUSTER_SIZE, 0);
     if (objsize) {
         if ((objsize - 1) & objsize) {    /* not a power of 2? */
@@ -407,9 +407,7 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
     acb->common.cb(acb->common.opaque, (acb->ret > 0 ? 0 : acb->ret));
     acb->status = 0;
 
-    if (!acb->cancelled) {
-        qemu_aio_release(acb);
-    }
+    qemu_aio_unref(acb);
 }
 
 /* TODO Convert to fine grained options */
@@ -538,25 +536,8 @@ static void qemu_rbd_close(BlockDriverState *bs)
     rados_shutdown(s->cluster);
 }
 
-/*
- * Cancel aio. Since we don't reference acb in a non qemu threads,
- * it is safe to access it here.
- */
-static void qemu_rbd_aio_cancel(BlockDriverAIOCB *blockacb)
-{
-    RBDAIOCB *acb = (RBDAIOCB *) blockacb;
-    acb->cancelled = 1;
-
-    while (acb->status == -EINPROGRESS) {
-        aio_poll(bdrv_get_aio_context(acb->common.bs), true);
-    }
-
-    qemu_aio_release(acb);
-}
-
 static const AIOCBInfo rbd_aiocb_info = {
     .aiocb_size = sizeof(RBDAIOCB),
-    .cancel = qemu_rbd_aio_cancel,
 };
 
 static void rbd_finish_bh(void *opaque)
@@ -608,16 +589,16 @@ static int rbd_aio_flush_wrapper(rbd_image_t image,
 #endif
 }
 
-static BlockDriverAIOCB *rbd_start_aio(BlockDriverState *bs,
-                                       int64_t sector_num,
-                                       QEMUIOVector *qiov,
-                                       int nb_sectors,
-                                       BlockDriverCompletionFunc *cb,
-                                       void *opaque,
-                                       RBDAIOCmd cmd)
+static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
+                                 int64_t sector_num,
+                                 QEMUIOVector *qiov,
+                                 int nb_sectors,
+                                 BlockCompletionFunc *cb,
+                                 void *opaque,
+                                 RBDAIOCmd cmd)
 {
     RBDAIOCB *acb;
-    RADOSCB *rcb;
+    RADOSCB *rcb = NULL;
     rbd_completion_t c;
     int64_t off, size;
     char *buf;
@@ -631,12 +612,14 @@ static BlockDriverAIOCB *rbd_start_aio(BlockDriverState *bs,
     if (cmd == RBD_AIO_DISCARD || cmd == RBD_AIO_FLUSH) {
         acb->bounce = NULL;
     } else {
-        acb->bounce = qemu_blockalign(bs, qiov->size);
+        acb->bounce = qemu_try_blockalign(bs, qiov->size);
+        if (acb->bounce == NULL) {
+            goto failed;
+        }
     }
     acb->ret = 0;
     acb->error = 0;
     acb->s = s;
-    acb->cancelled = 0;
     acb->bh = NULL;
     acb->status = -EINPROGRESS;
 
@@ -649,7 +632,7 @@ static BlockDriverAIOCB *rbd_start_aio(BlockDriverState *bs,
     off = sector_num * BDRV_SECTOR_SIZE;
     size = nb_sectors * BDRV_SECTOR_SIZE;
 
-    rcb = g_malloc(sizeof(RADOSCB));
+    rcb = g_new(RADOSCB, 1);
     rcb->done = 0;
     rcb->acb = acb;
     rcb->buf = buf;
@@ -688,36 +671,36 @@ failed_completion:
 failed:
     g_free(rcb);
     qemu_vfree(acb->bounce);
-    qemu_aio_release(acb);
+    qemu_aio_unref(acb);
     return NULL;
 }
 
-static BlockDriverAIOCB *qemu_rbd_aio_readv(BlockDriverState *bs,
-                                            int64_t sector_num,
-                                            QEMUIOVector *qiov,
-                                            int nb_sectors,
-                                            BlockDriverCompletionFunc *cb,
-                                            void *opaque)
+static BlockAIOCB *qemu_rbd_aio_readv(BlockDriverState *bs,
+                                      int64_t sector_num,
+                                      QEMUIOVector *qiov,
+                                      int nb_sectors,
+                                      BlockCompletionFunc *cb,
+                                      void *opaque)
 {
     return rbd_start_aio(bs, sector_num, qiov, nb_sectors, cb, opaque,
                          RBD_AIO_READ);
 }
 
-static BlockDriverAIOCB *qemu_rbd_aio_writev(BlockDriverState *bs,
-                                             int64_t sector_num,
-                                             QEMUIOVector *qiov,
-                                             int nb_sectors,
-                                             BlockDriverCompletionFunc *cb,
-                                             void *opaque)
+static BlockAIOCB *qemu_rbd_aio_writev(BlockDriverState *bs,
+                                       int64_t sector_num,
+                                       QEMUIOVector *qiov,
+                                       int nb_sectors,
+                                       BlockCompletionFunc *cb,
+                                       void *opaque)
 {
     return rbd_start_aio(bs, sector_num, qiov, nb_sectors, cb, opaque,
                          RBD_AIO_WRITE);
 }
 
 #ifdef LIBRBD_SUPPORTS_AIO_FLUSH
-static BlockDriverAIOCB *qemu_rbd_aio_flush(BlockDriverState *bs,
-                                            BlockDriverCompletionFunc *cb,
-                                            void *opaque)
+static BlockAIOCB *qemu_rbd_aio_flush(BlockDriverState *bs,
+                                      BlockCompletionFunc *cb,
+                                      void *opaque)
 {
     return rbd_start_aio(bs, 0, NULL, 0, cb, opaque, RBD_AIO_FLUSH);
 }
@@ -859,7 +842,7 @@ static int qemu_rbd_snap_list(BlockDriverState *bs,
     int max_snaps = RBD_MAX_SNAPS;
 
     do {
-        snaps = g_malloc(sizeof(*snaps) * max_snaps);
+        snaps = g_new(rbd_snap_info_t, max_snaps);
         snap_count = rbd_snap_list(s->image, snaps, &max_snaps);
         if (snap_count <= 0) {
             g_free(snaps);
@@ -870,7 +853,7 @@ static int qemu_rbd_snap_list(BlockDriverState *bs,
         goto done;
     }
 
-    sn_tab = g_malloc0(snap_count * sizeof(QEMUSnapshotInfo));
+    sn_tab = g_new0(QEMUSnapshotInfo, snap_count);
 
     for (i = 0; i < snap_count; i++) {
         const char *snap_name = snaps[i].name;
@@ -893,17 +876,29 @@ static int qemu_rbd_snap_list(BlockDriverState *bs,
 }
 
 #ifdef LIBRBD_SUPPORTS_DISCARD
-static BlockDriverAIOCB* qemu_rbd_aio_discard(BlockDriverState *bs,
-                                              int64_t sector_num,
-                                              int nb_sectors,
-                                              BlockDriverCompletionFunc *cb,
-                                              void *opaque)
+static BlockAIOCB* qemu_rbd_aio_discard(BlockDriverState *bs,
+                                        int64_t sector_num,
+                                        int nb_sectors,
+                                        BlockCompletionFunc *cb,
+                                        void *opaque)
 {
     return rbd_start_aio(bs, sector_num, NULL, nb_sectors, cb, opaque,
                          RBD_AIO_DISCARD);
 }
 #endif
 
+#ifdef LIBRBD_SUPPORTS_INVALIDATE
+static void qemu_rbd_invalidate_cache(BlockDriverState *bs,
+                                      Error **errp)
+{
+    BDRVRBDState *s = bs->opaque;
+    int r = rbd_invalidate_cache(s->image);
+    if (r < 0) {
+        error_setg_errno(errp, -r, "Failed to invalidate the cache");
+    }
+}
+#endif
+
 static QemuOptsList qemu_rbd_create_opts = {
     .name = "rbd-create-opts",
     .head = QTAILQ_HEAD_INITIALIZER(qemu_rbd_create_opts.head),
@@ -953,6 +948,9 @@ static BlockDriver bdrv_rbd = {
     .bdrv_snapshot_delete   = qemu_rbd_snap_remove,
     .bdrv_snapshot_list     = qemu_rbd_snap_list,
     .bdrv_snapshot_goto     = qemu_rbd_snap_rollback,
+#ifdef LIBRBD_SUPPORTS_INVALIDATE
+    .bdrv_invalidate_cache  = qemu_rbd_invalidate_cache,
+#endif
 };
 
 static void bdrv_rbd_init(void)
diff --git a/block/sheepdog.c b/block/sheepdog.c
index 8d9350c26..be3176fcb 100644
--- a/block/sheepdog.c
+++ b/block/sheepdog.c
@@ -103,6 +103,9 @@
 #define SD_INODE_SIZE (sizeof(SheepdogInode))
 #define CURRENT_VDI_ID 0
 
+#define LOCK_TYPE_NORMAL 0
+#define LOCK_TYPE_SHARED 1      /* for iSCSI multipath */
+
 typedef struct SheepdogReq {
     uint8_t proto_ver;
     uint8_t opcode;
@@ -166,7 +169,8 @@ typedef struct SheepdogVdiReq {
     uint8_t copy_policy;
     uint8_t reserved[2];
     uint32_t snapid;
-    uint32_t pad[3];
+    uint32_t type;
+    uint32_t pad[2];
 } SheepdogVdiReq;
 
 typedef struct SheepdogVdiRsp {
@@ -297,7 +301,7 @@ enum AIOCBState {
 };
 
 struct SheepdogAIOCB {
-    BlockDriverAIOCB common;
+    BlockAIOCB common;
 
     QEMUIOVector *qiov;
 
@@ -311,7 +315,6 @@ struct SheepdogAIOCB {
     void (*aio_done_func)(SheepdogAIOCB *);
 
     bool cancelable;
-    bool *finished;
     int nr_pending;
 };
 
@@ -442,10 +445,7 @@ static inline void free_aio_req(BDRVSheepdogState *s, AIOReq *aio_req)
 static void coroutine_fn sd_finish_aiocb(SheepdogAIOCB *acb)
 {
     qemu_coroutine_enter(acb->coroutine, NULL);
-    if (acb->finished) {
-        *acb->finished = true;
-    }
-    qemu_aio_release(acb);
+    qemu_aio_unref(acb);
 }
 
 /*
@@ -473,41 +473,38 @@ static bool sd_acb_cancelable(const SheepdogAIOCB *acb)
     return true;
 }
 
-static void sd_aio_cancel(BlockDriverAIOCB *blockacb)
+static void sd_aio_cancel(BlockAIOCB *blockacb)
 {
     SheepdogAIOCB *acb = (SheepdogAIOCB *)blockacb;
     BDRVSheepdogState *s = acb->common.bs->opaque;
     AIOReq *aioreq, *next;
-    bool finished = false;
-
-    acb->finished = &finished;
-    while (!finished) {
-        if (sd_acb_cancelable(acb)) {
-            /* Remove outstanding requests from pending and failed queues.  */
-            QLIST_FOREACH_SAFE(aioreq, &s->pending_aio_head, aio_siblings,
-                               next) {
-                if (aioreq->aiocb == acb) {
-                    free_aio_req(s, aioreq);
-                }
+
+    if (sd_acb_cancelable(acb)) {
+        /* Remove outstanding requests from pending and failed queues.  */
+        QLIST_FOREACH_SAFE(aioreq, &s->pending_aio_head, aio_siblings,
+                           next) {
+            if (aioreq->aiocb == acb) {
+                free_aio_req(s, aioreq);
             }
-            QLIST_FOREACH_SAFE(aioreq, &s->failed_aio_head, aio_siblings,
-                               next) {
-                if (aioreq->aiocb == acb) {
-                    free_aio_req(s, aioreq);
-                }
+        }
+        QLIST_FOREACH_SAFE(aioreq, &s->failed_aio_head, aio_siblings,
+                           next) {
+            if (aioreq->aiocb == acb) {
+                free_aio_req(s, aioreq);
             }
+        }
 
-            assert(acb->nr_pending == 0);
-            sd_finish_aiocb(acb);
-            return;
+        assert(acb->nr_pending == 0);
+        if (acb->common.cb) {
+            acb->common.cb(acb->common.opaque, -ECANCELED);
         }
-        aio_poll(s->aio_context, true);
+        sd_finish_aiocb(acb);
     }
 }
 
 static const AIOCBInfo sd_aiocb_info = {
-    .aiocb_size = sizeof(SheepdogAIOCB),
-    .cancel = sd_aio_cancel,
+    .aiocb_size     = sizeof(SheepdogAIOCB),
+    .cancel_async   = sd_aio_cancel,
 };
 
 static SheepdogAIOCB *sd_aio_setup(BlockDriverState *bs, QEMUIOVector *qiov,
@@ -524,7 +521,6 @@ static SheepdogAIOCB *sd_aio_setup(BlockDriverState *bs, QEMUIOVector *qiov,
 
     acb->aio_done_func = NULL;
     acb->cancelable = true;
-    acb->finished = NULL;
     acb->coroutine = qemu_coroutine_self();
     acb->ret = 0;
     acb->nr_pending = 0;
@@ -712,7 +708,6 @@ static void coroutine_fn send_pending_req(BDRVSheepdogState *s, uint64_t oid)
 
 static coroutine_fn void reconnect_to_sdog(void *opaque)
 {
-    Error *local_err = NULL;
     BDRVSheepdogState *s = opaque;
     AIOReq *aio_req, *next;
 
@@ -727,6 +722,7 @@ static coroutine_fn void reconnect_to_sdog(void *opaque)
 
     /* Try to reconnect the sheepdog server every one second. */
     while (s->fd < 0) {
+        Error *local_err = NULL;
         s->fd = get_sheep_fd(s, &local_err);
         if (s->fd < 0) {
             DPRINTF("Wait for connection to be established\n");
@@ -1090,6 +1086,7 @@ static int find_vdi_name(BDRVSheepdogState *s, const char *filename,
     memset(&hdr, 0, sizeof(hdr));
     if (lock) {
         hdr.opcode = SD_OP_LOCK_VDI;
+        hdr.type = LOCK_TYPE_NORMAL;
     } else {
         hdr.opcode = SD_OP_GET_VDI_INFO;
     }
@@ -1110,6 +1107,8 @@ static int find_vdi_name(BDRVSheepdogState *s, const char *filename,
                    sd_strerror(rsp->result), filename, snapid, tag);
         if (rsp->result == SD_RES_NO_VDI) {
             ret = -ENOENT;
+        } else if (rsp->result == SD_RES_VDI_LOCKED) {
+            ret = -EBUSY;
         } else {
             ret = -EIO;
         }
@@ -1682,7 +1681,7 @@ static int sd_create(const char *filename, QemuOpts *opts,
     uint32_t snapid;
     bool prealloc = false;
 
-    s = g_malloc0(sizeof(BDRVSheepdogState));
+    s = g_new0(BDRVSheepdogState, 1);
 
     memset(tag, 0, sizeof(tag));
     if (strstr(filename, "://")) {
@@ -1695,7 +1694,8 @@ static int sd_create(const char *filename, QemuOpts *opts,
         goto out;
     }
 
-    s->inode.vdi_size = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0);
+    s->inode.vdi_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                                 BDRV_SECTOR_SIZE);
     backing_file = qemu_opt_get_del(opts, BLOCK_OPT_BACKING_FILE);
     buf = qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC);
     if (!buf || !strcmp(buf, "off")) {
@@ -1793,6 +1793,7 @@ static void sd_close(BlockDriverState *bs)
     memset(&hdr, 0, sizeof(hdr));
 
     hdr.opcode = SD_OP_RELEASE_VDI;
+    hdr.type = LOCK_TYPE_NORMAL;
     hdr.base_vdi_id = s->inode.vdi_id;
     wlen = strlen(s->name) + 1;
     hdr.data_length = wlen;
@@ -2129,7 +2130,7 @@ static coroutine_fn int sd_co_writev(BlockDriverState *bs, int64_t sector_num,
 
     ret = sd_co_rw_vector(acb);
     if (ret <= 0) {
-        qemu_aio_release(acb);
+        qemu_aio_unref(acb);
         return ret;
     }
 
@@ -2150,7 +2151,7 @@ static coroutine_fn int sd_co_readv(BlockDriverState *bs, int64_t sector_num,
 
     ret = sd_co_rw_vector(acb);
     if (ret <= 0) {
-        qemu_aio_release(acb);
+        qemu_aio_unref(acb);
         return ret;
     }
 
@@ -2273,7 +2274,7 @@ static int sd_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
     uint32_t snapid = 0;
     int ret = 0;
 
-    old_s = g_malloc(sizeof(BDRVSheepdogState));
+    old_s = g_new(BDRVSheepdogState, 1);
 
     memcpy(old_s, s, sizeof(BDRVSheepdogState));
 
@@ -2357,7 +2358,7 @@ static int sd_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab)
         goto out;
     }
 
-    sn_tab = g_malloc0(nr * sizeof(*sn_tab));
+    sn_tab = g_new0(QEMUSnapshotInfo, nr);
 
     /* calculate a vdi id with hash function */
     hval = fnv_64a_buf(s->name, strlen(s->name), FNV1A_64_INIT);
@@ -2509,7 +2510,7 @@ static coroutine_fn int sd_co_discard(BlockDriverState *bs, int64_t sector_num,
 
     ret = sd_co_rw_vector(acb);
     if (ret <= 0) {
-        qemu_aio_release(acb);
+        qemu_aio_unref(acb);
         return ret;
     }
 
diff --git a/block/snapshot.c b/block/snapshot.c
index 85c52ff45..698e1a1d5 100644
--- a/block/snapshot.c
+++ b/block/snapshot.c
@@ -236,6 +236,10 @@ int bdrv_snapshot_delete(BlockDriverState *bs,
         error_setg(errp, "snapshot_id and name are both NULL");
         return -EINVAL;
     }
+
+    /* drain all pending i/o before deleting snapshot */
+    bdrv_drain_all();
+
     if (drv->bdrv_snapshot_delete) {
         return drv->bdrv_snapshot_delete(bs, snapshot_id, name, errp);
     }
diff --git a/block/ssh.c b/block/ssh.c
index cd2fd751f..f466cbf39 100644
--- a/block/ssh.c
+++ b/block/ssh.c
@@ -517,6 +517,11 @@ static int connect_to_ssh(BDRVSSHState *s, QDict *options,
     const char *host, *user, *path, *host_key_check;
     int port;
 
+    if (!qdict_haskey(options, "host")) {
+        ret = -EINVAL;
+        error_setg(errp, "No hostname was specified");
+        goto err;
+    }
     host = qdict_get_str(options, "host");
 
     if (qdict_haskey(options, "port")) {
@@ -525,6 +530,11 @@ static int connect_to_ssh(BDRVSSHState *s, QDict *options,
         port = 22;
     }
 
+    if (!qdict_haskey(options, "path")) {
+        ret = -EINVAL;
+        error_setg(errp, "No path was specified");
+        goto err;
+    }
     path = qdict_get_str(options, "path");
 
     if (qdict_haskey(options, "user")) {
@@ -700,7 +710,8 @@ static int ssh_create(const char *filename, QemuOpts *opts, Error **errp)
     ssh_state_init(&s);
 
     /* Get desired file size. */
-    total_size = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0);
+    total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                          BDRV_SECTOR_SIZE);
     DPRINTF("total_size=%" PRIi64, total_size);
 
     uri_options = qdict_new();
diff --git a/block/stream.c b/block/stream.c
index cdea3e8d0..a628901f6 100644
--- a/block/stream.c
+++ b/block/stream.c
@@ -79,9 +79,39 @@ static void close_unused_images(BlockDriverState *top, BlockDriverState *base,
     bdrv_refresh_limits(top, NULL);
 }
 
+typedef struct {
+    int ret;
+    bool reached_end;
+} StreamCompleteData;
+
+static void stream_complete(BlockJob *job, void *opaque)
+{
+    StreamBlockJob *s = container_of(job, StreamBlockJob, common);
+    StreamCompleteData *data = opaque;
+    BlockDriverState *base = s->base;
+
+    if (!block_job_is_cancelled(&s->common) && data->reached_end &&
+        data->ret == 0) {
+        const char *base_id = NULL, *base_fmt = NULL;
+        if (base) {
+            base_id = s->backing_file_str;
+            if (base->drv) {
+                base_fmt = base->drv->format_name;
+            }
+        }
+        data->ret = bdrv_change_backing_file(job->bs, base_id, base_fmt);
+        close_unused_images(job->bs, base, base_id);
+    }
+
+    g_free(s->backing_file_str);
+    block_job_completed(&s->common, data->ret);
+    g_free(data);
+}
+
 static void coroutine_fn stream_run(void *opaque)
 {
     StreamBlockJob *s = opaque;
+    StreamCompleteData *data;
     BlockDriverState *bs = s->common.bs;
     BlockDriverState *base = s->base;
     int64_t sector_num, end;
@@ -183,21 +213,13 @@ wait:
     /* Do not remove the backing file if an error was there but ignored.  */
     ret = error;
 
-    if (!block_job_is_cancelled(&s->common) && sector_num == end && ret == 0) {
-        const char *base_id = NULL, *base_fmt = NULL;
-        if (base) {
-            base_id = s->backing_file_str;
-            if (base->drv) {
-                base_fmt = base->drv->format_name;
-            }
-        }
-        ret = bdrv_change_backing_file(bs, base_id, base_fmt);
-        close_unused_images(bs, base, base_id);
-    }
-
     qemu_vfree(buf);
-    g_free(s->backing_file_str);
-    block_job_completed(&s->common, ret);
+
+    /* Modify backing chain and close BDSes in main loop */
+    data = g_malloc(sizeof(*data));
+    data->ret = ret;
+    data->reached_end = sector_num == end;
+    block_job_defer_to_main_loop(&s->common, stream_complete, data);
 }
 
 static void stream_set_speed(BlockJob *job, int64_t speed, Error **errp)
@@ -220,7 +242,7 @@ static const BlockJobDriver stream_job_driver = {
 void stream_start(BlockDriverState *bs, BlockDriverState *base,
                   const char *backing_file_str, int64_t speed,
                   BlockdevOnError on_error,
-                  BlockDriverCompletionFunc *cb,
+                  BlockCompletionFunc *cb,
                   void *opaque, Error **errp)
 {
     StreamBlockJob *s;
diff --git a/block/vdi.c b/block/vdi.c
index 197bd77c9..39070b75e 100644
--- a/block/vdi.c
+++ b/block/vdi.c
@@ -53,13 +53,6 @@
 #include "block/block_int.h"
 #include "qemu/module.h"
 #include "migration/migration.h"
-#ifdef __linux__
-#include <linux/fs.h>
-#include <sys/ioctl.h>
-#ifndef FS_NOCOW_FL
-#define FS_NOCOW_FL                     0x00800000 /* Do not cow file */
-#endif
-#endif
 
 #if defined(CONFIG_UUID)
 #include <uuid/uuid.h>
@@ -127,8 +120,18 @@ typedef unsigned char uuid_t[16];
 
 #define VDI_IS_ALLOCATED(X) ((X) < VDI_DISCARDED)
 
-/* max blocks in image is (0xffffffff / 4) */
-#define VDI_BLOCKS_IN_IMAGE_MAX  0x3fffffff
+/* The bmap will take up VDI_BLOCKS_IN_IMAGE_MAX * sizeof(uint32_t) bytes; since
+ * the bmap is read and written in a single operation, its size needs to be
+ * limited to INT_MAX; furthermore, when opening an image, the bmap size is
+ * rounded up to be aligned on BDRV_SECTOR_SIZE.
+ * Therefore this should satisfy the following:
+ * VDI_BLOCKS_IN_IMAGE_MAX * sizeof(uint32_t) + BDRV_SECTOR_SIZE == INT_MAX + 1
+ * (INT_MAX + 1 is the first value not representable as an int)
+ * This guarantees that any value below or equal to the constant will, when
+ * multiplied by sizeof(uint32_t) and rounded up to a BDRV_SECTOR_SIZE boundary,
+ * still be below or equal to INT_MAX. */
+#define VDI_BLOCKS_IN_IMAGE_MAX \
+    ((unsigned)((INT_MAX + 1u - BDRV_SECTOR_SIZE) / sizeof(uint32_t)))
 #define VDI_DISK_SIZE_MAX        ((uint64_t)VDI_BLOCKS_IN_IMAGE_MAX * \
                                   (uint64_t)DEFAULT_CLUSTER_SIZE)
 
@@ -144,12 +147,14 @@ static inline int uuid_is_null(const uuid_t uu)
     return memcmp(uu, null_uuid, sizeof(uuid_t)) == 0;
 }
 
+# if defined(CONFIG_VDI_DEBUG)
 static inline void uuid_unparse(const uuid_t uu, char *out)
 {
     snprintf(out, 37, UUID_FMT,
             uu[0], uu[1], uu[2], uu[3], uu[4], uu[5], uu[6], uu[7],
             uu[8], uu[9], uu[10], uu[11], uu[12], uu[13], uu[14], uu[15]);
 }
+# endif
 #endif
 
 typedef struct {
@@ -299,7 +304,12 @@ static int vdi_check(BlockDriverState *bs, BdrvCheckResult *res,
         return -ENOTSUP;
     }
 
-    bmap = g_malloc(s->header.blocks_in_image * sizeof(uint32_t));
+    bmap = g_try_new(uint32_t, s->header.blocks_in_image);
+    if (s->header.blocks_in_image && bmap == NULL) {
+        res->check_errors++;
+        return -ENOMEM;
+    }
+
     memset(bmap, 0xff, s->header.blocks_in_image * sizeof(uint32_t));
 
     /* Check block map and value of blocks_allocated. */
@@ -357,23 +367,23 @@ static int vdi_make_empty(BlockDriverState *bs)
 static int vdi_probe(const uint8_t *buf, int buf_size, const char *filename)
 {
     const VdiHeader *header = (const VdiHeader *)buf;
-    int result = 0;
+    int ret = 0;
 
     logout("\n");
 
     if (buf_size < sizeof(*header)) {
         /* Header too small, no VDI. */
     } else if (le32_to_cpu(header->signature) == VDI_SIGNATURE) {
-        result = 100;
+        ret = 100;
     }
 
-    if (result == 0) {
+    if (ret == 0) {
         logout("no vdi image\n");
     } else {
         logout("%s", header->text);
     }
 
-    return result;
+    return ret;
 }
 
 static int vdi_open(BlockDriverState *bs, QDict *options, int flags,
@@ -409,8 +419,7 @@ static int vdi_open(BlockDriverState *bs, QDict *options, int flags,
            We accept them but round the disk size to the next multiple of
            SECTOR_SIZE. */
         logout("odd disk size %" PRIu64 " B, round up\n", header.disk_size);
-        header.disk_size += SECTOR_SIZE - 1;
-        header.disk_size &= ~(SECTOR_SIZE - 1);
+        header.disk_size = ROUND_UP(header.disk_size, SECTOR_SIZE);
     }
 
     if (header.signature != VDI_SIGNATURE) {
@@ -477,8 +486,13 @@ static int vdi_open(BlockDriverState *bs, QDict *options, int flags,
     s->header = header;
 
     bmap_size = header.blocks_in_image * sizeof(uint32_t);
-    bmap_size = (bmap_size + SECTOR_SIZE - 1) / SECTOR_SIZE;
-    s->bmap = g_malloc(bmap_size * SECTOR_SIZE);
+    bmap_size = DIV_ROUND_UP(bmap_size, SECTOR_SIZE);
+    s->bmap = qemu_try_blockalign(bs->file, bmap_size * SECTOR_SIZE);
+    if (s->bmap == NULL) {
+        ret = -ENOMEM;
+        goto fail;
+    }
+
     ret = bdrv_read(bs->file, s->bmap_sector, (uint8_t *)s->bmap, bmap_size);
     if (ret < 0) {
         goto fail_free_bmap;
@@ -487,13 +501,13 @@ static int vdi_open(BlockDriverState *bs, QDict *options, int flags,
     /* Disable migration when vdi images are used */
     error_set(&s->migration_blocker,
               QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-              "vdi", bs->device_name, "live migration");
+              "vdi", bdrv_get_device_name(bs), "live migration");
     migrate_add_blocker(s->migration_blocker);
 
     return 0;
 
  fail_free_bmap:
-    g_free(s->bmap);
+    qemu_vfree(s->bmap);
 
  fail:
     return ret;
@@ -681,8 +695,7 @@ static int vdi_co_write(BlockDriverState *bs,
 
 static int vdi_create(const char *filename, QemuOpts *opts, Error **errp)
 {
-    int fd;
-    int result = 0;
+    int ret = 0;
     uint64_t bytes = 0;
     uint32_t blocks;
     size_t block_size = DEFAULT_CLUSTER_SIZE;
@@ -690,12 +703,16 @@ static int vdi_create(const char *filename, QemuOpts *opts, Error **errp)
     VdiHeader header;
     size_t i;
     size_t bmap_size;
-    bool nocow = false;
+    int64_t offset = 0;
+    Error *local_err = NULL;
+    BlockDriverState *bs = NULL;
+    uint32_t *bmap = NULL;
 
     logout("\n");
 
     /* Read out options. */
-    bytes = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0);
+    bytes = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                     BDRV_SECTOR_SIZE);
 #if defined(CONFIG_VDI_BLOCK_SIZE)
     /* TODO: Additional checks (SECTOR_SIZE * 2^n, ...). */
     block_size = qemu_opt_get_size_del(opts,
@@ -707,45 +724,33 @@ static int vdi_create(const char *filename, QemuOpts *opts, Error **errp)
         image_type = VDI_TYPE_STATIC;
     }
 #endif
-    nocow = qemu_opt_get_bool_del(opts, BLOCK_OPT_NOCOW, false);
 
     if (bytes > VDI_DISK_SIZE_MAX) {
-        result = -ENOTSUP;
+        ret = -ENOTSUP;
         error_setg(errp, "Unsupported VDI image size (size is 0x%" PRIx64
                           ", max supported is 0x%" PRIx64 ")",
                           bytes, VDI_DISK_SIZE_MAX);
         goto exit;
     }
 
-    fd = qemu_open(filename,
-                   O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_LARGEFILE,
-                   0644);
-    if (fd < 0) {
-        result = -errno;
+    ret = bdrv_create_file(filename, opts, &local_err);
+    if (ret < 0) {
+        error_propagate(errp, local_err);
         goto exit;
     }
-
-    if (nocow) {
-#ifdef __linux__
-        /* Set NOCOW flag to solve performance issue on fs like btrfs.
-         * This is an optimisation. The FS_IOC_SETFLAGS ioctl return value will
-         * be ignored since any failure of this operation should not block the
-         * left work.
-         */
-        int attr;
-        if (ioctl(fd, FS_IOC_GETFLAGS, &attr) == 0) {
-            attr |= FS_NOCOW_FL;
-            ioctl(fd, FS_IOC_SETFLAGS, &attr);
-        }
-#endif
+    ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
+                    NULL, &local_err);
+    if (ret < 0) {
+        error_propagate(errp, local_err);
+        goto exit;
     }
 
     /* We need enough blocks to store the given disk size,
        so always round up. */
-    blocks = (bytes + block_size - 1) / block_size;
+    blocks = DIV_ROUND_UP(bytes, block_size);
 
     bmap_size = blocks * sizeof(uint32_t);
-    bmap_size = ((bmap_size + SECTOR_SIZE - 1) & ~(SECTOR_SIZE -1));
+    bmap_size = ROUND_UP(bmap_size, SECTOR_SIZE);
 
     memset(&header, 0, sizeof(header));
     pstrcpy(header.text, sizeof(header.text), VDI_TEXT);
@@ -769,13 +774,20 @@ static int vdi_create(const char *filename, QemuOpts *opts, Error **errp)
     vdi_header_print(&header);
 #endif
     vdi_header_to_le(&header);
-    if (write(fd, &header, sizeof(header)) < 0) {
-        result = -errno;
-        goto close_and_exit;
+    ret = bdrv_pwrite_sync(bs, offset, &header, sizeof(header));
+    if (ret < 0) {
+        error_setg(errp, "Error writing header to %s", filename);
+        goto exit;
     }
+    offset += sizeof(header);
 
     if (bmap_size > 0) {
-        uint32_t *bmap = g_malloc0(bmap_size);
+        bmap = g_try_malloc0(bmap_size);
+        if (bmap == NULL) {
+            ret = -ENOMEM;
+            error_setg(errp, "Could not allocate bmap");
+            goto exit;
+        }
         for (i = 0; i < blocks; i++) {
             if (image_type == VDI_TYPE_STATIC) {
                 bmap[i] = i;
@@ -783,35 +795,33 @@ static int vdi_create(const char *filename, QemuOpts *opts, Error **errp)
                 bmap[i] = VDI_UNALLOCATED;
             }
         }
-        if (write(fd, bmap, bmap_size) < 0) {
-            result = -errno;
-            g_free(bmap);
-            goto close_and_exit;
+        ret = bdrv_pwrite_sync(bs, offset, bmap, bmap_size);
+        if (ret < 0) {
+            error_setg(errp, "Error writing bmap to %s", filename);
+            goto exit;
         }
-        g_free(bmap);
+        offset += bmap_size;
     }
 
     if (image_type == VDI_TYPE_STATIC) {
-        if (ftruncate(fd, sizeof(header) + bmap_size + blocks * block_size)) {
-            result = -errno;
-            goto close_and_exit;
+        ret = bdrv_truncate(bs, offset + blocks * block_size);
+        if (ret < 0) {
+            error_setg(errp, "Failed to statically allocate %s", filename);
+            goto exit;
         }
     }
 
-close_and_exit:
-    if ((close(fd) < 0) && !result) {
-        result = -errno;
-    }
-
 exit:
-    return result;
+    bdrv_unref(bs);
+    g_free(bmap);
+    return ret;
 }
 
 static void vdi_close(BlockDriverState *bs)
 {
     BDRVVdiState *s = bs->opaque;
 
-    g_free(s->bmap);
+    qemu_vfree(s->bmap);
 
     migrate_del_blocker(s->migration_blocker);
     error_free(s->migration_blocker);
diff --git a/block/vhdx-endian.c b/block/vhdx-endian.c
index fe879ed99..0640d3f4a 100644
--- a/block/vhdx-endian.c
+++ b/block/vhdx-endian.c
@@ -82,8 +82,6 @@ void vhdx_log_desc_le_import(VHDXLogDescriptor *d)
     assert(d != NULL);
 
     le32_to_cpus(&d->signature);
-    le32_to_cpus(&d->trailing_bytes);
-    le64_to_cpus(&d->leading_bytes);
     le64_to_cpus(&d->file_offset);
     le64_to_cpus(&d->sequence_number);
 }
@@ -99,6 +97,15 @@ void vhdx_log_desc_le_export(VHDXLogDescriptor *d)
     cpu_to_le64s(&d->sequence_number);
 }
 
+void vhdx_log_data_le_import(VHDXLogDataSector *d)
+{
+    assert(d != NULL);
+
+    le32_to_cpus(&d->data_signature);
+    le32_to_cpus(&d->sequence_high);
+    le32_to_cpus(&d->sequence_low);
+}
+
 void vhdx_log_data_le_export(VHDXLogDataSector *d)
 {
     assert(d != NULL);
diff --git a/block/vhdx-log.c b/block/vhdx-log.c
index a77c040ee..6547bec40 100644
--- a/block/vhdx-log.c
+++ b/block/vhdx-log.c
@@ -84,6 +84,7 @@ static int vhdx_log_peek_hdr(BlockDriverState *bs, VHDXLogEntries *log,
     if (ret < 0) {
         goto exit;
     }
+    vhdx_log_entry_hdr_le_import(hdr);
 
 exit:
     return ret;
@@ -211,7 +212,7 @@ static bool vhdx_log_hdr_is_valid(VHDXLogEntries *log, VHDXLogEntryHeader *hdr,
 {
     int valid = false;
 
-    if (memcmp(&hdr->signature, "loge", 4)) {
+    if (hdr->signature != VHDX_LOG_SIGNATURE) {
         goto exit;
     }
 
@@ -275,12 +276,12 @@ static bool vhdx_log_desc_is_valid(VHDXLogDescriptor *desc,
         goto exit;
     }
 
-    if (!memcmp(&desc->signature, "zero", 4)) {
+    if (desc->signature == VHDX_LOG_ZERO_SIGNATURE) {
         if (desc->zero_length % VHDX_LOG_SECTOR_SIZE == 0) {
             /* valid */
             ret = true;
         }
-    } else if (!memcmp(&desc->signature, "desc", 4)) {
+    } else if (desc->signature == VHDX_LOG_DESC_SIGNATURE) {
             /* valid */
             ret = true;
     }
@@ -327,13 +328,15 @@ static int vhdx_compute_desc_sectors(uint32_t desc_cnt)
  * passed into this function. Each descriptor will also be validated,
  * and error returned if any are invalid. */
 static int vhdx_log_read_desc(BlockDriverState *bs, BDRVVHDXState *s,
-                              VHDXLogEntries *log, VHDXLogDescEntries **buffer)
+                              VHDXLogEntries *log, VHDXLogDescEntries **buffer,
+                              bool convert_endian)
 {
     int ret = 0;
     uint32_t desc_sectors;
     uint32_t sectors_read;
     VHDXLogEntryHeader hdr;
     VHDXLogDescEntries *desc_entries = NULL;
+    VHDXLogDescriptor desc;
     int i;
 
     assert(*buffer == NULL);
@@ -342,14 +345,19 @@ static int vhdx_log_read_desc(BlockDriverState *bs, BDRVVHDXState *s,
     if (ret < 0) {
         goto exit;
     }
-    vhdx_log_entry_hdr_le_import(&hdr);
+
     if (vhdx_log_hdr_is_valid(log, &hdr, s) == false) {
         ret = -EINVAL;
         goto exit;
     }
 
     desc_sectors = vhdx_compute_desc_sectors(hdr.descriptor_count);
-    desc_entries = qemu_blockalign(bs, desc_sectors * VHDX_LOG_SECTOR_SIZE);
+    desc_entries = qemu_try_blockalign(bs->file,
+                                       desc_sectors * VHDX_LOG_SECTOR_SIZE);
+    if (desc_entries == NULL) {
+        ret = -ENOMEM;
+        goto exit;
+    }
 
     ret = vhdx_log_read_sectors(bs, log, &sectors_read, desc_entries,
                                 desc_sectors, false);
@@ -363,12 +371,19 @@ static int vhdx_log_read_desc(BlockDriverState *bs, BDRVVHDXState *s,
 
     /* put in proper endianness, and validate each desc */
     for (i = 0; i < hdr.descriptor_count; i++) {
-        vhdx_log_desc_le_import(&desc_entries->desc[i]);
-        if (vhdx_log_desc_is_valid(&desc_entries->desc[i], &hdr) == false) {
+        desc = desc_entries->desc[i];
+        vhdx_log_desc_le_import(&desc);
+        if (convert_endian) {
+            desc_entries->desc[i] = desc;
+        }
+        if (vhdx_log_desc_is_valid(&desc, &hdr) == false) {
             ret = -EINVAL;
             goto free_and_exit;
         }
     }
+    if (convert_endian) {
+        desc_entries->hdr = hdr;
+    }
 
     *buffer = desc_entries;
     goto exit;
@@ -403,7 +418,7 @@ static int vhdx_log_flush_desc(BlockDriverState *bs, VHDXLogDescriptor *desc,
 
     buffer = qemu_blockalign(bs, VHDX_LOG_SECTOR_SIZE);
 
-    if (!memcmp(&desc->signature, "desc", 4)) {
+    if (desc->signature == VHDX_LOG_DESC_SIGNATURE) {
         /* data sector */
         if (data == NULL) {
             ret = -EFAULT;
@@ -431,10 +446,15 @@ static int vhdx_log_flush_desc(BlockDriverState *bs, VHDXLogDescriptor *desc,
 
         memcpy(buffer+offset, &desc->trailing_bytes, 4);
 
-    } else if (!memcmp(&desc->signature, "zero", 4)) {
+    } else if (desc->signature == VHDX_LOG_ZERO_SIGNATURE) {
         /* write 'count' sectors of sector */
         memset(buffer, 0, VHDX_LOG_SECTOR_SIZE);
         count = desc->zero_length / VHDX_LOG_SECTOR_SIZE;
+    } else {
+        error_report("Invalid VHDX log descriptor entry signature 0x%" PRIx32,
+                      desc->signature);
+        ret = -EINVAL;
+        goto exit;
     }
 
     file_offset = desc->file_offset;
@@ -493,13 +513,13 @@ static int vhdx_log_flush(BlockDriverState *bs, BDRVVHDXState *s,
             goto exit;
         }
 
-        ret = vhdx_log_read_desc(bs, s, &logs->log, &desc_entries);
+        ret = vhdx_log_read_desc(bs, s, &logs->log, &desc_entries, true);
         if (ret < 0) {
             goto exit;
         }
 
         for (i = 0; i < desc_entries->hdr.descriptor_count; i++) {
-            if (!memcmp(&desc_entries->desc[i].signature, "desc", 4)) {
+            if (desc_entries->desc[i].signature == VHDX_LOG_DESC_SIGNATURE) {
                 /* data sector, so read a sector to flush */
                 ret = vhdx_log_read_sectors(bs, &logs->log, &sectors_read,
                                             data, 1, false);
@@ -510,6 +530,7 @@ static int vhdx_log_flush(BlockDriverState *bs, BDRVVHDXState *s,
                     ret = -EINVAL;
                     goto exit;
                 }
+                vhdx_log_data_le_import(data);
             }
 
             ret = vhdx_log_flush_desc(bs, &desc_entries->desc[i], data);
@@ -558,9 +579,6 @@ static int vhdx_validate_log_entry(BlockDriverState *bs, BDRVVHDXState *s,
         goto inc_and_exit;
     }
 
-    vhdx_log_entry_hdr_le_import(&hdr);
-
-
     if (vhdx_log_hdr_is_valid(log, &hdr, s) == false) {
         goto inc_and_exit;
     }
@@ -573,13 +591,13 @@ static int vhdx_validate_log_entry(BlockDriverState *bs, BDRVVHDXState *s,
 
     desc_sectors = vhdx_compute_desc_sectors(hdr.descriptor_count);
 
-    /* Read desc sectors, and calculate log checksum */
+    /* Read all log sectors, and calculate log checksum */
 
     total_sectors = hdr.entry_length / VHDX_LOG_SECTOR_SIZE;
 
 
     /* read_desc() will increment the read idx */
-    ret = vhdx_log_read_desc(bs, s, log, &desc_buffer);
+    ret = vhdx_log_read_desc(bs, s, log, &desc_buffer, false);
     if (ret < 0) {
         goto free_and_exit;
     }
@@ -602,7 +620,7 @@ static int vhdx_validate_log_entry(BlockDriverState *bs, BDRVVHDXState *s,
         }
     }
     crc ^= 0xffffffff;
-    if (crc != desc_buffer->hdr.checksum) {
+    if (crc != hdr.checksum) {
         goto free_and_exit;
     }
 
@@ -905,7 +923,7 @@ static int vhdx_log_write(BlockDriverState *bs, BDRVVHDXState *s,
     buffer = qemu_blockalign(bs, total_length);
     memcpy(buffer, &new_hdr, sizeof(new_hdr));
 
-    new_desc = (VHDXLogDescriptor *) (buffer + sizeof(new_hdr));
+    new_desc = buffer + sizeof(new_hdr);
     data_sector = buffer + (desc_sectors * VHDX_LOG_SECTOR_SIZE);
     data_tmp = data;
 
@@ -962,7 +980,6 @@ static int vhdx_log_write(BlockDriverState *bs, BDRVVHDXState *s,
      * last data sector */
     vhdx_update_checksum(buffer, total_length,
                          offsetof(VHDXLogEntryHeader, checksum));
-    cpu_to_le32s((uint32_t *)(buffer + 4));
 
     /* now write to the log */
     ret = vhdx_log_write_sectors(bs, &s->log, &sectors_written, buffer,
diff --git a/block/vhdx.c b/block/vhdx.c
index fedcf9f9c..12bfe75e5 100644
--- a/block/vhdx.c
+++ b/block/vhdx.c
@@ -99,7 +99,8 @@ static const MSGUID logical_sector_guid = { .data1 = 0x8141bf1d,
 /* Each parent type must have a valid GUID; this is for parent images
  * of type 'VHDX'.  If we were to allow e.g. a QCOW2 parent, we would
  * need to make up our own QCOW2 GUID type */
-static const MSGUID parent_vhdx_guid = { .data1 = 0xb04aefb7,
+static const MSGUID parent_vhdx_guid __attribute__((unused))
+                                     = { .data1 = 0xb04aefb7,
                                          .data2 = 0xd19e,
                                          .data3 = 0x4a81,
                                          .data4 = { 0xb7, 0x89, 0x25, 0xb8,
@@ -135,10 +136,8 @@ typedef struct VHDXSectorInfo {
  * buf: buffer pointer
  * size: size of buffer (must be > crc_offset+4)
  *
- * Note: The resulting checksum is in the CPU endianness, not necessarily
- *       in the file format endianness (LE).  Any header export to disk should
- *       make sure that vhdx_header_le_export() is used to convert to the
- *       correct endianness
+ * Note: The buffer should have all multi-byte data in little-endian format,
+ *       and the resulting checksum is in little endian format.
  */
 uint32_t vhdx_update_checksum(uint8_t *buf, size_t size, int crc_offset)
 {
@@ -149,6 +148,7 @@ uint32_t vhdx_update_checksum(uint8_t *buf, size_t size, int crc_offset)
 
     memset(buf + crc_offset, 0, sizeof(crc));
     crc =  crc32c(0xffffffff, buf, size);
+    cpu_to_le32s(&crc);
     memcpy(buf + crc_offset, &crc, sizeof(crc));
 
     return crc;
@@ -300,7 +300,7 @@ static int vhdx_write_header(BlockDriverState *bs_file, VHDXHeader *hdr,
 {
     uint8_t *buffer = NULL;
     int ret;
-    VHDXHeader header_le;
+    VHDXHeader *header_le;
 
     assert(bs_file != NULL);
     assert(hdr != NULL);
@@ -321,11 +321,12 @@ static int vhdx_write_header(BlockDriverState *bs_file, VHDXHeader *hdr,
     }
 
     /* overwrite the actual VHDXHeader portion */
-    memcpy(buffer, hdr, sizeof(VHDXHeader));
-    hdr->checksum = vhdx_update_checksum(buffer, VHDX_HEADER_SIZE,
-                                         offsetof(VHDXHeader, checksum));
-    vhdx_header_le_export(hdr, &header_le);
-    ret = bdrv_pwrite_sync(bs_file, offset, &header_le, sizeof(VHDXHeader));
+    header_le = (VHDXHeader *)buffer;
+    memcpy(header_le, hdr, sizeof(VHDXHeader));
+    vhdx_header_le_export(hdr, header_le);
+    vhdx_update_checksum(buffer, VHDX_HEADER_SIZE,
+                         offsetof(VHDXHeader, checksum));
+    ret = bdrv_pwrite_sync(bs_file, offset, header_le, sizeof(VHDXHeader));
 
 exit:
     qemu_vfree(buffer);
@@ -432,13 +433,14 @@ static void vhdx_parse_header(BlockDriverState *bs, BDRVVHDXState *s,
     }
     /* copy over just the relevant portion that we need */
     memcpy(header1, buffer, sizeof(VHDXHeader));
-    vhdx_header_le_import(header1);
 
-    if (vhdx_checksum_is_valid(buffer, VHDX_HEADER_SIZE, 4) &&
-        !memcmp(&header1->signature, "head", 4)             &&
-        header1->version == 1) {
-        h1_seq = header1->sequence_number;
-        h1_valid = true;
+    if (vhdx_checksum_is_valid(buffer, VHDX_HEADER_SIZE, 4)) {
+        vhdx_header_le_import(header1);
+        if (header1->signature == VHDX_HEADER_SIGNATURE &&
+            header1->version == 1) {
+            h1_seq = header1->sequence_number;
+            h1_valid = true;
+        }
     }
 
     ret = bdrv_pread(bs->file, VHDX_HEADER2_OFFSET, buffer, VHDX_HEADER_SIZE);
@@ -447,13 +449,14 @@ static void vhdx_parse_header(BlockDriverState *bs, BDRVVHDXState *s,
     }
     /* copy over just the relevant portion that we need */
     memcpy(header2, buffer, sizeof(VHDXHeader));
-    vhdx_header_le_import(header2);
 
-    if (vhdx_checksum_is_valid(buffer, VHDX_HEADER_SIZE, 4) &&
-        !memcmp(&header2->signature, "head", 4)             &&
-        header2->version == 1) {
-        h2_seq = header2->sequence_number;
-        h2_valid = true;
+    if (vhdx_checksum_is_valid(buffer, VHDX_HEADER_SIZE, 4)) {
+        vhdx_header_le_import(header2);
+        if (header2->signature == VHDX_HEADER_SIGNATURE &&
+            header2->version == 1) {
+            h2_seq = header2->sequence_number;
+            h2_valid = true;
+        }
     }
 
     /* If there is only 1 valid header (or no valid headers), we
@@ -519,15 +522,21 @@ static int vhdx_open_region_tables(BlockDriverState *bs, BDRVVHDXState *s)
         goto fail;
     }
     memcpy(&s->rt, buffer, sizeof(s->rt));
-    vhdx_region_header_le_import(&s->rt);
     offset += sizeof(s->rt);
 
-    if (!vhdx_checksum_is_valid(buffer, VHDX_HEADER_BLOCK_SIZE, 4) ||
-        memcmp(&s->rt.signature, "regi", 4)) {
+    if (!vhdx_checksum_is_valid(buffer, VHDX_HEADER_BLOCK_SIZE, 4)) {
         ret = -EINVAL;
         goto fail;
     }
 
+    vhdx_region_header_le_import(&s->rt);
+
+    if (s->rt.signature != VHDX_REGION_SIGNATURE) {
+        ret = -EINVAL;
+        goto fail;
+    }
+
+
     /* Per spec, maximum region table entry count is 2047 */
     if (s->rt.entry_count > 2047) {
         ret = -EINVAL;
@@ -630,7 +639,7 @@ static int vhdx_parse_metadata(BlockDriverState *bs, BDRVVHDXState *s)
 
     vhdx_metadata_header_le_import(&s->metadata_hdr);
 
-    if (memcmp(&s->metadata_hdr.signature, "metadata", 8)) {
+    if (s->metadata_hdr.signature != VHDX_METADATA_SIGNATURE) {
         ret = -EINVAL;
         goto exit;
     }
@@ -950,7 +959,11 @@ static int vhdx_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     /* s->bat is freed in vhdx_close() */
-    s->bat = qemu_blockalign(bs, s->bat_rt.length);
+    s->bat = qemu_try_blockalign(bs->file, s->bat_rt.length);
+    if (s->bat == NULL) {
+        ret = -ENOMEM;
+        goto fail;
+    }
 
     ret = bdrv_pread(bs->file, s->bat_offset, s->bat, s->bat_rt.length);
     if (ret < 0) {
@@ -991,7 +1004,7 @@ static int vhdx_open(BlockDriverState *bs, QDict *options, int flags,
     /* Disable migration when VHDX images are used */
     error_set(&s->migration_blocker,
             QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-            "vhdx", bs->device_name, "live migration");
+            "vhdx", bdrv_get_device_name(bs), "live migration");
     migrate_add_blocker(s->migration_blocker);
 
     return 0;
@@ -1369,7 +1382,7 @@ static int vhdx_create_new_headers(BlockDriverState *bs, uint64_t image_size,
     int ret = 0;
     VHDXHeader *hdr = NULL;
 
-    hdr = g_malloc0(sizeof(VHDXHeader));
+    hdr = g_new0(VHDXHeader, 1);
 
     hdr->signature       = VHDX_HEADER_SIGNATURE;
     hdr->sequence_number = g_random_int();
@@ -1395,6 +1408,12 @@ exit:
     return ret;
 }
 
+#define VHDX_METADATA_ENTRY_BUFFER_SIZE \
+                                    (sizeof(VHDXFileParameters)               +\
+                                     sizeof(VHDXVirtualDiskSize)              +\
+                                     sizeof(VHDXPage83Data)                   +\
+                                     sizeof(VHDXVirtualDiskLogicalSectorSize) +\
+                                     sizeof(VHDXVirtualDiskPhysicalSectorSize))
 
 /*
  * Create the Metadata entries.
@@ -1433,11 +1452,7 @@ static int vhdx_create_new_metadata(BlockDriverState *bs,
     VHDXVirtualDiskLogicalSectorSize  *mt_log_sector_size;
     VHDXVirtualDiskPhysicalSectorSize *mt_phys_sector_size;
 
-    entry_buffer = g_malloc0(sizeof(VHDXFileParameters)               +
-                             sizeof(VHDXVirtualDiskSize)              +
-                             sizeof(VHDXPage83Data)                   +
-                             sizeof(VHDXVirtualDiskLogicalSectorSize) +
-                             sizeof(VHDXVirtualDiskPhysicalSectorSize));
+    entry_buffer = g_malloc0(VHDX_METADATA_ENTRY_BUFFER_SIZE);
 
     mt_file_params = entry_buffer;
     offset += sizeof(VHDXFileParameters);
@@ -1518,7 +1533,7 @@ static int vhdx_create_new_metadata(BlockDriverState *bs,
     }
 
     ret = bdrv_pwrite(bs, metadata_offset + (64 * KiB), entry_buffer,
-                      VHDX_HEADER_BLOCK_SIZE);
+                      VHDX_METADATA_ENTRY_BUFFER_SIZE);
     if (ret < 0) {
         goto exit;
     }
@@ -1540,7 +1555,8 @@ exit:
  */
 static int vhdx_create_bat(BlockDriverState *bs, BDRVVHDXState *s,
                            uint64_t image_size, VHDXImageType type,
-                           bool use_zero_blocks, VHDXRegionTableEntry *rt_bat)
+                           bool use_zero_blocks, uint64_t file_offset,
+                           uint32_t length)
 {
     int ret = 0;
     uint64_t data_file_offset;
@@ -1555,7 +1571,7 @@ static int vhdx_create_bat(BlockDriverState *bs, BDRVVHDXState *s,
     /* this gives a data start after BAT/bitmap entries, and well
      * past any metadata entries (with a 4 MB buffer for future
      * expansion */
-    data_file_offset = rt_bat->file_offset + rt_bat->length + 5 * MiB;
+    data_file_offset = file_offset + length + 5 * MiB;
     total_sectors = image_size >> s->logical_sector_size_bits;
 
     if (type == VHDX_TYPE_DYNAMIC) {
@@ -1579,7 +1595,11 @@ static int vhdx_create_bat(BlockDriverState *bs, BDRVVHDXState *s,
                 use_zero_blocks ||
                 bdrv_has_zero_init(bs) == 0) {
         /* for a fixed file, the default BAT entry is not zero */
-        s->bat = g_malloc0(rt_bat->length);
+        s->bat = g_try_malloc0(length);
+        if (length && s->bat == NULL) {
+            ret = -ENOMEM;
+            goto exit;
+        }
         block_state = type == VHDX_TYPE_FIXED ? PAYLOAD_BLOCK_FULLY_PRESENT :
                                                 PAYLOAD_BLOCK_NOT_PRESENT;
         block_state = use_zero_blocks ? PAYLOAD_BLOCK_ZERO : block_state;
@@ -1594,7 +1614,7 @@ static int vhdx_create_bat(BlockDriverState *bs, BDRVVHDXState *s,
             cpu_to_le64s(&s->bat[sinfo.bat_idx]);
             sector_num += s->sectors_per_block;
         }
-        ret = bdrv_pwrite(bs, rt_bat->file_offset, s->bat, rt_bat->length);
+        ret = bdrv_pwrite(bs, file_offset, s->bat, length);
         if (ret < 0) {
             goto exit;
         }
@@ -1626,6 +1646,8 @@ static int vhdx_create_new_region_table(BlockDriverState *bs,
     int ret = 0;
     uint32_t offset = 0;
     void *buffer = NULL;
+    uint64_t bat_file_offset;
+    uint32_t bat_length;
     BDRVVHDXState *s = NULL;
     VHDXRegionTableHeader *region_table;
     VHDXRegionTableEntry *rt_bat;
@@ -1635,7 +1657,7 @@ static int vhdx_create_new_region_table(BlockDriverState *bs,
 
     /* Populate enough of the BDRVVHDXState to be able to use the
      * pre-existing BAT calculation, translation, and update functions */
-    s = g_malloc0(sizeof(BDRVVHDXState));
+    s = g_new0(BDRVVHDXState, 1);
 
     s->chunk_ratio = (VHDX_MAX_SECTORS_PER_BLOCK) *
                      (uint64_t) sector_size / (uint64_t) block_size;
@@ -1674,19 +1696,26 @@ static int vhdx_create_new_region_table(BlockDriverState *bs,
     rt_metadata->length      = 1 * MiB; /* min size, and more than enough */
     *metadata_offset = rt_metadata->file_offset;
 
+    bat_file_offset = rt_bat->file_offset;
+    bat_length = rt_bat->length;
+
+    vhdx_region_header_le_export(region_table);
+    vhdx_region_entry_le_export(rt_bat);
+    vhdx_region_entry_le_export(rt_metadata);
+
     vhdx_update_checksum(buffer, VHDX_HEADER_BLOCK_SIZE,
                          offsetof(VHDXRegionTableHeader, checksum));
 
 
     /* The region table gives us the data we need to create the BAT,
      * so do that now */
-    ret = vhdx_create_bat(bs, s, image_size, type, use_zero_blocks, rt_bat);
+    ret = vhdx_create_bat(bs, s, image_size, type, use_zero_blocks,
+                          bat_file_offset, bat_length);
+    if (ret < 0) {
+        goto exit;
+    }
 
     /* Now write out the region headers to disk */
-    vhdx_region_header_le_export(region_table);
-    vhdx_region_entry_le_export(rt_bat);
-    vhdx_region_entry_le_export(rt_metadata);
-
     ret = bdrv_pwrite(bs, VHDX_REGION_TABLE_OFFSET, buffer,
                       VHDX_HEADER_BLOCK_SIZE);
     if (ret < 0) {
@@ -1699,7 +1728,6 @@ static int vhdx_create_new_region_table(BlockDriverState *bs,
         goto exit;
     }
 
-
 exit:
     g_free(s);
     g_free(buffer);
@@ -1740,7 +1768,8 @@ static int vhdx_create(const char *filename, QemuOpts *opts, Error **errp)
     VHDXImageType image_type;
     Error *local_err = NULL;
 
-    image_size = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0);
+    image_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                          BDRV_SECTOR_SIZE);
     log_size = qemu_opt_get_size_del(opts, VHDX_BLOCK_OPT_LOG_SIZE, 0);
     block_size = qemu_opt_get_size_del(opts, VHDX_BLOCK_OPT_BLOCK_SIZE, 0);
     type = qemu_opt_get_del(opts, BLOCK_OPT_SUBFMT);
@@ -1849,7 +1878,6 @@ static int vhdx_create(const char *filename, QemuOpts *opts, Error **errp)
     }
 
 
-
 delete_and_exit:
     bdrv_unref(bs);
 exit:
diff --git a/block/vhdx.h b/block/vhdx.h
index 5370010c5..b4a12a081 100644
--- a/block/vhdx.h
+++ b/block/vhdx.h
@@ -435,6 +435,7 @@ void vhdx_header_le_import(VHDXHeader *h);
 void vhdx_header_le_export(VHDXHeader *orig_h, VHDXHeader *new_h);
 void vhdx_log_desc_le_import(VHDXLogDescriptor *d);
 void vhdx_log_desc_le_export(VHDXLogDescriptor *d);
+void vhdx_log_data_le_import(VHDXLogDataSector *d);
 void vhdx_log_data_le_export(VHDXLogDataSector *d);
 void vhdx_log_entry_hdr_le_import(VHDXLogEntryHeader *hdr);
 void vhdx_log_entry_hdr_le_export(VHDXLogEntryHeader *hdr);
diff --git a/block/vmdk.c b/block/vmdk.c
index 0517bbaf9..2cbfd3e72 100644
--- a/block/vmdk.c
+++ b/block/vmdk.c
@@ -106,6 +106,7 @@ typedef struct VmdkExtent {
     uint32_t l2_cache_counts[L2_CACHE_SIZE];
 
     int64_t cluster_sectors;
+    int64_t next_cluster_sector;
     char *type;
 } VmdkExtent;
 
@@ -124,7 +125,6 @@ typedef struct BDRVVmdkState {
 } BDRVVmdkState;
 
 typedef struct VmdkMetaData {
-    uint32_t offset;
     unsigned int l1_index;
     unsigned int l2_index;
     unsigned int l2_offset;
@@ -233,7 +233,7 @@ static void vmdk_free_last_extent(BlockDriverState *bs)
         return;
     }
     s->num_extents--;
-    s->extents = g_realloc(s->extents, s->num_extents * sizeof(VmdkExtent));
+    s->extents = g_renew(VmdkExtent, s->extents, s->num_extents);
 }
 
 static uint32_t vmdk_read_cid(BlockDriverState *bs, int parent)
@@ -397,6 +397,7 @@ static int vmdk_add_extent(BlockDriverState *bs,
 {
     VmdkExtent *extent;
     BDRVVmdkState *s = bs->opaque;
+    int64_t nb_sectors;
 
     if (cluster_sectors > 0x200000) {
         /* 0x200000 * 512Bytes = 1GB for one cluster is unrealistic */
@@ -412,8 +413,12 @@ static int vmdk_add_extent(BlockDriverState *bs,
         return -EFBIG;
     }
 
-    s->extents = g_realloc(s->extents,
-                              (s->num_extents + 1) * sizeof(VmdkExtent));
+    nb_sectors = bdrv_nb_sectors(file);
+    if (nb_sectors < 0) {
+        return nb_sectors;
+    }
+
+    s->extents = g_renew(VmdkExtent, s->extents, s->num_extents + 1);
     extent = &s->extents[s->num_extents];
     s->num_extents++;
 
@@ -427,6 +432,7 @@ static int vmdk_add_extent(BlockDriverState *bs,
     extent->l1_entry_sectors = l2_size * cluster_sectors;
     extent->l2_size = l2_size;
     extent->cluster_sectors = flat ? sectors : cluster_sectors;
+    extent->next_cluster_sector = ROUND_UP(nb_sectors, cluster_sectors);
 
     if (s->num_extents > 1) {
         extent->end_sector = (*(extent - 1)).end_sector + extent->sectors;
@@ -448,7 +454,11 @@ static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent,
 
     /* read the L1 table */
     l1_size = extent->l1_size * sizeof(uint32_t);
-    extent->l1_table = g_malloc(l1_size);
+    extent->l1_table = g_try_malloc(l1_size);
+    if (l1_size && extent->l1_table == NULL) {
+        return -ENOMEM;
+    }
+
     ret = bdrv_pread(extent->file,
                      extent->l1_table_offset,
                      extent->l1_table,
@@ -464,7 +474,11 @@ static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent,
     }
 
     if (extent->l1_backup_table_offset) {
-        extent->l1_backup_table = g_malloc(l1_size);
+        extent->l1_backup_table = g_try_malloc(l1_size);
+        if (l1_size && extent->l1_backup_table == NULL) {
+            ret = -ENOMEM;
+            goto fail_l1;
+        }
         ret = bdrv_pread(extent->file,
                          extent->l1_backup_table_offset,
                          extent->l1_backup_table,
@@ -481,7 +495,7 @@ static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent,
     }
 
     extent->l2_cache =
-        g_malloc(extent->l2_size * L2_CACHE_SIZE * sizeof(uint32_t));
+        g_new(uint32_t, extent->l2_size * L2_CACHE_SIZE);
     return 0;
  fail_l1b:
     g_free(extent->l1_backup_table);
@@ -643,7 +657,7 @@ static int vmdk_open_vmdk4(BlockDriverState *bs,
         snprintf(buf, sizeof(buf), "VMDK version %" PRId32,
                  le32_to_cpu(header.version));
         error_set(errp, QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
-                  bs->device_name, "vmdk", buf);
+                  bdrv_get_device_name(bs), "vmdk", buf);
         return -ENOTSUP;
     } else if (le32_to_cpu(header.version) == 3 && (flags & BDRV_O_RDWR)) {
         /* VMware KB 2064959 explains that version 3 added support for
@@ -669,8 +683,7 @@ static int vmdk_open_vmdk4(BlockDriverState *bs,
     if (le32_to_cpu(header.flags) & VMDK4_FLAG_RGD) {
         l1_backup_offset = le64_to_cpu(header.rgd_offset) << 9;
     }
-    if (bdrv_getlength(file) <
-            le64_to_cpu(header.grain_offset) * BDRV_SECTOR_SIZE) {
+    if (bdrv_nb_sectors(file) < le64_to_cpu(header.grain_offset)) {
         error_setg(errp, "File truncated, expecting at least %" PRId64 " bytes",
                    (int64_t)(le64_to_cpu(header.grain_offset)
                              * BDRV_SECTOR_SIZE));
@@ -821,6 +834,7 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
             ret = vmdk_add_extent(bs, extent_file, true, sectors,
                             0, 0, 0, 0, 0, &extent, errp);
             if (ret < 0) {
+                bdrv_unref(extent_file);
                 return ret;
             }
             extent->flat_start_offset = flat_offset << 9;
@@ -832,14 +846,15 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
             } else {
                 ret = vmdk_open_sparse(bs, extent_file, bs->open_flags, buf, errp);
             }
+            g_free(buf);
             if (ret) {
-                g_free(buf);
                 bdrv_unref(extent_file);
                 return ret;
             }
             extent = &s->extents[s->num_extents - 1];
         } else {
             error_setg(errp, "Unsupported extent type '%s'", type);
+            bdrv_unref(extent_file);
             return -ENOTSUP;
         }
         extent->type = g_strdup(type);
@@ -924,7 +939,7 @@ static int vmdk_open(BlockDriverState *bs, QDict *options, int flags,
     /* Disable migration when VMDK images are used */
     error_set(&s->migration_blocker,
               QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-              "vmdk", bs->device_name, "live migration");
+              "vmdk", bdrv_get_device_name(bs), "live migration");
     migrate_add_blocker(s->migration_blocker);
     g_free(buf);
     return 0;
@@ -952,57 +967,97 @@ static void vmdk_refresh_limits(BlockDriverState *bs, Error **errp)
     }
 }
 
+/**
+ * get_whole_cluster
+ *
+ * Copy backing file's cluster that covers @sector_num, otherwise write zero,
+ * to the cluster at @cluster_sector_num.
+ *
+ * If @skip_start_sector < @skip_end_sector, the relative range
+ * [@skip_start_sector, @skip_end_sector) is not copied or written, and leave
+ * it for call to write user data in the request.
+ */
 static int get_whole_cluster(BlockDriverState *bs,
-                VmdkExtent *extent,
-                uint64_t cluster_offset,
-                uint64_t offset,
-                bool allocate)
+                             VmdkExtent *extent,
+                             uint64_t cluster_sector_num,
+                             uint64_t sector_num,
+                             uint64_t skip_start_sector,
+                             uint64_t skip_end_sector)
 {
     int ret = VMDK_OK;
-    uint8_t *whole_grain = NULL;
+    int64_t cluster_bytes;
+    uint8_t *whole_grain;
+
+    /* For COW, align request sector_num to cluster start */
+    sector_num = QEMU_ALIGN_DOWN(sector_num, extent->cluster_sectors);
+    cluster_bytes = extent->cluster_sectors << BDRV_SECTOR_BITS;
+    whole_grain = qemu_blockalign(bs, cluster_bytes);
+
+    if (!bs->backing_hd) {
+        memset(whole_grain, 0,  skip_start_sector << BDRV_SECTOR_BITS);
+        memset(whole_grain + (skip_end_sector << BDRV_SECTOR_BITS), 0,
+               cluster_bytes - (skip_end_sector << BDRV_SECTOR_BITS));
+    }
 
+    assert(skip_end_sector <= extent->cluster_sectors);
     /* we will be here if it's first write on non-exist grain(cluster).
      * try to read from parent image, if exist */
-    if (bs->backing_hd) {
-        whole_grain =
-            qemu_blockalign(bs, extent->cluster_sectors << BDRV_SECTOR_BITS);
-        if (!vmdk_is_cid_valid(bs)) {
-            ret = VMDK_ERROR;
-            goto exit;
-        }
+    if (bs->backing_hd && !vmdk_is_cid_valid(bs)) {
+        ret = VMDK_ERROR;
+        goto exit;
+    }
 
-        /* floor offset to cluster */
-        offset -= offset % (extent->cluster_sectors * 512);
-        ret = bdrv_read(bs->backing_hd, offset >> 9, whole_grain,
-                extent->cluster_sectors);
+    /* Read backing data before skip range */
+    if (skip_start_sector > 0) {
+        if (bs->backing_hd) {
+            ret = bdrv_read(bs->backing_hd, sector_num,
+                            whole_grain, skip_start_sector);
+            if (ret < 0) {
+                ret = VMDK_ERROR;
+                goto exit;
+            }
+        }
+        ret = bdrv_write(extent->file, cluster_sector_num, whole_grain,
+                         skip_start_sector);
         if (ret < 0) {
             ret = VMDK_ERROR;
             goto exit;
         }
-
-        /* Write grain only into the active image */
-        ret = bdrv_write(extent->file, cluster_offset, whole_grain,
-                extent->cluster_sectors);
+    }
+    /* Read backing data after skip range */
+    if (skip_end_sector < extent->cluster_sectors) {
+        if (bs->backing_hd) {
+            ret = bdrv_read(bs->backing_hd, sector_num + skip_end_sector,
+                            whole_grain + (skip_end_sector << BDRV_SECTOR_BITS),
+                            extent->cluster_sectors - skip_end_sector);
+            if (ret < 0) {
+                ret = VMDK_ERROR;
+                goto exit;
+            }
+        }
+        ret = bdrv_write(extent->file, cluster_sector_num + skip_end_sector,
+                         whole_grain + (skip_end_sector << BDRV_SECTOR_BITS),
+                         extent->cluster_sectors - skip_end_sector);
         if (ret < 0) {
             ret = VMDK_ERROR;
             goto exit;
         }
     }
+
 exit:
     qemu_vfree(whole_grain);
     return ret;
 }
 
-static int vmdk_L2update(VmdkExtent *extent, VmdkMetaData *m_data)
+static int vmdk_L2update(VmdkExtent *extent, VmdkMetaData *m_data,
+                         uint32_t offset)
 {
-    uint32_t offset;
-    QEMU_BUILD_BUG_ON(sizeof(offset) != sizeof(m_data->offset));
-    offset = cpu_to_le32(m_data->offset);
+    offset = cpu_to_le32(offset);
     /* update L2 table */
     if (bdrv_pwrite_sync(
                 extent->file,
                 ((int64_t)m_data->l2_offset * 512)
-                    + (m_data->l2_index * sizeof(m_data->offset)),
+                    + (m_data->l2_index * sizeof(offset)),
                 &offset, sizeof(offset)) < 0) {
         return VMDK_ERROR;
     }
@@ -1012,7 +1067,7 @@ static int vmdk_L2update(VmdkExtent *extent, VmdkMetaData *m_data)
         if (bdrv_pwrite_sync(
                     extent->file,
                     ((int64_t)m_data->l2_offset * 512)
-                        + (m_data->l2_index * sizeof(m_data->offset)),
+                        + (m_data->l2_index * sizeof(offset)),
                     &offset, sizeof(offset)) < 0) {
             return VMDK_ERROR;
         }
@@ -1024,17 +1079,41 @@ static int vmdk_L2update(VmdkExtent *extent, VmdkMetaData *m_data)
     return VMDK_OK;
 }
 
+/**
+ * get_cluster_offset
+ *
+ * Look up cluster offset in extent file by sector number, and store in
+ * @cluster_offset.
+ *
+ * For flat extents, the start offset as parsed from the description file is
+ * returned.
+ *
+ * For sparse extents, look up in L1, L2 table. If allocate is true, return an
+ * offset for a new cluster and update L2 cache. If there is a backing file,
+ * COW is done before returning; otherwise, zeroes are written to the allocated
+ * cluster. Both COW and zero writing skips the sector range
+ * [@skip_start_sector, @skip_end_sector) passed in by caller, because caller
+ * has new data to write there.
+ *
+ * Returns: VMDK_OK if cluster exists and mapped in the image.
+ *          VMDK_UNALLOC if cluster is not mapped and @allocate is false.
+ *          VMDK_ERROR if failed.
+ */
 static int get_cluster_offset(BlockDriverState *bs,
-                                    VmdkExtent *extent,
-                                    VmdkMetaData *m_data,
-                                    uint64_t offset,
-                                    int allocate,
-                                    uint64_t *cluster_offset)
+                              VmdkExtent *extent,
+                              VmdkMetaData *m_data,
+                              uint64_t offset,
+                              bool allocate,
+                              uint64_t *cluster_offset,
+                              uint64_t skip_start_sector,
+                              uint64_t skip_end_sector)
 {
     unsigned int l1_index, l2_offset, l2_index;
     int min_index, i, j;
     uint32_t min_count, *l2_table;
     bool zeroed = false;
+    int64_t ret;
+    int64_t cluster_sector;
 
     if (m_data) {
         m_data->valid = 0;
@@ -1088,52 +1167,41 @@ static int get_cluster_offset(BlockDriverState *bs,
     extent->l2_cache_counts[min_index] = 1;
  found:
     l2_index = ((offset >> 9) / extent->cluster_sectors) % extent->l2_size;
-    *cluster_offset = le32_to_cpu(l2_table[l2_index]);
+    cluster_sector = le32_to_cpu(l2_table[l2_index]);
 
     if (m_data) {
         m_data->valid = 1;
         m_data->l1_index = l1_index;
         m_data->l2_index = l2_index;
-        m_data->offset = *cluster_offset;
         m_data->l2_offset = l2_offset;
         m_data->l2_cache_entry = &l2_table[l2_index];
     }
-    if (extent->has_zero_grain && *cluster_offset == VMDK_GTE_ZEROED) {
+    if (extent->has_zero_grain && cluster_sector == VMDK_GTE_ZEROED) {
         zeroed = true;
     }
 
-    if (!*cluster_offset || zeroed) {
+    if (!cluster_sector || zeroed) {
         if (!allocate) {
             return zeroed ? VMDK_ZEROED : VMDK_UNALLOC;
         }
 
-        /* Avoid the L2 tables update for the images that have snapshots. */
-        *cluster_offset = bdrv_getlength(extent->file);
-        if (!extent->compressed) {
-            bdrv_truncate(
-                extent->file,
-                *cluster_offset + (extent->cluster_sectors << 9)
-            );
-        }
-
-        *cluster_offset >>= 9;
-        l2_table[l2_index] = cpu_to_le32(*cluster_offset);
+        cluster_sector = extent->next_cluster_sector;
+        extent->next_cluster_sector += extent->cluster_sectors;
 
         /* First of all we write grain itself, to avoid race condition
          * that may to corrupt the image.
          * This problem may occur because of insufficient space on host disk
          * or inappropriate VM shutdown.
          */
-        if (get_whole_cluster(
-                bs, extent, *cluster_offset, offset, allocate) == -1) {
-            return VMDK_ERROR;
-        }
-
-        if (m_data) {
-            m_data->offset = *cluster_offset;
+        ret = get_whole_cluster(bs, extent,
+                                cluster_sector,
+                                offset >> BDRV_SECTOR_BITS,
+                                skip_start_sector, skip_end_sector);
+        if (ret) {
+            return ret;
         }
     }
-    *cluster_offset <<= 9;
+    *cluster_offset = cluster_sector << BDRV_SECTOR_BITS;
     return VMDK_OK;
 }
 
@@ -1168,7 +1236,8 @@ static int64_t coroutine_fn vmdk_co_get_block_status(BlockDriverState *bs,
     }
     qemu_co_mutex_lock(&s->lock);
     ret = get_cluster_offset(bs, extent, NULL,
-                            sector_num * 512, 0, &offset);
+                             sector_num * 512, false, &offset,
+                             0, 0);
     qemu_co_mutex_unlock(&s->lock);
 
     switch (ret) {
@@ -1321,9 +1390,9 @@ static int vmdk_read(BlockDriverState *bs, int64_t sector_num,
         if (!extent) {
             return -EIO;
         }
-        ret = get_cluster_offset(
-                            bs, extent, NULL,
-                            sector_num << 9, 0, &cluster_offset);
+        ret = get_cluster_offset(bs, extent, NULL,
+                                 sector_num << 9, false, &cluster_offset,
+                                 0, 0);
         extent_begin_sector = extent->end_sector - extent->sectors;
         extent_relative_sector_num = sector_num - extent_begin_sector;
         index_in_cluster = extent_relative_sector_num % extent->cluster_sectors;
@@ -1404,12 +1473,17 @@ static int vmdk_write(BlockDriverState *bs, int64_t sector_num,
         if (!extent) {
             return -EIO;
         }
-        ret = get_cluster_offset(
-                                bs,
-                                extent,
-                                &m_data,
-                                sector_num << 9, !extent->compressed,
-                                &cluster_offset);
+        extent_begin_sector = extent->end_sector - extent->sectors;
+        extent_relative_sector_num = sector_num - extent_begin_sector;
+        index_in_cluster = extent_relative_sector_num % extent->cluster_sectors;
+        n = extent->cluster_sectors - index_in_cluster;
+        if (n > nb_sectors) {
+            n = nb_sectors;
+        }
+        ret = get_cluster_offset(bs, extent, &m_data, sector_num << 9,
+                                 !(extent->compressed || zeroed),
+                                 &cluster_offset,
+                                 index_in_cluster, index_in_cluster + n);
         if (extent->compressed) {
             if (ret == VMDK_OK) {
                 /* Refuse write to allocated cluster for streamOptimized */
@@ -1418,24 +1492,13 @@ static int vmdk_write(BlockDriverState *bs, int64_t sector_num,
                 return -EIO;
             } else {
                 /* allocate */
-                ret = get_cluster_offset(
-                                        bs,
-                                        extent,
-                                        &m_data,
-                                        sector_num << 9, 1,
-                                        &cluster_offset);
+                ret = get_cluster_offset(bs, extent, &m_data, sector_num << 9,
+                                         true, &cluster_offset, 0, 0);
             }
         }
         if (ret == VMDK_ERROR) {
             return -EINVAL;
         }
-        extent_begin_sector = extent->end_sector - extent->sectors;
-        extent_relative_sector_num = sector_num - extent_begin_sector;
-        index_in_cluster = extent_relative_sector_num % extent->cluster_sectors;
-        n = extent->cluster_sectors - index_in_cluster;
-        if (n > nb_sectors) {
-            n = nb_sectors;
-        }
         if (zeroed) {
             /* Do zeroed write, buf is ignored */
             if (extent->has_zero_grain &&
@@ -1443,9 +1506,9 @@ static int vmdk_write(BlockDriverState *bs, int64_t sector_num,
                     n >= extent->cluster_sectors) {
                 n = extent->cluster_sectors;
                 if (!zero_dry_run) {
-                    m_data.offset = VMDK_GTE_ZEROED;
                     /* update L2 tables */
-                    if (vmdk_L2update(extent, &m_data) != VMDK_OK) {
+                    if (vmdk_L2update(extent, &m_data, VMDK_GTE_ZEROED)
+                            != VMDK_OK) {
                         return -EIO;
                     }
                 }
@@ -1461,7 +1524,9 @@ static int vmdk_write(BlockDriverState *bs, int64_t sector_num,
             }
             if (m_data.valid) {
                 /* update L2 tables */
-                if (vmdk_L2update(extent, &m_data) != VMDK_OK) {
+                if (vmdk_L2update(extent, &m_data,
+                                  cluster_offset >> BDRV_SECTOR_BITS)
+                        != VMDK_OK) {
                     return -EIO;
                 }
             }
@@ -1742,7 +1807,8 @@ static int vmdk_create(const char *filename, QemuOpts *opts, Error **errp)
         goto exit;
     }
     /* Read out options */
-    total_size = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0);
+    total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                          BDRV_SECTOR_SIZE);
     adapter_type = qemu_opt_get_del(opts, BLOCK_OPT_ADAPTER_TYPE);
     backing_file = qemu_opt_get_del(opts, BLOCK_OPT_BACKING_FILE);
     if (qemu_opt_get_bool_del(opts, BLOCK_OPT_COMPAT6, false)) {
@@ -1999,7 +2065,7 @@ static int vmdk_check(BlockDriverState *bs, BdrvCheckResult *result,
     BDRVVmdkState *s = bs->opaque;
     VmdkExtent *extent = NULL;
     int64_t sector_num = 0;
-    int64_t total_sectors = bdrv_getlength(bs) / BDRV_SECTOR_SIZE;
+    int64_t total_sectors = bdrv_nb_sectors(bs);
     int ret;
     uint64_t cluster_offset;
 
@@ -2020,7 +2086,7 @@ static int vmdk_check(BlockDriverState *bs, BdrvCheckResult *result,
         }
         ret = get_cluster_offset(bs, extent, NULL,
                                  sector_num << BDRV_SECTOR_BITS,
-                                 0, &cluster_offset);
+                                 false, &cluster_offset, 0, 0);
         if (ret == VMDK_ERROR) {
             fprintf(stderr,
                     "ERROR: could not get cluster_offset for sector %"
@@ -2071,23 +2137,29 @@ static ImageInfoSpecific *vmdk_get_specific_info(BlockDriverState *bs)
     return spec_info;
 }
 
+static bool vmdk_extents_type_eq(const VmdkExtent *a, const VmdkExtent *b)
+{
+    return a->flat == b->flat &&
+           a->compressed == b->compressed &&
+           (a->flat || a->cluster_sectors == b->cluster_sectors);
+}
+
 static int vmdk_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
 {
     int i;
     BDRVVmdkState *s = bs->opaque;
     assert(s->num_extents);
-    bdi->needs_compressed_writes = s->extents[0].compressed;
-    if (!s->extents[0].flat) {
-        bdi->cluster_size = s->extents[0].cluster_sectors << BDRV_SECTOR_BITS;
-    }
+
     /* See if we have multiple extents but they have different cases */
     for (i = 1; i < s->num_extents; i++) {
-        if (bdi->needs_compressed_writes != s->extents[i].compressed ||
-            (bdi->cluster_size && bdi->cluster_size !=
-                s->extents[i].cluster_sectors << BDRV_SECTOR_BITS)) {
+        if (!vmdk_extents_type_eq(&s->extents[0], &s->extents[i])) {
             return -ENOTSUP;
         }
     }
+    bdi->needs_compressed_writes = s->extents[0].compressed;
+    if (!s->extents[0].flat) {
+        bdi->cluster_size = s->extents[0].cluster_sectors << BDRV_SECTOR_BITS;
+    }
     return 0;
 }
 
diff --git a/block/vpc.c b/block/vpc.c
index 8b376a40b..38c4f02ff 100644
--- a/block/vpc.c
+++ b/block/vpc.c
@@ -29,13 +29,6 @@
 #if defined(CONFIG_UUID)
 #include <uuid/uuid.h>
 #endif
-#ifdef __linux__
-#include <linux/fs.h>
-#include <sys/ioctl.h>
-#ifndef FS_NOCOW_FL
-#define FS_NOCOW_FL                     0x00800000 /* Do not cow file */
-#endif
-#endif
 
 /**************************************************************/
 
@@ -214,7 +207,7 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
             "incorrect.\n", bs->filename);
 
     /* Write 'checksum' back to footer, or else will leave it with zero. */
-    footer->checksum = be32_to_cpu(checksum);
+    footer->checksum = cpu_to_be32(checksum);
 
     // The visible size of a image in Virtual PC depends on the geometry
     // rather than on the size stored in the footer (the size in the footer
@@ -276,7 +269,11 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
             goto fail;
         }
 
-        s->pagetable = qemu_blockalign(bs, s->max_table_entries * 4);
+        s->pagetable = qemu_try_blockalign(bs->file, s->max_table_entries * 4);
+        if (s->pagetable == NULL) {
+            ret = -ENOMEM;
+            goto fail;
+        }
 
         s->bat_offset = be64_to_cpu(dyndisk_header->table_offset);
 
@@ -323,7 +320,7 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
     /* Disable migration when VHD images are used */
     error_set(&s->migration_blocker,
               QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-              "vpc", bs->device_name, "live migration");
+              "vpc", bdrv_get_device_name(bs), "live migration");
     migrate_add_blocker(s->migration_blocker);
 
     return 0;
@@ -475,7 +472,7 @@ static int64_t alloc_block(BlockDriverState* bs, int64_t sector_num)
 
     // Write BAT entry to disk
     bat_offset = s->bat_offset + (4 * index);
-    bat_value = be32_to_cpu(s->pagetable[index]);
+    bat_value = cpu_to_be32(s->pagetable[index]);
     ret = bdrv_pwrite_sync(bs->file, bat_offset, &bat_value, 4);
     if (ret < 0)
         goto fail;
@@ -492,7 +489,7 @@ static int vpc_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
     BDRVVPCState *s = (BDRVVPCState *)bs->opaque;
     VHDFooter *footer = (VHDFooter *) s->footer_buf;
 
-    if (cpu_to_be32(footer->type) != VHD_FIXED) {
+    if (be32_to_cpu(footer->type) != VHD_FIXED) {
         bdi->cluster_size = s->block_size;
     }
 
@@ -509,7 +506,7 @@ static int vpc_read(BlockDriverState *bs, int64_t sector_num,
     int64_t sectors, sectors_per_block;
     VHDFooter *footer = (VHDFooter *) s->footer_buf;
 
-    if (cpu_to_be32(footer->type) == VHD_FIXED) {
+    if (be32_to_cpu(footer->type) == VHD_FIXED) {
         return bdrv_read(bs->file, sector_num, buf, nb_sectors);
     }
     while (nb_sectors > 0) {
@@ -558,7 +555,7 @@ static int vpc_write(BlockDriverState *bs, int64_t sector_num,
     int ret;
     VHDFooter *footer =  (VHDFooter *) s->footer_buf;
 
-    if (cpu_to_be32(footer->type) == VHD_FIXED) {
+    if (be32_to_cpu(footer->type) == VHD_FIXED) {
         return bdrv_write(bs->file, sector_num, buf, nb_sectors);
     }
     while (nb_sectors > 0) {
@@ -656,39 +653,41 @@ static int calculate_geometry(int64_t total_sectors, uint16_t* cyls,
     return 0;
 }
 
-static int create_dynamic_disk(int fd, uint8_t *buf, int64_t total_sectors)
+static int create_dynamic_disk(BlockDriverState *bs, uint8_t *buf,
+                               int64_t total_sectors)
 {
     VHDDynDiskHeader *dyndisk_header =
         (VHDDynDiskHeader *) buf;
     size_t block_size, num_bat_entries;
     int i;
-    int ret = -EIO;
+    int ret;
+    int64_t offset = 0;
 
     // Write the footer (twice: at the beginning and at the end)
     block_size = 0x200000;
     num_bat_entries = (total_sectors + block_size / 512) / (block_size / 512);
 
-    if (write(fd, buf, HEADER_SIZE) != HEADER_SIZE) {
+    ret = bdrv_pwrite_sync(bs, offset, buf, HEADER_SIZE);
+    if (ret) {
         goto fail;
     }
 
-    if (lseek(fd, 1536 + ((num_bat_entries * 4 + 511) & ~511), SEEK_SET) < 0) {
-        goto fail;
-    }
-    if (write(fd, buf, HEADER_SIZE) != HEADER_SIZE) {
+    offset = 1536 + ((num_bat_entries * 4 + 511) & ~511);
+    ret = bdrv_pwrite_sync(bs, offset, buf, HEADER_SIZE);
+    if (ret < 0) {
         goto fail;
     }
 
     // Write the initial BAT
-    if (lseek(fd, 3 * 512, SEEK_SET) < 0) {
-        goto fail;
-    }
+    offset = 3 * 512;
 
     memset(buf, 0xFF, 512);
     for (i = 0; i < (num_bat_entries * 4 + 511) / 512; i++) {
-        if (write(fd, buf, 512) != 512) {
+        ret = bdrv_pwrite_sync(bs, offset, buf, 512);
+        if (ret < 0) {
             goto fail;
         }
+        offset += 512;
     }
 
     // Prepare the Dynamic Disk Header
@@ -700,48 +699,44 @@ static int create_dynamic_disk(int fd, uint8_t *buf, int64_t total_sectors)
      * Note: The spec is actually wrong here for data_offset, it says
      * 0xFFFFFFFF, but MS tools expect all 64 bits to be set.
      */
-    dyndisk_header->data_offset = be64_to_cpu(0xFFFFFFFFFFFFFFFFULL);
-    dyndisk_header->table_offset = be64_to_cpu(3 * 512);
-    dyndisk_header->version = be32_to_cpu(0x00010000);
-    dyndisk_header->block_size = be32_to_cpu(block_size);
-    dyndisk_header->max_table_entries = be32_to_cpu(num_bat_entries);
+    dyndisk_header->data_offset = cpu_to_be64(0xFFFFFFFFFFFFFFFFULL);
+    dyndisk_header->table_offset = cpu_to_be64(3 * 512);
+    dyndisk_header->version = cpu_to_be32(0x00010000);
+    dyndisk_header->block_size = cpu_to_be32(block_size);
+    dyndisk_header->max_table_entries = cpu_to_be32(num_bat_entries);
 
-    dyndisk_header->checksum = be32_to_cpu(vpc_checksum(buf, 1024));
+    dyndisk_header->checksum = cpu_to_be32(vpc_checksum(buf, 1024));
 
     // Write the header
-    if (lseek(fd, 512, SEEK_SET) < 0) {
-        goto fail;
-    }
+    offset = 512;
 
-    if (write(fd, buf, 1024) != 1024) {
+    ret = bdrv_pwrite_sync(bs, offset, buf, 1024);
+    if (ret < 0) {
         goto fail;
     }
-    ret = 0;
 
  fail:
     return ret;
 }
 
-static int create_fixed_disk(int fd, uint8_t *buf, int64_t total_size)
+static int create_fixed_disk(BlockDriverState *bs, uint8_t *buf,
+                             int64_t total_size)
 {
-    int ret = -EIO;
+    int ret;
 
     /* Add footer to total size */
-    total_size += 512;
-    if (ftruncate(fd, total_size) != 0) {
-        ret = -errno;
-        goto fail;
-    }
-    if (lseek(fd, -512, SEEK_END) < 0) {
-        goto fail;
-    }
-    if (write(fd, buf, HEADER_SIZE) != HEADER_SIZE) {
-        goto fail;
+    total_size += HEADER_SIZE;
+
+    ret = bdrv_truncate(bs, total_size);
+    if (ret < 0) {
+        return ret;
     }
 
-    ret = 0;
+    ret = bdrv_pwrite_sync(bs, total_size - HEADER_SIZE, buf, HEADER_SIZE);
+    if (ret < 0) {
+        return ret;
+    }
 
- fail:
     return ret;
 }
 
@@ -750,7 +745,7 @@ static int vpc_create(const char *filename, QemuOpts *opts, Error **errp)
     uint8_t buf[1024];
     VHDFooter *footer = (VHDFooter *) buf;
     char *disk_type_param;
-    int fd, i;
+    int i;
     uint16_t cyls = 0;
     uint8_t heads = 0;
     uint8_t secs_per_cyl = 0;
@@ -758,10 +753,12 @@ static int vpc_create(const char *filename, QemuOpts *opts, Error **errp)
     int64_t total_size;
     int disk_type;
     int ret = -EIO;
-    bool nocow = false;
+    Error *local_err = NULL;
+    BlockDriverState *bs = NULL;
 
     /* Read out options */
-    total_size = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0);
+    total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                          BDRV_SECTOR_SIZE);
     disk_type_param = qemu_opt_get_del(opts, BLOCK_OPT_SUBFMT);
     if (disk_type_param) {
         if (!strcmp(disk_type_param, "dynamic")) {
@@ -775,28 +772,17 @@ static int vpc_create(const char *filename, QemuOpts *opts, Error **errp)
     } else {
         disk_type = VHD_DYNAMIC;
     }
-    nocow = qemu_opt_get_bool_del(opts, BLOCK_OPT_NOCOW, false);
 
-    /* Create the file */
-    fd = qemu_open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644);
-    if (fd < 0) {
-        ret = -EIO;
+    ret = bdrv_create_file(filename, opts, &local_err);
+    if (ret < 0) {
+        error_propagate(errp, local_err);
         goto out;
     }
-
-    if (nocow) {
-#ifdef __linux__
-        /* Set NOCOW flag to solve performance issue on fs like btrfs.
-         * This is an optimisation. The FS_IOC_SETFLAGS ioctl return value will
-         * be ignored since any failure of this operation should not block the
-         * left work.
-         */
-        int attr;
-        if (ioctl(fd, FS_IOC_GETFLAGS, &attr) == 0) {
-            attr |= FS_NOCOW_FL;
-            ioctl(fd, FS_IOC_SETFLAGS, &attr);
-        }
-#endif
+    ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
+                    NULL, &local_err);
+    if (ret < 0) {
+        error_propagate(errp, local_err);
+        goto out;
     }
 
     /*
@@ -810,7 +796,7 @@ static int vpc_create(const char *filename, QemuOpts *opts, Error **errp)
                                &secs_per_cyl))
         {
             ret = -EFBIG;
-            goto fail;
+            goto out;
         }
     }
 
@@ -824,46 +810,45 @@ static int vpc_create(const char *filename, QemuOpts *opts, Error **errp)
     memcpy(footer->creator_app, "qemu", 4);
     memcpy(footer->creator_os, "Wi2k", 4);
 
-    footer->features = be32_to_cpu(0x02);
-    footer->version = be32_to_cpu(0x00010000);
+    footer->features = cpu_to_be32(0x02);
+    footer->version = cpu_to_be32(0x00010000);
     if (disk_type == VHD_DYNAMIC) {
-        footer->data_offset = be64_to_cpu(HEADER_SIZE);
+        footer->data_offset = cpu_to_be64(HEADER_SIZE);
     } else {
-        footer->data_offset = be64_to_cpu(0xFFFFFFFFFFFFFFFFULL);
+        footer->data_offset = cpu_to_be64(0xFFFFFFFFFFFFFFFFULL);
     }
-    footer->timestamp = be32_to_cpu(time(NULL) - VHD_TIMESTAMP_BASE);
+    footer->timestamp = cpu_to_be32(time(NULL) - VHD_TIMESTAMP_BASE);
 
     /* Version of Virtual PC 2007 */
-    footer->major = be16_to_cpu(0x0005);
-    footer->minor = be16_to_cpu(0x0003);
+    footer->major = cpu_to_be16(0x0005);
+    footer->minor = cpu_to_be16(0x0003);
     if (disk_type == VHD_DYNAMIC) {
-        footer->orig_size = be64_to_cpu(total_sectors * 512);
-        footer->size = be64_to_cpu(total_sectors * 512);
+        footer->orig_size = cpu_to_be64(total_sectors * 512);
+        footer->size = cpu_to_be64(total_sectors * 512);
     } else {
-        footer->orig_size = be64_to_cpu(total_size);
-        footer->size = be64_to_cpu(total_size);
+        footer->orig_size = cpu_to_be64(total_size);
+        footer->size = cpu_to_be64(total_size);
     }
-    footer->cyls = be16_to_cpu(cyls);
+    footer->cyls = cpu_to_be16(cyls);
     footer->heads = heads;
     footer->secs_per_cyl = secs_per_cyl;
 
-    footer->type = be32_to_cpu(disk_type);
+    footer->type = cpu_to_be32(disk_type);
 
 #if defined(CONFIG_UUID)
     uuid_generate(footer->uuid);
 #endif
 
-    footer->checksum = be32_to_cpu(vpc_checksum(buf, HEADER_SIZE));
+    footer->checksum = cpu_to_be32(vpc_checksum(buf, HEADER_SIZE));
 
     if (disk_type == VHD_DYNAMIC) {
-        ret = create_dynamic_disk(fd, buf, total_sectors);
+        ret = create_dynamic_disk(bs, buf, total_sectors);
     } else {
-        ret = create_fixed_disk(fd, buf, total_size);
+        ret = create_fixed_disk(bs, buf, total_size);
     }
 
-fail:
-    qemu_close(fd);
 out:
+    bdrv_unref(bs);
     g_free(disk_type_param);
     return ret;
 }
@@ -873,7 +858,7 @@ static int vpc_has_zero_init(BlockDriverState *bs)
     BDRVVPCState *s = bs->opaque;
     VHDFooter *footer =  (VHDFooter *) s->footer_buf;
 
-    if (cpu_to_be32(footer->type) == VHD_FIXED) {
+    if (be32_to_cpu(footer->type) == VHD_FIXED) {
         return bdrv_has_zero_init(bs->file);
     } else {
         return 1;
diff --git a/block/vvfat.c b/block/vvfat.c
index 70176b161..cefe3a4f7 100644
--- a/block/vvfat.c
+++ b/block/vvfat.c
@@ -52,10 +52,6 @@
 
 #define DLOG(a) a
 
-#undef stderr
-#define stderr STDERR
-FILE* stderr = NULL;
-
 static void checkpoint(void);
 
 #ifdef __MINGW32__
@@ -732,7 +728,7 @@ static int read_directory(BDRVVVFATState* s, int mapping_index)
 	if(first_cluster == 0 && (is_dotdot || is_dot))
 	    continue;
 
-	buffer=(char*)g_malloc(length);
+	buffer = g_malloc(length);
 	snprintf(buffer,length,"%s/%s",dirname,entry->d_name);
 
 	if(stat(buffer,&st)<0) {
@@ -767,7 +763,7 @@ static int read_directory(BDRVVVFATState* s, int mapping_index)
 
 	/* create mapping for this file */
 	if(!is_dot && !is_dotdot && (S_ISDIR(st.st_mode) || st.st_size)) {
-	    s->current_mapping=(mapping_t*)array_get_next(&(s->mapping));
+	    s->current_mapping = array_get_next(&(s->mapping));
 	    s->current_mapping->begin=0;
 	    s->current_mapping->end=st.st_size;
 	    /*
@@ -811,12 +807,12 @@ static int read_directory(BDRVVVFATState* s, int mapping_index)
     }
 
      /* reget the mapping, since s->mapping was possibly realloc()ed */
-    mapping = (mapping_t*)array_get(&(s->mapping), mapping_index);
+    mapping = array_get(&(s->mapping), mapping_index);
     first_cluster += (s->directory.next - mapping->info.dir.first_dir_index)
 	* 0x20 / s->cluster_size;
     mapping->end = first_cluster;
 
-    direntry = (direntry_t*)array_get(&(s->directory), mapping->dir_index);
+    direntry = array_get(&(s->directory), mapping->dir_index);
     set_begin_of_direntry(direntry, mapping->begin);
 
     return 0;
@@ -1082,11 +1078,6 @@ static int vvfat_open(BlockDriverState *bs, QDict *options, int flags,
     vvv = s;
 #endif
 
-DLOG(if (stderr == NULL) {
-    stderr = fopen("vvfat.log", "a");
-    setbuf(stderr, NULL);
-})
-
     opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
     qemu_opts_absorb_qdict(opts, options, &local_err);
     if (local_err) {
@@ -1191,7 +1182,7 @@ DLOG(if (stderr == NULL) {
     if (s->qcow) {
         error_set(&s->migration_blocker,
                   QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-                  "vvfat (rw)", bs->device_name, "live migration");
+                  "vvfat (rw)", bdrv_get_device_name(bs), "live migration");
         migrate_add_blocker(s->migration_blocker);
     }
 
@@ -2948,9 +2939,9 @@ static int enable_write_target(BDRVVVFATState *s, Error **errp)
     unlink(s->qcow_filename);
 #endif
 
-    bdrv_set_backing_hd(s->bs, bdrv_new("", &error_abort));
+    bdrv_set_backing_hd(s->bs, bdrv_new());
     s->bs->backing_hd->drv = &vvfat_write_target;
-    s->bs->backing_hd->opaque = g_malloc(sizeof(void*));
+    s->bs->backing_hd->opaque = g_new(void *, 1);
     *(void**)s->bs->backing_hd->opaque = s;
 
     return 0;
diff --git a/block/win32-aio.c b/block/win32-aio.c
index 8e417f70a..64e86827b 100644
--- a/block/win32-aio.c
+++ b/block/win32-aio.c
@@ -44,7 +44,7 @@ struct QEMUWin32AIOState {
 };
 
 typedef struct QEMUWin32AIOCB {
-    BlockDriverAIOCB common;
+    BlockAIOCB common;
     struct QEMUWin32AIOState *ctx;
     int nbytes;
     OVERLAPPED ov;
@@ -88,7 +88,7 @@ static void win32_aio_process_completion(QEMUWin32AIOState *s,
 
 
     waiocb->common.cb(waiocb->common.opaque, ret);
-    qemu_aio_release(waiocb);
+    qemu_aio_unref(waiocb);
 }
 
 static void win32_aio_completion_cb(EventNotifier *e)
@@ -106,28 +106,14 @@ static void win32_aio_completion_cb(EventNotifier *e)
     }
 }
 
-static void win32_aio_cancel(BlockDriverAIOCB *blockacb)
-{
-    QEMUWin32AIOCB *waiocb = (QEMUWin32AIOCB *)blockacb;
-
-    /*
-     * CancelIoEx is only supported in Vista and newer.  For now, just
-     * wait for completion.
-     */
-    while (!HasOverlappedIoCompleted(&waiocb->ov)) {
-        aio_poll(bdrv_get_aio_context(blockacb->bs), true);
-    }
-}
-
 static const AIOCBInfo win32_aiocb_info = {
     .aiocb_size         = sizeof(QEMUWin32AIOCB),
-    .cancel             = win32_aio_cancel,
 };
 
-BlockDriverAIOCB *win32_aio_submit(BlockDriverState *bs,
+BlockAIOCB *win32_aio_submit(BlockDriverState *bs,
         QEMUWin32AIOState *aio, HANDLE hfile,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque, int type)
+        BlockCompletionFunc *cb, void *opaque, int type)
 {
     struct QEMUWin32AIOCB *waiocb;
     uint64_t offset = sector_num * 512;
@@ -139,7 +125,10 @@ BlockDriverAIOCB *win32_aio_submit(BlockDriverState *bs,
     waiocb->is_read = (type == QEMU_AIO_READ);
 
     if (qiov->niov > 1) {
-        waiocb->buf = qemu_blockalign(bs, qiov->size);
+        waiocb->buf = qemu_try_blockalign(bs, qiov->size);
+        if (waiocb->buf == NULL) {
+            goto out;
+        }
         if (type & QEMU_AIO_WRITE) {
             iov_to_buf(qiov->iov, qiov->niov, 0, waiocb->buf, qiov->size);
         }
@@ -168,7 +157,8 @@ BlockDriverAIOCB *win32_aio_submit(BlockDriverState *bs,
 
 out_dec_count:
     aio->count--;
-    qemu_aio_release(waiocb);
+out:
+    qemu_aio_unref(waiocb);
     return NULL;
 }
 
diff --git a/blockdev-nbd.c b/blockdev-nbd.c
index b3a24740b..06f901ef6 100644
--- a/blockdev-nbd.c
+++ b/blockdev-nbd.c
@@ -108,7 +108,7 @@ void qmp_nbd_server_add(const char *device, bool has_writable, bool writable,
 
     nbd_export_set_name(exp, device);
 
-    n = g_malloc0(sizeof(NBDCloseNotifier));
+    n = g_new0(NBDCloseNotifier, 1);
     n->n.notify = nbd_close_notifier;
     n->exp = exp;
     bdrv_add_close_notifier(bs, &n->n);
diff --git a/blockdev.c b/blockdev.c
index 48bd9a37b..57910b82c 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -30,6 +30,7 @@
  * THE SOFTWARE.
  */
 
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "hw/block/block.h"
 #include "block/blockjob.h"
@@ -39,14 +40,13 @@
 #include "qapi/qmp/types.h"
 #include "qapi-visit.h"
 #include "qapi/qmp-output-visitor.h"
+#include "qapi/util.h"
 #include "sysemu/sysemu.h"
 #include "block/block_int.h"
 #include "qmp-commands.h"
 #include "trace.h"
 #include "sysemu/arch_init.h"
 
-static QTAILQ_HEAD(drivelist, DriveInfo) drives = QTAILQ_HEAD_INITIALIZER(drives);
-
 static const char *const if_name[IF_COUNT] = {
     [IF_NONE] = "none",
     [IF_IDE] = "ide",
@@ -59,7 +59,7 @@ static const char *const if_name[IF_COUNT] = {
     [IF_XEN] = "xen",
 };
 
-static const int if_max_devs[IF_COUNT] = {
+static int if_max_devs[IF_COUNT] = {
     /*
      * Do not change these numbers!  They govern how drive option
      * index maps to unit and bus.  That mapping is ABI.
@@ -78,6 +78,32 @@ static const int if_max_devs[IF_COUNT] = {
     [IF_SCSI] = 7,
 };
 
+/**
+ * Boards may call this to offer board-by-board overrides
+ * of the default, global values.
+ */
+void override_max_devs(BlockInterfaceType type, int max_devs)
+{
+    BlockBackend *blk;
+    DriveInfo *dinfo;
+
+    if (max_devs <= 0) {
+        return;
+    }
+
+    for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
+        dinfo = blk_legacy_dinfo(blk);
+        if (dinfo->type == type) {
+            fprintf(stderr, "Cannot override units-per-bus property of"
+                    " the %s interface, because a drive of that type has"
+                    " already been added.\n", if_name[type]);
+            g_assert_not_reached();
+        }
+    }
+
+    if_max_devs[type] = max_devs;
+}
+
 /*
  * We automatically delete the drive when a device using it gets
  * unplugged.  Questionable feature, but we can't just drop it.
@@ -85,31 +111,54 @@ static const int if_max_devs[IF_COUNT] = {
  * automatic deletion, and generic qdev code calls blockdev_auto_del()
  * when deletion is actually safe.
  */
-void blockdev_mark_auto_del(BlockDriverState *bs)
+void blockdev_mark_auto_del(BlockBackend *blk)
 {
-    DriveInfo *dinfo = drive_get_by_blockdev(bs);
+    DriveInfo *dinfo = blk_legacy_dinfo(blk);
+    BlockDriverState *bs = blk_bs(blk);
+    AioContext *aio_context;
 
-    if (dinfo && !dinfo->enable_auto_del) {
+    if (!dinfo) {
         return;
     }
 
+    aio_context = bdrv_get_aio_context(bs);
+    aio_context_acquire(aio_context);
+
     if (bs->job) {
         block_job_cancel(bs->job);
     }
-    if (dinfo) {
-        dinfo->auto_del = 1;
-    }
+
+    aio_context_release(aio_context);
+
+    dinfo->auto_del = 1;
 }
 
-void blockdev_auto_del(BlockDriverState *bs)
+void blockdev_auto_del(BlockBackend *blk)
 {
-    DriveInfo *dinfo = drive_get_by_blockdev(bs);
+    DriveInfo *dinfo = blk_legacy_dinfo(blk);
 
     if (dinfo && dinfo->auto_del) {
-        drive_del(dinfo);
+        blk_unref(blk);
     }
 }
 
+/**
+ * Returns the current mapping of how many units per bus
+ * a particular interface can support.
+ *
+ *  A positive integer indicates n units per bus.
+ *  0 implies the mapping has not been established.
+ * -1 indicates an invalid BlockInterfaceType was given.
+ */
+int drive_get_max_devs(BlockInterfaceType type)
+{
+    if (type >= IF_IDE && type < IF_COUNT) {
+        return if_max_devs[type];
+    }
+
+    return -1;
+}
+
 static int drive_index_to_bus_id(BlockInterfaceType type, int index)
 {
     int max_devs = if_max_devs[type];
@@ -151,20 +200,43 @@ QemuOpts *drive_add(BlockInterfaceType type, int index, const char *file,
 
 DriveInfo *drive_get(BlockInterfaceType type, int bus, int unit)
 {
+    BlockBackend *blk;
     DriveInfo *dinfo;
 
-    /* seek interface, bus and unit */
-
-    QTAILQ_FOREACH(dinfo, &drives, next) {
-        if (dinfo->type == type &&
-	    dinfo->bus == bus &&
-	    dinfo->unit == unit)
+    for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
+        dinfo = blk_legacy_dinfo(blk);
+        if (dinfo && dinfo->type == type
+            && dinfo->bus == bus && dinfo->unit == unit) {
             return dinfo;
+        }
     }
 
     return NULL;
 }
 
+bool drive_check_orphaned(void)
+{
+    BlockBackend *blk;
+    DriveInfo *dinfo;
+    bool rs = false;
+
+    for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
+        dinfo = blk_legacy_dinfo(blk);
+        /* If dinfo->bdrv->dev is NULL, it has no device attached. */
+        /* Unless this is a default drive, this may be an oversight. */
+        if (!blk_get_attached_dev(blk) && !dinfo->is_default &&
+            dinfo->type != IF_NONE) {
+            fprintf(stderr, "Warning: Orphaned drive without device: "
+                    "id=%s,file=%s,if=%s,bus=%d,unit=%d\n",
+                    blk_name(blk), blk_bs(blk)->filename, if_name[dinfo->type],
+                    dinfo->bus, dinfo->unit);
+            rs = true;
+        }
+    }
+
+    return rs;
+}
+
 DriveInfo *drive_get_by_index(BlockInterfaceType type, int index)
 {
     return drive_get(type,
@@ -175,13 +247,15 @@ DriveInfo *drive_get_by_index(BlockInterfaceType type, int index)
 int drive_get_max_bus(BlockInterfaceType type)
 {
     int max_bus;
+    BlockBackend *blk;
     DriveInfo *dinfo;
 
     max_bus = -1;
-    QTAILQ_FOREACH(dinfo, &drives, next) {
-        if(dinfo->type == type &&
-           dinfo->bus > max_bus)
+    for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
+        dinfo = blk_legacy_dinfo(blk);
+        if (dinfo && dinfo->type == type && dinfo->bus > max_bus) {
             max_bus = dinfo->bus;
+        }
     }
     return max_bus;
 }
@@ -196,36 +270,11 @@ DriveInfo *drive_get_next(BlockInterfaceType type)
     return drive_get(type, 0, next_block_unit[type]++);
 }
 
-DriveInfo *drive_get_by_blockdev(BlockDriverState *bs)
-{
-    DriveInfo *dinfo;
-
-    QTAILQ_FOREACH(dinfo, &drives, next) {
-        if (dinfo->bdrv == bs) {
-            return dinfo;
-        }
-    }
-    return NULL;
-}
-
 static void bdrv_format_print(void *opaque, const char *name)
 {
     error_printf(" %s", name);
 }
 
-void drive_del(DriveInfo *dinfo)
-{
-    if (dinfo->opts) {
-        qemu_opts_del(dinfo->opts);
-    }
-
-    bdrv_unref(dinfo->bdrv);
-    g_free(dinfo->id);
-    QTAILQ_REMOVE(&drives, dinfo, next);
-    g_free(dinfo->serial);
-    g_free(dinfo);
-}
-
 typedef struct {
     QEMUBH *bh;
     BlockDriverState *bs;
@@ -274,25 +323,6 @@ static int parse_block_error_action(const char *buf, bool is_read, Error **errp)
     }
 }
 
-static inline int parse_enum_option(const char *lookup[], const char *buf,
-                                    int max, int def, Error **errp)
-{
-    int i;
-
-    if (!buf) {
-        return def;
-    }
-
-    for (i = 0; i < max; i++) {
-        if (!strcmp(buf, lookup[i])) {
-            return i;
-        }
-    }
-
-    error_setg(errp, "invalid parameter value: %s", buf);
-    return def;
-}
-
 static bool check_throttle_config(ThrottleConfig *cfg, Error **errp)
 {
     if (throttle_conflicting(cfg)) {
@@ -312,14 +342,15 @@ static bool check_throttle_config(ThrottleConfig *cfg, Error **errp)
 typedef enum { MEDIA_DISK, MEDIA_CDROM } DriveMediaType;
 
 /* Takes the ownership of bs_opts */
-static DriveInfo *blockdev_init(const char *file, QDict *bs_opts,
-                                Error **errp)
+static BlockBackend *blockdev_init(const char *file, QDict *bs_opts,
+                                   Error **errp)
 {
     const char *buf;
     int ro = 0;
     int bdrv_flags = 0;
     int on_read_error, on_write_error;
-    DriveInfo *dinfo;
+    BlockBackend *blk;
+    BlockDriverState *bs;
     ThrottleConfig cfg;
     int snapshot = 0;
     bool copy_on_read;
@@ -456,11 +487,11 @@ static DriveInfo *blockdev_init(const char *file, QDict *bs_opts,
     }
 
     detect_zeroes =
-        parse_enum_option(BlockdevDetectZeroesOptions_lookup,
-                          qemu_opt_get(opts, "detect-zeroes"),
-                          BLOCKDEV_DETECT_ZEROES_OPTIONS_MAX,
-                          BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF,
-                          &error);
+        qapi_enum_parse(BlockdevDetectZeroesOptions_lookup,
+                        qemu_opt_get(opts, "detect-zeroes"),
+                        BLOCKDEV_DETECT_ZEROES_OPTIONS_MAX,
+                        BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF,
+                        &error);
     if (error) {
         error_propagate(errp, error);
         goto early_err;
@@ -474,24 +505,21 @@ static DriveInfo *blockdev_init(const char *file, QDict *bs_opts,
     }
 
     /* init */
-    dinfo = g_malloc0(sizeof(*dinfo));
-    dinfo->id = g_strdup(qemu_opts_id(opts));
-    dinfo->bdrv = bdrv_new(dinfo->id, &error);
-    if (error) {
-        error_propagate(errp, error);
-        goto bdrv_new_err;
+    blk = blk_new_with_bs(qemu_opts_id(opts), errp);
+    if (!blk) {
+        goto early_err;
     }
-    dinfo->bdrv->open_flags = snapshot ? BDRV_O_SNAPSHOT : 0;
-    dinfo->bdrv->read_only = ro;
-    dinfo->bdrv->detect_zeroes = detect_zeroes;
-    QTAILQ_INSERT_TAIL(&drives, dinfo, next);
+    bs = blk_bs(blk);
+    bs->open_flags = snapshot ? BDRV_O_SNAPSHOT : 0;
+    bs->read_only = ro;
+    bs->detect_zeroes = detect_zeroes;
 
-    bdrv_set_on_error(dinfo->bdrv, on_read_error, on_write_error);
+    bdrv_set_on_error(bs, on_read_error, on_write_error);
 
     /* disk I/O throttling */
     if (throttle_enabled(&cfg)) {
-        bdrv_io_limits_enable(dinfo->bdrv);
-        bdrv_set_io_limits(dinfo->bdrv, &cfg);
+        bdrv_io_limits_enable(bs);
+        bdrv_set_io_limits(bs, &cfg);
     }
 
     if (!file || !*file) {
@@ -500,7 +528,7 @@ static DriveInfo *blockdev_init(const char *file, QDict *bs_opts,
         } else {
             QDECREF(bs_opts);
             qemu_opts_del(opts);
-            return dinfo;
+            return blk;
         }
     }
     if (snapshot) {
@@ -520,29 +548,27 @@ static DriveInfo *blockdev_init(const char *file, QDict *bs_opts,
     bdrv_flags |= ro ? 0 : BDRV_O_RDWR;
 
     QINCREF(bs_opts);
-    ret = bdrv_open(&dinfo->bdrv, file, NULL, bs_opts, bdrv_flags, drv, &error);
+    ret = bdrv_open(&bs, file, NULL, bs_opts, bdrv_flags, drv, &error);
+    assert(bs == blk_bs(blk));
 
     if (ret < 0) {
         error_setg(errp, "could not open disk image %s: %s",
-                   file ?: dinfo->id, error_get_pretty(error));
+                   file ?: blk_name(blk), error_get_pretty(error));
         error_free(error);
         goto err;
     }
 
-    if (bdrv_key_required(dinfo->bdrv))
+    if (bdrv_key_required(bs)) {
         autostart = 0;
+    }
 
     QDECREF(bs_opts);
     qemu_opts_del(opts);
 
-    return dinfo;
+    return blk;
 
 err:
-    bdrv_unref(dinfo->bdrv);
-    QTAILQ_REMOVE(&drives, dinfo, next);
-bdrv_new_err:
-    g_free(dinfo->id);
-    g_free(dinfo);
+    blk_unref(blk);
 early_err:
     qemu_opts_del(opts);
 err_no_opts:
@@ -550,12 +576,22 @@ err_no_opts:
     return NULL;
 }
 
-static void qemu_opt_rename(QemuOpts *opts, const char *from, const char *to)
+static void qemu_opt_rename(QemuOpts *opts, const char *from, const char *to,
+                            Error **errp)
 {
     const char *value;
 
     value = qemu_opt_get(opts, from);
     if (value) {
+        if (qemu_opt_find(opts, to)) {
+            error_setg(errp, "'%s' and its alias '%s' can't be used at the "
+                       "same time", to, from);
+            return;
+        }
+    }
+
+    /* rename all items in opts */
+    while ((value = qemu_opt_get(opts, from))) {
         qemu_opt_set(opts, to, value);
         qemu_opt_unset(opts, from);
     }
@@ -645,6 +681,7 @@ QemuOptsList qemu_legacy_drive_opts = {
 DriveInfo *drive_new(QemuOpts *all_opts, BlockInterfaceType block_default_type)
 {
     const char *value;
+    BlockBackend *blk;
     DriveInfo *dinfo = NULL;
     QDict *bs_opts;
     QemuOpts *legacy_opts;
@@ -659,28 +696,43 @@ DriveInfo *drive_new(QemuOpts *all_opts, BlockInterfaceType block_default_type)
     const char *serial;
     const char *filename;
     Error *local_err = NULL;
+    int i;
 
     /* Change legacy command line options into QMP ones */
-    qemu_opt_rename(all_opts, "iops", "throttling.iops-total");
-    qemu_opt_rename(all_opts, "iops_rd", "throttling.iops-read");
-    qemu_opt_rename(all_opts, "iops_wr", "throttling.iops-write");
+    static const struct {
+        const char *from;
+        const char *to;
+    } opt_renames[] = {
+        { "iops",           "throttling.iops-total" },
+        { "iops_rd",        "throttling.iops-read" },
+        { "iops_wr",        "throttling.iops-write" },
 
-    qemu_opt_rename(all_opts, "bps", "throttling.bps-total");
-    qemu_opt_rename(all_opts, "bps_rd", "throttling.bps-read");
-    qemu_opt_rename(all_opts, "bps_wr", "throttling.bps-write");
+        { "bps",            "throttling.bps-total" },
+        { "bps_rd",         "throttling.bps-read" },
+        { "bps_wr",         "throttling.bps-write" },
 
-    qemu_opt_rename(all_opts, "iops_max", "throttling.iops-total-max");
-    qemu_opt_rename(all_opts, "iops_rd_max", "throttling.iops-read-max");
-    qemu_opt_rename(all_opts, "iops_wr_max", "throttling.iops-write-max");
+        { "iops_max",       "throttling.iops-total-max" },
+        { "iops_rd_max",    "throttling.iops-read-max" },
+        { "iops_wr_max",    "throttling.iops-write-max" },
 
-    qemu_opt_rename(all_opts, "bps_max", "throttling.bps-total-max");
-    qemu_opt_rename(all_opts, "bps_rd_max", "throttling.bps-read-max");
-    qemu_opt_rename(all_opts, "bps_wr_max", "throttling.bps-write-max");
+        { "bps_max",        "throttling.bps-total-max" },
+        { "bps_rd_max",     "throttling.bps-read-max" },
+        { "bps_wr_max",     "throttling.bps-write-max" },
 
-    qemu_opt_rename(all_opts,
-                    "iops_size", "throttling.iops-size");
+        { "iops_size",      "throttling.iops-size" },
 
-    qemu_opt_rename(all_opts, "readonly", "read-only");
+        { "readonly",       "read-only" },
+    };
+
+    for (i = 0; i < ARRAY_SIZE(opt_renames); i++) {
+        qemu_opt_rename(all_opts, opt_renames[i].from, opt_renames[i].to,
+                        &local_err);
+        if (local_err) {
+            error_report("%s", error_get_pretty(local_err));
+            error_free(local_err);
+            return NULL;
+        }
+    }
 
     value = qemu_opt_get(all_opts, "cache");
     if (value) {
@@ -927,9 +979,9 @@ DriveInfo *drive_new(QemuOpts *all_opts, BlockInterfaceType block_default_type)
     }
 
     /* Actual block device init: Functionality shared with blockdev-add */
-    dinfo = blockdev_init(filename, bs_opts, &local_err);
+    blk = blockdev_init(filename, bs_opts, &local_err);
     bs_opts = NULL;
-    if (dinfo == NULL) {
+    if (!blk) {
         if (local_err) {
             error_report("%s", error_get_pretty(local_err));
             error_free(local_err);
@@ -939,8 +991,8 @@ DriveInfo *drive_new(QemuOpts *all_opts, BlockInterfaceType block_default_type)
         assert(!local_err);
     }
 
-    /* Set legacy DriveInfo fields */
-    dinfo->enable_auto_del = true;
+    /* Create legacy DriveInfo */
+    dinfo = g_malloc0(sizeof(*dinfo));
     dinfo->opts = all_opts;
 
     dinfo->cyls = cyls;
@@ -952,9 +1004,10 @@ DriveInfo *drive_new(QemuOpts *all_opts, BlockInterfaceType block_default_type)
     dinfo->bus = bus_id;
     dinfo->unit = unit_id;
     dinfo->devaddr = devaddr;
-
     dinfo->serial = g_strdup(serial);
 
+    blk_set_legacy_dinfo(blk, dinfo);
+
     switch(type) {
     case IF_IDE:
     case IF_SCSI:
@@ -1094,7 +1147,7 @@ SnapshotInfo *qmp_blockdev_snapshot_delete_internal_sync(const char *device,
         return NULL;
     }
 
-    info = g_malloc0(sizeof(SnapshotInfo));
+    info = g_new0(SnapshotInfo, 1);
     info->id = g_strdup(sn.id_str);
     info->name = g_strdup(sn.name);
     info->date_nsec = sn.date_nsec;
@@ -1547,19 +1600,21 @@ exit:
 }
 
 
-static void eject_device(BlockDriverState *bs, int force, Error **errp)
+static void eject_device(BlockBackend *blk, int force, Error **errp)
 {
+    BlockDriverState *bs = blk_bs(blk);
+
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_EJECT, errp)) {
         return;
     }
-    if (!bdrv_dev_has_removable_media(bs)) {
+    if (!blk_dev_has_removable_media(blk)) {
         error_setg(errp, "Device '%s' is not removable",
                    bdrv_get_device_name(bs));
         return;
     }
 
-    if (bdrv_dev_is_medium_locked(bs) && !bdrv_dev_is_tray_open(bs)) {
-        bdrv_dev_eject_request(bs, force);
+    if (blk_dev_is_medium_locked(blk) && !blk_dev_is_tray_open(blk)) {
+        blk_dev_eject_request(blk, force);
         if (!force) {
             error_setg(errp, "Device '%s' is locked",
                        bdrv_get_device_name(bs));
@@ -1572,15 +1627,15 @@ static void eject_device(BlockDriverState *bs, int force, Error **errp)
 
 void qmp_eject(const char *device, bool has_force, bool force, Error **errp)
 {
-    BlockDriverState *bs;
+    BlockBackend *blk;
 
-    bs = bdrv_find(device);
-    if (!bs) {
+    blk = blk_by_name(device);
+    if (!blk) {
         error_set(errp, QERR_DEVICE_NOT_FOUND, device);
         return;
     }
 
-    eject_device(bs, force, errp);
+    eject_device(blk, force, errp);
 }
 
 void qmp_block_passwd(bool has_device, const char *device,
@@ -1639,16 +1694,18 @@ static void qmp_bdrv_open_encrypted(BlockDriverState *bs, const char *filename,
 void qmp_change_blockdev(const char *device, const char *filename,
                          const char *format, Error **errp)
 {
+    BlockBackend *blk;
     BlockDriverState *bs;
     BlockDriver *drv = NULL;
     int bdrv_flags;
     Error *err = NULL;
 
-    bs = bdrv_find(device);
-    if (!bs) {
+    blk = blk_by_name(device);
+    if (!blk) {
         error_set(errp, QERR_DEVICE_NOT_FOUND, device);
         return;
     }
+    bs = blk_bs(blk);
 
     if (format) {
         drv = bdrv_find_whitelisted_format(format, bs->read_only);
@@ -1658,7 +1715,7 @@ void qmp_change_blockdev(const char *device, const char *filename,
         }
     }
 
-    eject_device(bs, 0, &err);
+    eject_device(blk, 0, &err);
     if (err) {
         error_propagate(errp, err);
         return;
@@ -1756,17 +1813,31 @@ void qmp_block_set_io_throttle(const char *device, int64_t bps, int64_t bps_rd,
 int do_drive_del(Monitor *mon, const QDict *qdict, QObject **ret_data)
 {
     const char *id = qdict_get_str(qdict, "id");
+    BlockBackend *blk;
     BlockDriverState *bs;
+    AioContext *aio_context;
     Error *local_err = NULL;
 
-    bs = bdrv_find(id);
-    if (!bs) {
+    blk = blk_by_name(id);
+    if (!blk) {
         error_report("Device '%s' not found", id);
         return -1;
     }
+    bs = blk_bs(blk);
+
+    if (!blk_legacy_dinfo(blk)) {
+        error_report("Deleting device added with blockdev-add"
+                     " is not supported");
+        return -1;
+    }
+
+    aio_context = bdrv_get_aio_context(bs);
+    aio_context_acquire(aio_context);
+
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_DRIVE_DEL, &local_err)) {
         error_report("%s", error_get_pretty(local_err));
         error_free(local_err);
+        aio_context_release(aio_context);
         return -1;
     }
 
@@ -1780,16 +1851,16 @@ int do_drive_del(Monitor *mon, const QDict *qdict, QObject **ret_data)
      * can be removed.  If this is a drive with no device backing
      * then we can just get rid of the block driver state right here.
      */
-    if (bdrv_get_attached_dev(bs)) {
-        bdrv_make_anon(bs);
-
+    if (blk_get_attached_dev(blk)) {
+        blk_hide_on_behalf_of_do_drive_del(blk);
         /* Further I/O must not pause the guest */
         bdrv_set_on_error(bs, BLOCKDEV_ON_ERROR_REPORT,
                           BLOCKDEV_ON_ERROR_REPORT);
     } else {
-        drive_del(drive_get_by_blockdev(bs));
+        blk_unref(blk);
     }
 
+    aio_context_release(aio_context);
     return 0;
 }
 
@@ -1799,6 +1870,7 @@ void qmp_block_resize(bool has_device, const char *device,
 {
     Error *local_err = NULL;
     BlockDriverState *bs;
+    AioContext *aio_context;
     int ret;
 
     bs = bdrv_lookup_bs(has_device ? device : NULL,
@@ -1809,19 +1881,22 @@ void qmp_block_resize(bool has_device, const char *device,
         return;
     }
 
+    aio_context = bdrv_get_aio_context(bs);
+    aio_context_acquire(aio_context);
+
     if (!bdrv_is_first_non_filter(bs)) {
         error_set(errp, QERR_FEATURE_DISABLED, "resize");
-        return;
+        goto out;
     }
 
     if (size < 0) {
         error_set(errp, QERR_INVALID_PARAMETER_VALUE, "size", "a >0 size");
-        return;
+        goto out;
     }
 
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_RESIZE, NULL)) {
         error_set(errp, QERR_DEVICE_IN_USE, device);
-        return;
+        goto out;
     }
 
     /* complete all in-flight operations before resizing the device */
@@ -1847,10 +1922,18 @@ void qmp_block_resize(bool has_device, const char *device,
         error_setg_errno(errp, -ret, "Could not resize");
         break;
     }
+
+out:
+    aio_context_release(aio_context);
 }
 
 static void block_job_cb(void *opaque, int ret)
 {
+    /* Note that this function may be executed from another AioContext besides
+     * the QEMU main loop.  If you need to access anything that assumes the
+     * QEMU global mutex, use a BH or introduce a mutex.
+     */
+
     BlockDriverState *bs = opaque;
     const char *msg = NULL;
 
@@ -1880,6 +1963,7 @@ void qmp_block_stream(const char *device,
 {
     BlockDriverState *bs;
     BlockDriverState *base_bs = NULL;
+    AioContext *aio_context;
     Error *local_err = NULL;
     const char *base_name = NULL;
 
@@ -1893,16 +1977,20 @@ void qmp_block_stream(const char *device,
         return;
     }
 
+    aio_context = bdrv_get_aio_context(bs);
+    aio_context_acquire(aio_context);
+
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_STREAM, errp)) {
-        return;
+        goto out;
     }
 
     if (has_base) {
         base_bs = bdrv_find_backing_image(bs, base);
         if (base_bs == NULL) {
             error_set(errp, QERR_BASE_NOT_FOUND, base);
-            return;
+            goto out;
         }
+        assert(bdrv_get_aio_context(base_bs) == aio_context);
         base_name = base;
     }
 
@@ -1911,7 +1999,7 @@ void qmp_block_stream(const char *device,
     if (base_bs == NULL && has_backing_file) {
         error_setg(errp, "backing file specified, but streaming the "
                          "entire chain");
-        return;
+        goto out;
     }
 
     /* backing_file string overrides base bs filename */
@@ -1921,10 +2009,13 @@ void qmp_block_stream(const char *device,
                  on_error, block_job_cb, bs, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
-        return;
+        goto out;
     }
 
     trace_qmp_block_stream(bs, bs->job);
+
+out:
+    aio_context_release(aio_context);
 }
 
 void qmp_block_commit(const char *device,
@@ -1936,6 +2027,7 @@ void qmp_block_commit(const char *device,
 {
     BlockDriverState *bs;
     BlockDriverState *base_bs, *top_bs;
+    AioContext *aio_context;
     Error *local_err = NULL;
     /* This will be part of the QMP command, if/when the
      * BlockdevOnError change for blkmirror makes it in
@@ -1946,9 +2038,6 @@ void qmp_block_commit(const char *device,
         speed = 0;
     }
 
-    /* drain all i/o before commits */
-    bdrv_drain_all();
-
     /* Important Note:
      *  libvirt relies on the DeviceNotFound error class in order to probe for
      *  live commit feature versions; for this to work, we must make sure to
@@ -1960,8 +2049,14 @@ void qmp_block_commit(const char *device,
         return;
     }
 
+    aio_context = bdrv_get_aio_context(bs);
+    aio_context_acquire(aio_context);
+
+    /* drain all i/o before commits */
+    bdrv_drain_all();
+
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_COMMIT, errp)) {
-        return;
+        goto out;
     }
 
     /* default top_bs is the active layer */
@@ -1975,9 +2070,11 @@ void qmp_block_commit(const char *device,
 
     if (top_bs == NULL) {
         error_setg(errp, "Top image file %s not found", top ? top : "NULL");
-        return;
+        goto out;
     }
 
+    assert(bdrv_get_aio_context(top_bs) == aio_context);
+
     if (has_base && base) {
         base_bs = bdrv_find_backing_image(top_bs, base);
     } else {
@@ -1986,20 +2083,22 @@ void qmp_block_commit(const char *device,
 
     if (base_bs == NULL) {
         error_set(errp, QERR_BASE_NOT_FOUND, base ? base : "NULL");
-        return;
+        goto out;
     }
 
+    assert(bdrv_get_aio_context(base_bs) == aio_context);
+
     /* Do not allow attempts to commit an image into itself */
     if (top_bs == base_bs) {
         error_setg(errp, "cannot commit an image into itself");
-        return;
+        goto out;
     }
 
     if (top_bs == bs) {
         if (has_backing_file) {
             error_setg(errp, "'backing-file' specified,"
                              " but 'top' is the active layer");
-            return;
+            goto out;
         }
         commit_active_start(bs, base_bs, speed, on_error, block_job_cb,
                             bs, &local_err);
@@ -2009,8 +2108,11 @@ void qmp_block_commit(const char *device,
     }
     if (local_err != NULL) {
         error_propagate(errp, local_err);
-        return;
+        goto out;
     }
+
+out:
+    aio_context_release(aio_context);
 }
 
 void qmp_drive_backup(const char *device, const char *target,
@@ -2025,6 +2127,7 @@ void qmp_drive_backup(const char *device, const char *target,
     BlockDriverState *bs;
     BlockDriverState *target_bs;
     BlockDriverState *source = NULL;
+    AioContext *aio_context;
     BlockDriver *drv = NULL;
     Error *local_err = NULL;
     int flags;
@@ -2050,9 +2153,12 @@ void qmp_drive_backup(const char *device, const char *target,
         return;
     }
 
+    aio_context = bdrv_get_aio_context(bs);
+    aio_context_acquire(aio_context);
+
     if (!bdrv_is_inserted(bs)) {
         error_set(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
-        return;
+        goto out;
     }
 
     if (!has_format) {
@@ -2062,12 +2168,12 @@ void qmp_drive_backup(const char *device, const char *target,
         drv = bdrv_find_format(format);
         if (!drv) {
             error_set(errp, QERR_INVALID_BLOCK_FORMAT, format);
-            return;
+            goto out;
         }
     }
 
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_BACKUP_SOURCE, errp)) {
-        return;
+        goto out;
     }
 
     flags = bs->open_flags | BDRV_O_RDWR;
@@ -2087,7 +2193,7 @@ void qmp_drive_backup(const char *device, const char *target,
     size = bdrv_getlength(bs);
     if (size < 0) {
         error_setg_errno(errp, -size, "bdrv_getlength failed");
-        return;
+        goto out;
     }
 
     if (mode != NEW_IMAGE_MODE_EXISTING) {
@@ -2104,23 +2210,28 @@ void qmp_drive_backup(const char *device, const char *target,
 
     if (local_err) {
         error_propagate(errp, local_err);
-        return;
+        goto out;
     }
 
     target_bs = NULL;
     ret = bdrv_open(&target_bs, target, NULL, NULL, flags, drv, &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
-        return;
+        goto out;
     }
 
+    bdrv_set_aio_context(target_bs, aio_context);
+
     backup_start(bs, target_bs, speed, sync, on_source_error, on_target_error,
                  block_job_cb, bs, &local_err);
     if (local_err != NULL) {
         bdrv_unref(target_bs);
         error_propagate(errp, local_err);
-        return;
+        goto out;
     }
+
+out:
+    aio_context_release(aio_context);
 }
 
 BlockDeviceInfoList *qmp_query_named_block_nodes(Error **errp)
@@ -2145,6 +2256,7 @@ void qmp_drive_mirror(const char *device, const char *target,
 {
     BlockDriverState *bs;
     BlockDriverState *source, *target_bs;
+    AioContext *aio_context;
     BlockDriver *drv = NULL;
     Error *local_err = NULL;
     QDict *options = NULL;
@@ -2172,11 +2284,12 @@ void qmp_drive_mirror(const char *device, const char *target,
     }
 
     if (granularity != 0 && (granularity < 512 || granularity > 1048576 * 64)) {
-        error_set(errp, QERR_INVALID_PARAMETER, device);
+        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "granularity",
+                  "a value in range [512B, 64MB]");
         return;
     }
     if (granularity & (granularity - 1)) {
-        error_set(errp, QERR_INVALID_PARAMETER, device);
+        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "granularity", "power of 2");
         return;
     }
 
@@ -2186,9 +2299,12 @@ void qmp_drive_mirror(const char *device, const char *target,
         return;
     }
 
+    aio_context = bdrv_get_aio_context(bs);
+    aio_context_acquire(aio_context);
+
     if (!bdrv_is_inserted(bs)) {
         error_set(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
-        return;
+        goto out;
     }
 
     if (!has_format) {
@@ -2198,12 +2314,12 @@ void qmp_drive_mirror(const char *device, const char *target,
         drv = bdrv_find_format(format);
         if (!drv) {
             error_set(errp, QERR_INVALID_BLOCK_FORMAT, format);
-            return;
+            goto out;
         }
     }
 
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_MIRROR, errp)) {
-        return;
+        goto out;
     }
 
     flags = bs->open_flags | BDRV_O_RDWR;
@@ -2218,29 +2334,36 @@ void qmp_drive_mirror(const char *device, const char *target,
     size = bdrv_getlength(bs);
     if (size < 0) {
         error_setg_errno(errp, -size, "bdrv_getlength failed");
-        return;
+        goto out;
     }
 
     if (has_replaces) {
         BlockDriverState *to_replace_bs;
+        AioContext *replace_aio_context;
+        int64_t replace_size;
 
         if (!has_node_name) {
             error_setg(errp, "a node-name must be provided when replacing a"
                              " named node of the graph");
-            return;
+            goto out;
         }
 
         to_replace_bs = check_to_replace_node(replaces, &local_err);
 
         if (!to_replace_bs) {
             error_propagate(errp, local_err);
-            return;
+            goto out;
         }
 
-        if (size != bdrv_getlength(to_replace_bs)) {
+        replace_aio_context = bdrv_get_aio_context(to_replace_bs);
+        aio_context_acquire(replace_aio_context);
+        replace_size = bdrv_getlength(to_replace_bs);
+        aio_context_release(replace_aio_context);
+
+        if (size != replace_size) {
             error_setg(errp, "cannot replace image with a mirror image of "
                              "different size");
-            return;
+            goto out;
         }
     }
 
@@ -2269,7 +2392,7 @@ void qmp_drive_mirror(const char *device, const char *target,
 
     if (local_err) {
         error_propagate(errp, local_err);
-        return;
+        goto out;
     }
 
     if (has_node_name) {
@@ -2285,9 +2408,11 @@ void qmp_drive_mirror(const char *device, const char *target,
                     flags | BDRV_O_NO_BACKING, drv, &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
-        return;
+        goto out;
     }
 
+    bdrv_set_aio_context(target_bs, aio_context);
+
     /* pass the node name to replace to mirror start since it's loose coupling
      * and will allow to check whether the node still exist at mirror completion
      */
@@ -2299,24 +2424,42 @@ void qmp_drive_mirror(const char *device, const char *target,
     if (local_err != NULL) {
         bdrv_unref(target_bs);
         error_propagate(errp, local_err);
-        return;
+        goto out;
     }
+
+out:
+    aio_context_release(aio_context);
 }
 
-static BlockJob *find_block_job(const char *device)
+/* Get the block job for a given device name and acquire its AioContext */
+static BlockJob *find_block_job(const char *device, AioContext **aio_context)
 {
     BlockDriverState *bs;
 
     bs = bdrv_find(device);
-    if (!bs || !bs->job) {
-        return NULL;
+    if (!bs) {
+        goto notfound;
+    }
+
+    *aio_context = bdrv_get_aio_context(bs);
+    aio_context_acquire(*aio_context);
+
+    if (!bs->job) {
+        aio_context_release(*aio_context);
+        goto notfound;
     }
+
     return bs->job;
+
+notfound:
+    *aio_context = NULL;
+    return NULL;
 }
 
 void qmp_block_job_set_speed(const char *device, int64_t speed, Error **errp)
 {
-    BlockJob *job = find_block_job(device);
+    AioContext *aio_context;
+    BlockJob *job = find_block_job(device, &aio_context);
 
     if (!job) {
         error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
@@ -2324,34 +2467,40 @@ void qmp_block_job_set_speed(const char *device, int64_t speed, Error **errp)
     }
 
     block_job_set_speed(job, speed, errp);
+    aio_context_release(aio_context);
 }
 
 void qmp_block_job_cancel(const char *device,
                           bool has_force, bool force, Error **errp)
 {
-    BlockJob *job = find_block_job(device);
-
-    if (!has_force) {
-        force = false;
-    }
+    AioContext *aio_context;
+    BlockJob *job = find_block_job(device, &aio_context);
 
     if (!job) {
         error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
         return;
     }
+
+    if (!has_force) {
+        force = false;
+    }
+
     if (job->paused && !force) {
         error_setg(errp, "The block job for device '%s' is currently paused",
                    device);
-        return;
+        goto out;
     }
 
     trace_qmp_block_job_cancel(job);
     block_job_cancel(job);
+out:
+    aio_context_release(aio_context);
 }
 
 void qmp_block_job_pause(const char *device, Error **errp)
 {
-    BlockJob *job = find_block_job(device);
+    AioContext *aio_context;
+    BlockJob *job = find_block_job(device, &aio_context);
 
     if (!job) {
         error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
@@ -2360,11 +2509,13 @@ void qmp_block_job_pause(const char *device, Error **errp)
 
     trace_qmp_block_job_pause(job);
     block_job_pause(job);
+    aio_context_release(aio_context);
 }
 
 void qmp_block_job_resume(const char *device, Error **errp)
 {
-    BlockJob *job = find_block_job(device);
+    AioContext *aio_context;
+    BlockJob *job = find_block_job(device, &aio_context);
 
     if (!job) {
         error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
@@ -2373,11 +2524,13 @@ void qmp_block_job_resume(const char *device, Error **errp)
 
     trace_qmp_block_job_resume(job);
     block_job_resume(job);
+    aio_context_release(aio_context);
 }
 
 void qmp_block_job_complete(const char *device, Error **errp)
 {
-    BlockJob *job = find_block_job(device);
+    AioContext *aio_context;
+    BlockJob *job = find_block_job(device, &aio_context);
 
     if (!job) {
         error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
@@ -2386,6 +2539,7 @@ void qmp_block_job_complete(const char *device, Error **errp)
 
     trace_qmp_block_job_complete(job);
     block_job_complete(job, errp);
+    aio_context_release(aio_context);
 }
 
 void qmp_change_backing_file(const char *device,
@@ -2470,7 +2624,7 @@ void qmp_change_backing_file(const char *device,
 void qmp_blockdev_add(BlockdevOptions *options, Error **errp)
 {
     QmpOutputVisitor *ov = qmp_output_visitor_new();
-    DriveInfo *dinfo;
+    BlockBackend *blk;
     QObject *obj;
     QDict *qdict;
     Error *local_err = NULL;
@@ -2508,14 +2662,14 @@ void qmp_blockdev_add(BlockdevOptions *options, Error **errp)
 
     qdict_flatten(qdict);
 
-    dinfo = blockdev_init(NULL, qdict, &local_err);
+    blk = blockdev_init(NULL, qdict, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         goto fail;
     }
 
-    if (bdrv_key_required(dinfo->bdrv)) {
-        drive_del(dinfo);
+    if (bdrv_key_required(blk_bs(blk))) {
+        blk_unref(blk);
         error_setg(errp, "blockdev-add doesn't support encrypted devices");
         goto fail;
     }
@@ -2524,26 +2678,27 @@ fail:
     qmp_output_visitor_cleanup(ov);
 }
 
-static void do_qmp_query_block_jobs_one(void *opaque, BlockDriverState *bs)
+BlockJobInfoList *qmp_query_block_jobs(Error **errp)
 {
-    BlockJobInfoList **prev = opaque;
-    BlockJob *job = bs->job;
+    BlockJobInfoList *head = NULL, **p_next = &head;
+    BlockDriverState *bs;
+
+    for (bs = bdrv_next(NULL); bs; bs = bdrv_next(bs)) {
+        AioContext *aio_context = bdrv_get_aio_context(bs);
 
-    if (job) {
-        BlockJobInfoList *elem = g_new0(BlockJobInfoList, 1);
-        elem->value = block_job_query(bs->job);
-        (*prev)->next = elem;
-        *prev = elem;
+        aio_context_acquire(aio_context);
+
+        if (bs->job) {
+            BlockJobInfoList *elem = g_new0(BlockJobInfoList, 1);
+            elem->value = block_job_query(bs->job);
+            *p_next = elem;
+            p_next = &elem->next;
+        }
+
+        aio_context_release(aio_context);
     }
-}
 
-BlockJobInfoList *qmp_query_block_jobs(Error **errp)
-{
-    /* Dummy is a fake list element for holding the head pointer */
-    BlockJobInfoList dummy = {};
-    BlockJobInfoList *prev = &dummy;
-    bdrv_iterate(do_qmp_query_block_jobs_one, &prev);
-    return dummy.next;
+    return head;
 }
 
 QemuOptsList qemu_common_drive_opts = {
diff --git a/blockjob.c b/blockjob.c
index ca0b4e25d..ba2255d91 100644
--- a/blockjob.c
+++ b/blockjob.c
@@ -36,7 +36,7 @@
 #include "qapi-event.h"
 
 void *block_job_create(const BlockJobDriver *driver, BlockDriverState *bs,
-                       int64_t speed, BlockDriverCompletionFunc *cb,
+                       int64_t speed, BlockCompletionFunc *cb,
                        void *opaque, Error **errp)
 {
     BlockJob *job;
@@ -50,6 +50,7 @@ void *block_job_create(const BlockJobDriver *driver, BlockDriverState *bs,
     error_setg(&job->blocker, "block device is in use by block job: %s",
                BlockJobType_lookup[driver->job_type]);
     bdrv_op_block_all(bs, job->blocker);
+    bdrv_op_unblock(bs, BLOCK_OP_TYPE_DATAPLANE, job->blocker);
 
     job->driver        = driver;
     job->bs            = bs;
@@ -107,7 +108,8 @@ void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp)
 void block_job_complete(BlockJob *job, Error **errp)
 {
     if (job->paused || job->cancelled || !job->driver->complete) {
-        error_set(errp, QERR_BLOCK_JOB_NOT_READY, job->bs->device_name);
+        error_set(errp, QERR_BLOCK_JOB_NOT_READY,
+                  bdrv_get_device_name(job->bs));
         return;
     }
 
@@ -152,27 +154,30 @@ void block_job_iostatus_reset(BlockJob *job)
     }
 }
 
-struct BlockCancelData {
+struct BlockFinishData {
     BlockJob *job;
-    BlockDriverCompletionFunc *cb;
+    BlockCompletionFunc *cb;
     void *opaque;
     bool cancelled;
     int ret;
 };
 
-static void block_job_cancel_cb(void *opaque, int ret)
+static void block_job_finish_cb(void *opaque, int ret)
 {
-    struct BlockCancelData *data = opaque;
+    struct BlockFinishData *data = opaque;
 
     data->cancelled = block_job_is_cancelled(data->job);
     data->ret = ret;
     data->cb(data->opaque, ret);
 }
 
-int block_job_cancel_sync(BlockJob *job)
+static int block_job_finish_sync(BlockJob *job,
+                                 void (*finish)(BlockJob *, Error **errp),
+                                 Error **errp)
 {
-    struct BlockCancelData data;
+    struct BlockFinishData data;
     BlockDriverState *bs = job->bs;
+    Error *local_err = NULL;
 
     assert(bs->job == job);
 
@@ -183,15 +188,37 @@ int block_job_cancel_sync(BlockJob *job)
     data.cb = job->cb;
     data.opaque = job->opaque;
     data.ret = -EINPROGRESS;
-    job->cb = block_job_cancel_cb;
+    job->cb = block_job_finish_cb;
     job->opaque = &data;
-    block_job_cancel(job);
+    finish(job, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return -EBUSY;
+    }
     while (data.ret == -EINPROGRESS) {
         aio_poll(bdrv_get_aio_context(bs), true);
     }
     return (data.cancelled && data.ret == 0) ? -ECANCELED : data.ret;
 }
 
+/* A wrapper around block_job_cancel() taking an Error ** parameter so it may be
+ * used with block_job_finish_sync() without the need for (rather nasty)
+ * function pointer casts there. */
+static void block_job_cancel_err(BlockJob *job, Error **errp)
+{
+    block_job_cancel(job);
+}
+
+int block_job_cancel_sync(BlockJob *job)
+{
+    return block_job_finish_sync(job, &block_job_cancel_err, NULL);
+}
+
+int block_job_complete_sync(BlockJob *job, Error **errp)
+{
+    return block_job_finish_sync(job, &block_job_complete, errp);
+}
+
 void block_job_sleep_ns(BlockJob *job, QEMUClockType type, int64_t ns)
 {
     assert(job->busy);
@@ -205,7 +232,7 @@ void block_job_sleep_ns(BlockJob *job, QEMUClockType type, int64_t ns)
     if (block_job_is_paused(job)) {
         qemu_coroutine_yield();
     } else {
-        co_sleep_ns(type, ns);
+        co_aio_sleep_ns(bdrv_get_aio_context(job->bs), type, ns);
     }
     job->busy = true;
 }
@@ -235,6 +262,7 @@ BlockJobInfo *block_job_query(BlockJob *job)
     info->offset    = job->offset;
     info->speed     = job->speed;
     info->io_status = job->iostatus;
+    info->ready     = job->ready;
     return info;
 }
 
@@ -270,6 +298,8 @@ void block_job_event_completed(BlockJob *job, const char *msg)
 
 void block_job_event_ready(BlockJob *job)
 {
+    job->ready = true;
+
     qapi_event_send_block_job_ready(job->driver->job_type,
                                     bdrv_get_device_name(job->bs),
                                     job->len,
@@ -313,3 +343,48 @@ BlockErrorAction block_job_error_action(BlockJob *job, BlockDriverState *bs,
     }
     return action;
 }
+
+typedef struct {
+    BlockJob *job;
+    QEMUBH *bh;
+    AioContext *aio_context;
+    BlockJobDeferToMainLoopFn *fn;
+    void *opaque;
+} BlockJobDeferToMainLoopData;
+
+static void block_job_defer_to_main_loop_bh(void *opaque)
+{
+    BlockJobDeferToMainLoopData *data = opaque;
+    AioContext *aio_context;
+
+    qemu_bh_delete(data->bh);
+
+    /* Prevent race with block_job_defer_to_main_loop() */
+    aio_context_acquire(data->aio_context);
+
+    /* Fetch BDS AioContext again, in case it has changed */
+    aio_context = bdrv_get_aio_context(data->job->bs);
+    aio_context_acquire(aio_context);
+
+    data->fn(data->job, data->opaque);
+
+    aio_context_release(aio_context);
+
+    aio_context_release(data->aio_context);
+
+    g_free(data);
+}
+
+void block_job_defer_to_main_loop(BlockJob *job,
+                                  BlockJobDeferToMainLoopFn *fn,
+                                  void *opaque)
+{
+    BlockJobDeferToMainLoopData *data = g_malloc(sizeof(*data));
+    data->job = job;
+    data->bh = qemu_bh_new(block_job_defer_to_main_loop_bh, data);
+    data->aio_context = bdrv_get_aio_context(job->bs);
+    data->fn = fn;
+    data->opaque = opaque;
+
+    qemu_bh_schedule(data->bh);
+}
diff --git a/bootdevice.c b/bootdevice.c
new file mode 100644
index 000000000..b29970c7a
--- /dev/null
+++ b/bootdevice.c
@@ -0,0 +1,258 @@
+/*
+ * QEMU Boot Device Implement
+ *
+ * Copyright (c) 2014 HUAWEI TECHNOLOGIES CO.,LTD.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "sysemu/sysemu.h"
+#include "qapi/visitor.h"
+#include "qemu/error-report.h"
+
+typedef struct FWBootEntry FWBootEntry;
+
+struct FWBootEntry {
+    QTAILQ_ENTRY(FWBootEntry) link;
+    int32_t bootindex;
+    DeviceState *dev;
+    char *suffix;
+};
+
+static QTAILQ_HEAD(, FWBootEntry) fw_boot_order =
+    QTAILQ_HEAD_INITIALIZER(fw_boot_order);
+
+void check_boot_index(int32_t bootindex, Error **errp)
+{
+    FWBootEntry *i;
+
+    if (bootindex >= 0) {
+        QTAILQ_FOREACH(i, &fw_boot_order, link) {
+            if (i->bootindex == bootindex) {
+                error_setg(errp, "The bootindex %d has already been used",
+                           bootindex);
+                return;
+            }
+        }
+    }
+}
+
+void del_boot_device_path(DeviceState *dev, const char *suffix)
+{
+    FWBootEntry *i;
+
+    if (dev == NULL) {
+        return;
+    }
+
+    QTAILQ_FOREACH(i, &fw_boot_order, link) {
+        if ((!suffix || !g_strcmp0(i->suffix, suffix)) &&
+             i->dev == dev) {
+            QTAILQ_REMOVE(&fw_boot_order, i, link);
+            g_free(i->suffix);
+            g_free(i);
+
+            break;
+        }
+    }
+}
+
+void add_boot_device_path(int32_t bootindex, DeviceState *dev,
+                          const char *suffix)
+{
+    FWBootEntry *node, *i;
+
+    if (bootindex < 0) {
+        del_boot_device_path(dev, suffix);
+        return;
+    }
+
+    assert(dev != NULL || suffix != NULL);
+
+    del_boot_device_path(dev, suffix);
+
+    node = g_malloc0(sizeof(FWBootEntry));
+    node->bootindex = bootindex;
+    node->suffix = g_strdup(suffix);
+    node->dev = dev;
+
+    QTAILQ_FOREACH(i, &fw_boot_order, link) {
+        if (i->bootindex == bootindex) {
+            error_report("Two devices with same boot index %d", bootindex);
+            exit(1);
+        } else if (i->bootindex < bootindex) {
+            continue;
+        }
+        QTAILQ_INSERT_BEFORE(i, node, link);
+        return;
+    }
+    QTAILQ_INSERT_TAIL(&fw_boot_order, node, link);
+}
+
+DeviceState *get_boot_device(uint32_t position)
+{
+    uint32_t counter = 0;
+    FWBootEntry *i = NULL;
+    DeviceState *res = NULL;
+
+    if (!QTAILQ_EMPTY(&fw_boot_order)) {
+        QTAILQ_FOREACH(i, &fw_boot_order, link) {
+            if (counter == position) {
+                res = i->dev;
+                break;
+            }
+            counter++;
+        }
+    }
+    return res;
+}
+
+/*
+ * This function returns null terminated string that consist of new line
+ * separated device paths.
+ *
+ * memory pointed by "size" is assigned total length of the array in bytes
+ *
+ */
+char *get_boot_devices_list(size_t *size, bool ignore_suffixes)
+{
+    FWBootEntry *i;
+    size_t total = 0;
+    char *list = NULL;
+
+    QTAILQ_FOREACH(i, &fw_boot_order, link) {
+        char *devpath = NULL, *bootpath;
+        size_t len;
+
+        if (i->dev) {
+            devpath = qdev_get_fw_dev_path(i->dev);
+            assert(devpath);
+        }
+
+        if (i->suffix && !ignore_suffixes && devpath) {
+            size_t bootpathlen = strlen(devpath) + strlen(i->suffix) + 1;
+
+            bootpath = g_malloc(bootpathlen);
+            snprintf(bootpath, bootpathlen, "%s%s", devpath, i->suffix);
+            g_free(devpath);
+        } else if (devpath) {
+            bootpath = devpath;
+        } else if (!ignore_suffixes) {
+            assert(i->suffix);
+            bootpath = g_strdup(i->suffix);
+        } else {
+            bootpath = g_strdup("");
+        }
+
+        if (total) {
+            list[total-1] = '\n';
+        }
+        len = strlen(bootpath) + 1;
+        list = g_realloc(list, total + len);
+        memcpy(&list[total], bootpath, len);
+        total += len;
+        g_free(bootpath);
+    }
+
+    *size = total;
+
+    if (boot_strict && *size > 0) {
+        list[total-1] = '\n';
+        list = g_realloc(list, total + 5);
+        memcpy(&list[total], "HALT", 5);
+        *size = total + 5;
+    }
+    return list;
+}
+
+typedef struct {
+    int32_t *bootindex;
+    const char *suffix;
+    DeviceState *dev;
+} BootIndexProperty;
+
+static void device_get_bootindex(Object *obj, Visitor *v, void *opaque,
+                                 const char *name, Error **errp)
+{
+    BootIndexProperty *prop = opaque;
+    visit_type_int32(v, prop->bootindex, name, errp);
+}
+
+static void device_set_bootindex(Object *obj, Visitor *v, void *opaque,
+                                 const char *name, Error **errp)
+{
+    BootIndexProperty *prop = opaque;
+    int32_t boot_index;
+    Error *local_err = NULL;
+
+    visit_type_int32(v, &boot_index, name, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    /* check whether bootindex is present in fw_boot_order list  */
+    check_boot_index(boot_index, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    /* change bootindex to a new one */
+    *prop->bootindex = boot_index;
+
+    add_boot_device_path(*prop->bootindex, prop->dev, prop->suffix);
+
+out:
+    if (local_err) {
+        error_propagate(errp, local_err);
+    }
+}
+
+static void property_release_bootindex(Object *obj, const char *name,
+                                       void *opaque)
+
+{
+    BootIndexProperty *prop = opaque;
+
+    del_boot_device_path(prop->dev, prop->suffix);
+    g_free(prop);
+}
+
+void device_add_bootindex_property(Object *obj, int32_t *bootindex,
+                                   const char *name, const char *suffix,
+                                   DeviceState *dev, Error **errp)
+{
+    Error *local_err = NULL;
+    BootIndexProperty *prop = g_malloc0(sizeof(*prop));
+
+    prop->bootindex = bootindex;
+    prop->suffix = suffix;
+    prop->dev = dev;
+
+    object_property_add(obj, name, "int32",
+                        device_get_bootindex,
+                        device_set_bootindex,
+                        property_release_bootindex,
+                        prop, &local_err);
+
+    if (local_err) {
+        error_propagate(errp, local_err);
+        g_free(prop);
+        return;
+    }
+    /* initialize devices' bootindex property to -1 */
+    object_property_set_int(obj, -1, name, NULL);
+}
diff --git a/configure b/configure
index f7685b565..47048f008 100755
--- a/configure
+++ b/configure
@@ -326,7 +326,7 @@ seccomp=""
 glusterfs=""
 glusterfs_discard="no"
 glusterfs_zerofill="no"
-virtio_blk_data_plane=""
+archipelago=""
 gtk=""
 gtkabi=""
 vte=""
@@ -388,6 +388,7 @@ cpp="${CPP-$cc -E}"
 objcopy="${OBJCOPY-${cross_prefix}objcopy}"
 ld="${LD-${cross_prefix}ld}"
 libtool="${LIBTOOL-${cross_prefix}libtool}"
+nm="${NM-${cross_prefix}nm}"
 strip="${STRIP-${cross_prefix}strip}"
 windres="${WINDRES-${cross_prefix}windres}"
 pkg_config_exe="${PKG_CONFIG-${cross_prefix}pkg-config}"
@@ -1087,9 +1088,12 @@ for opt do
   ;;
   --enable-glusterfs) glusterfs="yes"
   ;;
-  --disable-virtio-blk-data-plane) virtio_blk_data_plane="no"
+  --disable-archipelago) archipelago="no"
   ;;
-  --enable-virtio-blk-data-plane) virtio_blk_data_plane="yes"
+  --enable-archipelago) archipelago="yes"
+  ;;
+  --disable-virtio-blk-data-plane|--enable-virtio-blk-data-plane)
+      echo "$0: $opt is obsolete, virtio-blk data-plane is always on" >&2
   ;;
   --disable-gtk) gtk="no"
   ;;
@@ -1344,7 +1348,7 @@ Advanced options (experts only):
   --enable-linux-aio       enable Linux AIO support
   --disable-cap-ng         disable libcap-ng support
   --enable-cap-ng          enable libcap-ng support
-  --disable-attr           disables attr and xattr support
+  --disable-attr           disable attr and xattr support
   --enable-attr            enable attr and xattr support
   --disable-blobs          disable installing provided firmware blobs
   --enable-docs            enable documentation build
@@ -1375,20 +1379,22 @@ Advanced options (experts only):
   --with-vss-sdk=SDK-path  enable Windows VSS support in QEMU Guest Agent
   --with-win-sdk=SDK-path  path to Windows Platform SDK (to build VSS .tlb)
   --disable-seccomp        disable seccomp support
-  --enable-seccomp         enables seccomp support
+  --enable-seccomp         enable seccomp support
   --with-coroutine=BACKEND coroutine backend. Supported options:
                            gthread, ucontext, sigaltstack, windows
   --disable-coroutine-pool disable coroutine freelist (worse performance)
   --enable-coroutine-pool  enable coroutine freelist (better performance)
   --enable-glusterfs       enable GlusterFS backend
   --disable-glusterfs      disable GlusterFS backend
+  --enable-archipelago     enable Archipelago backend
+  --disable-archipelago    disable Archipelago backend
   --enable-gcov            enable test coverage analysis with gcov
   --gcov=GCOV              use specified gcov [$gcov_tool]
   --disable-tpm            disable TPM support
   --enable-tpm             enable TPM support
   --disable-libssh2        disable ssh block device support
   --enable-libssh2         enable ssh block device support
-  --disable-vhdx           disables support for the Microsoft VHDX image format
+  --disable-vhdx           disable support for the Microsoft VHDX image format
   --enable-vhdx            enable support for the Microsoft VHDX image format
   --disable-quorum         disable quorum block filter support
   --enable-quorum          enable quorum block filter support
@@ -1723,6 +1729,7 @@ fi
 
 cat > $TMPC <<EOF
 #include <sys/socket.h>
+#include <linux/ip.h>
 int main(void) { return sizeof(struct mmsghdr); }
 EOF
 if compile_prog "" "" ; then
@@ -1816,7 +1823,8 @@ fi
 # libseccomp check
 
 if test "$seccomp" != "no" ; then
-    if $pkg_config --atleast-version=2.1.0 libseccomp; then
+    if test "$cpu" = "i386" || test "$cpu" = "x86_64" &&
+        $pkg_config --atleast-version=2.1.1 libseccomp; then
         libs_softmmu="$libs_softmmu `$pkg_config --libs libseccomp`"
         QEMU_CFLAGS="$QEMU_CFLAGS `$pkg_config --cflags libseccomp`"
 	seccomp="yes"
@@ -2708,6 +2716,12 @@ for i in $glib_modules; do
     fi
 done
 
+# g_test_trap_subprocess added in 2.38. Used by some tests.
+glib_subprocess=yes
+if ! $pkg_config --atleast-version=2.38 glib-2.0; then
+    glib_subprocess=no
+fi
+
 ##########################################
 # SHA command probe for modules
 if test "$modules" = yes; then
@@ -2729,7 +2743,7 @@ fi
 if test "$pixman" = ""; then
   if test "$want_tools" = "no" -a "$softmmu" = "no"; then
     pixman="none"
-  elif $pkg_config pixman-1 > /dev/null 2>&1; then
+  elif $pkg_config --atleast-version=0.21.8 pixman-1 > /dev/null 2>&1; then
     pixman="system"
   else
     pixman="internal"
@@ -2745,11 +2759,12 @@ if test "$pixman" = "none"; then
   pixman_cflags=
   pixman_libs=
 elif test "$pixman" = "system"; then
+  # pixman version has been checked above
   pixman_cflags=`$pkg_config --cflags pixman-1`
   pixman_libs=`$pkg_config --libs pixman-1`
 else
   if test ! -d ${source_path}/pixman/pixman; then
-    error_exit "pixman not present. Your options:" \
+    error_exit "pixman >= 0.21.8 not present. Your options:" \
         "  (1) Preferred: Install the pixman devel package (any recent" \
         "      distro should have packages as Xorg needs pixman too)." \
         "  (2) Fetch the pixman submodule, using:" \
@@ -2928,16 +2943,6 @@ else
 fi
 
 ##########################################
-# adjust virtio-blk-data-plane based on linux-aio
-
-if test "$virtio_blk_data_plane" = "yes" -a \
-	"$linux_aio" != "yes" ; then
-  error_exit "virtio-blk-data-plane requires Linux AIO, please try --enable-linux-aio"
-elif test -z "$virtio_blk_data_plane" ; then
-  virtio_blk_data_plane=$linux_aio
-fi
-
-##########################################
 # attr probe
 
 if test "$attr" != "no" ; then
@@ -3071,6 +3076,33 @@ EOF
   fi
 fi
 
+
+##########################################
+# archipelago probe
+if test "$archipelago" != "no" ; then
+    cat > $TMPC <<EOF
+#include <stdio.h>
+#include <xseg/xseg.h>
+#include <xseg/protocol.h>
+int main(void) {
+    xseg_initialize();
+    return 0;
+}
+EOF
+    archipelago_libs=-lxseg
+    if compile_prog "" "$archipelago_libs"; then
+        archipelago="yes"
+        libs_tools="$archipelago_libs $libs_tools"
+        libs_softmmu="$archipelago_libs $libs_softmmu"
+    else
+      if test "$archipelago" = "yes" ; then
+        feature_not_found "Archipelago backend support" "Install libxseg devel"
+      fi
+      archipelago="no"
+    fi
+fi
+
+
 ##########################################
 # glusterfs probe
 if test "$glusterfs" != "no" ; then
@@ -3086,7 +3118,8 @@ if test "$glusterfs" != "no" ; then
     fi
   else
     if test "$glusterfs" = "yes" ; then
-      feature_not_found "GlusterFS backend support" "Install glusterfs-api devel"
+      feature_not_found "GlusterFS backend support" \
+          "Install glusterfs-api devel >= 3"
     fi
     glusterfs="no"
   fi
@@ -3276,6 +3309,21 @@ if compile_prog "" "" ; then
   fallocate_punch_hole=yes
 fi
 
+# check for posix_fallocate
+posix_fallocate=no
+cat > $TMPC << EOF
+#include <fcntl.h>
+
+int main(void)
+{
+    posix_fallocate(0, 0, 0);
+    return 0;
+}
+EOF
+if compile_prog "" "" ; then
+    posix_fallocate=yes
+fi
+
 # check for sync_file_range
 sync_file_range=no
 cat > $TMPC << EOF
@@ -3420,6 +3468,37 @@ if compile_prog "" "" ; then
   sendfile=yes
 fi
 
+# check for timerfd support (glibc 2.8 and newer)
+timerfd=no
+cat > $TMPC << EOF
+#include <sys/timerfd.h>
+
+int main(void)
+{
+    return(timerfd_create(CLOCK_REALTIME, 0));
+}
+EOF
+if compile_prog "" "" ; then
+  timerfd=yes
+fi
+
+# check for setns and unshare support
+setns=no
+cat > $TMPC << EOF
+#include <sched.h>
+
+int main(void)
+{
+    int ret;
+    ret = setns(0, 0);
+    ret = unshare(0);
+    return ret;
+}
+EOF
+if compile_prog "" "" ; then
+  setns=yes
+fi
+
 # Check if tools are available to build documentation.
 if test "$docs" != "no" ; then
   if has makeinfo && has pod2man; then
@@ -3531,7 +3610,8 @@ EOF
     spice_server_version=$($pkg_config --modversion spice-server)
   else
     if test "$spice" = "yes" ; then
-      feature_not_found "spice" "Install spice-server and spice-protocol devel"
+      feature_not_found "spice" \
+          "Install spice-server(>=0.12.0) and spice-protocol(>=0.12.3) devel"
     fi
     spice="no"
   fi
@@ -3562,7 +3642,7 @@ EOF
         smartcard_nss="yes"
     else
         if test "$smartcard_nss" = "yes"; then
-            feature_not_found "nss"
+            feature_not_found "nss" "Install nss devel >= 3.12.8"
         fi
         smartcard_nss="no"
     fi
@@ -3578,7 +3658,7 @@ if test "$libusb" != "no" ; then
         libs_softmmu="$libs_softmmu $libusb_libs"
     else
         if test "$libusb" = "yes"; then
-            feature_not_found "libusb" "Install libusb devel"
+            feature_not_found "libusb" "Install libusb devel >= 1.0.13"
         fi
         libusb="no"
     fi
@@ -3892,12 +3972,11 @@ else
 fi
 
 ########################################
-# check if we have valgrind/valgrind.h and valgrind/memcheck.h
+# check if we have valgrind/valgrind.h
 
 valgrind_h=no
 cat > $TMPC << EOF
 #include <valgrind/valgrind.h>
-#include <valgrind/memcheck.h>
 int main(void) {
   return 0;
 }
@@ -4003,7 +4082,7 @@ if test "$libnfs" != "no" ; then
     LIBS="$LIBS $libnfs_libs"
   else
     if test "$libnfs" = "yes" ; then
-      feature_not_found "libnfs"
+      feature_not_found "libnfs" "Install libnfs devel >= 1.9.3"
     fi
     libnfs="no"
   fi
@@ -4133,9 +4212,9 @@ EOF
   fi
 fi
 
-# add pixman flags after all config tests are done
-QEMU_CFLAGS="$QEMU_CFLAGS $pixman_cflags $fdt_cflags"
-libs_softmmu="$libs_softmmu $pixman_libs"
+# prepend pixman and ftd flags after all config tests are done
+QEMU_CFLAGS="$pixman_cflags $fdt_cflags $QEMU_CFLAGS"
+libs_softmmu="$pixman_libs $libs_softmmu"
 
 echo "Install prefix    $prefix"
 echo "BIOS directory    `eval echo $qemu_datadir`"
@@ -4250,7 +4329,7 @@ echo "seccomp support   $seccomp"
 echo "coroutine backend $coroutine"
 echo "coroutine pool    $coroutine_pool"
 echo "GlusterFS support $glusterfs"
-echo "virtio-blk-data-plane $virtio_blk_data_plane"
+echo "Archipelago support $archipelago"
 echo "gcov              $gcov_tool"
 echo "gcov enabled      $gcov"
 echo "TPM support       $tpm"
@@ -4459,6 +4538,9 @@ fi
 if test "$fallocate_punch_hole" = "yes" ; then
   echo "CONFIG_FALLOCATE_PUNCH_HOLE=y" >> $config_host_mak
 fi
+if test "$posix_fallocate" = "yes" ; then
+  echo "CONFIG_POSIX_FALLOCATE=y" >> $config_host_mak
+fi
 if test "$sync_file_range" = "yes" ; then
   echo "CONFIG_SYNC_FILE_RANGE=y" >> $config_host_mak
 fi
@@ -4486,6 +4568,12 @@ fi
 if test "$sendfile" = "yes" ; then
   echo "CONFIG_SENDFILE=y" >> $config_host_mak
 fi
+if test "$timerfd" = "yes" ; then
+  echo "CONFIG_TIMERFD=y" >> $config_host_mak
+fi
+if test "$setns" = "yes" ; then
+  echo "CONFIG_SETNS=y" >> $config_host_mak
+fi
 if test "$inotify" = "yes" ; then
   echo "CONFIG_INOTIFY=y" >> $config_host_mak
 fi
@@ -4510,6 +4598,9 @@ if test "$bluez" = "yes" ; then
   echo "CONFIG_BLUEZ=y" >> $config_host_mak
   echo "BLUEZ_CFLAGS=$bluez_cflags" >> $config_host_mak
 fi
+if test "glib_subprocess" = "yes" ; then
+  echo "CONFIG_HAS_GLIB_SUBPROCESS_TESTS=y" >> $config_host_mak
+fi
 echo "GLIB_CFLAGS=$glib_cflags" >> $config_host_mak
 if test "$gtk" = "yes" ; then
   echo "CONFIG_GTK=y" >> $config_host_mak
@@ -4688,6 +4779,11 @@ if test "$glusterfs_zerofill" = "yes" ; then
   echo "CONFIG_GLUSTERFS_ZEROFILL=y" >> $config_host_mak
 fi
 
+if test "$archipelago" = "yes" ; then
+  echo "CONFIG_ARCHIPELAGO=m" >> $config_host_mak
+  echo "ARCHIPELAGO_LIBS=$archipelago_libs" >> $config_host_mak
+fi
+
 if test "$libssh2" = "yes" ; then
   echo "CONFIG_LIBSSH2=m" >> $config_host_mak
   echo "LIBSSH2_CFLAGS=$libssh2_cflags" >> $config_host_mak
@@ -4698,10 +4794,6 @@ if test "$quorum" = "yes" ; then
   echo "CONFIG_QUORUM=y" >> $config_host_mak
 fi
 
-if test "$virtio_blk_data_plane" = "yes" ; then
-  echo 'CONFIG_VIRTIO_BLK_DATA_PLANE=$(CONFIG_VIRTIO)' >> $config_host_mak
-fi
-
 if test "$vhdx" = "yes" ; then
   echo "CONFIG_VHDX=y" >> $config_host_mak
 fi
@@ -4808,6 +4900,7 @@ echo "AS=$as" >> $config_host_mak
 echo "CPP=$cpp" >> $config_host_mak
 echo "OBJCOPY=$objcopy" >> $config_host_mak
 echo "LD=$ld" >> $config_host_mak
+echo "NM=$nm" >> $config_host_mak
 echo "WINDRES=$windres" >> $config_host_mak
 echo "LIBTOOL=$libtool" >> $config_host_mak
 echo "CFLAGS=$CFLAGS" >> $config_host_mak
@@ -4816,6 +4909,7 @@ echo "QEMU_CFLAGS=$QEMU_CFLAGS" >> $config_host_mak
 echo "QEMU_INCLUDES=$QEMU_INCLUDES" >> $config_host_mak
 if test "$sparse" = "yes" ; then
   echo "CC           := REAL_CC=\"\$(CC)\" cgcc"       >> $config_host_mak
+  echo "CXX          := REAL_CC=\"\$(CXX)\" cgcc"      >> $config_host_mak
   echo "HOST_CC      := REAL_CC=\"\$(HOST_CC)\" cgcc"  >> $config_host_mak
   echo "QEMU_CFLAGS  += -Wbitwise -Wno-transparent-union -Wno-old-initializer -Wno-non-pointer-null" >> $config_host_mak
 fi
@@ -4936,7 +5030,7 @@ case "$target_name" in
   aarch64)
     TARGET_BASE_ARCH=arm
     bflt="yes"
-    gdb_xml_files="aarch64-core.xml aarch64-fpu.xml"
+    gdb_xml_files="aarch64-core.xml aarch64-fpu.xml arm-core.xml arm-vfp.xml arm-vfp3.xml arm-neon.xml"
   ;;
   cris)
   ;;
@@ -4965,6 +5059,8 @@ case "$target_name" in
     TARGET_BASE_ARCH=mips
     echo "TARGET_ABI_MIPSN64=y" >> $config_target_mak
   ;;
+  tricore)
+  ;;
   moxie)
   ;;
   or32)
@@ -5013,6 +5109,7 @@ case "$target_name" in
     echo "TARGET_ABI32=y" >> $config_target_mak
   ;;
   s390x)
+    gdb_xml_files="s390x-core64.xml s390-acr.xml s390-fpr.xml"
   ;;
   unicore32)
   ;;
@@ -5292,10 +5389,6 @@ for rom in seabios vgabios ; do
     echo "LD=$ld" >> $config_mak
 done
 
-if test "$docs" = "yes" ; then
-  mkdir -p QMP
-fi
-
 # set up qemu-iotests in this build directory
 iotests_common_env="tests/qemu-iotests/common.env"
 iotests_check="tests/qemu-iotests/check"
diff --git a/coroutine-sigaltstack.c b/coroutine-sigaltstack.c
index 3de0bb33b..63519fffc 100644
--- a/coroutine-sigaltstack.c
+++ b/coroutine-sigaltstack.c
@@ -155,7 +155,7 @@ Coroutine *qemu_coroutine_new(void)
     stack_t oss;
     sigset_t sigs;
     sigset_t osigs;
-    jmp_buf old_env;
+    sigjmp_buf old_env;
 
     /* The way to manipulate stack is with the sigaltstack function. We
      * prepare a stack, with it delivering a signal to ourselves and then
diff --git a/cpu-exec.c b/cpu-exec.c
index 38e5f02a3..3913de020 100644
--- a/cpu-exec.c
+++ b/cpu-exec.c
@@ -18,10 +18,114 @@
  */
 #include "config.h"
 #include "cpu.h"
+#include "trace.h"
 #include "disas/disas.h"
 #include "tcg.h"
 #include "qemu/atomic.h"
 #include "sysemu/qtest.h"
+#include "qemu/timer.h"
+
+/* -icount align implementation. */
+
+typedef struct SyncClocks {
+    int64_t diff_clk;
+    int64_t last_cpu_icount;
+    int64_t realtime_clock;
+} SyncClocks;
+
+#if !defined(CONFIG_USER_ONLY)
+/* Allow the guest to have a max 3ms advance.
+ * The difference between the 2 clocks could therefore
+ * oscillate around 0.
+ */
+#define VM_CLOCK_ADVANCE 3000000
+#define THRESHOLD_REDUCE 1.5
+#define MAX_DELAY_PRINT_RATE 2000000000LL
+#define MAX_NB_PRINTS 100
+
+static void align_clocks(SyncClocks *sc, const CPUState *cpu)
+{
+    int64_t cpu_icount;
+
+    if (!icount_align_option) {
+        return;
+    }
+
+    cpu_icount = cpu->icount_extra + cpu->icount_decr.u16.low;
+    sc->diff_clk += cpu_icount_to_ns(sc->last_cpu_icount - cpu_icount);
+    sc->last_cpu_icount = cpu_icount;
+
+    if (sc->diff_clk > VM_CLOCK_ADVANCE) {
+#ifndef _WIN32
+        struct timespec sleep_delay, rem_delay;
+        sleep_delay.tv_sec = sc->diff_clk / 1000000000LL;
+        sleep_delay.tv_nsec = sc->diff_clk % 1000000000LL;
+        if (nanosleep(&sleep_delay, &rem_delay) < 0) {
+            sc->diff_clk -= (sleep_delay.tv_sec - rem_delay.tv_sec) * 1000000000LL;
+            sc->diff_clk -= sleep_delay.tv_nsec - rem_delay.tv_nsec;
+        } else {
+            sc->diff_clk = 0;
+        }
+#else
+        Sleep(sc->diff_clk / SCALE_MS);
+        sc->diff_clk = 0;
+#endif
+    }
+}
+
+static void print_delay(const SyncClocks *sc)
+{
+    static float threshold_delay;
+    static int64_t last_realtime_clock;
+    static int nb_prints;
+
+    if (icount_align_option &&
+        sc->realtime_clock - last_realtime_clock >= MAX_DELAY_PRINT_RATE &&
+        nb_prints < MAX_NB_PRINTS) {
+        if ((-sc->diff_clk / (float)1000000000LL > threshold_delay) ||
+            (-sc->diff_clk / (float)1000000000LL <
+             (threshold_delay - THRESHOLD_REDUCE))) {
+            threshold_delay = (-sc->diff_clk / 1000000000LL) + 1;
+            printf("Warning: The guest is now late by %.1f to %.1f seconds\n",
+                   threshold_delay - 1,
+                   threshold_delay);
+            nb_prints++;
+            last_realtime_clock = sc->realtime_clock;
+        }
+    }
+}
+
+static void init_delay_params(SyncClocks *sc,
+                              const CPUState *cpu)
+{
+    if (!icount_align_option) {
+        return;
+    }
+    sc->realtime_clock = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
+    sc->diff_clk = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) -
+                   sc->realtime_clock +
+                   cpu_get_clock_offset();
+    sc->last_cpu_icount = cpu->icount_extra + cpu->icount_decr.u16.low;
+    if (sc->diff_clk < max_delay) {
+        max_delay = sc->diff_clk;
+    }
+    if (sc->diff_clk > max_advance) {
+        max_advance = sc->diff_clk;
+    }
+
+    /* Print every 2s max if the guest is late. We limit the number
+       of printed messages to NB_PRINT_MAX(currently 100) */
+    print_delay(sc);
+}
+#else
+static void align_clocks(SyncClocks *sc, const CPUState *cpu)
+{
+}
+
+static void init_delay_params(SyncClocks *sc, const CPUState *cpu)
+{
+}
+#endif /* CONFIG USER ONLY */
 
 void cpu_loop_exit(CPUState *cpu)
 {
@@ -65,6 +169,9 @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, uint8_t *tb_ptr)
 #endif /* DEBUG_DISAS */
 
     next_tb = tcg_qemu_tb_exec(env, tb_ptr);
+    trace_exec_tb_exit((void *) (next_tb & ~TB_EXIT_MASK),
+                       next_tb & TB_EXIT_MASK);
+
     if ((next_tb & TB_EXIT_MASK) > TB_EXIT_IDX1) {
         /* We didn't start executing this TB (eg because the instruction
          * counter hit zero); we must restore the guest PC to the address
@@ -105,6 +212,7 @@ static void cpu_exec_nocache(CPUArchState *env, int max_cycles,
                      max_cycles);
     cpu->current_tb = tb;
     /* execute the generated code */
+    trace_exec_tb_nocache(tb, tb->pc);
     cpu_tb_exec(cpu, tb->tc_ptr);
     cpu->current_tb = NULL;
     tb_phys_invalidate(tb, -1);
@@ -187,16 +295,10 @@ static inline TranslationBlock *tb_find_fast(CPUArchState *env)
     return tb;
 }
 
-static CPUDebugExcpHandler *debug_excp_handler;
-
-void cpu_set_debug_excp_handler(CPUDebugExcpHandler *handler)
-{
-    debug_excp_handler = handler;
-}
-
 static void cpu_handle_debug_exception(CPUArchState *env)
 {
     CPUState *cpu = ENV_GET_CPU(env);
+    CPUClass *cc = CPU_GET_CLASS(cpu);
     CPUWatchpoint *wp;
 
     if (!cpu->watchpoint_hit) {
@@ -204,9 +306,8 @@ static void cpu_handle_debug_exception(CPUArchState *env)
             wp->flags &= ~BP_WATCHPOINT_HIT;
         }
     }
-    if (debug_excp_handler) {
-        debug_excp_handler(env);
-    }
+
+    cc->debug_excp_handler(cpu);
 }
 
 /* main execution loop */
@@ -216,10 +317,7 @@ volatile sig_atomic_t exit_request;
 int cpu_exec(CPUArchState *env)
 {
     CPUState *cpu = ENV_GET_CPU(env);
-#if !(defined(CONFIG_USER_ONLY) && \
-      (defined(TARGET_M68K) || defined(TARGET_PPC) || defined(TARGET_S390X)))
     CPUClass *cc = CPU_GET_CLASS(cpu);
-#endif
 #ifdef TARGET_I386
     X86CPU *x86_cpu = X86_CPU(cpu);
 #endif
@@ -227,6 +325,8 @@ int cpu_exec(CPUArchState *env)
     TranslationBlock *tb;
     uint8_t *tc_ptr;
     uintptr_t next_tb;
+    SyncClocks sc;
+
     /* This must be volatile so it is not trashed by longjmp() */
     volatile bool have_tb_lock = false;
 
@@ -252,37 +352,16 @@ int cpu_exec(CPUArchState *env)
         cpu->exit_request = 1;
     }
 
-#if defined(TARGET_I386)
-    /* put eflags in CPU temporary format */
-    CC_SRC = env->eflags & (CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C);
-    env->df = 1 - (2 * ((env->eflags >> 10) & 1));
-    CC_OP = CC_OP_EFLAGS;
-    env->eflags &= ~(DF_MASK | CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C);
-#elif defined(TARGET_SPARC)
-#elif defined(TARGET_M68K)
-    env->cc_op = CC_OP_FLAGS;
-    env->cc_dest = env->sr & 0xf;
-    env->cc_x = (env->sr >> 4) & 1;
-#elif defined(TARGET_ALPHA)
-#elif defined(TARGET_ARM)
-#elif defined(TARGET_UNICORE32)
-#elif defined(TARGET_PPC)
-    env->reserve_addr = -1;
-#elif defined(TARGET_LM32)
-#elif defined(TARGET_MICROBLAZE)
-#elif defined(TARGET_MIPS)
-#elif defined(TARGET_MOXIE)
-#elif defined(TARGET_OPENRISC)
-#elif defined(TARGET_SH4)
-#elif defined(TARGET_CRIS)
-#elif defined(TARGET_S390X)
-#elif defined(TARGET_XTENSA)
-    /* XXXXX */
-#else
-#error unsupported target CPU
-#endif
+    cc->cpu_exec_enter(cpu);
     cpu->exception_index = -1;
 
+    /* Calculate difference between guest clock and host clock.
+     * This delay includes the delay of the last cycle, so
+     * what we have to do is sleep until it is 0. As for the
+     * advance/delay we gain here, we try to fix it next time.
+     */
+    init_delay_params(&sc, cpu);
+
     /* prepare setjmp context for exception handling */
     for(;;) {
         if (sigsetjmp(cpu->jmp_env, 0) == 0) {
@@ -325,16 +404,12 @@ int cpu_exec(CPUArchState *env)
                         cpu->exception_index = EXCP_DEBUG;
                         cpu_loop_exit(cpu);
                     }
-#if defined(TARGET_ARM) || defined(TARGET_SPARC) || defined(TARGET_MIPS) || \
-    defined(TARGET_PPC) || defined(TARGET_ALPHA) || defined(TARGET_CRIS) || \
-    defined(TARGET_MICROBLAZE) || defined(TARGET_LM32) || defined(TARGET_UNICORE32)
                     if (interrupt_request & CPU_INTERRUPT_HALT) {
                         cpu->interrupt_request &= ~CPU_INTERRUPT_HALT;
                         cpu->halted = 1;
                         cpu->exception_index = EXCP_HLT;
                         cpu_loop_exit(cpu);
                     }
-#endif
 #if defined(TARGET_I386)
                     if (interrupt_request & CPU_INTERRUPT_INIT) {
                         cpu_svm_check_intercept_param(env, SVM_EXIT_INIT, 0);
@@ -347,251 +422,15 @@ int cpu_exec(CPUArchState *env)
                         cpu_reset(cpu);
                     }
 #endif
-#if defined(TARGET_I386)
-#if !defined(CONFIG_USER_ONLY)
-                    if (interrupt_request & CPU_INTERRUPT_POLL) {
-                        cpu->interrupt_request &= ~CPU_INTERRUPT_POLL;
-                        apic_poll_irq(x86_cpu->apic_state);
-                    }
-#endif
-                    if (interrupt_request & CPU_INTERRUPT_SIPI) {
-                            do_cpu_sipi(x86_cpu);
-                    } else if (env->hflags2 & HF2_GIF_MASK) {
-                        if ((interrupt_request & CPU_INTERRUPT_SMI) &&
-                            !(env->hflags & HF_SMM_MASK)) {
-                            cpu_svm_check_intercept_param(env, SVM_EXIT_SMI,
-                                                          0);
-                            cpu->interrupt_request &= ~CPU_INTERRUPT_SMI;
-                            do_smm_enter(x86_cpu);
-                            next_tb = 0;
-                        } else if ((interrupt_request & CPU_INTERRUPT_NMI) &&
-                                   !(env->hflags2 & HF2_NMI_MASK)) {
-                            cpu->interrupt_request &= ~CPU_INTERRUPT_NMI;
-                            env->hflags2 |= HF2_NMI_MASK;
-                            do_interrupt_x86_hardirq(env, EXCP02_NMI, 1);
-                            next_tb = 0;
-                        } else if (interrupt_request & CPU_INTERRUPT_MCE) {
-                            cpu->interrupt_request &= ~CPU_INTERRUPT_MCE;
-                            do_interrupt_x86_hardirq(env, EXCP12_MCHK, 0);
-                            next_tb = 0;
-                        } else if ((interrupt_request & CPU_INTERRUPT_HARD) &&
-                                   (((env->hflags2 & HF2_VINTR_MASK) && 
-                                     (env->hflags2 & HF2_HIF_MASK)) ||
-                                    (!(env->hflags2 & HF2_VINTR_MASK) && 
-                                     (env->eflags & IF_MASK && 
-                                      !(env->hflags & HF_INHIBIT_IRQ_MASK))))) {
-                            int intno;
-                            cpu_svm_check_intercept_param(env, SVM_EXIT_INTR,
-                                                          0);
-                            cpu->interrupt_request &= ~(CPU_INTERRUPT_HARD |
-                                                        CPU_INTERRUPT_VIRQ);
-                            intno = cpu_get_pic_interrupt(env);
-                            qemu_log_mask(CPU_LOG_TB_IN_ASM, "Servicing hardware INT=0x%02x\n", intno);
-                            do_interrupt_x86_hardirq(env, intno, 1);
-                            /* ensure that no TB jump will be modified as
-                               the program flow was changed */
-                            next_tb = 0;
-#if !defined(CONFIG_USER_ONLY)
-                        } else if ((interrupt_request & CPU_INTERRUPT_VIRQ) &&
-                                   (env->eflags & IF_MASK) && 
-                                   !(env->hflags & HF_INHIBIT_IRQ_MASK)) {
-                            int intno;
-                            /* FIXME: this should respect TPR */
-                            cpu_svm_check_intercept_param(env, SVM_EXIT_VINTR,
-                                                          0);
-                            intno = ldl_phys(cpu->as,
-                                             env->vm_vmcb
-                                             + offsetof(struct vmcb,
-                                                        control.int_vector));
-                            qemu_log_mask(CPU_LOG_TB_IN_ASM, "Servicing virtual hardware INT=0x%02x\n", intno);
-                            do_interrupt_x86_hardirq(env, intno, 1);
-                            cpu->interrupt_request &= ~CPU_INTERRUPT_VIRQ;
-                            next_tb = 0;
-#endif
-                        }
-                    }
-#elif defined(TARGET_PPC)
-                    if (interrupt_request & CPU_INTERRUPT_HARD) {
-                        ppc_hw_interrupt(env);
-                        if (env->pending_interrupts == 0) {
-                            cpu->interrupt_request &= ~CPU_INTERRUPT_HARD;
-                        }
-                        next_tb = 0;
-                    }
-#elif defined(TARGET_LM32)
-                    if ((interrupt_request & CPU_INTERRUPT_HARD)
-                        && (env->ie & IE_IE)) {
-                        cpu->exception_index = EXCP_IRQ;
-                        cc->do_interrupt(cpu);
-                        next_tb = 0;
-                    }
-#elif defined(TARGET_MICROBLAZE)
-                    if ((interrupt_request & CPU_INTERRUPT_HARD)
-                        && (env->sregs[SR_MSR] & MSR_IE)
-                        && !(env->sregs[SR_MSR] & (MSR_EIP | MSR_BIP))
-                        && !(env->iflags & (D_FLAG | IMM_FLAG))) {
-                        cpu->exception_index = EXCP_IRQ;
-                        cc->do_interrupt(cpu);
-                        next_tb = 0;
-                    }
-#elif defined(TARGET_MIPS)
-                    if ((interrupt_request & CPU_INTERRUPT_HARD) &&
-                        cpu_mips_hw_interrupts_pending(env)) {
-                        /* Raise it */
-                        cpu->exception_index = EXCP_EXT_INTERRUPT;
-                        env->error_code = 0;
-                        cc->do_interrupt(cpu);
-                        next_tb = 0;
-                    }
-#elif defined(TARGET_OPENRISC)
-                    {
-                        int idx = -1;
-                        if ((interrupt_request & CPU_INTERRUPT_HARD)
-                            && (env->sr & SR_IEE)) {
-                            idx = EXCP_INT;
-                        }
-                        if ((interrupt_request & CPU_INTERRUPT_TIMER)
-                            && (env->sr & SR_TEE)) {
-                            idx = EXCP_TICK;
-                        }
-                        if (idx >= 0) {
-                            cpu->exception_index = idx;
-                            cc->do_interrupt(cpu);
-                            next_tb = 0;
-                        }
-                    }
-#elif defined(TARGET_SPARC)
-                    if (interrupt_request & CPU_INTERRUPT_HARD) {
-                        if (cpu_interrupts_enabled(env) &&
-                            env->interrupt_index > 0) {
-                            int pil = env->interrupt_index & 0xf;
-                            int type = env->interrupt_index & 0xf0;
-
-                            if (((type == TT_EXTINT) &&
-                                  cpu_pil_allowed(env, pil)) ||
-                                  type != TT_EXTINT) {
-                                cpu->exception_index = env->interrupt_index;
-                                cc->do_interrupt(cpu);
-                                next_tb = 0;
-                            }
-                        }
-                    }
-#elif defined(TARGET_ARM)
-                    if (interrupt_request & CPU_INTERRUPT_FIQ
-                        && !(env->daif & PSTATE_F)) {
-                        cpu->exception_index = EXCP_FIQ;
-                        cc->do_interrupt(cpu);
-                        next_tb = 0;
-                    }
-                    /* ARMv7-M interrupt return works by loading a magic value
-                       into the PC.  On real hardware the load causes the
-                       return to occur.  The qemu implementation performs the
-                       jump normally, then does the exception return when the
-                       CPU tries to execute code at the magic address.
-                       This will cause the magic PC value to be pushed to
-                       the stack if an interrupt occurred at the wrong time.
-                       We avoid this by disabling interrupts when
-                       pc contains a magic address.  */
-                    if (interrupt_request & CPU_INTERRUPT_HARD
-                        && ((IS_M(env) && env->regs[15] < 0xfffffff0)
-                            || !(env->daif & PSTATE_I))) {
-                        cpu->exception_index = EXCP_IRQ;
-                        cc->do_interrupt(cpu);
-                        next_tb = 0;
-                    }
-#elif defined(TARGET_UNICORE32)
-                    if (interrupt_request & CPU_INTERRUPT_HARD
-                        && !(env->uncached_asr & ASR_I)) {
-                        cpu->exception_index = UC32_EXCP_INTR;
-                        cc->do_interrupt(cpu);
-                        next_tb = 0;
-                    }
-#elif defined(TARGET_SH4)
-                    if (interrupt_request & CPU_INTERRUPT_HARD) {
-                        cc->do_interrupt(cpu);
-                        next_tb = 0;
-                    }
-#elif defined(TARGET_ALPHA)
-                    {
-                        int idx = -1;
-                        /* ??? This hard-codes the OSF/1 interrupt levels.  */
-                        switch (env->pal_mode ? 7 : env->ps & PS_INT_MASK) {
-                        case 0 ... 3:
-                            if (interrupt_request & CPU_INTERRUPT_HARD) {
-                                idx = EXCP_DEV_INTERRUPT;
-                            }
-                            /* FALLTHRU */
-                        case 4:
-                            if (interrupt_request & CPU_INTERRUPT_TIMER) {
-                                idx = EXCP_CLK_INTERRUPT;
-                            }
-                            /* FALLTHRU */
-                        case 5:
-                            if (interrupt_request & CPU_INTERRUPT_SMP) {
-                                idx = EXCP_SMP_INTERRUPT;
-                            }
-                            /* FALLTHRU */
-                        case 6:
-                            if (interrupt_request & CPU_INTERRUPT_MCHK) {
-                                idx = EXCP_MCHK;
-                            }
-                        }
-                        if (idx >= 0) {
-                            cpu->exception_index = idx;
-                            env->error_code = 0;
-                            cc->do_interrupt(cpu);
-                            next_tb = 0;
-                        }
-                    }
-#elif defined(TARGET_CRIS)
-                    if (interrupt_request & CPU_INTERRUPT_HARD
-                        && (env->pregs[PR_CCS] & I_FLAG)
-                        && !env->locked_irq) {
-                        cpu->exception_index = EXCP_IRQ;
-                        cc->do_interrupt(cpu);
+                    /* The target hook has 3 exit conditions:
+                       False when the interrupt isn't processed,
+                       True when it is, and we should restart on a new TB,
+                       and via longjmp via cpu_loop_exit.  */
+                    if (cc->cpu_exec_interrupt(cpu, interrupt_request)) {
                         next_tb = 0;
                     }
-                    if (interrupt_request & CPU_INTERRUPT_NMI) {
-                        unsigned int m_flag_archval;
-                        if (env->pregs[PR_VR] < 32) {
-                            m_flag_archval = M_FLAG_V10;
-                        } else {
-                            m_flag_archval = M_FLAG_V32;
-                        }
-                        if ((env->pregs[PR_CCS] & m_flag_archval)) {
-                            cpu->exception_index = EXCP_NMI;
-                            cc->do_interrupt(cpu);
-                            next_tb = 0;
-                        }
-                    }
-#elif defined(TARGET_M68K)
-                    if (interrupt_request & CPU_INTERRUPT_HARD
-                        && ((env->sr & SR_I) >> SR_I_SHIFT)
-                            < env->pending_level) {
-                        /* Real hardware gets the interrupt vector via an
-                           IACK cycle at this point.  Current emulated
-                           hardware doesn't rely on this, so we
-                           provide/save the vector when the interrupt is
-                           first signalled.  */
-                        cpu->exception_index = env->pending_vector;
-                        do_interrupt_m68k_hardirq(env);
-                        next_tb = 0;
-                    }
-#elif defined(TARGET_S390X) && !defined(CONFIG_USER_ONLY)
-                    if ((interrupt_request & CPU_INTERRUPT_HARD) &&
-                        (env->psw.mask & PSW_MASK_EXT)) {
-                        cc->do_interrupt(cpu);
-                        next_tb = 0;
-                    }
-#elif defined(TARGET_XTENSA)
-                    if (interrupt_request & CPU_INTERRUPT_HARD) {
-                        cpu->exception_index = EXC_IRQ;
-                        cc->do_interrupt(cpu);
-                        next_tb = 0;
-                    }
-#endif
-                   /* Don't use the cached interrupt_request value,
-                      do_interrupt may have updated the EXITTB flag. */
+                    /* Don't use the cached interrupt_request value,
+                       do_interrupt may have updated the EXITTB flag. */
                     if (cpu->interrupt_request & CPU_INTERRUPT_EXITTB) {
                         cpu->interrupt_request &= ~CPU_INTERRUPT_EXITTB;
                         /* ensure that no TB jump will be modified as
@@ -637,6 +476,7 @@ int cpu_exec(CPUArchState *env)
                 cpu->current_tb = tb;
                 barrier();
                 if (likely(!cpu->exit_request)) {
+                    trace_exec_tb(tb, tb->pc);
                     tc_ptr = tb->tc_ptr;
                     /* execute the generated code */
                     next_tb = cpu_tb_exec(cpu, tc_ptr);
@@ -672,6 +512,7 @@ int cpu_exec(CPUArchState *env)
                             if (insns_left > 0) {
                                 /* Execute remaining instructions.  */
                                 cpu_exec_nocache(env, insns_left, tb);
+                                align_clocks(&sc, cpu);
                             }
                             cpu->exception_index = EXCP_INTERRUPT;
                             next_tb = 0;
@@ -684,6 +525,9 @@ int cpu_exec(CPUArchState *env)
                     }
                 }
                 cpu->current_tb = NULL;
+                /* Try to align the host and virtual clocks
+                   if the guest is in advance */
+                align_clocks(&sc, cpu);
                 /* reset soft MMU for next block (it can currently
                    only be set by a memory fault) */
             } /* for(;;) */
@@ -692,10 +536,7 @@ int cpu_exec(CPUArchState *env)
              * local variables as longjmp is marked 'noreturn'. */
             cpu = current_cpu;
             env = cpu->env_ptr;
-#if !(defined(CONFIG_USER_ONLY) && \
-      (defined(TARGET_M68K) || defined(TARGET_PPC) || defined(TARGET_S390X)))
             cc = CPU_GET_CLASS(cpu);
-#endif
 #ifdef TARGET_I386
             x86_cpu = X86_CPU(cpu);
 #endif
@@ -706,35 +547,7 @@ int cpu_exec(CPUArchState *env)
         }
     } /* for(;;) */
 
-
-#if defined(TARGET_I386)
-    /* restore flags in standard format */
-    env->eflags = env->eflags | cpu_cc_compute_all(env, CC_OP)
-        | (env->df & DF_MASK);
-#elif defined(TARGET_ARM)
-    /* XXX: Save/restore host fpu exception state?.  */
-#elif defined(TARGET_UNICORE32)
-#elif defined(TARGET_SPARC)
-#elif defined(TARGET_PPC)
-#elif defined(TARGET_LM32)
-#elif defined(TARGET_M68K)
-    cpu_m68k_flush_flags(env, env->cc_op);
-    env->cc_op = CC_OP_FLAGS;
-    env->sr = (env->sr & 0xffe0)
-              | env->cc_dest | (env->cc_x << 4);
-#elif defined(TARGET_MICROBLAZE)
-#elif defined(TARGET_MIPS)
-#elif defined(TARGET_MOXIE)
-#elif defined(TARGET_OPENRISC)
-#elif defined(TARGET_SH4)
-#elif defined(TARGET_ALPHA)
-#elif defined(TARGET_CRIS)
-#elif defined(TARGET_S390X)
-#elif defined(TARGET_XTENSA)
-    /* XXXXX */
-#else
-#error unsupported target CPU
-#endif
+    cc->cpu_exec_exit(cpu);
 
     /* fail safe : never use current_cpu outside cpu_exec() */
     current_cpu = NULL;
diff --git a/cpus.c b/cpus.c
index 5e7f2cf3c..0c33458bb 100644
--- a/cpus.c
+++ b/cpus.c
@@ -40,6 +40,7 @@
 #include "qemu/bitmap.h"
 #include "qemu/seqlock.h"
 #include "qapi-event.h"
+#include "hw/nmi.h"
 
 #ifndef _WIN32
 #include "qemu/compatfd.h"
@@ -64,6 +65,8 @@
 #endif /* CONFIG_LINUX */
 
 static CPUState *next_cpu;
+int64_t max_delay;
+int64_t max_advance;
 
 bool cpu_is_stopped(CPUState *cpu)
 {
@@ -102,17 +105,12 @@ static bool all_cpu_threads_idle(void)
 
 /* Protected by TimersState seqlock */
 
-/* Compensate for varying guest execution speed.  */
-static int64_t qemu_icount_bias;
-static int64_t vm_clock_warp_start;
+static int64_t vm_clock_warp_start = -1;
 /* Conversion factor from emulated instructions to virtual clock ticks.  */
 static int icount_time_shift;
 /* Arbitrarily pick 1MIPS as the minimum allowable speed.  */
 #define MAX_ICOUNT_SHIFT 10
 
-/* Only written by TCG thread */
-static int64_t qemu_icount;
-
 static QEMUTimer *icount_rt_timer;
 static QEMUTimer *icount_vm_timer;
 static QEMUTimer *icount_warp_timer;
@@ -129,6 +127,11 @@ typedef struct TimersState {
     int64_t cpu_clock_offset;
     int32_t cpu_ticks_enabled;
     int64_t dummy;
+
+    /* Compensate for varying guest execution speed.  */
+    int64_t qemu_icount_bias;
+    /* Only written by TCG thread */
+    int64_t qemu_icount;
 } TimersState;
 
 static TimersState timers_state;
@@ -139,14 +142,14 @@ static int64_t cpu_get_icount_locked(void)
     int64_t icount;
     CPUState *cpu = current_cpu;
 
-    icount = qemu_icount;
+    icount = timers_state.qemu_icount;
     if (cpu) {
         if (!cpu_can_do_io(cpu)) {
             fprintf(stderr, "Bad clock read\n");
         }
         icount -= (cpu->icount_decr.u16.low + cpu->icount_extra);
     }
-    return qemu_icount_bias + (icount << icount_time_shift);
+    return timers_state.qemu_icount_bias + cpu_icount_to_ns(icount);
 }
 
 int64_t cpu_get_icount(void)
@@ -162,6 +165,11 @@ int64_t cpu_get_icount(void)
     return icount;
 }
 
+int64_t cpu_icount_to_ns(int64_t icount)
+{
+    return icount << icount_time_shift;
+}
+
 /* return the host CPU cycle counter and handle stop/restart */
 /* Caller must hold the BQL */
 int64_t cpu_get_ticks(void)
@@ -214,6 +222,23 @@ int64_t cpu_get_clock(void)
     return ti;
 }
 
+/* return the offset between the host clock and virtual CPU clock */
+int64_t cpu_get_clock_offset(void)
+{
+    int64_t ti;
+    unsigned start;
+
+    do {
+        start = seqlock_read_begin(&timers_state.vm_clock_seqlock);
+        ti = timers_state.cpu_clock_offset;
+        if (!timers_state.cpu_ticks_enabled) {
+            ti -= get_clock();
+        }
+    } while (seqlock_read_retry(&timers_state.vm_clock_seqlock, start));
+
+    return -ti;
+}
+
 /* enable cpu_get_ticks()
  * Caller must hold BQL which server as mutex for vm_clock_seqlock.
  */
@@ -284,7 +309,8 @@ static void icount_adjust(void)
         icount_time_shift++;
     }
     last_delta = delta;
-    qemu_icount_bias = cur_icount - (qemu_icount << icount_time_shift);
+    timers_state.qemu_icount_bias = cur_icount
+                              - (timers_state.qemu_icount << icount_time_shift);
     seqlock_write_unlock(&timers_state.vm_clock_seqlock);
 }
 
@@ -333,7 +359,7 @@ static void icount_warp_rt(void *opaque)
             int64_t delta = cur_time - cur_icount;
             warp_delta = MIN(warp_delta, delta);
         }
-        qemu_icount_bias += warp_delta;
+        timers_state.qemu_icount_bias += warp_delta;
     }
     vm_clock_warp_start = -1;
     seqlock_write_unlock(&timers_state.vm_clock_seqlock);
@@ -351,7 +377,7 @@ void qtest_clock_warp(int64_t dest)
         int64_t deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
         int64_t warp = qemu_soonest_timeout(dest - clock, deadline);
         seqlock_write_lock(&timers_state.vm_clock_seqlock);
-        qemu_icount_bias += warp;
+        timers_state.qemu_icount_bias += warp;
         seqlock_write_unlock(&timers_state.vm_clock_seqlock);
 
         qemu_clock_run_timers(QEMU_CLOCK_VIRTUAL);
@@ -428,6 +454,25 @@ void qemu_clock_warp(QEMUClockType type)
     }
 }
 
+static bool icount_state_needed(void *opaque)
+{
+    return use_icount;
+}
+
+/*
+ * This is a subsection for icount migration.
+ */
+static const VMStateDescription icount_vmstate_timers = {
+    .name = "timer/icount",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT64(qemu_icount_bias, TimersState),
+        VMSTATE_INT64(qemu_icount, TimersState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_timers = {
     .name = "timer",
     .version_id = 2,
@@ -437,23 +482,48 @@ static const VMStateDescription vmstate_timers = {
         VMSTATE_INT64(dummy, TimersState),
         VMSTATE_INT64_V(cpu_clock_offset, TimersState, 2),
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (VMStateSubsection[]) {
+        {
+            .vmsd = &icount_vmstate_timers,
+            .needed = icount_state_needed,
+        }, {
+            /* empty */
+        }
     }
 };
 
-void configure_icount(const char *option)
+void cpu_ticks_init(void)
 {
     seqlock_init(&timers_state.vm_clock_seqlock, NULL);
     vmstate_register(NULL, 0, &vmstate_timers, &timers_state);
+}
+
+void configure_icount(QemuOpts *opts, Error **errp)
+{
+    const char *option;
+    char *rem_str = NULL;
+
+    option = qemu_opt_get(opts, "shift");
     if (!option) {
+        if (qemu_opt_get(opts, "align") != NULL) {
+            error_setg(errp, "Please specify shift option when using align");
+        }
         return;
     }
-
+    icount_align_option = qemu_opt_get_bool(opts, "align", false);
     icount_warp_timer = timer_new_ns(QEMU_CLOCK_REALTIME,
                                           icount_warp_rt, NULL);
     if (strcmp(option, "auto") != 0) {
-        icount_time_shift = strtol(option, NULL, 0);
+        errno = 0;
+        icount_time_shift = strtol(option, &rem_str, 0);
+        if (errno != 0 || *rem_str != '\0' || !strlen(option)) {
+            error_setg(errp, "icount: Invalid shift value");
+        }
         use_icount = 1;
         return;
+    } else if (icount_align_option) {
+        error_setg(errp, "shift=auto and align=on are incompatible");
     }
 
     use_icount = 2;
@@ -523,6 +593,15 @@ void cpu_synchronize_all_post_init(void)
     }
 }
 
+void cpu_clean_all_dirty(void)
+{
+    CPUState *cpu;
+
+    CPU_FOREACH(cpu) {
+        cpu_clean_state(cpu);
+    }
+}
+
 static int do_vm_stop(RunState state)
 {
     int ret = 0;
@@ -1250,7 +1329,8 @@ static int tcg_cpu_exec(CPUArchState *env)
         int64_t count;
         int64_t deadline;
         int decr;
-        qemu_icount -= (cpu->icount_decr.u16.low + cpu->icount_extra);
+        timers_state.qemu_icount -= (cpu->icount_decr.u16.low
+                                    + cpu->icount_extra);
         cpu->icount_decr.u16.low = 0;
         cpu->icount_extra = 0;
         deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
@@ -1265,7 +1345,7 @@ static int tcg_cpu_exec(CPUArchState *env)
         }
 
         count = qemu_icount_round(deadline);
-        qemu_icount += count;
+        timers_state.qemu_icount += count;
         decr = (count > 0xffff) ? 0xffff : count;
         count -= decr;
         cpu->icount_decr.u16.low = decr;
@@ -1278,7 +1358,8 @@ static int tcg_cpu_exec(CPUArchState *env)
     if (use_icount) {
         /* Fold pending instructions back into the
            instruction counter, and clear the interrupt flag.  */
-        qemu_icount -= (cpu->icount_decr.u16.low + cpu->icount_extra);
+        timers_state.qemu_icount -= (cpu->icount_decr.u16.low
+                        + cpu->icount_extra);
         cpu->icount_decr.u32 = 0;
         cpu->icount_extra = 0;
     }
@@ -1342,6 +1423,9 @@ CpuInfoList *qmp_query_cpus(Error **errp)
 #elif defined(TARGET_MIPS)
         MIPSCPU *mips_cpu = MIPS_CPU(cpu);
         CPUMIPSState *env = &mips_cpu->env;
+#elif defined(TARGET_TRICORE)
+        TriCoreCPU *tricore_cpu = TRICORE_CPU(cpu);
+        CPUTriCoreState *env = &tricore_cpu->env;
 #endif
 
         cpu_synchronize_state(cpu);
@@ -1366,6 +1450,9 @@ CpuInfoList *qmp_query_cpus(Error **errp)
 #elif defined(TARGET_MIPS)
         info->value->has_PC = true;
         info->value->PC = env->active_tc.PC;
+#elif defined(TARGET_TRICORE)
+        info->value->has_PC = true;
+        info->value->PC = env->PC;
 #endif
 
         /* XXX: waiting for the qapi to support GSList */
@@ -1469,21 +1556,24 @@ void qmp_inject_nmi(Error **errp)
             apic_deliver_nmi(cpu->apic_state);
         }
     }
-#elif defined(TARGET_S390X)
-    CPUState *cs;
-    S390CPU *cpu;
-
-    CPU_FOREACH(cs) {
-        cpu = S390_CPU(cs);
-        if (cpu->env.cpu_num == monitor_get_cpu_index()) {
-            if (s390_cpu_restart(S390_CPU(cs)) == -1) {
-                error_set(errp, QERR_UNSUPPORTED);
-                return;
-            }
-            break;
-        }
-    }
 #else
-    error_set(errp, QERR_UNSUPPORTED);
+    nmi_monitor_handle(monitor_get_cpu_index(), errp);
 #endif
 }
+
+void dump_drift_info(FILE *f, fprintf_function cpu_fprintf)
+{
+    if (!use_icount) {
+        return;
+    }
+
+    cpu_fprintf(f, "Host - Guest clock  %"PRIi64" ms\n",
+                (cpu_get_clock() - cpu_get_icount())/SCALE_MS);
+    if (icount_align_option) {
+        cpu_fprintf(f, "Max guest delay     %"PRIi64" ms\n", -max_delay/SCALE_MS);
+        cpu_fprintf(f, "Max guest advance   %"PRIi64" ms\n", max_advance/SCALE_MS);
+    } else {
+        cpu_fprintf(f, "Max guest delay     NA\n");
+        cpu_fprintf(f, "Max guest advance   NA\n");
+    }
+}
diff --git a/cputlb.c b/cputlb.c
index afd3705ff..a55518a0d 100644
--- a/cputlb.c
+++ b/cputlb.c
@@ -60,8 +60,10 @@ void tlb_flush(CPUState *cpu, int flush_global)
     cpu->current_tb = NULL;
 
     memset(env->tlb_table, -1, sizeof(env->tlb_table));
+    memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table));
     memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
 
+    env->vtlb_index = 0;
     env->tlb_flush_addr = -1;
     env->tlb_flush_mask = 0;
     tlb_flush_count++;
@@ -108,6 +110,14 @@ void tlb_flush_page(CPUState *cpu, target_ulong addr)
         tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
     }
 
+    /* check whether there are entries that need to be flushed in the vtlb */
+    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
+        int k;
+        for (k = 0; k < CPU_VTLB_SIZE; k++) {
+            tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], addr);
+        }
+    }
+
     tb_flush_jmp_cache(cpu, addr);
 }
 
@@ -172,6 +182,11 @@ void cpu_tlb_reset_dirty_all(ram_addr_t start1, ram_addr_t length)
                 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
                                       start1, length);
             }
+
+            for (i = 0; i < CPU_VTLB_SIZE; i++) {
+                tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i],
+                                      start1, length);
+            }
         }
     }
 }
@@ -195,6 +210,13 @@ void tlb_set_dirty(CPUArchState *env, target_ulong vaddr)
     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
         tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
     }
+
+    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
+        int k;
+        for (k = 0; k < CPU_VTLB_SIZE; k++) {
+            tlb_set_dirty1(&env->tlb_v_table[mmu_idx][k], vaddr);
+        }
+    }
 }
 
 /* Our TLB does not support large pages, so remember the area covered by
@@ -235,6 +257,7 @@ void tlb_set_page(CPUState *cpu, target_ulong vaddr,
     uintptr_t addend;
     CPUTLBEntry *te;
     hwaddr iotlb, xlat, sz;
+    unsigned vidx = env->vtlb_index++ % CPU_VTLB_SIZE;
 
     assert(size >= TARGET_PAGE_SIZE);
     if (size != TARGET_PAGE_SIZE) {
@@ -267,8 +290,14 @@ void tlb_set_page(CPUState *cpu, target_ulong vaddr,
                                             prot, &address);
 
     index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-    env->iotlb[mmu_idx][index] = iotlb - vaddr;
     te = &env->tlb_table[mmu_idx][index];
+
+    /* do not discard the translation in te, evict it into a victim tlb */
+    env->tlb_v_table[mmu_idx][vidx] = *te;
+    env->iotlb_v[mmu_idx][vidx] = env->iotlb[mmu_idx][index];
+
+    /* refill the tlb */
+    env->iotlb[mmu_idx][index] = iotlb - vaddr;
     te->addend = addend - vaddr;
     if (prot & PAGE_READ) {
         te->addr_read = address;
diff --git a/default-configs/mips-softmmu.mak b/default-configs/mips-softmmu.mak
index 71177efdf..2a80b04dc 100644
--- a/default-configs/mips-softmmu.mak
+++ b/default-configs/mips-softmmu.mak
@@ -32,6 +32,5 @@ CONFIG_G364FB=y
 CONFIG_I8259=y
 CONFIG_JAZZ_LED=y
 CONFIG_MC146818RTC=y
-CONFIG_VT82C686=y
 CONFIG_ISA_TESTDEV=y
 CONFIG_EMPTY_SLOT=y
diff --git a/default-configs/mips64-softmmu.mak b/default-configs/mips64-softmmu.mak
index 617301b75..f1f933bc4 100644
--- a/default-configs/mips64-softmmu.mak
+++ b/default-configs/mips64-softmmu.mak
@@ -32,6 +32,5 @@ CONFIG_G364FB=y
 CONFIG_I8259=y
 CONFIG_JAZZ_LED=y
 CONFIG_MC146818RTC=y
-CONFIG_VT82C686=y
 CONFIG_ISA_TESTDEV=y
 CONFIG_EMPTY_SLOT=y
diff --git a/default-configs/mipsel-softmmu.mak b/default-configs/mipsel-softmmu.mak
index 532a9aefb..7708185f6 100644
--- a/default-configs/mipsel-softmmu.mak
+++ b/default-configs/mipsel-softmmu.mak
@@ -32,6 +32,5 @@ CONFIG_G364FB=y
 CONFIG_I8259=y
 CONFIG_JAZZ_LED=y
 CONFIG_MC146818RTC=y
-CONFIG_VT82C686=y
 CONFIG_ISA_TESTDEV=y
 CONFIG_EMPTY_SLOT=y
diff --git a/default-configs/ppc-softmmu.mak b/default-configs/ppc-softmmu.mak
index 33f8d84e6..d725b2331 100644
--- a/default-configs/ppc-softmmu.mak
+++ b/default-configs/ppc-softmmu.mak
@@ -45,8 +45,8 @@ CONFIG_PREP=y
 CONFIG_MAC=y
 CONFIG_E500=y
 CONFIG_OPENPIC_KVM=$(and $(CONFIG_E500),$(CONFIG_KVM))
+CONFIG_ETSEC=y
+CONFIG_LIBDECNUMBER=y
 # For PReP
 CONFIG_MC146818RTC=y
-CONFIG_ETSEC=y
 CONFIG_ISA_TESTDEV=y
-CONFIG_LIBDECNUMBER=y
diff --git a/default-configs/ppc64-softmmu.mak b/default-configs/ppc64-softmmu.mak
index 37a15b7ce..bd30d6934 100644
--- a/default-configs/ppc64-softmmu.mak
+++ b/default-configs/ppc64-softmmu.mak
@@ -46,6 +46,8 @@ CONFIG_PREP=y
 CONFIG_MAC=y
 CONFIG_E500=y
 CONFIG_OPENPIC_KVM=$(and $(CONFIG_E500),$(CONFIG_KVM))
+CONFIG_ETSEC=y
+CONFIG_LIBDECNUMBER=y
 # For pSeries
 CONFIG_XICS=$(CONFIG_PSERIES)
 CONFIG_XICS_KVM=$(and $(CONFIG_PSERIES),$(CONFIG_KVM))
@@ -58,4 +60,3 @@ CONFIG_I82374=y
 CONFIG_I8257=y
 CONFIG_MC146818RTC=y
 CONFIG_ISA_TESTDEV=y
-CONFIG_LIBDECNUMBER=y
diff --git a/default-configs/tricore-softmmu.mak b/default-configs/tricore-softmmu.mak
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/default-configs/tricore-softmmu.mak
diff --git a/device-hotplug.c b/device-hotplug.c
index e6a1ffb9f..9e38cc480 100644
--- a/device-hotplug.c
+++ b/device-hotplug.c
@@ -24,6 +24,7 @@
 
 #include "hw/hw.h"
 #include "hw/boards.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "qemu/config-file.h"
 #include "sysemu/sysemu.h"
@@ -76,6 +77,6 @@ void drive_hot_add(Monitor *mon, const QDict *qdict)
 
 err:
     if (dinfo) {
-        drive_del(dinfo);
+        blk_unref(blk_by_legacy_dinfo(dinfo));
     }
 }
diff --git a/device_tree.c b/device_tree.c
index ca8350481..df9eed9cb 100644
--- a/device_tree.c
+++ b/device_tree.c
@@ -20,6 +20,7 @@
 
 #include "config.h"
 #include "qemu-common.h"
+#include "qemu/error-report.h"
 #include "sysemu/device_tree.h"
 #include "sysemu/sysemu.h"
 #include "hw/loader.h"
@@ -59,13 +60,13 @@ void *create_device_tree(int *sizep)
     }
     ret = fdt_open_into(fdt, fdt, *sizep);
     if (ret) {
-        fprintf(stderr, "Unable to copy device tree in memory\n");
+        error_report("Unable to copy device tree in memory");
         exit(1);
     }
 
     return fdt;
 fail:
-    fprintf(stderr, "%s Couldn't create dt: %s\n", __func__, fdt_strerror(ret));
+    error_report("%s Couldn't create dt: %s", __func__, fdt_strerror(ret));
     exit(1);
 }
 
@@ -79,8 +80,8 @@ void *load_device_tree(const char *filename_path, int *sizep)
     *sizep = 0;
     dt_size = get_image_size(filename_path);
     if (dt_size < 0) {
-        printf("Unable to get size of device tree file '%s'\n",
-            filename_path);
+        error_report("Unable to get size of device tree file '%s'",
+                     filename_path);
         goto fail;
     }
 
@@ -92,21 +93,21 @@ void *load_device_tree(const char *filename_path, int *sizep)
 
     dt_file_load_size = load_image(filename_path, fdt);
     if (dt_file_load_size < 0) {
-        printf("Unable to open device tree file '%s'\n",
-               filename_path);
+        error_report("Unable to open device tree file '%s'",
+                     filename_path);
         goto fail;
     }
 
     ret = fdt_open_into(fdt, fdt, dt_size);
     if (ret) {
-        printf("Unable to copy device tree in memory\n");
+        error_report("Unable to copy device tree in memory");
         goto fail;
     }
 
     /* Check sanity of device tree */
     if (fdt_check_header(fdt)) {
-        printf ("Device tree file loaded into memory is invalid: %s\n",
-            filename_path);
+        error_report("Device tree file loaded into memory is invalid: %s",
+                     filename_path);
         goto fail;
     }
     *sizep = dt_size;
@@ -123,8 +124,8 @@ static int findnode_nofail(void *fdt, const char *node_path)
 
     offset = fdt_path_offset(fdt, node_path);
     if (offset < 0) {
-        fprintf(stderr, "%s Couldn't find node %s: %s\n", __func__, node_path,
-                fdt_strerror(offset));
+        error_report("%s Couldn't find node %s: %s", __func__, node_path,
+                     fdt_strerror(offset));
         exit(1);
     }
 
@@ -138,8 +139,8 @@ int qemu_fdt_setprop(void *fdt, const char *node_path,
 
     r = fdt_setprop(fdt, findnode_nofail(fdt, node_path), property, val, size);
     if (r < 0) {
-        fprintf(stderr, "%s: Couldn't set %s/%s: %s\n", __func__, node_path,
-                property, fdt_strerror(r));
+        error_report("%s: Couldn't set %s/%s: %s", __func__, node_path,
+                     property, fdt_strerror(r));
         exit(1);
     }
 
@@ -153,8 +154,8 @@ int qemu_fdt_setprop_cell(void *fdt, const char *node_path,
 
     r = fdt_setprop_cell(fdt, findnode_nofail(fdt, node_path), property, val);
     if (r < 0) {
-        fprintf(stderr, "%s: Couldn't set %s/%s = %#08x: %s\n", __func__,
-                node_path, property, val, fdt_strerror(r));
+        error_report("%s: Couldn't set %s/%s = %#08x: %s", __func__,
+                     node_path, property, val, fdt_strerror(r));
         exit(1);
     }
 
@@ -175,8 +176,8 @@ int qemu_fdt_setprop_string(void *fdt, const char *node_path,
 
     r = fdt_setprop_string(fdt, findnode_nofail(fdt, node_path), property, string);
     if (r < 0) {
-        fprintf(stderr, "%s: Couldn't set %s/%s = %s: %s\n", __func__,
-                node_path, property, string, fdt_strerror(r));
+        error_report("%s: Couldn't set %s/%s = %s: %s", __func__,
+                     node_path, property, string, fdt_strerror(r));
         exit(1);
     }
 
@@ -193,8 +194,8 @@ const void *qemu_fdt_getprop(void *fdt, const char *node_path,
     }
     r = fdt_getprop(fdt, findnode_nofail(fdt, node_path), property, lenp);
     if (!r) {
-        fprintf(stderr, "%s: Couldn't get %s/%s: %s\n", __func__,
-                node_path, property, fdt_strerror(*lenp));
+        error_report("%s: Couldn't get %s/%s: %s", __func__,
+                     node_path, property, fdt_strerror(*lenp));
         exit(1);
     }
     return r;
@@ -206,8 +207,8 @@ uint32_t qemu_fdt_getprop_cell(void *fdt, const char *node_path,
     int len;
     const uint32_t *p = qemu_fdt_getprop(fdt, node_path, property, &len);
     if (len != 4) {
-        fprintf(stderr, "%s: %s/%s not 4 bytes long (not a cell?)\n",
-                __func__, node_path, property);
+        error_report("%s: %s/%s not 4 bytes long (not a cell?)",
+                     __func__, node_path, property);
         exit(1);
     }
     return be32_to_cpu(*p);
@@ -219,8 +220,8 @@ uint32_t qemu_fdt_get_phandle(void *fdt, const char *path)
 
     r = fdt_get_phandle(fdt, findnode_nofail(fdt, path));
     if (r == 0) {
-        fprintf(stderr, "%s: Couldn't get phandle for %s: %s\n", __func__,
-                path, fdt_strerror(r));
+        error_report("%s: Couldn't get phandle for %s: %s", __func__,
+                     path, fdt_strerror(r));
         exit(1);
     }
 
@@ -265,8 +266,8 @@ int qemu_fdt_nop_node(void *fdt, const char *node_path)
 
     r = fdt_nop_node(fdt, findnode_nofail(fdt, node_path));
     if (r < 0) {
-        fprintf(stderr, "%s: Couldn't nop node %s: %s\n", __func__, node_path,
-                fdt_strerror(r));
+        error_report("%s: Couldn't nop node %s: %s", __func__, node_path,
+                     fdt_strerror(r));
         exit(1);
     }
 
@@ -294,8 +295,8 @@ int qemu_fdt_add_subnode(void *fdt, const char *name)
 
     retval = fdt_add_subnode(fdt, parent, basename);
     if (retval < 0) {
-        fprintf(stderr, "FDT: Failed to create subnode %s: %s\n", name,
-                fdt_strerror(retval));
+        error_report("FDT: Failed to create subnode %s: %s", name,
+                     fdt_strerror(retval));
         exit(1);
     }
 
diff --git a/disas/arm-a64.cc b/disas/arm-a64.cc
index 162be0c42..ca29f6f25 100644
--- a/disas/arm-a64.cc
+++ b/disas/arm-a64.cc
@@ -39,7 +39,7 @@ public:
     ~QEMUDisassembler() { }
 
 protected:
-    void ProcessOutput(Instruction *instr) {
+    virtual void ProcessOutput(const Instruction *instr) {
         fprintf(stream_, "%08" PRIx32 "      %s",
                 instr->InstructionBits(), GetOutput());
     }
diff --git a/disas/libvixl/README b/disas/libvixl/README
index a0ecac3dd..cba31b458 100644
--- a/disas/libvixl/README
+++ b/disas/libvixl/README
@@ -2,7 +2,7 @@
 The code in this directory is a subset of libvixl:
  https://github.com/armvixl/vixl
 (specifically, it is the set of files needed for disassembly only,
-taken from libvixl 1.4).
+taken from libvixl 1.6).
 Bugfixes should preferably be sent upstream initially.
 
 The disassembler does not currently support the entire A64 instruction
diff --git a/disas/libvixl/a64/assembler-a64.h b/disas/libvixl/a64/assembler-a64.h
index 1e2947b28..16a704b7d 100644
--- a/disas/libvixl/a64/assembler-a64.h
+++ b/disas/libvixl/a64/assembler-a64.h
@@ -28,9 +28,11 @@
 #define VIXL_A64_ASSEMBLER_A64_H_
 
 #include <list>
+#include <stack>
 
 #include "globals.h"
 #include "utils.h"
+#include "code-buffer.h"
 #include "a64/instructions-a64.h"
 
 namespace vixl {
@@ -167,6 +169,11 @@ class CPURegister {
     return type_ == kFPRegister;
   }
 
+  bool IsW() const { return IsValidRegister() && Is32Bits(); }
+  bool IsX() const { return IsValidRegister() && Is64Bits(); }
+  bool IsS() const { return IsValidFPRegister() && Is32Bits(); }
+  bool IsD() const { return IsValidFPRegister() && Is64Bits(); }
+
   const Register& W() const;
   const Register& X() const;
   const FPRegister& S() const;
@@ -190,12 +197,12 @@ class CPURegister {
 
 class Register : public CPURegister {
  public:
-  explicit Register() : CPURegister() {}
+  Register() : CPURegister() {}
   inline explicit Register(const CPURegister& other)
       : CPURegister(other.code(), other.size(), other.type()) {
     VIXL_ASSERT(IsValidRegister());
   }
-  explicit Register(unsigned code, unsigned size)
+  Register(unsigned code, unsigned size)
       : CPURegister(code, size, kRegister) {}
 
   bool IsValid() const {
@@ -535,7 +542,7 @@ class Operand {
 class MemOperand {
  public:
   explicit MemOperand(Register base,
-                      ptrdiff_t offset = 0,
+                      int64_t offset = 0,
                       AddrMode addrmode = Offset);
   explicit MemOperand(Register base,
                       Register regoffset,
@@ -551,7 +558,7 @@ class MemOperand {
 
   const Register& base() const { return base_; }
   const Register& regoffset() const { return regoffset_; }
-  ptrdiff_t offset() const { return offset_; }
+  int64_t offset() const { return offset_; }
   AddrMode addrmode() const { return addrmode_; }
   Shift shift() const { return shift_; }
   Extend extend() const { return extend_; }
@@ -564,7 +571,7 @@ class MemOperand {
  private:
   Register base_;
   Register regoffset_;
-  ptrdiff_t offset_;
+  int64_t offset_;
   AddrMode addrmode_;
   Shift shift_;
   Extend extend_;
@@ -574,71 +581,233 @@ class MemOperand {
 
 class Label {
  public:
-  Label() : is_bound_(false), link_(NULL), target_(NULL) {}
+  Label() : location_(kLocationUnbound) {}
   ~Label() {
     // If the label has been linked to, it needs to be bound to a target.
     VIXL_ASSERT(!IsLinked() || IsBound());
   }
 
-  inline Instruction* link() const { return link_; }
-  inline Instruction* target() const { return target_; }
+  inline bool IsBound() const { return location_ >= 0; }
+  inline bool IsLinked() const { return !links_.empty(); }
 
-  inline bool IsBound() const { return is_bound_; }
-  inline bool IsLinked() const { return link_ != NULL; }
+ private:
+  // The list of linked instructions is stored in a stack-like structure. We
+  // don't use std::stack directly because it's slow for the common case where
+  // only one or two instructions refer to a label, and labels themselves are
+  // short-lived. This class behaves like std::stack, but the first few links
+  // are preallocated (configured by kPreallocatedLinks).
+  //
+  // If more than N links are required, this falls back to std::stack.
+  class LinksStack {
+   public:
+    LinksStack() : size_(0), links_extended_(NULL) {}
+    ~LinksStack() {
+      delete links_extended_;
+    }
 
-  inline void set_link(Instruction* new_link) { link_ = new_link; }
+    size_t size() const {
+      return size_;
+    }
 
-  static const int kEndOfChain = 0;
+    bool empty() const {
+      return size_ == 0;
+    }
 
- private:
-  // Indicates if the label has been bound, ie its location is fixed.
-  bool is_bound_;
-  // Branches instructions branching to this label form a chained list, with
-  // their offset indicating where the next instruction is located.
-  // link_ points to the latest branch instruction generated branching to this
-  // branch.
-  // If link_ is not NULL, the label has been linked to.
-  Instruction* link_;
+    void push(ptrdiff_t value) {
+      if (size_ < kPreallocatedLinks) {
+        links_[size_] = value;
+      } else {
+        if (links_extended_ == NULL) {
+          links_extended_ = new std::stack<ptrdiff_t>();
+        }
+        VIXL_ASSERT(size_ == (links_extended_->size() + kPreallocatedLinks));
+        links_extended_->push(value);
+      }
+      size_++;
+    }
+
+    ptrdiff_t top() const {
+      return (size_ <= kPreallocatedLinks) ? links_[size_ - 1]
+                                           : links_extended_->top();
+    }
+
+    void pop() {
+      size_--;
+      if (size_ >= kPreallocatedLinks) {
+        links_extended_->pop();
+        VIXL_ASSERT(size_ == (links_extended_->size() + kPreallocatedLinks));
+      }
+    }
+
+   private:
+    static const size_t kPreallocatedLinks = 4;
+
+    size_t size_;
+    ptrdiff_t links_[kPreallocatedLinks];
+    std::stack<ptrdiff_t> * links_extended_;
+  };
+
+  inline ptrdiff_t location() const { return location_; }
+
+  inline void Bind(ptrdiff_t location) {
+    // Labels can only be bound once.
+    VIXL_ASSERT(!IsBound());
+    location_ = location;
+  }
+
+  inline void AddLink(ptrdiff_t instruction) {
+    // If a label is bound, the assembler already has the information it needs
+    // to write the instruction, so there is no need to add it to links_.
+    VIXL_ASSERT(!IsBound());
+    links_.push(instruction);
+  }
+
+  inline ptrdiff_t GetAndRemoveNextLink() {
+    VIXL_ASSERT(IsLinked());
+    ptrdiff_t link = links_.top();
+    links_.pop();
+    return link;
+  }
+
+  // The offsets of the instructions that have linked to this label.
+  LinksStack links_;
   // The label location.
-  Instruction* target_;
+  ptrdiff_t location_;
+
+  static const ptrdiff_t kLocationUnbound = -1;
+
+  // It is not safe to copy labels, so disable the copy constructor by declaring
+  // it private (without an implementation).
+  Label(const Label&);
 
+  // The Assembler class is responsible for binding and linking labels, since
+  // the stored offsets need to be consistent with the Assembler's buffer.
   friend class Assembler;
 };
 
 
-// TODO: Obtain better values for these, based on real-world data.
-const int kLiteralPoolCheckInterval = 4 * KBytes;
-const int kRecommendedLiteralPoolRange = 2 * kLiteralPoolCheckInterval;
+// A literal is a 32-bit or 64-bit piece of data stored in the instruction
+// stream and loaded through a pc relative load. The same literal can be
+// referred to by multiple instructions but a literal can only reside at one
+// place in memory. A literal can be used by a load before or after being
+// placed in memory.
+//
+// Internally an offset of 0 is associated with a literal which has been
+// neither used nor placed. Then two possibilities arise:
+//  1) the label is placed, the offset (stored as offset + 1) is used to
+//     resolve any subsequent load using the label.
+//  2) the label is not placed and offset is the offset of the last load using
+//     the literal (stored as -offset -1). If multiple loads refer to this
+//     literal then the last load holds the offset of the preceding load and
+//     all loads form a chain. Once the offset is placed all the loads in the
+//     chain are resolved and future loads fall back to possibility 1.
+class RawLiteral {
+ public:
+  RawLiteral() : size_(0), offset_(0), raw_value_(0) {}
+
+  size_t size() {
+    VIXL_STATIC_ASSERT(kDRegSizeInBytes == kXRegSizeInBytes);
+    VIXL_STATIC_ASSERT(kSRegSizeInBytes == kWRegSizeInBytes);
+    VIXL_ASSERT((size_ == kXRegSizeInBytes) || (size_ == kWRegSizeInBytes));
+    return size_;
+  }
+  uint64_t raw_value64() {
+    VIXL_ASSERT(size_ == kXRegSizeInBytes);
+    return raw_value_;
+  }
+  uint32_t raw_value32() {
+    VIXL_ASSERT(size_ == kWRegSizeInBytes);
+    VIXL_ASSERT(is_uint32(raw_value_) || is_int32(raw_value_));
+    return static_cast<uint32_t>(raw_value_);
+  }
+  bool IsUsed() { return offset_ < 0; }
+  bool IsPlaced() { return offset_ > 0; }
+
+ protected:
+  ptrdiff_t offset() {
+    VIXL_ASSERT(IsPlaced());
+    return offset_ - 1;
+  }
+  void set_offset(ptrdiff_t offset) {
+    VIXL_ASSERT(offset >= 0);
+    VIXL_ASSERT(IsWordAligned(offset));
+    VIXL_ASSERT(!IsPlaced());
+    offset_ = offset + 1;
+  }
+  ptrdiff_t last_use() {
+    VIXL_ASSERT(IsUsed());
+    return -offset_ - 1;
+  }
+  void set_last_use(ptrdiff_t offset) {
+    VIXL_ASSERT(offset >= 0);
+    VIXL_ASSERT(IsWordAligned(offset));
+    VIXL_ASSERT(!IsPlaced());
+    offset_ = -offset - 1;
+  }
 
+  size_t size_;
+  ptrdiff_t offset_;
+  uint64_t raw_value_;
 
-// Control whether a branch over the literal pool should also be emitted. This
-// is needed if the literal pool has to be emitted in the middle of the JITted
-// code.
-enum LiteralPoolEmitOption {
-  JumpRequired,
-  NoJumpRequired
+  friend class Assembler;
 };
 
 
-// Literal pool entry.
-class Literal {
+template <typename T>
+class Literal : public RawLiteral {
  public:
-  Literal(Instruction* pc, uint64_t imm, unsigned size)
-      : pc_(pc), value_(imm), size_(size) {}
+  explicit Literal(T value) {
+    size_ = sizeof(value);
+    memcpy(&raw_value_, &value, sizeof(value));
+  }
+};
 
- private:
-  Instruction* pc_;
-  int64_t value_;
-  unsigned size_;
 
-  friend class Assembler;
+// Control whether or not position-independent code should be emitted.
+enum PositionIndependentCodeOption {
+  // All code generated will be position-independent; all branches and
+  // references to labels generated with the Label class will use PC-relative
+  // addressing.
+  PositionIndependentCode,
+
+  // Allow VIXL to generate code that refers to absolute addresses. With this
+  // option, it will not be possible to copy the code buffer and run it from a
+  // different address; code must be generated in its final location.
+  PositionDependentCode,
+
+  // Allow VIXL to assume that the bottom 12 bits of the address will be
+  // constant, but that the top 48 bits may change. This allows `adrp` to
+  // function in systems which copy code between pages, but otherwise maintain
+  // 4KB page alignment.
+  PageOffsetDependentCode
+};
+
+
+// Control how scaled- and unscaled-offset loads and stores are generated.
+enum LoadStoreScalingOption {
+  // Prefer scaled-immediate-offset instructions, but emit unscaled-offset,
+  // register-offset, pre-index or post-index instructions if necessary.
+  PreferScaledOffset,
+
+  // Prefer unscaled-immediate-offset instructions, but emit scaled-offset,
+  // register-offset, pre-index or post-index instructions if necessary.
+  PreferUnscaledOffset,
+
+  // Require scaled-immediate-offset instructions.
+  RequireScaledOffset,
+
+  // Require unscaled-immediate-offset instructions.
+  RequireUnscaledOffset
 };
 
 
 // Assembler.
 class Assembler {
  public:
-  Assembler(byte* buffer, unsigned buffer_size);
+  Assembler(size_t capacity,
+            PositionIndependentCodeOption pic = PositionIndependentCode);
+  Assembler(byte* buffer, size_t capacity,
+            PositionIndependentCodeOption pic = PositionIndependentCode);
 
   // The destructor asserts that one of the following is true:
   //  * The Assembler object has not been used.
@@ -650,9 +819,6 @@ class Assembler {
 
   // Start generating code from the beginning of the buffer, discarding any code
   // and data that has already been emitted into the buffer.
-  //
-  // In order to avoid any accidental transfer of state, Reset ASSERTs that the
-  // constant pool is not blocked.
   void Reset();
 
   // Finalize a code buffer of generated instructions. This function must be
@@ -662,12 +828,49 @@ class Assembler {
   // Label.
   // Bind a label to the current PC.
   void bind(Label* label);
-  int UpdateAndGetByteOffsetTo(Label* label);
-  inline int UpdateAndGetInstructionOffsetTo(Label* label) {
-    VIXL_ASSERT(Label::kEndOfChain == 0);
-    return UpdateAndGetByteOffsetTo(label) >> kInstructionSizeLog2;
+
+  // Bind a label to a specified offset from the start of the buffer.
+  void BindToOffset(Label* label, ptrdiff_t offset);
+
+  // Place a literal at the current PC.
+  void place(RawLiteral* literal);
+
+  ptrdiff_t CursorOffset() const {
+    return buffer_->CursorOffset();
+  }
+
+  ptrdiff_t BufferEndOffset() const {
+    return static_cast<ptrdiff_t>(buffer_->capacity());
+  }
+
+  // Return the address of an offset in the buffer.
+  template <typename T>
+  inline T GetOffsetAddress(ptrdiff_t offset) {
+    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
+    return buffer_->GetOffsetAddress<T>(offset);
+  }
+
+  // Return the address of a bound label.
+  template <typename T>
+  inline T GetLabelAddress(const Label * label) {
+    VIXL_ASSERT(label->IsBound());
+    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
+    return GetOffsetAddress<T>(label->location());
+  }
+
+  // Return the address of the cursor.
+  template <typename T>
+  inline T GetCursorAddress() {
+    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
+    return GetOffsetAddress<T>(CursorOffset());
   }
 
+  // Return the address of the start of the buffer.
+  template <typename T>
+  inline T GetStartAddress() {
+    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
+    return GetOffsetAddress<T>(0);
+  }
 
   // Instruction set functions.
 
@@ -733,6 +936,12 @@ class Assembler {
   // Calculate the address of a PC offset.
   void adr(const Register& rd, int imm21);
 
+  // Calculate the page address of a label.
+  void adrp(const Register& rd, Label* label);
+
+  // Calculate the page address of a PC offset.
+  void adrp(const Register& rd, int imm21);
+
   // Data Processing instructions.
   // Add.
   void add(const Register& rd,
@@ -1112,31 +1321,76 @@ class Assembler {
 
   // Memory instructions.
   // Load integer or FP register.
-  void ldr(const CPURegister& rt, const MemOperand& src);
+  void ldr(const CPURegister& rt, const MemOperand& src,
+           LoadStoreScalingOption option = PreferScaledOffset);
 
   // Store integer or FP register.
-  void str(const CPURegister& rt, const MemOperand& dst);
+  void str(const CPURegister& rt, const MemOperand& dst,
+           LoadStoreScalingOption option = PreferScaledOffset);
 
   // Load word with sign extension.
-  void ldrsw(const Register& rt, const MemOperand& src);
+  void ldrsw(const Register& rt, const MemOperand& src,
+             LoadStoreScalingOption option = PreferScaledOffset);
 
   // Load byte.
-  void ldrb(const Register& rt, const MemOperand& src);
+  void ldrb(const Register& rt, const MemOperand& src,
+            LoadStoreScalingOption option = PreferScaledOffset);
 
   // Store byte.
-  void strb(const Register& rt, const MemOperand& dst);
+  void strb(const Register& rt, const MemOperand& dst,
+            LoadStoreScalingOption option = PreferScaledOffset);
 
   // Load byte with sign extension.
-  void ldrsb(const Register& rt, const MemOperand& src);
+  void ldrsb(const Register& rt, const MemOperand& src,
+             LoadStoreScalingOption option = PreferScaledOffset);
 
   // Load half-word.
-  void ldrh(const Register& rt, const MemOperand& src);
+  void ldrh(const Register& rt, const MemOperand& src,
+            LoadStoreScalingOption option = PreferScaledOffset);
 
   // Store half-word.
-  void strh(const Register& rt, const MemOperand& dst);
+  void strh(const Register& rt, const MemOperand& dst,
+            LoadStoreScalingOption option = PreferScaledOffset);
 
   // Load half-word with sign extension.
-  void ldrsh(const Register& rt, const MemOperand& src);
+  void ldrsh(const Register& rt, const MemOperand& src,
+             LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Load integer or FP register (with unscaled offset).
+  void ldur(const CPURegister& rt, const MemOperand& src,
+            LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Store integer or FP register (with unscaled offset).
+  void stur(const CPURegister& rt, const MemOperand& src,
+            LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Load word with sign extension.
+  void ldursw(const Register& rt, const MemOperand& src,
+              LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Load byte (with unscaled offset).
+  void ldurb(const Register& rt, const MemOperand& src,
+             LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Store byte (with unscaled offset).
+  void sturb(const Register& rt, const MemOperand& dst,
+             LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Load byte with sign extension (and unscaled offset).
+  void ldursb(const Register& rt, const MemOperand& src,
+              LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Load half-word (with unscaled offset).
+  void ldurh(const Register& rt, const MemOperand& src,
+             LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Store half-word (with unscaled offset).
+  void sturh(const Register& rt, const MemOperand& dst,
+             LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Load half-word with sign extension (and unscaled offset).
+  void ldursh(const Register& rt, const MemOperand& src,
+              LoadStoreScalingOption option = PreferUnscaledOffset);
 
   // Load integer or FP register pair.
   void ldp(const CPURegister& rt, const CPURegister& rt2,
@@ -1157,14 +1411,90 @@ class Assembler {
   void stnp(const CPURegister& rt, const CPURegister& rt2,
             const MemOperand& dst);
 
-  // Load literal to register.
-  void ldr(const Register& rt, uint64_t imm);
+  // Load integer or FP register from literal pool.
+  void ldr(const CPURegister& rt, RawLiteral* literal);
+
+  // Load word with sign extension from literal pool.
+  void ldrsw(const Register& rt, RawLiteral* literal);
 
-  // Load double precision floating point literal to FP register.
-  void ldr(const FPRegister& ft, double imm);
+  // Load integer or FP register from pc + imm19 << 2.
+  void ldr(const CPURegister& rt, int imm19);
+
+  // Load word with sign extension from pc + imm19 << 2.
+  void ldrsw(const Register& rt, int imm19);
+
+  // Store exclusive byte.
+  void stxrb(const Register& rs, const Register& rt, const MemOperand& dst);
+
+  // Store exclusive half-word.
+  void stxrh(const Register& rs, const Register& rt, const MemOperand& dst);
+
+  // Store exclusive register.
+  void stxr(const Register& rs, const Register& rt, const MemOperand& dst);
+
+  // Load exclusive byte.
+  void ldxrb(const Register& rt, const MemOperand& src);
+
+  // Load exclusive half-word.
+  void ldxrh(const Register& rt, const MemOperand& src);
+
+  // Load exclusive register.
+  void ldxr(const Register& rt, const MemOperand& src);
+
+  // Store exclusive register pair.
+  void stxp(const Register& rs,
+            const Register& rt,
+            const Register& rt2,
+            const MemOperand& dst);
+
+  // Load exclusive register pair.
+  void ldxp(const Register& rt, const Register& rt2, const MemOperand& src);
+
+  // Store-release exclusive byte.
+  void stlxrb(const Register& rs, const Register& rt, const MemOperand& dst);
+
+  // Store-release exclusive half-word.
+  void stlxrh(const Register& rs, const Register& rt, const MemOperand& dst);
+
+  // Store-release exclusive register.
+  void stlxr(const Register& rs, const Register& rt, const MemOperand& dst);
+
+  // Load-acquire exclusive byte.
+  void ldaxrb(const Register& rt, const MemOperand& src);
+
+  // Load-acquire exclusive half-word.
+  void ldaxrh(const Register& rt, const MemOperand& src);
+
+  // Load-acquire exclusive register.
+  void ldaxr(const Register& rt, const MemOperand& src);
+
+  // Store-release exclusive register pair.
+  void stlxp(const Register& rs,
+             const Register& rt,
+             const Register& rt2,
+             const MemOperand& dst);
+
+  // Load-acquire exclusive register pair.
+  void ldaxp(const Register& rt, const Register& rt2, const MemOperand& src);
+
+  // Store-release byte.
+  void stlrb(const Register& rt, const MemOperand& dst);
+
+  // Store-release half-word.
+  void stlrh(const Register& rt, const MemOperand& dst);
+
+  // Store-release register.
+  void stlr(const Register& rt, const MemOperand& dst);
+
+  // Load-acquire byte.
+  void ldarb(const Register& rt, const MemOperand& src);
+
+  // Load-acquire half-word.
+  void ldarh(const Register& rt, const MemOperand& src);
+
+  // Load-acquire register.
+  void ldar(const Register& rt, const MemOperand& src);
 
-  // Load single precision floating point literal to FP register.
-  void ldr(const FPRegister& ft, float imm);
 
   // Move instructions. The default shift of -1 indicates that the move
   // instruction will calculate an appropriate 16-bit immediate and left shift
@@ -1214,6 +1544,9 @@ class Assembler {
   // System hint.
   void hint(SystemHint code);
 
+  // Clear exclusive monitor.
+  void clrex(int imm4 = 0xf);
+
   // Data memory barrier.
   void dmb(BarrierDomain domain, BarrierType type);
 
@@ -1375,25 +1708,26 @@ class Assembler {
   inline void dci(Instr raw_inst) { Emit(raw_inst); }
 
   // Emit 32 bits of data into the instruction stream.
-  inline void dc32(uint32_t data) { EmitData(&data, sizeof(data)); }
+  inline void dc32(uint32_t data) {
+    VIXL_ASSERT(buffer_monitor_ > 0);
+    buffer_->Emit32(data);
+  }
 
   // Emit 64 bits of data into the instruction stream.
-  inline void dc64(uint64_t data) { EmitData(&data, sizeof(data)); }
+  inline void dc64(uint64_t data) {
+    VIXL_ASSERT(buffer_monitor_ > 0);
+    buffer_->Emit64(data);
+  }
 
   // Copy a string into the instruction stream, including the terminating NULL
-  // character. The instruction pointer (pc_) is then aligned correctly for
+  // character. The instruction pointer is then aligned correctly for
   // subsequent instructions.
-  void EmitStringData(const char * string) {
+  void EmitString(const char * string) {
     VIXL_ASSERT(string != NULL);
+    VIXL_ASSERT(buffer_monitor_ > 0);
 
-    size_t len = strlen(string) + 1;
-    EmitData(string, len);
-
-    // Pad with NULL characters until pc_ is aligned.
-    const char pad[] = {'\0', '\0', '\0', '\0'};
-    VIXL_STATIC_ASSERT(sizeof(pad) == kInstructionSize);
-    Instruction* next_pc = AlignUp(pc_, kInstructionSize);
-    EmitData(&pad, next_pc - pc_);
+    buffer_->EmitString(string);
+    buffer_->Align();
   }
 
   // Code generation helpers.
@@ -1429,6 +1763,11 @@ class Assembler {
     return rt2.code() << Rt2_offset;
   }
 
+  static Instr Rs(CPURegister rs) {
+    VIXL_ASSERT(rs.code() != kSPRegInternalCode);
+    return rs.code() << Rs_offset;
+  }
+
   // These encoding functions allow the stack pointer to be encoded, and
   // disallow the zero register.
   static Instr RdSP(Register rd) {
@@ -1619,6 +1958,11 @@ class Assembler {
     return imm7 << ImmHint_offset;
   }
 
+  static Instr CRm(int imm4) {
+    VIXL_ASSERT(is_uint4(imm4));
+    return imm4 << CRm_offset;
+  }
+
   static Instr ImmBarrierDomain(int imm2) {
     VIXL_ASSERT(is_uint2(imm2));
     return imm2 << ImmBarrierDomain_offset;
@@ -1659,55 +2003,73 @@ class Assembler {
     return scale << FPScale_offset;
   }
 
-  // Size of the code generated in bytes
-  uint64_t SizeOfCodeGenerated() const {
-    VIXL_ASSERT((pc_ >= buffer_) && (pc_ < (buffer_ + buffer_size_)));
-    return pc_ - buffer_;
-  }
-
   // Size of the code generated since label to the current position.
-  uint64_t SizeOfCodeGeneratedSince(Label* label) const {
+  size_t SizeOfCodeGeneratedSince(Label* label) const {
     VIXL_ASSERT(label->IsBound());
-    VIXL_ASSERT((pc_ >= label->target()) && (pc_ < (buffer_ + buffer_size_)));
-    return pc_ - label->target();
+    return buffer_->OffsetFrom(label->location());
   }
 
+  size_t BufferCapacity() const { return buffer_->capacity(); }
 
-  inline void BlockLiteralPool() {
-    literal_pool_monitor_++;
-  }
+  size_t RemainingBufferSpace() const { return buffer_->RemainingBytes(); }
 
-  inline void ReleaseLiteralPool() {
-    if (--literal_pool_monitor_ == 0) {
-      // Has the literal pool been blocked for too long?
-      VIXL_ASSERT(literals_.empty() ||
-             (pc_ < (literals_.back()->pc_ + kMaxLoadLiteralRange)));
+  void EnsureSpaceFor(size_t amount) {
+    if (buffer_->RemainingBytes() < amount) {
+      size_t capacity = buffer_->capacity();
+      size_t size = buffer_->CursorOffset();
+      do {
+        // TODO(all): refine.
+        capacity *= 2;
+      } while ((capacity - size) <  amount);
+      buffer_->Grow(capacity);
     }
   }
 
-  inline bool IsLiteralPoolBlocked() {
-    return literal_pool_monitor_ != 0;
+#ifdef DEBUG
+  void AcquireBuffer() {
+    VIXL_ASSERT(buffer_monitor_ >= 0);
+    buffer_monitor_++;
   }
 
-  void CheckLiteralPool(LiteralPoolEmitOption option = JumpRequired);
-  void EmitLiteralPool(LiteralPoolEmitOption option = NoJumpRequired);
-  size_t LiteralPoolSize();
+  void ReleaseBuffer() {
+    buffer_monitor_--;
+    VIXL_ASSERT(buffer_monitor_ >= 0);
+  }
+#endif
 
- protected:
-  inline const Register& AppropriateZeroRegFor(const CPURegister& reg) const {
+  inline PositionIndependentCodeOption pic() {
+    return pic_;
+  }
+
+  inline bool AllowPageOffsetDependentCode() {
+    return (pic() == PageOffsetDependentCode) ||
+           (pic() == PositionDependentCode);
+  }
+
+  static inline const Register& AppropriateZeroRegFor(const CPURegister& reg) {
     return reg.Is64Bits() ? xzr : wzr;
   }
 
 
+ protected:
   void LoadStore(const CPURegister& rt,
                  const MemOperand& addr,
-                 LoadStoreOp op);
-  static bool IsImmLSUnscaled(ptrdiff_t offset);
-  static bool IsImmLSScaled(ptrdiff_t offset, LSDataSize size);
+                 LoadStoreOp op,
+                 LoadStoreScalingOption option = PreferScaledOffset);
+  static bool IsImmLSUnscaled(int64_t offset);
+  static bool IsImmLSScaled(int64_t offset, LSDataSize size);
+
+  void LoadStorePair(const CPURegister& rt,
+                     const CPURegister& rt2,
+                     const MemOperand& addr,
+                     LoadStorePairOp op);
+  static bool IsImmLSPair(int64_t offset, LSDataSize size);
 
+  // TODO(all): The third parameter should be passed by reference but gcc 4.8.2
+  // reports a bogus uninitialised warning then.
   void Logical(const Register& rd,
                const Register& rn,
-               const Operand& operand,
+               const Operand operand,
                LogicalOp op);
   void LogicalImmediate(const Register& rd,
                         const Register& rn,
@@ -1717,9 +2079,9 @@ class Assembler {
                         LogicalOp op);
   static bool IsImmLogical(uint64_t value,
                            unsigned width,
-                           unsigned* n,
-                           unsigned* imm_s,
-                           unsigned* imm_r);
+                           unsigned* n = NULL,
+                           unsigned* imm_s = NULL,
+                           unsigned* imm_r = NULL);
 
   void ConditionalCompare(const Register& rn,
                           const Operand& operand,
@@ -1768,6 +2130,7 @@ class Assembler {
     const CPURegister& rt, const CPURegister& rt2);
   static LoadStorePairNonTemporalOp StorePairNonTemporalOpFor(
     const CPURegister& rt, const CPURegister& rt2);
+  static LoadLiteralOp LoadLiteralOpFor(const CPURegister& rt);
 
 
  private:
@@ -1786,10 +2149,6 @@ class Assembler {
                                 const Operand& operand,
                                 FlagsUpdate S,
                                 Instr op);
-  void LoadStorePair(const CPURegister& rt,
-                     const CPURegister& rt2,
-                     const MemOperand& addr,
-                     LoadStorePairOp op);
   void LoadStorePairNonTemporal(const CPURegister& rt,
                                 const CPURegister& rt2,
                                 const MemOperand& addr,
@@ -1821,75 +2180,110 @@ class Assembler {
                                const FPRegister& fa,
                                FPDataProcessing3SourceOp op);
 
-  void RecordLiteral(int64_t imm, unsigned size);
+  // Link the current (not-yet-emitted) instruction to the specified label, then
+  // return an offset to be encoded in the instruction. If the label is not yet
+  // bound, an offset of 0 is returned.
+  ptrdiff_t LinkAndGetByteOffsetTo(Label * label);
+  ptrdiff_t LinkAndGetInstructionOffsetTo(Label * label);
+  ptrdiff_t LinkAndGetPageOffsetTo(Label * label);
 
-  // Emit the instruction at pc_.
-  void Emit(Instr instruction) {
-    VIXL_STATIC_ASSERT(sizeof(*pc_) == 1);
-    VIXL_STATIC_ASSERT(sizeof(instruction) == kInstructionSize);
-    VIXL_ASSERT((pc_ + sizeof(instruction)) <= (buffer_ + buffer_size_));
+  // A common implementation for the LinkAndGet<Type>OffsetTo helpers.
+  template <int element_shift>
+  ptrdiff_t LinkAndGetOffsetTo(Label* label);
 
-#ifdef DEBUG
-    finalized_ = false;
-#endif
+  // Literal load offset are in words (32-bit).
+  ptrdiff_t LinkAndGetWordOffsetTo(RawLiteral* literal);
 
-    memcpy(pc_, &instruction, sizeof(instruction));
-    pc_ += sizeof(instruction);
-    CheckBufferSpace();
+  // Emit the instruction in buffer_.
+  void Emit(Instr instruction) {
+    VIXL_STATIC_ASSERT(sizeof(instruction) == kInstructionSize);
+    VIXL_ASSERT(buffer_monitor_ > 0);
+    buffer_->Emit32(instruction);
   }
 
-  // Emit data inline in the instruction stream.
-  void EmitData(void const * data, unsigned size) {
-    VIXL_STATIC_ASSERT(sizeof(*pc_) == 1);
-    VIXL_ASSERT((pc_ + size) <= (buffer_ + buffer_size_));
+  // Buffer where the code is emitted.
+  CodeBuffer* buffer_;
+  PositionIndependentCodeOption pic_;
 
 #ifdef DEBUG
-    finalized_ = false;
+  int64_t buffer_monitor_;
 #endif
+};
 
-    // TODO: Record this 'instruction' as data, so that it can be disassembled
-    // correctly.
-    memcpy(pc_, data, size);
-    pc_ += size;
-    CheckBufferSpace();
-  }
-
-  inline void CheckBufferSpace() {
-    VIXL_ASSERT(pc_ < (buffer_ + buffer_size_));
-    if (pc_ > next_literal_pool_check_) {
-      CheckLiteralPool();
-    }
-  }
 
-  // The buffer into which code and relocation info are generated.
-  Instruction* buffer_;
-  // Buffer size, in bytes.
-  unsigned buffer_size_;
-  Instruction* pc_;
-  std::list<Literal*> literals_;
-  Instruction* next_literal_pool_check_;
-  unsigned literal_pool_monitor_;
+// All Assembler emits MUST acquire/release the underlying code buffer. The
+// helper scope below will do so and optionally ensure the buffer is big enough
+// to receive the emit. It is possible to request the scope not to perform any
+// checks (kNoCheck) if for example it is known in advance the buffer size is
+// adequate or there is some other size checking mechanism in place.
+class CodeBufferCheckScope {
+ public:
+  // Tell whether or not the scope needs to ensure the associated CodeBuffer
+  // has enough space for the requested size.
+  enum CheckPolicy {
+    kNoCheck,
+    kCheck
+  };
 
-  friend class BlockLiteralPoolScope;
+  // Tell whether or not the scope should assert the amount of code emitted
+  // within the scope is consistent with the requested amount.
+  enum AssertPolicy {
+    kNoAssert,    // No assert required.
+    kExactSize,   // The code emitted must be exactly size bytes.
+    kMaximumSize  // The code emitted must be at most size bytes.
+  };
 
+  CodeBufferCheckScope(Assembler* assm,
+                       size_t size,
+                       CheckPolicy check_policy = kCheck,
+                       AssertPolicy assert_policy = kMaximumSize)
+      : assm_(assm) {
+    if (check_policy == kCheck) assm->EnsureSpaceFor(size);
 #ifdef DEBUG
-  bool finalized_;
+    assm->bind(&start_);
+    size_ = size;
+    assert_policy_ = assert_policy;
+    assm->AcquireBuffer();
+#else
+    USE(assert_policy);
 #endif
-};
+  }
 
-class BlockLiteralPoolScope {
- public:
-  explicit BlockLiteralPoolScope(Assembler* assm) : assm_(assm) {
-    assm_->BlockLiteralPool();
+  // This is a shortcut for CodeBufferCheckScope(assm, 0, kNoCheck, kNoAssert).
+  explicit CodeBufferCheckScope(Assembler* assm) : assm_(assm) {
+#ifdef DEBUG
+    size_ = 0;
+    assert_policy_ = kNoAssert;
+    assm->AcquireBuffer();
+#endif
   }
 
-  ~BlockLiteralPoolScope() {
-    assm_->ReleaseLiteralPool();
+  ~CodeBufferCheckScope() {
+#ifdef DEBUG
+    assm_->ReleaseBuffer();
+    switch (assert_policy_) {
+      case kNoAssert: break;
+      case kExactSize:
+        VIXL_ASSERT(assm_->SizeOfCodeGeneratedSince(&start_) == size_);
+        break;
+      case kMaximumSize:
+        VIXL_ASSERT(assm_->SizeOfCodeGeneratedSince(&start_) <= size_);
+        break;
+      default:
+        VIXL_UNREACHABLE();
+    }
+#endif
   }
 
- private:
+ protected:
   Assembler* assm_;
+#ifdef DEBUG
+  Label start_;
+  size_t size_;
+  AssertPolicy assert_policy_;
+#endif
 };
+
 }  // namespace vixl
 
 #endif  // VIXL_A64_ASSEMBLER_A64_H_
diff --git a/disas/libvixl/a64/constants-a64.h b/disas/libvixl/a64/constants-a64.h
index 99677c1be..7a14f85f5 100644
--- a/disas/libvixl/a64/constants-a64.h
+++ b/disas/libvixl/a64/constants-a64.h
@@ -46,13 +46,13 @@ R(24) R(25) R(26) R(27) R(28) R(29) R(30) R(31)
 
 #define INSTRUCTION_FIELDS_LIST(V_)                                            \
 /* Register fields */                                                          \
-V_(Rd, 4, 0, Bits)                        /* Destination register.     */      \
-V_(Rn, 9, 5, Bits)                        /* First source register.    */      \
-V_(Rm, 20, 16, Bits)                      /* Second source register.   */      \
-V_(Ra, 14, 10, Bits)                      /* Third source register.    */      \
-V_(Rt, 4, 0, Bits)                        /* Load dest / store source. */      \
-V_(Rt2, 14, 10, Bits)                     /* Load second dest /        */      \
-                                         /* store second source.      */       \
+V_(Rd, 4, 0, Bits)                        /* Destination register.        */   \
+V_(Rn, 9, 5, Bits)                        /* First source register.       */   \
+V_(Rm, 20, 16, Bits)                      /* Second source register.      */   \
+V_(Ra, 14, 10, Bits)                      /* Third source register.       */   \
+V_(Rt, 4, 0, Bits)                        /* Load/store register.         */   \
+V_(Rt2, 14, 10, Bits)                     /* Load/store second register.  */   \
+V_(Rs, 20, 16, Bits)                      /* Exclusive access status.     */   \
 V_(PrefetchMode, 4, 0, Bits)                                                   \
                                                                                \
 /* Common bits */                                                              \
@@ -126,6 +126,13 @@ V_(SysOp1, 18, 16, Bits)                                                       \
 V_(SysOp2, 7, 5, Bits)                                                         \
 V_(CRn, 15, 12, Bits)                                                          \
 V_(CRm, 11, 8, Bits)                                                           \
+                                                                               \
+/* Load-/store-exclusive */                                                    \
+V_(LdStXLoad, 22, 22, Bits)                                                    \
+V_(LdStXNotExclusive, 23, 23, Bits)                                            \
+V_(LdStXAcquireRelease, 15, 15, Bits)                                          \
+V_(LdStXSizeLog2, 31, 30, Bits)                                                \
+V_(LdStXPair, 21, 21, Bits)                                                    \
 
 
 #define SYSTEM_REGISTER_FIELDS_LIST(V_, M_)                                    \
@@ -585,6 +592,13 @@ enum MemBarrierOp {
   ISB             = MemBarrierFixed | 0x00000040
 };
 
+enum SystemExclusiveMonitorOp {
+  SystemExclusiveMonitorFixed = 0xD503305F,
+  SystemExclusiveMonitorFMask = 0xFFFFF0FF,
+  SystemExclusiveMonitorMask  = 0xFFFFF0FF,
+  CLREX                       = SystemExclusiveMonitorFixed
+};
+
 // Any load or store.
 enum LoadStoreAnyOp {
   LoadStoreAnyFMask = 0x0a000000,
@@ -702,7 +716,7 @@ enum LoadStoreUnscaledOffsetOp {
 
 // Load/store (post, pre, offset and unsigned.)
 enum LoadStoreOp {
-  LoadStoreOpMask   = 0xC4C00000,
+  LoadStoreOpMask = 0xC4C00000,
   #define LOAD_STORE(A, B, C, D)  \
   A##B##_##C = D
   LOAD_STORE_OP_LIST(LOAD_STORE),
@@ -756,6 +770,44 @@ enum LoadStoreRegisterOffset {
   #undef LOAD_STORE_REGISTER_OFFSET
 };
 
+enum LoadStoreExclusive {
+  LoadStoreExclusiveFixed = 0x08000000,
+  LoadStoreExclusiveFMask = 0x3F000000,
+  LoadStoreExclusiveMask  = 0xFFE08000,
+  STXRB_w  = LoadStoreExclusiveFixed | 0x00000000,
+  STXRH_w  = LoadStoreExclusiveFixed | 0x40000000,
+  STXR_w   = LoadStoreExclusiveFixed | 0x80000000,
+  STXR_x   = LoadStoreExclusiveFixed | 0xC0000000,
+  LDXRB_w  = LoadStoreExclusiveFixed | 0x00400000,
+  LDXRH_w  = LoadStoreExclusiveFixed | 0x40400000,
+  LDXR_w   = LoadStoreExclusiveFixed | 0x80400000,
+  LDXR_x   = LoadStoreExclusiveFixed | 0xC0400000,
+  STXP_w   = LoadStoreExclusiveFixed | 0x80200000,
+  STXP_x   = LoadStoreExclusiveFixed | 0xC0200000,
+  LDXP_w   = LoadStoreExclusiveFixed | 0x80600000,
+  LDXP_x   = LoadStoreExclusiveFixed | 0xC0600000,
+  STLXRB_w = LoadStoreExclusiveFixed | 0x00008000,
+  STLXRH_w = LoadStoreExclusiveFixed | 0x40008000,
+  STLXR_w  = LoadStoreExclusiveFixed | 0x80008000,
+  STLXR_x  = LoadStoreExclusiveFixed | 0xC0008000,
+  LDAXRB_w = LoadStoreExclusiveFixed | 0x00408000,
+  LDAXRH_w = LoadStoreExclusiveFixed | 0x40408000,
+  LDAXR_w  = LoadStoreExclusiveFixed | 0x80408000,
+  LDAXR_x  = LoadStoreExclusiveFixed | 0xC0408000,
+  STLXP_w  = LoadStoreExclusiveFixed | 0x80208000,
+  STLXP_x  = LoadStoreExclusiveFixed | 0xC0208000,
+  LDAXP_w  = LoadStoreExclusiveFixed | 0x80608000,
+  LDAXP_x  = LoadStoreExclusiveFixed | 0xC0608000,
+  STLRB_w  = LoadStoreExclusiveFixed | 0x00808000,
+  STLRH_w  = LoadStoreExclusiveFixed | 0x40808000,
+  STLR_w   = LoadStoreExclusiveFixed | 0x80808000,
+  STLR_x   = LoadStoreExclusiveFixed | 0xC0808000,
+  LDARB_w  = LoadStoreExclusiveFixed | 0x00C08000,
+  LDARH_w  = LoadStoreExclusiveFixed | 0x40C08000,
+  LDAR_w   = LoadStoreExclusiveFixed | 0x80C08000,
+  LDAR_x   = LoadStoreExclusiveFixed | 0xC0C08000
+};
+
 // Conditional compare.
 enum ConditionalCompareOp {
   ConditionalCompareMask = 0x60000000,
diff --git a/disas/libvixl/a64/cpu-a64.h b/disas/libvixl/a64/cpu-a64.h
index dfd8f015c..59b7974a1 100644
--- a/disas/libvixl/a64/cpu-a64.h
+++ b/disas/libvixl/a64/cpu-a64.h
@@ -28,6 +28,7 @@
 #define VIXL_CPU_A64_H
 
 #include "globals.h"
+#include "instructions-a64.h"
 
 namespace vixl {
 
@@ -42,6 +43,32 @@ class CPU {
   // safely run.
   static void EnsureIAndDCacheCoherency(void *address, size_t length);
 
+  // Handle tagged pointers.
+  template <typename T>
+  static T SetPointerTag(T pointer, uint64_t tag) {
+    VIXL_ASSERT(is_uintn(kAddressTagWidth, tag));
+
+    // Use C-style casts to get static_cast behaviour for integral types (T),
+    // and reinterpret_cast behaviour for other types.
+
+    uint64_t raw = (uint64_t)pointer;
+    VIXL_STATIC_ASSERT(sizeof(pointer) == sizeof(raw));
+
+    raw = (raw & ~kAddressTagMask) | (tag << kAddressTagOffset);
+    return (T)raw;
+  }
+
+  template <typename T>
+  static uint64_t GetPointerTag(T pointer) {
+    // Use C-style casts to get static_cast behaviour for integral types (T),
+    // and reinterpret_cast behaviour for other types.
+
+    uint64_t raw = (uint64_t)pointer;
+    VIXL_STATIC_ASSERT(sizeof(pointer) == sizeof(raw));
+
+    return (raw & kAddressTagMask) >> kAddressTagOffset;
+  }
+
  private:
   // Return the content of the cache type register.
   static uint32_t GetCacheType();
diff --git a/disas/libvixl/a64/decoder-a64.cc b/disas/libvixl/a64/decoder-a64.cc
index 8450eb3b4..82591ca30 100644
--- a/disas/libvixl/a64/decoder-a64.cc
+++ b/disas/libvixl/a64/decoder-a64.cc
@@ -29,8 +29,8 @@
 #include "a64/decoder-a64.h"
 
 namespace vixl {
-// Top-level instruction decode function.
-void Decoder::Decode(Instruction *instr) {
+
+void Decoder::DecodeInstruction(const Instruction *instr) {
   if (instr->Bits(28, 27) == 0) {
     VisitUnallocated(instr);
   } else {
@@ -109,20 +109,17 @@ void Decoder::Decode(Instruction *instr) {
 }
 
 void Decoder::AppendVisitor(DecoderVisitor* new_visitor) {
-  visitors_.remove(new_visitor);
-  visitors_.push_front(new_visitor);
+  visitors_.push_back(new_visitor);
 }
 
 
 void Decoder::PrependVisitor(DecoderVisitor* new_visitor) {
-  visitors_.remove(new_visitor);
-  visitors_.push_back(new_visitor);
+  visitors_.push_front(new_visitor);
 }
 
 
 void Decoder::InsertVisitorBefore(DecoderVisitor* new_visitor,
                                   DecoderVisitor* registered_visitor) {
-  visitors_.remove(new_visitor);
   std::list<DecoderVisitor*>::iterator it;
   for (it = visitors_.begin(); it != visitors_.end(); it++) {
     if (*it == registered_visitor) {
@@ -139,7 +136,6 @@ void Decoder::InsertVisitorBefore(DecoderVisitor* new_visitor,
 
 void Decoder::InsertVisitorAfter(DecoderVisitor* new_visitor,
                                  DecoderVisitor* registered_visitor) {
-  visitors_.remove(new_visitor);
   std::list<DecoderVisitor*>::iterator it;
   for (it = visitors_.begin(); it != visitors_.end(); it++) {
     if (*it == registered_visitor) {
@@ -160,7 +156,7 @@ void Decoder::RemoveVisitor(DecoderVisitor* visitor) {
 }
 
 
-void Decoder::DecodePCRelAddressing(Instruction* instr) {
+void Decoder::DecodePCRelAddressing(const Instruction* instr) {
   VIXL_ASSERT(instr->Bits(27, 24) == 0x0);
   // We know bit 28 is set, as <b28:b27> = 0 is filtered out at the top level
   // decode.
@@ -169,11 +165,11 @@ void Decoder::DecodePCRelAddressing(Instruction* instr) {
 }
 
 
-void Decoder::DecodeBranchSystemException(Instruction* instr) {
+void Decoder::DecodeBranchSystemException(const Instruction* instr) {
   VIXL_ASSERT((instr->Bits(27, 24) == 0x4) ||
-         (instr->Bits(27, 24) == 0x5) ||
-         (instr->Bits(27, 24) == 0x6) ||
-         (instr->Bits(27, 24) == 0x7) );
+              (instr->Bits(27, 24) == 0x5) ||
+              (instr->Bits(27, 24) == 0x6) ||
+              (instr->Bits(27, 24) == 0x7) );
 
   switch (instr->Bits(31, 29)) {
     case 0:
@@ -270,18 +266,17 @@ void Decoder::DecodeBranchSystemException(Instruction* instr) {
 }
 
 
-void Decoder::DecodeLoadStore(Instruction* instr) {
+void Decoder::DecodeLoadStore(const Instruction* instr) {
   VIXL_ASSERT((instr->Bits(27, 24) == 0x8) ||
-         (instr->Bits(27, 24) == 0x9) ||
-         (instr->Bits(27, 24) == 0xC) ||
-         (instr->Bits(27, 24) == 0xD) );
+              (instr->Bits(27, 24) == 0x9) ||
+              (instr->Bits(27, 24) == 0xC) ||
+              (instr->Bits(27, 24) == 0xD) );
 
   if (instr->Bit(24) == 0) {
     if (instr->Bit(28) == 0) {
       if (instr->Bit(29) == 0) {
         if (instr->Bit(26) == 0) {
-          // TODO: VisitLoadStoreExclusive.
-          VisitUnimplemented(instr);
+          VisitLoadStoreExclusive(instr);
         } else {
           DecodeAdvSIMDLoadStore(instr);
         }
@@ -389,7 +384,7 @@ void Decoder::DecodeLoadStore(Instruction* instr) {
 }
 
 
-void Decoder::DecodeLogical(Instruction* instr) {
+void Decoder::DecodeLogical(const Instruction* instr) {
   VIXL_ASSERT(instr->Bits(27, 24) == 0x2);
 
   if (instr->Mask(0x80400000) == 0x00400000) {
@@ -408,7 +403,7 @@ void Decoder::DecodeLogical(Instruction* instr) {
 }
 
 
-void Decoder::DecodeBitfieldExtract(Instruction* instr) {
+void Decoder::DecodeBitfieldExtract(const Instruction* instr) {
   VIXL_ASSERT(instr->Bits(27, 24) == 0x3);
 
   if ((instr->Mask(0x80400000) == 0x80000000) ||
@@ -433,7 +428,7 @@ void Decoder::DecodeBitfieldExtract(Instruction* instr) {
 }
 
 
-void Decoder::DecodeAddSubImmediate(Instruction* instr) {
+void Decoder::DecodeAddSubImmediate(const Instruction* instr) {
   VIXL_ASSERT(instr->Bits(27, 24) == 0x1);
   if (instr->Bit(23) == 1) {
     VisitUnallocated(instr);
@@ -443,7 +438,7 @@ void Decoder::DecodeAddSubImmediate(Instruction* instr) {
 }
 
 
-void Decoder::DecodeDataProcessing(Instruction* instr) {
+void Decoder::DecodeDataProcessing(const Instruction* instr) {
   VIXL_ASSERT((instr->Bits(27, 24) == 0xA) ||
               (instr->Bits(27, 24) == 0xB));
 
@@ -558,7 +553,7 @@ void Decoder::DecodeDataProcessing(Instruction* instr) {
 }
 
 
-void Decoder::DecodeFP(Instruction* instr) {
+void Decoder::DecodeFP(const Instruction* instr) {
   VIXL_ASSERT((instr->Bits(27, 24) == 0xE) ||
               (instr->Bits(27, 24) == 0xF));
 
@@ -685,14 +680,14 @@ void Decoder::DecodeFP(Instruction* instr) {
 }
 
 
-void Decoder::DecodeAdvSIMDLoadStore(Instruction* instr) {
+void Decoder::DecodeAdvSIMDLoadStore(const Instruction* instr) {
   // TODO: Implement Advanced SIMD load/store instruction decode.
   VIXL_ASSERT(instr->Bits(29, 25) == 0x6);
   VisitUnimplemented(instr);
 }
 
 
-void Decoder::DecodeAdvSIMDDataProcessing(Instruction* instr) {
+void Decoder::DecodeAdvSIMDDataProcessing(const Instruction* instr) {
   // TODO: Implement Advanced SIMD data processing instruction decode.
   VIXL_ASSERT(instr->Bits(27, 25) == 0x7);
   VisitUnimplemented(instr);
@@ -700,7 +695,7 @@ void Decoder::DecodeAdvSIMDDataProcessing(Instruction* instr) {
 
 
 #define DEFINE_VISITOR_CALLERS(A)                                              \
-  void Decoder::Visit##A(Instruction *instr) {                                 \
+  void Decoder::Visit##A(const Instruction *instr) {                           \
     VIXL_ASSERT(instr->Mask(A##FMask) == A##Fixed);                            \
     std::list<DecoderVisitor*>::iterator it;                                   \
     for (it = visitors_.begin(); it != visitors_.end(); it++) {                \
diff --git a/disas/libvixl/a64/decoder-a64.h b/disas/libvixl/a64/decoder-a64.h
index bbbbd8124..172594c89 100644
--- a/disas/libvixl/a64/decoder-a64.h
+++ b/disas/libvixl/a64/decoder-a64.h
@@ -59,6 +59,7 @@
   V(LoadStorePreIndex)              \
   V(LoadStoreRegisterOffset)        \
   V(LoadStoreUnsignedOffset)        \
+  V(LoadStoreExclusive)             \
   V(LogicalShifted)                 \
   V(AddSubShifted)                  \
   V(AddSubExtended)                 \
@@ -87,112 +88,152 @@ namespace vixl {
 // must provide implementations for all of these functions.
 class DecoderVisitor {
  public:
-  #define DECLARE(A) virtual void Visit##A(Instruction* instr) = 0;
+  enum VisitorConstness {
+    kConstVisitor,
+    kNonConstVisitor
+  };
+  explicit DecoderVisitor(VisitorConstness constness = kConstVisitor)
+      : constness_(constness) {}
+
+  virtual ~DecoderVisitor() {}
+
+  #define DECLARE(A) virtual void Visit##A(const Instruction* instr) = 0;
   VISITOR_LIST(DECLARE)
   #undef DECLARE
 
-  virtual ~DecoderVisitor() {}
+  bool IsConstVisitor() const { return constness_ == kConstVisitor; }
+  Instruction* MutableInstruction(const Instruction* instr) {
+    VIXL_ASSERT(!IsConstVisitor());
+    return const_cast<Instruction*>(instr);
+  }
 
  private:
-  // Visitors are registered in a list.
-  std::list<DecoderVisitor*> visitors_;
-
-  friend class Decoder;
+  VisitorConstness constness_;
 };
 
 
-class Decoder: public DecoderVisitor {
+class Decoder {
  public:
   Decoder() {}
 
-  // Top-level instruction decoder function. Decodes an instruction and calls
-  // the visitor functions registered with the Decoder class.
-  void Decode(Instruction *instr);
+  // Top-level wrappers around the actual decoding function.
+  void Decode(const Instruction* instr) {
+    std::list<DecoderVisitor*>::iterator it;
+    for (it = visitors_.begin(); it != visitors_.end(); it++) {
+      VIXL_ASSERT((*it)->IsConstVisitor());
+    }
+    DecodeInstruction(instr);
+  }
+  void Decode(Instruction* instr) {
+    DecodeInstruction(const_cast<const Instruction*>(instr));
+  }
 
   // Register a new visitor class with the decoder.
   // Decode() will call the corresponding visitor method from all registered
   // visitor classes when decoding reaches the leaf node of the instruction
   // decode tree.
-  // Visitors are called in the order.
-  // A visitor can only be registered once.
-  // Registering an already registered visitor will update its position.
+  // Visitors are called in order.
+  // A visitor can be registered multiple times.
   //
   //   d.AppendVisitor(V1);
   //   d.AppendVisitor(V2);
-  //   d.PrependVisitor(V2);            // Move V2 at the start of the list.
-  //   d.InsertVisitorBefore(V3, V2);
-  //   d.AppendVisitor(V4);
-  //   d.AppendVisitor(V4);             // No effect.
+  //   d.PrependVisitor(V2);
+  //   d.AppendVisitor(V3);
   //
   //   d.Decode(i);
   //
-  // will call in order visitor methods in V3, V2, V1, V4.
+  // will call in order visitor methods in V2, V1, V2, V3.
   void AppendVisitor(DecoderVisitor* visitor);
   void PrependVisitor(DecoderVisitor* visitor);
+  // These helpers register `new_visitor` before or after the first instance of
+  // `registered_visiter` in the list.
+  // So if
+  //   V1, V2, V1, V2
+  // are registered in this order in the decoder, calls to
+  //   d.InsertVisitorAfter(V3, V1);
+  //   d.InsertVisitorBefore(V4, V2);
+  // will yield the order
+  //   V1, V3, V4, V2, V1, V2
+  //
+  // For more complex modifications of the order of registered visitors, one can
+  // directly access and modify the list of visitors via the `visitors()'
+  // accessor.
   void InsertVisitorBefore(DecoderVisitor* new_visitor,
                            DecoderVisitor* registered_visitor);
   void InsertVisitorAfter(DecoderVisitor* new_visitor,
                           DecoderVisitor* registered_visitor);
 
-  // Remove a previously registered visitor class from the list of visitors
-  // stored by the decoder.
+  // Remove all instances of a previously registered visitor class from the list
+  // of visitors stored by the decoder.
   void RemoveVisitor(DecoderVisitor* visitor);
 
-  #define DECLARE(A) void Visit##A(Instruction* instr);
+  #define DECLARE(A) void Visit##A(const Instruction* instr);
   VISITOR_LIST(DECLARE)
   #undef DECLARE
 
+
+  std::list<DecoderVisitor*>* visitors() { return &visitors_; }
+
  private:
+  // Decodes an instruction and calls the visitor functions registered with the
+  // Decoder class.
+  void DecodeInstruction(const Instruction* instr);
+
   // Decode the PC relative addressing instruction, and call the corresponding
   // visitors.
   // On entry, instruction bits 27:24 = 0x0.
-  void DecodePCRelAddressing(Instruction* instr);
+  void DecodePCRelAddressing(const Instruction* instr);
 
   // Decode the add/subtract immediate instruction, and call the correspoding
   // visitors.
   // On entry, instruction bits 27:24 = 0x1.
-  void DecodeAddSubImmediate(Instruction* instr);
+  void DecodeAddSubImmediate(const Instruction* instr);
 
   // Decode the branch, system command, and exception generation parts of
   // the instruction tree, and call the corresponding visitors.
   // On entry, instruction bits 27:24 = {0x4, 0x5, 0x6, 0x7}.
-  void DecodeBranchSystemException(Instruction* instr);
+  void DecodeBranchSystemException(const Instruction* instr);
 
   // Decode the load and store parts of the instruction tree, and call
   // the corresponding visitors.
   // On entry, instruction bits 27:24 = {0x8, 0x9, 0xC, 0xD}.
-  void DecodeLoadStore(Instruction* instr);
+  void DecodeLoadStore(const Instruction* instr);
 
   // Decode the logical immediate and move wide immediate parts of the
   // instruction tree, and call the corresponding visitors.
   // On entry, instruction bits 27:24 = 0x2.
-  void DecodeLogical(Instruction* instr);
+  void DecodeLogical(const Instruction* instr);
 
   // Decode the bitfield and extraction parts of the instruction tree,
   // and call the corresponding visitors.
   // On entry, instruction bits 27:24 = 0x3.
-  void DecodeBitfieldExtract(Instruction* instr);
+  void DecodeBitfieldExtract(const Instruction* instr);
 
   // Decode the data processing parts of the instruction tree, and call the
   // corresponding visitors.
   // On entry, instruction bits 27:24 = {0x1, 0xA, 0xB}.
-  void DecodeDataProcessing(Instruction* instr);
+  void DecodeDataProcessing(const Instruction* instr);
 
   // Decode the floating point parts of the instruction tree, and call the
   // corresponding visitors.
   // On entry, instruction bits 27:24 = {0xE, 0xF}.
-  void DecodeFP(Instruction* instr);
+  void DecodeFP(const Instruction* instr);
 
   // Decode the Advanced SIMD (NEON) load/store part of the instruction tree,
   // and call the corresponding visitors.
   // On entry, instruction bits 29:25 = 0x6.
-  void DecodeAdvSIMDLoadStore(Instruction* instr);
+  void DecodeAdvSIMDLoadStore(const Instruction* instr);
 
   // Decode the Advanced SIMD (NEON) data processing part of the instruction
   // tree, and call the corresponding visitors.
   // On entry, instruction bits 27:25 = 0x7.
-  void DecodeAdvSIMDDataProcessing(Instruction* instr);
+  void DecodeAdvSIMDDataProcessing(const Instruction* instr);
+
+ private:
+  // Visitors are registered in a list.
+  std::list<DecoderVisitor*> visitors_;
 };
+
 }  // namespace vixl
 
 #endif  // VIXL_A64_DECODER_A64_H_
diff --git a/disas/libvixl/a64/disasm-a64.cc b/disas/libvixl/a64/disasm-a64.cc
index f81ce4bbb..e4a74aa57 100644
--- a/disas/libvixl/a64/disasm-a64.cc
+++ b/disas/libvixl/a64/disasm-a64.cc
@@ -24,6 +24,7 @@
 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+#include <cstdlib>
 #include "a64/disasm-a64.h"
 
 namespace vixl {
@@ -56,7 +57,7 @@ char* Disassembler::GetOutput() {
 }
 
 
-void Disassembler::VisitAddSubImmediate(Instruction* instr) {
+void Disassembler::VisitAddSubImmediate(const Instruction* instr) {
   bool rd_is_zr = RdIsZROrSP(instr);
   bool stack_op = (rd_is_zr || RnIsZROrSP(instr)) &&
                   (instr->ImmAddSub() == 0) ? true : false;
@@ -101,7 +102,7 @@ void Disassembler::VisitAddSubImmediate(Instruction* instr) {
 }
 
 
-void Disassembler::VisitAddSubShifted(Instruction* instr) {
+void Disassembler::VisitAddSubShifted(const Instruction* instr) {
   bool rd_is_zr = RdIsZROrSP(instr);
   bool rn_is_zr = RnIsZROrSP(instr);
   const char *mnemonic = "";
@@ -148,7 +149,7 @@ void Disassembler::VisitAddSubShifted(Instruction* instr) {
 }
 
 
-void Disassembler::VisitAddSubExtended(Instruction* instr) {
+void Disassembler::VisitAddSubExtended(const Instruction* instr) {
   bool rd_is_zr = RdIsZROrSP(instr);
   const char *mnemonic = "";
   Extend mode = static_cast<Extend>(instr->ExtendMode());
@@ -186,7 +187,7 @@ void Disassembler::VisitAddSubExtended(Instruction* instr) {
 }
 
 
-void Disassembler::VisitAddSubWithCarry(Instruction* instr) {
+void Disassembler::VisitAddSubWithCarry(const Instruction* instr) {
   bool rn_is_zr = RnIsZROrSP(instr);
   const char *mnemonic = "";
   const char *form = "'Rd, 'Rn, 'Rm";
@@ -221,7 +222,7 @@ void Disassembler::VisitAddSubWithCarry(Instruction* instr) {
 }
 
 
-void Disassembler::VisitLogicalImmediate(Instruction* instr) {
+void Disassembler::VisitLogicalImmediate(const Instruction* instr) {
   bool rd_is_zr = RdIsZROrSP(instr);
   bool rn_is_zr = RnIsZROrSP(instr);
   const char *mnemonic = "";
@@ -293,7 +294,7 @@ bool Disassembler::IsMovzMovnImm(unsigned reg_size, uint64_t value) {
 }
 
 
-void Disassembler::VisitLogicalShifted(Instruction* instr) {
+void Disassembler::VisitLogicalShifted(const Instruction* instr) {
   bool rd_is_zr = RdIsZROrSP(instr);
   bool rn_is_zr = RnIsZROrSP(instr);
   const char *mnemonic = "";
@@ -344,7 +345,7 @@ void Disassembler::VisitLogicalShifted(Instruction* instr) {
 }
 
 
-void Disassembler::VisitConditionalCompareRegister(Instruction* instr) {
+void Disassembler::VisitConditionalCompareRegister(const Instruction* instr) {
   const char *mnemonic = "";
   const char *form = "'Rn, 'Rm, 'INzcv, 'Cond";
 
@@ -359,7 +360,7 @@ void Disassembler::VisitConditionalCompareRegister(Instruction* instr) {
 }
 
 
-void Disassembler::VisitConditionalCompareImmediate(Instruction* instr) {
+void Disassembler::VisitConditionalCompareImmediate(const Instruction* instr) {
   const char *mnemonic = "";
   const char *form = "'Rn, 'IP, 'INzcv, 'Cond";
 
@@ -374,7 +375,7 @@ void Disassembler::VisitConditionalCompareImmediate(Instruction* instr) {
 }
 
 
-void Disassembler::VisitConditionalSelect(Instruction* instr) {
+void Disassembler::VisitConditionalSelect(const Instruction* instr) {
   bool rnm_is_zr = (RnIsZROrSP(instr) && RmIsZROrSP(instr));
   bool rn_is_rm = (instr->Rn() == instr->Rm());
   const char *mnemonic = "";
@@ -427,7 +428,7 @@ void Disassembler::VisitConditionalSelect(Instruction* instr) {
 }
 
 
-void Disassembler::VisitBitfield(Instruction* instr) {
+void Disassembler::VisitBitfield(const Instruction* instr) {
   unsigned s = instr->ImmS();
   unsigned r = instr->ImmR();
   unsigned rd_size_minus_1 =
@@ -505,7 +506,7 @@ void Disassembler::VisitBitfield(Instruction* instr) {
 }
 
 
-void Disassembler::VisitExtract(Instruction* instr) {
+void Disassembler::VisitExtract(const Instruction* instr) {
   const char *mnemonic = "";
   const char *form = "'Rd, 'Rn, 'Rm, 'IExtract";
 
@@ -526,16 +527,16 @@ void Disassembler::VisitExtract(Instruction* instr) {
 }
 
 
-void Disassembler::VisitPCRelAddressing(Instruction* instr) {
+void Disassembler::VisitPCRelAddressing(const Instruction* instr) {
   switch (instr->Mask(PCRelAddressingMask)) {
     case ADR: Format(instr, "adr", "'Xd, 'AddrPCRelByte"); break;
-    // ADRP is not implemented.
+    case ADRP: Format(instr, "adrp", "'Xd, 'AddrPCRelPage"); break;
     default: Format(instr, "unimplemented", "(PCRelAddressing)");
   }
 }
 
 
-void Disassembler::VisitConditionalBranch(Instruction* instr) {
+void Disassembler::VisitConditionalBranch(const Instruction* instr) {
   switch (instr->Mask(ConditionalBranchMask)) {
     case B_cond: Format(instr, "b.'CBrn", "'BImmCond"); break;
     default: VIXL_UNREACHABLE();
@@ -543,7 +544,8 @@ void Disassembler::VisitConditionalBranch(Instruction* instr) {
 }
 
 
-void Disassembler::VisitUnconditionalBranchToRegister(Instruction* instr) {
+void Disassembler::VisitUnconditionalBranchToRegister(
+    const Instruction* instr) {
   const char *mnemonic = "unimplemented";
   const char *form = "'Xn";
 
@@ -563,7 +565,7 @@ void Disassembler::VisitUnconditionalBranchToRegister(Instruction* instr) {
 }
 
 
-void Disassembler::VisitUnconditionalBranch(Instruction* instr) {
+void Disassembler::VisitUnconditionalBranch(const Instruction* instr) {
   const char *mnemonic = "";
   const char *form = "'BImmUncn";
 
@@ -576,7 +578,7 @@ void Disassembler::VisitUnconditionalBranch(Instruction* instr) {
 }
 
 
-void Disassembler::VisitDataProcessing1Source(Instruction* instr) {
+void Disassembler::VisitDataProcessing1Source(const Instruction* instr) {
   const char *mnemonic = "";
   const char *form = "'Rd, 'Rn";
 
@@ -597,7 +599,7 @@ void Disassembler::VisitDataProcessing1Source(Instruction* instr) {
 }
 
 
-void Disassembler::VisitDataProcessing2Source(Instruction* instr) {
+void Disassembler::VisitDataProcessing2Source(const Instruction* instr) {
   const char *mnemonic = "unimplemented";
   const char *form = "'Rd, 'Rn, 'Rm";
 
@@ -618,7 +620,7 @@ void Disassembler::VisitDataProcessing2Source(Instruction* instr) {
 }
 
 
-void Disassembler::VisitDataProcessing3Source(Instruction* instr) {
+void Disassembler::VisitDataProcessing3Source(const Instruction* instr) {
   bool ra_is_zr = RaIsZROrSP(instr);
   const char *mnemonic = "";
   const char *form = "'Xd, 'Wn, 'Wm, 'Xa";
@@ -696,7 +698,7 @@ void Disassembler::VisitDataProcessing3Source(Instruction* instr) {
 }
 
 
-void Disassembler::VisitCompareBranch(Instruction* instr) {
+void Disassembler::VisitCompareBranch(const Instruction* instr) {
   const char *mnemonic = "";
   const char *form = "'Rt, 'BImmCmpa";
 
@@ -711,7 +713,7 @@ void Disassembler::VisitCompareBranch(Instruction* instr) {
 }
 
 
-void Disassembler::VisitTestBranch(Instruction* instr) {
+void Disassembler::VisitTestBranch(const Instruction* instr) {
   const char *mnemonic = "";
   // If the top bit of the immediate is clear, the tested register is
   // disassembled as Wt, otherwise Xt. As the top bit of the immediate is
@@ -728,7 +730,7 @@ void Disassembler::VisitTestBranch(Instruction* instr) {
 }
 
 
-void Disassembler::VisitMoveWideImmediate(Instruction* instr) {
+void Disassembler::VisitMoveWideImmediate(const Instruction* instr) {
   const char *mnemonic = "";
   const char *form = "'Rd, 'IMoveImm";
 
@@ -767,7 +769,7 @@ void Disassembler::VisitMoveWideImmediate(Instruction* instr) {
   V(LDR_s, "ldr", "'St")      \
   V(LDR_d, "ldr", "'Dt")
 
-void Disassembler::VisitLoadStorePreIndex(Instruction* instr) {
+void Disassembler::VisitLoadStorePreIndex(const Instruction* instr) {
   const char *mnemonic = "unimplemented";
   const char *form = "(LoadStorePreIndex)";
 
@@ -781,7 +783,7 @@ void Disassembler::VisitLoadStorePreIndex(Instruction* instr) {
 }
 
 
-void Disassembler::VisitLoadStorePostIndex(Instruction* instr) {
+void Disassembler::VisitLoadStorePostIndex(const Instruction* instr) {
   const char *mnemonic = "unimplemented";
   const char *form = "(LoadStorePostIndex)";
 
@@ -795,7 +797,7 @@ void Disassembler::VisitLoadStorePostIndex(Instruction* instr) {
 }
 
 
-void Disassembler::VisitLoadStoreUnsignedOffset(Instruction* instr) {
+void Disassembler::VisitLoadStoreUnsignedOffset(const Instruction* instr) {
   const char *mnemonic = "unimplemented";
   const char *form = "(LoadStoreUnsignedOffset)";
 
@@ -810,7 +812,7 @@ void Disassembler::VisitLoadStoreUnsignedOffset(Instruction* instr) {
 }
 
 
-void Disassembler::VisitLoadStoreRegisterOffset(Instruction* instr) {
+void Disassembler::VisitLoadStoreRegisterOffset(const Instruction* instr) {
   const char *mnemonic = "unimplemented";
   const char *form = "(LoadStoreRegisterOffset)";
 
@@ -825,7 +827,7 @@ void Disassembler::VisitLoadStoreRegisterOffset(Instruction* instr) {
 }
 
 
-void Disassembler::VisitLoadStoreUnscaledOffset(Instruction* instr) {
+void Disassembler::VisitLoadStoreUnscaledOffset(const Instruction* instr) {
   const char *mnemonic = "unimplemented";
   const char *form = "'Wt, ['Xns'ILS]";
   const char *form_x = "'Xt, ['Xns'ILS]";
@@ -856,7 +858,7 @@ void Disassembler::VisitLoadStoreUnscaledOffset(Instruction* instr) {
 }
 
 
-void Disassembler::VisitLoadLiteral(Instruction* instr) {
+void Disassembler::VisitLoadLiteral(const Instruction* instr) {
   const char *mnemonic = "ldr";
   const char *form = "(LoadLiteral)";
 
@@ -865,6 +867,11 @@ void Disassembler::VisitLoadLiteral(Instruction* instr) {
     case LDR_x_lit: form = "'Xt, 'ILLiteral 'LValue"; break;
     case LDR_s_lit: form = "'St, 'ILLiteral 'LValue"; break;
     case LDR_d_lit: form = "'Dt, 'ILLiteral 'LValue"; break;
+    case LDRSW_x_lit: {
+      mnemonic = "ldrsw";
+      form = "'Xt, 'ILLiteral 'LValue";
+      break;
+    }
     default: mnemonic = "unimplemented";
   }
   Format(instr, mnemonic, form);
@@ -882,7 +889,7 @@ void Disassembler::VisitLoadLiteral(Instruction* instr) {
   V(STP_d, "stp", "'Dt, 'Dt2", "8")     \
   V(LDP_d, "ldp", "'Dt, 'Dt2", "8")
 
-void Disassembler::VisitLoadStorePairPostIndex(Instruction* instr) {
+void Disassembler::VisitLoadStorePairPostIndex(const Instruction* instr) {
   const char *mnemonic = "unimplemented";
   const char *form = "(LoadStorePairPostIndex)";
 
@@ -896,7 +903,7 @@ void Disassembler::VisitLoadStorePairPostIndex(Instruction* instr) {
 }
 
 
-void Disassembler::VisitLoadStorePairPreIndex(Instruction* instr) {
+void Disassembler::VisitLoadStorePairPreIndex(const Instruction* instr) {
   const char *mnemonic = "unimplemented";
   const char *form = "(LoadStorePairPreIndex)";
 
@@ -910,7 +917,7 @@ void Disassembler::VisitLoadStorePairPreIndex(Instruction* instr) {
 }
 
 
-void Disassembler::VisitLoadStorePairOffset(Instruction* instr) {
+void Disassembler::VisitLoadStorePairOffset(const Instruction* instr) {
   const char *mnemonic = "unimplemented";
   const char *form = "(LoadStorePairOffset)";
 
@@ -924,7 +931,7 @@ void Disassembler::VisitLoadStorePairOffset(Instruction* instr) {
 }
 
 
-void Disassembler::VisitLoadStorePairNonTemporal(Instruction* instr) {
+void Disassembler::VisitLoadStorePairNonTemporal(const Instruction* instr) {
   const char *mnemonic = "unimplemented";
   const char *form;
 
@@ -943,7 +950,50 @@ void Disassembler::VisitLoadStorePairNonTemporal(Instruction* instr) {
 }
 
 
-void Disassembler::VisitFPCompare(Instruction* instr) {
+void Disassembler::VisitLoadStoreExclusive(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form;
+
+  switch (instr->Mask(LoadStoreExclusiveMask)) {
+    case STXRB_w: mnemonic = "stxrb"; form = "'Ws, 'Wt, ['Xns]"; break;
+    case STXRH_w: mnemonic = "stxrh"; form = "'Ws, 'Wt, ['Xns]"; break;
+    case STXR_w: mnemonic = "stxr"; form = "'Ws, 'Wt, ['Xns]"; break;
+    case STXR_x: mnemonic = "stxr"; form = "'Ws, 'Xt, ['Xns]"; break;
+    case LDXRB_w: mnemonic = "ldxrb"; form = "'Wt, ['Xns]"; break;
+    case LDXRH_w: mnemonic = "ldxrh"; form = "'Wt, ['Xns]"; break;
+    case LDXR_w: mnemonic = "ldxr"; form = "'Wt, ['Xns]"; break;
+    case LDXR_x: mnemonic = "ldxr"; form = "'Xt, ['Xns]"; break;
+    case STXP_w: mnemonic = "stxp"; form = "'Ws, 'Wt, 'Wt2, ['Xns]"; break;
+    case STXP_x: mnemonic = "stxp"; form = "'Ws, 'Xt, 'Xt2, ['Xns]"; break;
+    case LDXP_w: mnemonic = "ldxp"; form = "'Wt, 'Wt2, ['Xns]"; break;
+    case LDXP_x: mnemonic = "ldxp"; form = "'Xt, 'Xt2, ['Xns]"; break;
+    case STLXRB_w: mnemonic = "stlxrb"; form = "'Ws, 'Wt, ['Xns]"; break;
+    case STLXRH_w: mnemonic = "stlxrh"; form = "'Ws, 'Wt, ['Xns]"; break;
+    case STLXR_w: mnemonic = "stlxr"; form = "'Ws, 'Wt, ['Xns]"; break;
+    case STLXR_x: mnemonic = "stlxr"; form = "'Ws, 'Xt, ['Xns]"; break;
+    case LDAXRB_w: mnemonic = "ldaxrb"; form = "'Wt, ['Xns]"; break;
+    case LDAXRH_w: mnemonic = "ldaxrh"; form = "'Wt, ['Xns]"; break;
+    case LDAXR_w: mnemonic = "ldaxr"; form = "'Wt, ['Xns]"; break;
+    case LDAXR_x: mnemonic = "ldaxr"; form = "'Xt, ['Xns]"; break;
+    case STLXP_w: mnemonic = "stlxp"; form = "'Ws, 'Wt, 'Wt2, ['Xns]"; break;
+    case STLXP_x: mnemonic = "stlxp"; form = "'Ws, 'Xt, 'Xt2, ['Xns]"; break;
+    case LDAXP_w: mnemonic = "ldaxp"; form = "'Wt, 'Wt2, ['Xns]"; break;
+    case LDAXP_x: mnemonic = "ldaxp"; form = "'Xt, 'Xt2, ['Xns]"; break;
+    case STLRB_w: mnemonic = "stlrb"; form = "'Wt, ['Xns]"; break;
+    case STLRH_w: mnemonic = "stlrh"; form = "'Wt, ['Xns]"; break;
+    case STLR_w: mnemonic = "stlr"; form = "'Wt, ['Xns]"; break;
+    case STLR_x: mnemonic = "stlr"; form = "'Xt, ['Xns]"; break;
+    case LDARB_w: mnemonic = "ldarb"; form = "'Wt, ['Xns]"; break;
+    case LDARH_w: mnemonic = "ldarh"; form = "'Wt, ['Xns]"; break;
+    case LDAR_w: mnemonic = "ldar"; form = "'Wt, ['Xns]"; break;
+    case LDAR_x: mnemonic = "ldar"; form = "'Xt, ['Xns]"; break;
+    default: form = "(LoadStoreExclusive)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPCompare(const Instruction* instr) {
   const char *mnemonic = "unimplemented";
   const char *form = "'Fn, 'Fm";
   const char *form_zero = "'Fn, #0.0";
@@ -959,7 +1009,7 @@ void Disassembler::VisitFPCompare(Instruction* instr) {
 }
 
 
-void Disassembler::VisitFPConditionalCompare(Instruction* instr) {
+void Disassembler::VisitFPConditionalCompare(const Instruction* instr) {
   const char *mnemonic = "unmplemented";
   const char *form = "'Fn, 'Fm, 'INzcv, 'Cond";
 
@@ -974,7 +1024,7 @@ void Disassembler::VisitFPConditionalCompare(Instruction* instr) {
 }
 
 
-void Disassembler::VisitFPConditionalSelect(Instruction* instr) {
+void Disassembler::VisitFPConditionalSelect(const Instruction* instr) {
   const char *mnemonic = "";
   const char *form = "'Fd, 'Fn, 'Fm, 'Cond";
 
@@ -987,7 +1037,7 @@ void Disassembler::VisitFPConditionalSelect(Instruction* instr) {
 }
 
 
-void Disassembler::VisitFPDataProcessing1Source(Instruction* instr) {
+void Disassembler::VisitFPDataProcessing1Source(const Instruction* instr) {
   const char *mnemonic = "unimplemented";
   const char *form = "'Fd, 'Fn";
 
@@ -1015,7 +1065,7 @@ void Disassembler::VisitFPDataProcessing1Source(Instruction* instr) {
 }
 
 
-void Disassembler::VisitFPDataProcessing2Source(Instruction* instr) {
+void Disassembler::VisitFPDataProcessing2Source(const Instruction* instr) {
   const char *mnemonic = "";
   const char *form = "'Fd, 'Fn, 'Fm";
 
@@ -1039,7 +1089,7 @@ void Disassembler::VisitFPDataProcessing2Source(Instruction* instr) {
 }
 
 
-void Disassembler::VisitFPDataProcessing3Source(Instruction* instr) {
+void Disassembler::VisitFPDataProcessing3Source(const Instruction* instr) {
   const char *mnemonic = "";
   const char *form = "'Fd, 'Fn, 'Fm, 'Fa";
 
@@ -1058,7 +1108,7 @@ void Disassembler::VisitFPDataProcessing3Source(Instruction* instr) {
 }
 
 
-void Disassembler::VisitFPImmediate(Instruction* instr) {
+void Disassembler::VisitFPImmediate(const Instruction* instr) {
   const char *mnemonic = "";
   const char *form = "(FPImmediate)";
 
@@ -1071,7 +1121,7 @@ void Disassembler::VisitFPImmediate(Instruction* instr) {
 }
 
 
-void Disassembler::VisitFPIntegerConvert(Instruction* instr) {
+void Disassembler::VisitFPIntegerConvert(const Instruction* instr) {
   const char *mnemonic = "unimplemented";
   const char *form = "(FPIntegerConvert)";
   const char *form_rf = "'Rd, 'Fn";
@@ -1127,7 +1177,7 @@ void Disassembler::VisitFPIntegerConvert(Instruction* instr) {
 }
 
 
-void Disassembler::VisitFPFixedPointConvert(Instruction* instr) {
+void Disassembler::VisitFPFixedPointConvert(const Instruction* instr) {
   const char *mnemonic = "";
   const char *form = "'Rd, 'Fn, 'IFPFBits";
   const char *form_fr = "'Fd, 'Rn, 'IFPFBits";
@@ -1155,14 +1205,22 @@ void Disassembler::VisitFPFixedPointConvert(Instruction* instr) {
 }
 
 
-void Disassembler::VisitSystem(Instruction* instr) {
+void Disassembler::VisitSystem(const Instruction* instr) {
   // Some system instructions hijack their Op and Cp fields to represent a
   // range of immediates instead of indicating a different instruction. This
   // makes the decoding tricky.
   const char *mnemonic = "unimplemented";
   const char *form = "(System)";
 
-  if (instr->Mask(SystemSysRegFMask) == SystemSysRegFixed) {
+  if (instr->Mask(SystemExclusiveMonitorFMask) == SystemExclusiveMonitorFixed) {
+    switch (instr->Mask(SystemExclusiveMonitorMask)) {
+      case CLREX: {
+        mnemonic = "clrex";
+        form = (instr->CRm() == 0xf) ? NULL : "'IX";
+        break;
+      }
+    }
+  } else if (instr->Mask(SystemSysRegFMask) == SystemSysRegFixed) {
     switch (instr->Mask(SystemSysRegMask)) {
       case MRS: {
         mnemonic = "mrs";
@@ -1184,7 +1242,6 @@ void Disassembler::VisitSystem(Instruction* instr) {
       }
     }
   } else if (instr->Mask(SystemHintFMask) == SystemHintFixed) {
-    VIXL_ASSERT(instr->Mask(SystemHintMask) == HINT);
     switch (instr->ImmHint()) {
       case NOP: {
         mnemonic = "nop";
@@ -1216,7 +1273,7 @@ void Disassembler::VisitSystem(Instruction* instr) {
 }
 
 
-void Disassembler::VisitException(Instruction* instr) {
+void Disassembler::VisitException(const Instruction* instr) {
   const char *mnemonic = "unimplemented";
   const char *form = "'IDebug";
 
@@ -1235,22 +1292,75 @@ void Disassembler::VisitException(Instruction* instr) {
 }
 
 
-void Disassembler::VisitUnimplemented(Instruction* instr) {
+void Disassembler::VisitUnimplemented(const Instruction* instr) {
   Format(instr, "unimplemented", "(Unimplemented)");
 }
 
 
-void Disassembler::VisitUnallocated(Instruction* instr) {
+void Disassembler::VisitUnallocated(const Instruction* instr) {
   Format(instr, "unallocated", "(Unallocated)");
 }
 
 
-void Disassembler::ProcessOutput(Instruction* /*instr*/) {
+void Disassembler::ProcessOutput(const Instruction* /*instr*/) {
   // The base disasm does nothing more than disassembling into a buffer.
 }
 
 
-void Disassembler::Format(Instruction* instr, const char* mnemonic,
+void Disassembler::AppendRegisterNameToOutput(const Instruction* instr,
+                                              const CPURegister& reg) {
+  USE(instr);
+  VIXL_ASSERT(reg.IsValid());
+  char reg_char;
+
+  if (reg.IsRegister()) {
+    reg_char = reg.Is64Bits() ? 'x' : 'w';
+  } else {
+    VIXL_ASSERT(reg.IsFPRegister());
+    reg_char = reg.Is64Bits() ? 'd' : 's';
+  }
+
+  if (reg.IsFPRegister() || !(reg.Aliases(sp) || reg.Aliases(xzr))) {
+    // A normal register: w0 - w30, x0 - x30, s0 - s31, d0 - d31.
+    AppendToOutput("%c%d", reg_char, reg.code());
+  } else if (reg.Aliases(sp)) {
+    // Disassemble w31/x31 as stack pointer wsp/sp.
+    AppendToOutput("%s", reg.Is64Bits() ? "sp" : "wsp");
+  } else {
+    // Disassemble w31/x31 as zero register wzr/xzr.
+    AppendToOutput("%czr", reg_char);
+  }
+}
+
+
+void Disassembler::AppendPCRelativeOffsetToOutput(const Instruction* instr,
+                                                  int64_t offset) {
+  USE(instr);
+  char sign = (offset < 0) ? '-' : '+';
+  AppendToOutput("#%c0x%" PRIx64, sign, std::abs(offset));
+}
+
+
+void Disassembler::AppendAddressToOutput(const Instruction* instr,
+                                         const void* addr) {
+  USE(instr);
+  AppendToOutput("(addr %p)", addr);
+}
+
+
+void Disassembler::AppendCodeAddressToOutput(const Instruction* instr,
+                                             const void* addr) {
+  AppendAddressToOutput(instr, addr);
+}
+
+
+void Disassembler::AppendDataAddressToOutput(const Instruction* instr,
+                                             const void* addr) {
+  AppendAddressToOutput(instr, addr);
+}
+
+
+void Disassembler::Format(const Instruction* instr, const char* mnemonic,
                           const char* format) {
   VIXL_ASSERT(mnemonic != NULL);
   ResetOutput();
@@ -1264,7 +1374,7 @@ void Disassembler::Format(Instruction* instr, const char* mnemonic,
 }
 
 
-void Disassembler::Substitute(Instruction* instr, const char* string) {
+void Disassembler::Substitute(const Instruction* instr, const char* string) {
   char chr = *string++;
   while (chr != '\0') {
     if (chr == '\'') {
@@ -1277,7 +1387,8 @@ void Disassembler::Substitute(Instruction* instr, const char* string) {
 }
 
 
-int Disassembler::SubstituteField(Instruction* instr, const char* format) {
+int Disassembler::SubstituteField(const Instruction* instr,
+                                  const char* format) {
   switch (format[0]) {
     case 'R':  // Register. X or W, selected by sf bit.
     case 'F':  // FP Register. S or D, selected by type field.
@@ -1303,7 +1414,7 @@ int Disassembler::SubstituteField(Instruction* instr, const char* format) {
 }
 
 
-int Disassembler::SubstituteRegisterField(Instruction* instr,
+int Disassembler::SubstituteRegisterField(const Instruction* instr,
                                           const char* format) {
   unsigned reg_num = 0;
   unsigned field_len = 2;
@@ -1312,6 +1423,7 @@ int Disassembler::SubstituteRegisterField(Instruction* instr,
     case 'n': reg_num = instr->Rn(); break;
     case 'm': reg_num = instr->Rm(); break;
     case 'a': reg_num = instr->Ra(); break;
+    case 's': reg_num = instr->Rs(); break;
     case 't': {
       if (format[2] == '2') {
         reg_num = instr->Rt2();
@@ -1329,34 +1441,47 @@ int Disassembler::SubstituteRegisterField(Instruction* instr,
     field_len = 3;
   }
 
-  char reg_type;
+  CPURegister::RegisterType reg_type;
+  unsigned reg_size;
+
   if (format[0] == 'R') {
     // Register type is R: use sf bit to choose X and W.
-    reg_type = instr->SixtyFourBits() ? 'x' : 'w';
+    reg_type = CPURegister::kRegister;
+    reg_size = instr->SixtyFourBits() ? kXRegSize : kWRegSize;
   } else if (format[0] == 'F') {
     // Floating-point register: use type field to choose S or D.
-    reg_type = ((instr->FPType() & 1) == 0) ? 's' : 'd';
+    reg_type = CPURegister::kFPRegister;
+    reg_size = ((instr->FPType() & 1) == 0) ? kSRegSize : kDRegSize;
   } else {
-    // Register type is specified. Make it lower case.
-    reg_type = format[0] + 0x20;
+    // The register type is specified.
+    switch (format[0]) {
+      case 'W':
+        reg_type = CPURegister::kRegister; reg_size = kWRegSize; break;
+      case 'X':
+        reg_type = CPURegister::kRegister; reg_size = kXRegSize; break;
+      case 'S':
+        reg_type = CPURegister::kFPRegister; reg_size = kSRegSize; break;
+      case 'D':
+        reg_type = CPURegister::kFPRegister; reg_size = kDRegSize; break;
+      default:
+        VIXL_UNREACHABLE();
+        reg_type = CPURegister::kRegister;
+        reg_size = kXRegSize;
+    }
   }
 
-  if ((reg_num != kZeroRegCode) || (reg_type == 's') || (reg_type == 'd')) {
-    // A normal register: w0 - w30, x0 - x30, s0 - s31, d0 - d31.
-    AppendToOutput("%c%d", reg_type, reg_num);
-  } else if (format[2] == 's') {
-    // Disassemble w31/x31 as stack pointer wsp/sp.
-    AppendToOutput("%s", (reg_type == 'w') ? "wsp" : "sp");
-  } else {
-    // Disassemble w31/x31 as zero register wzr/xzr.
-    AppendToOutput("%czr", reg_type);
+  if ((reg_type == CPURegister::kRegister) &&
+      (reg_num == kZeroRegCode) && (format[2] == 's')) {
+    reg_num = kSPRegInternalCode;
   }
 
+  AppendRegisterNameToOutput(instr, CPURegister(reg_num, reg_size, reg_type));
+
   return field_len;
 }
 
 
-int Disassembler::SubstituteImmediateField(Instruction* instr,
+int Disassembler::SubstituteImmediateField(const Instruction* instr,
                                            const char* format) {
   VIXL_ASSERT(format[0] == 'I');
 
@@ -1406,8 +1531,7 @@ int Disassembler::SubstituteImmediateField(Instruction* instr,
     }
     case 'C': {  // ICondB - Immediate Conditional Branch.
       int64_t offset = instr->ImmCondBranch() << 2;
-      char sign = (offset >= 0) ? '+' : '-';
-      AppendToOutput("#%c0x%" PRIx64, sign, offset);
+      AppendPCRelativeOffsetToOutput(instr, offset);
       return 6;
     }
     case 'A': {  // IAddSub.
@@ -1458,6 +1582,10 @@ int Disassembler::SubstituteImmediateField(Instruction* instr,
       AppendToOutput("#0x%" PRIx64, instr->ImmException());
       return 6;
     }
+    case 'X': {  // IX - CLREX instruction.
+      AppendToOutput("#0x%" PRIx64, instr->CRm());
+      return 2;
+    }
     default: {
       VIXL_UNIMPLEMENTED();
       return 0;
@@ -1466,7 +1594,7 @@ int Disassembler::SubstituteImmediateField(Instruction* instr,
 }
 
 
-int Disassembler::SubstituteBitfieldImmediateField(Instruction* instr,
+int Disassembler::SubstituteBitfieldImmediateField(const Instruction* instr,
                                                    const char* format) {
   VIXL_ASSERT((format[0] == 'I') && (format[1] == 'B'));
   unsigned r = instr->ImmR();
@@ -1501,7 +1629,7 @@ int Disassembler::SubstituteBitfieldImmediateField(Instruction* instr,
 }
 
 
-int Disassembler::SubstituteLiteralField(Instruction* instr,
+int Disassembler::SubstituteLiteralField(const Instruction* instr,
                                          const char* format) {
   VIXL_ASSERT(strncmp(format, "LValue", 6) == 0);
   USE(format);
@@ -1509,16 +1637,21 @@ int Disassembler::SubstituteLiteralField(Instruction* instr,
   switch (instr->Mask(LoadLiteralMask)) {
     case LDR_w_lit:
     case LDR_x_lit:
+    case LDRSW_x_lit:
     case LDR_s_lit:
-    case LDR_d_lit: AppendToOutput("(addr %p)", instr->LiteralAddress()); break;
-    default: VIXL_UNREACHABLE();
+    case LDR_d_lit:
+      AppendDataAddressToOutput(instr, instr->LiteralAddress());
+      break;
+    default:
+      VIXL_UNREACHABLE();
   }
 
   return 6;
 }
 
 
-int Disassembler::SubstituteShiftField(Instruction* instr, const char* format) {
+int Disassembler::SubstituteShiftField(const Instruction* instr,
+                                       const char* format) {
   VIXL_ASSERT(format[0] == 'H');
   VIXL_ASSERT(instr->ShiftDP() <= 0x3);
 
@@ -1541,7 +1674,7 @@ int Disassembler::SubstituteShiftField(Instruction* instr, const char* format) {
 }
 
 
-int Disassembler::SubstituteConditionField(Instruction* instr,
+int Disassembler::SubstituteConditionField(const Instruction* instr,
                                            const char* format) {
   VIXL_ASSERT(format[0] == 'C');
   const char* condition_code[] = { "eq", "ne", "hs", "lo",
@@ -1562,28 +1695,28 @@ int Disassembler::SubstituteConditionField(Instruction* instr,
 }
 
 
-int Disassembler::SubstitutePCRelAddressField(Instruction* instr,
+int Disassembler::SubstitutePCRelAddressField(const Instruction* instr,
                                               const char* format) {
-  USE(format);
-  VIXL_ASSERT(strncmp(format, "AddrPCRel", 9) == 0);
-
-  int offset = instr->ImmPCRel();
+  VIXL_ASSERT((strcmp(format, "AddrPCRelByte") == 0) ||   // Used by `adr`.
+              (strcmp(format, "AddrPCRelPage") == 0));    // Used by `adrp`.
 
-  // Only ADR (AddrPCRelByte) is supported.
-  VIXL_ASSERT(strcmp(format, "AddrPCRelByte") == 0);
+  int64_t offset = instr->ImmPCRel();
+  const Instruction * base = instr;
 
-  char sign = '+';
-  if (offset < 0) {
-    offset = -offset;
-    sign = '-';
+  if (format[9] == 'P') {
+    offset *= kPageSize;
+    base = AlignDown(base, kPageSize);
   }
-  VIXL_STATIC_ASSERT(sizeof(*instr) == 1);
-  AppendToOutput("#%c0x%x (addr %p)", sign, offset, instr + offset);
+
+  const void* target = reinterpret_cast<const void*>(base + offset);
+  AppendPCRelativeOffsetToOutput(instr, offset);
+  AppendToOutput(" ");
+  AppendAddressToOutput(instr, target);
   return 13;
 }
 
 
-int Disassembler::SubstituteBranchTargetField(Instruction* instr,
+int Disassembler::SubstituteBranchTargetField(const Instruction* instr,
                                               const char* format) {
   VIXL_ASSERT(strncmp(format, "BImm", 4) == 0);
 
@@ -1600,18 +1733,18 @@ int Disassembler::SubstituteBranchTargetField(Instruction* instr,
     default: VIXL_UNIMPLEMENTED();
   }
   offset <<= kInstructionSizeLog2;
-  char sign = '+';
-  if (offset < 0) {
-    offset = -offset;
-    sign = '-';
-  }
+  const void* target_address = reinterpret_cast<const void*>(instr + offset);
   VIXL_STATIC_ASSERT(sizeof(*instr) == 1);
-  AppendToOutput("#%c0x%" PRIx64 " (addr %p)", sign, offset, instr + offset);
+
+  AppendPCRelativeOffsetToOutput(instr, offset);
+  AppendToOutput(" ");
+  AppendCodeAddressToOutput(instr, target_address);
+
   return 8;
 }
 
 
-int Disassembler::SubstituteExtendField(Instruction* instr,
+int Disassembler::SubstituteExtendField(const Instruction* instr,
                                         const char* format) {
   VIXL_ASSERT(strncmp(format, "Ext", 3) == 0);
   VIXL_ASSERT(instr->ExtendMode() <= 7);
@@ -1638,7 +1771,7 @@ int Disassembler::SubstituteExtendField(Instruction* instr,
 }
 
 
-int Disassembler::SubstituteLSRegOffsetField(Instruction* instr,
+int Disassembler::SubstituteLSRegOffsetField(const Instruction* instr,
                                              const char* format) {
   VIXL_ASSERT(strncmp(format, "Offsetreg", 9) == 0);
   const char* extend_mode[] = { "undefined", "undefined", "uxtw", "lsl",
@@ -1667,7 +1800,7 @@ int Disassembler::SubstituteLSRegOffsetField(Instruction* instr,
 }
 
 
-int Disassembler::SubstitutePrefetchField(Instruction* instr,
+int Disassembler::SubstitutePrefetchField(const Instruction* instr,
                                           const char* format) {
   VIXL_ASSERT(format[0] == 'P');
   USE(format);
@@ -1682,7 +1815,7 @@ int Disassembler::SubstitutePrefetchField(Instruction* instr,
   return 6;
 }
 
-int Disassembler::SubstituteBarrierField(Instruction* instr,
+int Disassembler::SubstituteBarrierField(const Instruction* instr,
                                          const char* format) {
   VIXL_ASSERT(format[0] == 'M');
   USE(format);
@@ -1714,7 +1847,7 @@ void Disassembler::AppendToOutput(const char* format, ...) {
 }
 
 
-void PrintDisassembler::ProcessOutput(Instruction* instr) {
+void PrintDisassembler::ProcessOutput(const Instruction* instr) {
   fprintf(stream_, "0x%016" PRIx64 "  %08" PRIx32 "\t\t%s\n",
           reinterpret_cast<uint64_t>(instr),
           instr->InstructionBits(),
diff --git a/disas/libvixl/a64/disasm-a64.h b/disas/libvixl/a64/disasm-a64.h
index 3a56e1551..db043375c 100644
--- a/disas/libvixl/a64/disasm-a64.h
+++ b/disas/libvixl/a64/disasm-a64.h
@@ -31,6 +31,7 @@
 #include "utils.h"
 #include "instructions-a64.h"
 #include "decoder-a64.h"
+#include "assembler-a64.h"
 
 namespace vixl {
 
@@ -42,50 +43,85 @@ class Disassembler: public DecoderVisitor {
   char* GetOutput();
 
   // Declare all Visitor functions.
-  #define DECLARE(A)  void Visit##A(Instruction* instr);
+  #define DECLARE(A)  void Visit##A(const Instruction* instr);
   VISITOR_LIST(DECLARE)
   #undef DECLARE
 
  protected:
-  virtual void ProcessOutput(Instruction* instr);
+  virtual void ProcessOutput(const Instruction* instr);
+
+  // Default output functions.  The functions below implement a default way of
+  // printing elements in the disassembly. A sub-class can override these to
+  // customize the disassembly output.
+
+  // Prints the name of a register.
+  virtual void AppendRegisterNameToOutput(const Instruction* instr,
+                                          const CPURegister& reg);
+
+  // Prints a PC-relative offset. This is used for example when disassembling
+  // branches to immediate offsets.
+  virtual void AppendPCRelativeOffsetToOutput(const Instruction* instr,
+                                              int64_t offset);
+
+  // Prints an address, in the general case. It can be code or data. This is
+  // used for example to print the target address of an ADR instruction.
+  virtual void AppendAddressToOutput(const Instruction* instr,
+                                     const void* addr);
+
+  // Prints the address of some code.
+  // This is used for example to print the target address of a branch to an
+  // immediate offset.
+  // A sub-class can for example override this method to lookup the address and
+  // print an appropriate name.
+  virtual void AppendCodeAddressToOutput(const Instruction* instr,
+                                         const void* addr);
+
+  // Prints the address of some data.
+  // This is used for example to print the source address of a load literal
+  // instruction.
+  virtual void AppendDataAddressToOutput(const Instruction* instr,
+                                         const void* addr);
 
  private:
-  void Format(Instruction* instr, const char* mnemonic, const char* format);
-  void Substitute(Instruction* instr, const char* string);
-  int SubstituteField(Instruction* instr, const char* format);
-  int SubstituteRegisterField(Instruction* instr, const char* format);
-  int SubstituteImmediateField(Instruction* instr, const char* format);
-  int SubstituteLiteralField(Instruction* instr, const char* format);
-  int SubstituteBitfieldImmediateField(Instruction* instr, const char* format);
-  int SubstituteShiftField(Instruction* instr, const char* format);
-  int SubstituteExtendField(Instruction* instr, const char* format);
-  int SubstituteConditionField(Instruction* instr, const char* format);
-  int SubstitutePCRelAddressField(Instruction* instr, const char* format);
-  int SubstituteBranchTargetField(Instruction* instr, const char* format);
-  int SubstituteLSRegOffsetField(Instruction* instr, const char* format);
-  int SubstitutePrefetchField(Instruction* instr, const char* format);
-  int SubstituteBarrierField(Instruction* instr, const char* format);
-
-  inline bool RdIsZROrSP(Instruction* instr) const {
+  void Format(
+      const Instruction* instr, const char* mnemonic, const char* format);
+  void Substitute(const Instruction* instr, const char* string);
+  int SubstituteField(const Instruction* instr, const char* format);
+  int SubstituteRegisterField(const Instruction* instr, const char* format);
+  int SubstituteImmediateField(const Instruction* instr, const char* format);
+  int SubstituteLiteralField(const Instruction* instr, const char* format);
+  int SubstituteBitfieldImmediateField(
+      const Instruction* instr, const char* format);
+  int SubstituteShiftField(const Instruction* instr, const char* format);
+  int SubstituteExtendField(const Instruction* instr, const char* format);
+  int SubstituteConditionField(const Instruction* instr, const char* format);
+  int SubstitutePCRelAddressField(const Instruction* instr, const char* format);
+  int SubstituteBranchTargetField(const Instruction* instr, const char* format);
+  int SubstituteLSRegOffsetField(const Instruction* instr, const char* format);
+  int SubstitutePrefetchField(const Instruction* instr, const char* format);
+  int SubstituteBarrierField(const Instruction* instr, const char* format);
+
+  inline bool RdIsZROrSP(const Instruction* instr) const {
     return (instr->Rd() == kZeroRegCode);
   }
 
-  inline bool RnIsZROrSP(Instruction* instr) const {
+  inline bool RnIsZROrSP(const Instruction* instr) const {
     return (instr->Rn() == kZeroRegCode);
   }
 
-  inline bool RmIsZROrSP(Instruction* instr) const {
+  inline bool RmIsZROrSP(const Instruction* instr) const {
     return (instr->Rm() == kZeroRegCode);
   }
 
-  inline bool RaIsZROrSP(Instruction* instr) const {
+  inline bool RaIsZROrSP(const Instruction* instr) const {
     return (instr->Ra() == kZeroRegCode);
   }
 
   bool IsMovzMovnImm(unsigned reg_size, uint64_t value);
 
+ protected:
   void ResetOutput();
-  void AppendToOutput(const char* string, ...);
+  void AppendToOutput(const char* string, ...) PRINTF_CHECK(2, 3);
 
   char* buffer_;
   uint32_t buffer_pos_;
@@ -97,10 +133,10 @@ class Disassembler: public DecoderVisitor {
 class PrintDisassembler: public Disassembler {
  public:
   explicit PrintDisassembler(FILE* stream) : stream_(stream) { }
-  ~PrintDisassembler() { }
+  virtual ~PrintDisassembler() { }
 
  protected:
-  virtual void ProcessOutput(Instruction* instr);
+  virtual void ProcessOutput(const Instruction* instr);
 
  private:
   FILE *stream_;
diff --git a/disas/libvixl/a64/instructions-a64.cc b/disas/libvixl/a64/instructions-a64.cc
index c4eb7c451..1f08c781e 100644
--- a/disas/libvixl/a64/instructions-a64.cc
+++ b/disas/libvixl/a64/instructions-a64.cc
@@ -57,7 +57,7 @@ static uint64_t RepeatBitsAcrossReg(unsigned reg_size,
 // Logical immediates can't encode zero, so a return value of zero is used to
 // indicate a failure case. Specifically, where the constraints on imm_s are
 // not met.
-uint64_t Instruction::ImmLogical() {
+uint64_t Instruction::ImmLogical() const {
   unsigned reg_size = SixtyFourBits() ? kXRegSize : kWRegSize;
   int64_t n = BitN();
   int64_t imm_s = ImmSetBits();
@@ -108,7 +108,7 @@ uint64_t Instruction::ImmLogical() {
 }
 
 
-float Instruction::ImmFP32() {
+float Instruction::ImmFP32() const {
   //  ImmFP: abcdefgh (8 bits)
   // Single: aBbb.bbbc.defg.h000.0000.0000.0000.0000 (32 bits)
   // where B is b ^ 1
@@ -122,7 +122,7 @@ float Instruction::ImmFP32() {
 }
 
 
-double Instruction::ImmFP64() {
+double Instruction::ImmFP64() const {
   //  ImmFP: abcdefgh (8 bits)
   // Double: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
   //         0000.0000.0000.0000.0000.0000.0000.0000 (64 bits)
@@ -148,18 +148,25 @@ LSDataSize CalcLSPairDataSize(LoadStorePairOp op) {
 }
 
 
-Instruction* Instruction::ImmPCOffsetTarget() {
+const Instruction* Instruction::ImmPCOffsetTarget() const {
+  const Instruction * base = this;
   ptrdiff_t offset;
   if (IsPCRelAddressing()) {
-    // PC-relative addressing. Only ADR is supported.
+    // ADR and ADRP.
     offset = ImmPCRel();
+    if (Mask(PCRelAddressingMask) == ADRP) {
+      base = AlignDown(base, kPageSize);
+      offset *= kPageSize;
+    } else {
+      VIXL_ASSERT(Mask(PCRelAddressingMask) == ADR);
+    }
   } else {
     // All PC-relative branches.
     VIXL_ASSERT(BranchType() != UnknownBranchType);
     // Relative branch offsets are instruction-size-aligned.
     offset = ImmBranch() << kInstructionSizeLog2;
   }
-  return this + offset;
+  return base + offset;
 }
 
 
@@ -175,7 +182,7 @@ inline int Instruction::ImmBranch() const {
 }
 
 
-void Instruction::SetImmPCOffsetTarget(Instruction* target) {
+void Instruction::SetImmPCOffsetTarget(const Instruction* target) {
   if (IsPCRelAddressing()) {
     SetPCRelImmTarget(target);
   } else {
@@ -184,17 +191,23 @@ void Instruction::SetImmPCOffsetTarget(Instruction* target) {
 }
 
 
-void Instruction::SetPCRelImmTarget(Instruction* target) {
-  // ADRP is not supported, so 'this' must point to an ADR instruction.
-  VIXL_ASSERT(Mask(PCRelAddressingMask) == ADR);
-
-  Instr imm = Assembler::ImmPCRelAddress(target - this);
+void Instruction::SetPCRelImmTarget(const Instruction* target) {
+  int32_t imm21;
+  if ((Mask(PCRelAddressingMask) == ADR)) {
+    imm21 = target - this;
+  } else {
+    VIXL_ASSERT(Mask(PCRelAddressingMask) == ADRP);
+    uintptr_t this_page = reinterpret_cast<uintptr_t>(this) / kPageSize;
+    uintptr_t target_page = reinterpret_cast<uintptr_t>(target) / kPageSize;
+    imm21 = target_page - this_page;
+  }
+  Instr imm = Assembler::ImmPCRelAddress(imm21);
 
   SetInstructionBits(Mask(~ImmPCRel_mask) | imm);
 }
 
 
-void Instruction::SetBranchImmTarget(Instruction* target) {
+void Instruction::SetBranchImmTarget(const Instruction* target) {
   VIXL_ASSERT(((target - this) & 3) == 0);
   Instr branch_imm = 0;
   uint32_t imm_mask = 0;
@@ -226,9 +239,9 @@ void Instruction::SetBranchImmTarget(Instruction* target) {
 }
 
 
-void Instruction::SetImmLLiteral(Instruction* source) {
-  VIXL_ASSERT(((source - this) & 3) == 0);
-  int offset = (source - this) >> kLiteralEntrySizeLog2;
+void Instruction::SetImmLLiteral(const Instruction* source) {
+  VIXL_ASSERT(IsWordAligned(source));
+  ptrdiff_t offset = (source - this) >> kLiteralEntrySizeLog2;
   Instr imm = Assembler::ImmLLiteral(offset);
   Instr mask = ImmLLiteral_mask;
 
diff --git a/disas/libvixl/a64/instructions-a64.h b/disas/libvixl/a64/instructions-a64.h
index a4240d7d3..29f972291 100644
--- a/disas/libvixl/a64/instructions-a64.h
+++ b/disas/libvixl/a64/instructions-a64.h
@@ -41,6 +41,11 @@ const unsigned kLiteralEntrySize = 4;
 const unsigned kLiteralEntrySizeLog2 = 2;
 const unsigned kMaxLoadLiteralRange = 1 * MBytes;
 
+// This is the nominal page size (as used by the adrp instruction); the actual
+// size of the memory pages allocated by the kernel is likely to differ.
+const unsigned kPageSize = 4 * KBytes;
+const unsigned kPageSizeLog2 = 12;
+
 const unsigned kWRegSize = 32;
 const unsigned kWRegSizeLog2 = 5;
 const unsigned kWRegSizeInBytes = kWRegSize / 8;
@@ -79,36 +84,18 @@ const unsigned kZeroRegCode = 31;
 const unsigned kSPRegInternalCode = 63;
 const unsigned kRegCodeMask = 0x1f;
 
+const unsigned kAddressTagOffset = 56;
+const unsigned kAddressTagWidth = 8;
+const uint64_t kAddressTagMask =
+    ((UINT64_C(1) << kAddressTagWidth) - 1) << kAddressTagOffset;
+VIXL_STATIC_ASSERT(kAddressTagMask == UINT64_C(0xff00000000000000));
+
 // AArch64 floating-point specifics. These match IEEE-754.
 const unsigned kDoubleMantissaBits = 52;
 const unsigned kDoubleExponentBits = 11;
 const unsigned kFloatMantissaBits = 23;
 const unsigned kFloatExponentBits = 8;
 
-const float kFP32PositiveInfinity = rawbits_to_float(0x7f800000);
-const float kFP32NegativeInfinity = rawbits_to_float(0xff800000);
-const double kFP64PositiveInfinity =
-    rawbits_to_double(UINT64_C(0x7ff0000000000000));
-const double kFP64NegativeInfinity =
-    rawbits_to_double(UINT64_C(0xfff0000000000000));
-
-// This value is a signalling NaN as both a double and as a float (taking the
-// least-significant word).
-static const double kFP64SignallingNaN =
-    rawbits_to_double(UINT64_C(0x7ff000007f800001));
-static const float kFP32SignallingNaN = rawbits_to_float(0x7f800001);
-
-// A similar value, but as a quiet NaN.
-static const double kFP64QuietNaN =
-    rawbits_to_double(UINT64_C(0x7ff800007fc00001));
-static const float kFP32QuietNaN = rawbits_to_float(0x7fc00001);
-
-// The default NaN values (for FPCR.DN=1).
-static const double kFP64DefaultNaN =
-    rawbits_to_double(UINT64_C(0x7ff8000000000000));
-static const float kFP32DefaultNaN = rawbits_to_float(0x7fc00000);
-
-
 enum LSDataSize {
   LSByte        = 0,
   LSHalfword    = 1,
@@ -191,9 +178,9 @@ class Instruction {
     return signed_bitextract_32(width-1, 0, offset);
   }
 
-  uint64_t ImmLogical();
-  float ImmFP32();
-  double ImmFP64();
+  uint64_t ImmLogical() const;
+  float ImmFP32() const;
+  double ImmFP64() const;
 
   inline LSDataSize SizeLSPair() const {
     return CalcLSPairDataSize(
@@ -301,46 +288,49 @@ class Instruction {
 
   // Find the target of this instruction. 'this' may be a branch or a
   // PC-relative addressing instruction.
-  Instruction* ImmPCOffsetTarget();
+  const Instruction* ImmPCOffsetTarget() const;
 
   // Patch a PC-relative offset to refer to 'target'. 'this' may be a branch or
   // a PC-relative addressing instruction.
-  void SetImmPCOffsetTarget(Instruction* target);
+  void SetImmPCOffsetTarget(const Instruction* target);
   // Patch a literal load instruction to load from 'source'.
-  void SetImmLLiteral(Instruction* source);
+  void SetImmLLiteral(const Instruction* source);
 
-  inline uint8_t* LiteralAddress() {
+  inline uint8_t* LiteralAddress() const {
     int offset = ImmLLiteral() << kLiteralEntrySizeLog2;
-    return reinterpret_cast<uint8_t*>(this) + offset;
+    const uint8_t* address = reinterpret_cast<const uint8_t*>(this) + offset;
+    // Note that the result is safely mutable only if the backing buffer is
+    // safely mutable.
+    return const_cast<uint8_t*>(address);
   }
 
-  inline uint32_t Literal32() {
+  inline uint32_t Literal32() const {
     uint32_t literal;
     memcpy(&literal, LiteralAddress(), sizeof(literal));
 
     return literal;
   }
 
-  inline uint64_t Literal64() {
+  inline uint64_t Literal64() const {
     uint64_t literal;
     memcpy(&literal, LiteralAddress(), sizeof(literal));
 
     return literal;
   }
 
-  inline float LiteralFP32() {
+  inline float LiteralFP32() const {
     return rawbits_to_float(Literal32());
   }
 
-  inline double LiteralFP64() {
+  inline double LiteralFP64() const {
     return rawbits_to_double(Literal64());
   }
 
-  inline Instruction* NextInstruction() {
+  inline const Instruction* NextInstruction() const {
     return this + kInstructionSize;
   }
 
-  inline Instruction* InstructionAtOffset(int64_t offset) {
+  inline const Instruction* InstructionAtOffset(int64_t offset) const {
     VIXL_ASSERT(IsWordAligned(this + offset));
     return this + offset;
   }
@@ -349,11 +339,15 @@ class Instruction {
     return reinterpret_cast<Instruction*>(src);
   }
 
+  template<typename T> static inline const Instruction* CastConst(T src) {
+    return reinterpret_cast<const Instruction*>(src);
+  }
+
  private:
   inline int ImmBranch() const;
 
-  void SetPCRelImmTarget(Instruction* target);
-  void SetBranchImmTarget(Instruction* target);
+  void SetPCRelImmTarget(const Instruction* target);
+  void SetBranchImmTarget(const Instruction* target);
 };
 }  // namespace vixl
 
diff --git a/disas/libvixl/code-buffer.h b/disas/libvixl/code-buffer.h
new file mode 100644
index 000000000..da6233dd8
--- /dev/null
+++ b/disas/libvixl/code-buffer.h
@@ -0,0 +1,113 @@
+// Copyright 2014, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_CODE_BUFFER_H
+#define VIXL_CODE_BUFFER_H
+
+#include <string.h>
+#include "globals.h"
+
+namespace vixl {
+
+class CodeBuffer {
+ public:
+  explicit CodeBuffer(size_t capacity = 4 * KBytes);
+  CodeBuffer(void* buffer, size_t capacity);
+  ~CodeBuffer();
+
+  void Reset();
+
+  ptrdiff_t OffsetFrom(ptrdiff_t offset) const {
+    ptrdiff_t cursor_offset = cursor_ - buffer_;
+    VIXL_ASSERT((offset >= 0) && (offset <= cursor_offset));
+    return cursor_offset - offset;
+  }
+
+  ptrdiff_t CursorOffset() const {
+    return OffsetFrom(0);
+  }
+
+  template <typename T>
+  T GetOffsetAddress(ptrdiff_t offset) const {
+    VIXL_ASSERT((offset >= 0) && (offset <= (cursor_ - buffer_)));
+    return reinterpret_cast<T>(buffer_ + offset);
+  }
+
+  size_t RemainingBytes() const {
+    VIXL_ASSERT((cursor_ >= buffer_) && (cursor_ <= (buffer_ + capacity_)));
+    return (buffer_ + capacity_) - cursor_;
+  }
+
+  // A code buffer can emit:
+  //  * 32-bit data: instruction and constant.
+  //  * 64-bit data: constant.
+  //  * string: debug info.
+  void Emit32(uint32_t data) { Emit(data); }
+
+  void Emit64(uint64_t data) { Emit(data); }
+
+  void EmitString(const char* string);
+
+  // Align to kInstructionSize.
+  void Align();
+
+  size_t capacity() const { return capacity_; }
+
+  bool IsManaged() const { return managed_; }
+
+  void Grow(size_t new_capacity);
+
+  bool IsDirty() const { return dirty_; }
+
+  void SetClean() { dirty_ = false; }
+
+ private:
+  template <typename T>
+  void Emit(T value) {
+    VIXL_ASSERT(RemainingBytes() >= sizeof(value));
+    dirty_ = true;
+    memcpy(cursor_, &value, sizeof(value));
+    cursor_ += sizeof(value);
+  }
+
+  // Backing store of the buffer.
+  byte* buffer_;
+  // If true the backing store is allocated and deallocated by the buffer. The
+  // backing store can then grow on demand. If false the backing store is
+  // provided by the user and cannot be resized internally.
+  bool managed_;
+  // Pointer to the next location to be written.
+  byte* cursor_;
+  // True if there has been any write since the buffer was created or cleaned.
+  bool dirty_;
+  // Capacity in bytes of the backing store.
+  size_t capacity_;
+};
+
+}  // namespace vixl
+
+#endif  // VIXL_CODE_BUFFER_H
+
diff --git a/disas/libvixl/platform.h b/disas/libvixl/platform.h
index b5c2085b9..de2b110cc 100644
--- a/disas/libvixl/platform.h
+++ b/disas/libvixl/platform.h
@@ -28,14 +28,10 @@
 #define PLATFORM_H
 
 // Define platform specific functionalities.
+#include <signal.h>
 
 namespace vixl {
-#ifdef USE_SIMULATOR
-// Currently we assume running the simulator implies running on x86 hardware.
-inline void HostBreakpoint() { asm("int3"); }
-#else
-inline void HostBreakpoint() { asm("brk"); }
-#endif
+inline void HostBreakpoint() { raise(SIGINT); }
 }  // namespace vixl
 
 #endif
diff --git a/disas/libvixl/utils.cc b/disas/libvixl/utils.cc
index c9c05d1e1..21965d7a1 100644
--- a/disas/libvixl/utils.cc
+++ b/disas/libvixl/utils.cc
@@ -124,4 +124,15 @@ int CountSetBits(uint64_t value, int width) {
 
   return value;
 }
+
+
+uint64_t LowestSetBit(uint64_t value) {
+  return value & -value;
+}
+
+
+bool IsPowerOf2(int64_t value) {
+  return (value != 0) && ((value & (value - 1)) == 0);
+}
+
 }  // namespace vixl
diff --git a/disas/libvixl/utils.h b/disas/libvixl/utils.h
index 83c928c8e..1540c3060 100644
--- a/disas/libvixl/utils.h
+++ b/disas/libvixl/utils.h
@@ -33,6 +33,14 @@
 
 namespace vixl {
 
+// Macros for compile-time format checking.
+#if defined(__GNUC__)
+#define PRINTF_CHECK(format_index, varargs_index) \
+  __attribute__((format(printf, format_index, varargs_index)))
+#else
+#define PRINTF_CHECK(format_index, varargs_index)
+#endif
+
 // Check number width.
 inline bool is_intn(unsigned n, int64_t x) {
   VIXL_ASSERT((0 < n) && (n < 64));
@@ -155,35 +163,46 @@ int CountLeadingZeros(uint64_t value, int width);
 int CountLeadingSignBits(int64_t value, int width);
 int CountTrailingZeros(uint64_t value, int width);
 int CountSetBits(uint64_t value, int width);
+uint64_t LowestSetBit(uint64_t value);
+bool IsPowerOf2(int64_t value);
 
 // Pointer alignment
 // TODO: rename/refactor to make it specific to instructions.
 template<typename T>
 bool IsWordAligned(T pointer) {
   VIXL_ASSERT(sizeof(pointer) == sizeof(intptr_t));   // NOLINT(runtime/sizeof)
-  return (reinterpret_cast<intptr_t>(pointer) & 3) == 0;
+  return ((intptr_t)(pointer) & 3) == 0;
 }
 
 // Increment a pointer until it has the specified alignment.
 template<class T>
 T AlignUp(T pointer, size_t alignment) {
-  VIXL_STATIC_ASSERT(sizeof(pointer) == sizeof(uintptr_t));
-  uintptr_t pointer_raw = reinterpret_cast<uintptr_t>(pointer);
+  // Use C-style casts to get static_cast behaviour for integral types (T), and
+  // reinterpret_cast behaviour for other types.
+
+  uintptr_t pointer_raw = (uintptr_t)pointer;
+  VIXL_STATIC_ASSERT(sizeof(pointer) == sizeof(pointer_raw));
+
   size_t align_step = (alignment - pointer_raw) % alignment;
   VIXL_ASSERT((pointer_raw + align_step) % alignment == 0);
-  return reinterpret_cast<T>(pointer_raw + align_step);
+
+  return (T)(pointer_raw + align_step);
 }
 
 // Decrement a pointer until it has the specified alignment.
 template<class T>
 T AlignDown(T pointer, size_t alignment) {
-  VIXL_STATIC_ASSERT(sizeof(pointer) == sizeof(uintptr_t));
-  uintptr_t pointer_raw = reinterpret_cast<uintptr_t>(pointer);
+  // Use C-style casts to get static_cast behaviour for integral types (T), and
+  // reinterpret_cast behaviour for other types.
+
+  uintptr_t pointer_raw = (uintptr_t)pointer;
+  VIXL_STATIC_ASSERT(sizeof(pointer) == sizeof(pointer_raw));
+
   size_t align_step = pointer_raw % alignment;
   VIXL_ASSERT((pointer_raw - align_step) % alignment == 0);
-  return reinterpret_cast<T>(pointer_raw - align_step);
-}
 
+  return (T)(pointer_raw - align_step);
+}
 
 }  // namespace vixl
 
diff --git a/disas/mips.c b/disas/mips.c
index 2106b574c..2614c52a4 100644
--- a/disas/mips.c
+++ b/disas/mips.c
@@ -119,6 +119,8 @@ see <http://www.gnu.org/licenses/>.  */
 #define OP_SH_IMMEDIATE		0
 #define OP_MASK_DELTA		0xffff
 #define OP_SH_DELTA		0
+#define OP_MASK_DELTA_R6        0x1ff
+#define OP_SH_DELTA_R6          7
 #define OP_MASK_FUNCT		0x3f
 #define OP_SH_FUNCT		0
 #define OP_MASK_SPEC		0x3f
@@ -218,6 +220,28 @@ see <http://www.gnu.org/licenses/>.  */
 #define OP_SH_MTACC_D		13
 #define OP_MASK_MTACC_D		0x3
 
+/* MSA */
+#define OP_MASK_1BIT            0x1
+#define OP_SH_1BIT              16
+#define OP_MASK_2BIT            0x3
+#define OP_SH_2BIT              16
+#define OP_MASK_3BIT            0x7
+#define OP_SH_3BIT              16
+#define OP_MASK_4BIT            0xf
+#define OP_SH_4BIT              16
+#define OP_MASK_5BIT            0x1f
+#define OP_SH_5BIT              16
+#define OP_MASK_10BIT           0x3ff
+#define OP_SH_10BIT             11
+#define OP_MASK_MSACR11         0x1f
+#define OP_SH_MSACR11           11
+#define OP_MASK_MSACR6          0x1f
+#define OP_SH_MSACR6            6
+#define OP_MASK_GPR             0x1f
+#define OP_SH_GPR               6
+#define OP_MASK_1_TO_4          0x3
+#define OP_SH_1_TO_4            6
+
 #define	OP_OP_COP0		0x10
 #define	OP_OP_COP1		0x11
 #define	OP_OP_COP2		0x12
@@ -405,6 +429,12 @@ struct mips_opcode
    "+3" UDI immediate bits 6-20
    "+4" UDI immediate bits 6-25
 
+   R6 immediates/displacements :
+   (adding suffix to 'o' to avoid adding new characters)
+   "+o"  9 bits immediate/displacement (shift = 7)
+   "+o1" 18 bits immediate/displacement (shift = 0)
+   "+o2" 19 bits immediate/displacement (shift = 0)
+
    Other:
    "()" parens surrounding optional value
    ","  separates operands
@@ -502,6 +532,9 @@ struct mips_opcode
 /* Instruction writes MDMX accumulator. */
 #define INSN2_WRITE_MDMX_ACC	    0x00000004
 
+/* Reads the general purpose register in OP_*_RD.  */
+#define INSN2_READ_GPR_D    0x00000200
+
 /* Instruction is actually a macro.  It should be ignored by the
    disassembler, and requires special treatment by the assembler.  */
 #define INSN_MACRO                  0xffffffff
@@ -521,6 +554,8 @@ struct mips_opcode
 #define INSN_ISA64                0x00000040
 #define INSN_ISA32R2              0x00000080
 #define INSN_ISA64R2              0x00000100
+#define INSN_ISA32R6              0x00000200
+#define INSN_ISA64R6              0x00000400
 
 /* Masks used for MIPS-defined ASEs.  */
 #define INSN_ASE_MASK		  0x0000f000
@@ -557,7 +592,12 @@ struct mips_opcode
 #define INSN_5500		  0x02000000
 
 /* MDMX ASE */
-#define INSN_MDMX                 0x04000000
+#define INSN_MDMX                 0x00000000    /* Deprecated */
+
+/* MIPS MSA Extension */
+#define INSN_MSA                  0x04000000
+#define INSN_MSA64                0x04000000
+
 /* MT ASE */
 #define INSN_MT                   0x08000000
 /* SmartMIPS ASE  */
@@ -585,6 +625,8 @@ struct mips_opcode
 #define       ISA_MIPS32R2    (ISA_MIPS32 | INSN_ISA32R2)
 #define       ISA_MIPS64R2    (ISA_MIPS64 | INSN_ISA32R2 | INSN_ISA64R2)
 
+#define       ISA_MIPS32R6    (ISA_MIPS32R2 | INSN_ISA32R6)
+#define       ISA_MIPS64R6    (ISA_MIPS64R2 | INSN_ISA32R6 | INSN_ISA64R6)
 
 /* CPU defines, use instead of hardcoding processor number. Keep this
    in sync with bfd/archures.c in order for machine selection to work.  */
@@ -1121,6 +1163,8 @@ extern const int bfd_mips16_num_opcodes;
 #define I64     INSN_ISA64
 #define I33	INSN_ISA32R2
 #define I65	INSN_ISA64R2
+#define I32R6   INSN_ISA32R6
+#define I64R6   INSN_ISA64R6
 
 /* MIPS64 MIPS-3D ASE support.  */
 #define I16     INSN_MIPS16
@@ -1190,6 +1234,17 @@ extern const int bfd_mips16_num_opcodes;
 /* MIPS MT ASE support.  */
 #define MT32	INSN_MT
 
+/* MSA */
+#define MSA     INSN_MSA
+#define MSA64   INSN_MSA64
+#define WR_VD   INSN_WRITE_FPR_D    /* Reuse INSN_WRITE_FPR_D */
+#define RD_VD   WR_VD               /* Reuse WR_VD */
+#define RD_VT   INSN_READ_FPR_T     /* Reuse INSN_READ_FPR_T */
+#define RD_VS   INSN_READ_FPR_S     /* Reuse INSN_READ_FPR_S */
+#define RD_d    INSN2_READ_GPR_D    /* Reuse INSN2_READ_GPR_D */
+
+#define RD_rd6  0
+
 /* The order of overloaded instructions matters.  Label arguments and
    register arguments look the same. Instructions that can have either
    for arguments must apear in the correct order in this table for the
@@ -1209,6 +1264,681 @@ const struct mips_opcode mips_builtin_opcodes[] =
    them first.  The assemblers uses a hash table based on the
    instruction name anyhow.  */
 /* name,    args,	match,	    mask,	pinfo,          	membership */
+{"lwpc",    "s,+o2",    0xec080000, 0xfc180000, WR_d,                 0, I32R6},
+{"lwupc",   "s,+o2",    0xec100000, 0xfc180000, WR_d,                 0, I64R6},
+{"ldpc",    "s,+o1",    0xec180000, 0xfc1c0000, WR_d,                 0, I64R6},
+{"addiupc", "s,+o2",    0xec000000, 0xfc180000, WR_d,                 0, I32R6},
+{"auipc",   "s,u",      0xec1e0000, 0xfc1f0000, WR_d,                 0, I32R6},
+{"aluipc",  "s,u",      0xec1f0000, 0xfc1f0000, WR_d,                 0, I32R6},
+{"daui",    "s,t,u",    0x74000000, 0xfc000000, RD_s|WR_t,            0, I64R6},
+{"dahi",    "s,u",      0x04060000, 0xfc1f0000, RD_s,                 0, I64R6},
+{"dati",    "s,u",      0x041e0000, 0xfc1f0000, RD_s,                 0, I64R6},
+{"lsa",     "d,s,t",    0x00000005, 0xfc00073f, WR_d|RD_s|RD_t,       0, I32R6},
+{"dlsa",    "d,s,t",    0x00000015, 0xfc00073f, WR_d|RD_s|RD_t,       0, I64R6},
+{"clz",     "U,s",      0x00000050, 0xfc1f07ff, WR_d|RD_s,            0, I32R6},
+{"clo",     "U,s",      0x00000051, 0xfc1f07ff, WR_d|RD_s,            0, I32R6},
+{"dclz",    "U,s",      0x00000052, 0xfc1f07ff, WR_d|RD_s,            0, I64R6},
+{"dclo",    "U,s",      0x00000053, 0xfc1f07ff, WR_d|RD_s,            0, I64R6},
+{"sdbbp",   "B",        0x0000000e, 0xfc00003f, TRAP,                 0, I32R6},
+{"mul",     "d,s,t",    0x00000098, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I32R6},
+{"muh",     "d,s,t",    0x000000d8, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I32R6},
+{"mulu",    "d,s,t",    0x00000099, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I32R6},
+{"muhu",    "d,s,t",    0x000000d9, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I32R6},
+{"div",     "d,s,t",    0x0000009a, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I32R6},
+{"mod",     "d,s,t",    0x000000da, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I32R6},
+{"divu",    "d,s,t",    0x0000009b, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I32R6},
+{"modu",    "d,s,t",    0x000000db, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I32R6},
+{"dmul",    "d,s,t",    0x0000009c, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I64R6},
+{"dmuh",    "d,s,t",    0x000000dc, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I64R6},
+{"dmulu",   "d,s,t",    0x0000009d, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I64R6},
+{"dmuhu",   "d,s,t",    0x000000dd, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I64R6},
+{"ddiv",    "d,s,t",    0x0000009e, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I64R6},
+{"dmod",    "d,s,t",    0x000000de, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I64R6},
+{"ddivu",   "d,s,t",    0x0000009f, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I64R6},
+{"dmodu",   "d,s,t",    0x000000df, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I64R6},
+{"ll",      "t,o(b)",   0x7c000036, 0xfc00007f, LDD|RD_b|WR_t,        0, I32R6},
+{"sc",      "t,o(b)",   0x7c000026, 0xfc00007f, LDD|RD_b|WR_t,        0, I32R6},
+{"lld",     "t,o(b)",   0x7c000037, 0xfc00007f, LDD|RD_b|WR_t,        0, I64R6},
+{"scd",     "t,o(b)",   0x7c000027, 0xfc00007f, LDD|RD_b|WR_t,        0, I64R6},
+{"pref",    "h,o(b)",   0x7c000035, 0xfc00007f, RD_b,                 0, I32R6},
+{"cache",   "k,o(b)",   0x7c000025, 0xfc00007f, RD_b,                 0, I32R6},
+{"seleqz",  "d,v,t",    0x00000035, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I32R6},
+{"selnez",  "d,v,t",    0x00000037, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I32R6},
+{"maddf.s", "D,S,T",    0x46000018, 0xffe0003f, WR_D|RD_S|RD_T|FP_S,  0, I32R6},
+{"maddf.d", "D,S,T",    0x46200018, 0xffe0003f, WR_D|RD_S|RD_T|FP_D,  0, I32R6},
+{"msubf.s", "D,S,T",    0x46000019, 0xffe0003f, WR_D|RD_S|RD_T|FP_S,  0, I32R6},
+{"msubf.d", "D,S,T",    0x46200019, 0xffe0003f, WR_D|RD_S|RD_T|FP_D,  0, I32R6},
+{"max.s",   "D,S,T",    0x4600001e, 0xffe0003f, WR_D|RD_S|RD_T|FP_S,  0, I32R6},
+{"max.d",   "D,S,T",    0x4620001e, 0xffe0003f, WR_D|RD_S|RD_T|FP_D,  0, I32R6},
+{"maxa.s",  "D,S,T",    0x4600001f, 0xffe0003f, WR_D|RD_S|RD_T|FP_S,  0, I32R6},
+{"maxa.d",  "D,S,T",    0x4620001f, 0xffe0003f, WR_D|RD_S|RD_T|FP_D,  0, I32R6},
+{"rint.s",  "D,S",      0x4600001a, 0xffff003f, WR_D|RD_S|FP_S,       0, I32R6},
+{"rint.d",  "D,S",      0x4620001a, 0xffff003f, WR_D|RD_S|FP_D,       0, I32R6},
+{"class.s", "D,S",      0x4600001b, 0xffff003f, WR_D|RD_S|FP_S,       0, I32R6},
+{"class.d", "D,S",      0x4620001b, 0xffff003f, WR_D|RD_S|FP_D,       0, I32R6},
+{"min.s",   "D,S,T",    0x4600001c, 0xffe0003f, WR_D|RD_S|RD_T|FP_S,  0, I32R6},
+{"min.d",   "D,S,T",    0x4620001c, 0xffe0003f, WR_D|RD_S|RD_T|FP_D,  0, I32R6},
+{"mina.s",  "D,S,T",    0x4600001d, 0xffe0003f, WR_D|RD_S|RD_T|FP_S,  0, I32R6},
+{"mina.d",  "D,S,T",    0x4620001d, 0xffe0003f, WR_D|RD_S|RD_T|FP_D,  0, I32R6},
+{"sel.s",   "D,S,T",    0x46000010, 0xffe0003f, WR_D|RD_S|RD_T|FP_S,  0, I32R6},
+{"sel.d",   "D,S,T",    0x46200010, 0xffe0003f, WR_D|RD_S|RD_T|FP_D,  0, I32R6},
+{"seleqz.s", "D,S,T",   0x46000014, 0xffe0003f, WR_D|RD_S|RD_T|FP_S,  0, I32R6},
+{"seleqz.d", "D,S,T",   0x46200014, 0xffe0003f, WR_D|RD_S|RD_T|FP_D,  0, I32R6},
+{"selnez.s", "D,S,T",   0x46000017, 0xffe0003f, WR_D|RD_S|RD_T|FP_S,  0, I32R6},
+{"selnez.d", "D,S,T",   0x46200017, 0xffe0003f, WR_D|RD_S|RD_T|FP_D,  0, I32R6},
+{"align",   "d,v,t",    0x7c000220, 0xfc00073f, WR_d|RD_s|RD_t,       0, I32R6},
+{"dalign",  "d,v,t",    0x7c000224, 0xfc00063f, WR_d|RD_s|RD_t,       0, I64R6},
+{"bitswap", "d,w",      0x7c000020, 0xffe007ff, WR_d|RD_t,            0, I32R6},
+{"dbitswap","d,w",      0x7c000024, 0xffe007ff, WR_d|RD_t,            0, I64R6},
+{"balc",    "+p",       0xe8000000, 0xfc000000, UBD|WR_31,            0, I32R6},
+{"bc",      "+p",       0xc8000000, 0xfc000000, UBD|WR_31,            0, I32R6},
+{"jic",     "t,o",      0xd8000000, 0xffe00000, UBD|RD_t,             0, I32R6},
+{"beqzc",   "s,+p",     0xd8000000, 0xfc000000, CBD|RD_s,             0, I32R6},
+{"jialc",   "t,o",      0xf8000000, 0xffe00000, UBD|RD_t,             0, I32R6},
+{"bnezc",   "s,+p",     0xf8000000, 0xfc000000, CBD|RD_s,             0, I32R6},
+{"beqzalc", "s,t,p",    0x20000000, 0xffe00000, CBD|RD_s|RD_t,        0, I32R6},
+{"bovc",    "s,t,p",    0x20000000, 0xfc000000, CBD|RD_s|RD_t,        0, I32R6},
+{"beqc",    "s,t,p",    0x20000000, 0xfc000000, CBD|RD_s|RD_t,        0, I32R6},
+{"bnezalc", "s,t,p",    0x60000000, 0xffe00000, CBD|RD_s|RD_t,        0, I32R6},
+{"bnvc",    "s,t,p",    0x60000000, 0xfc000000, CBD|RD_s|RD_t,        0, I32R6},
+{"bnec",    "s,t,p",    0x60000000, 0xfc000000, CBD|RD_s|RD_t,        0, I32R6},
+{"blezc",   "s,t,p",    0x58000000, 0xffe00000, CBD|RD_s|RD_t,        0, I32R6},
+{"bgezc",   "s,t,p",    0x58000000, 0xfc000000, CBD|RD_s|RD_t,        0, I32R6},
+{"bgec",    "s,t,p",    0x58000000, 0xfc000000, CBD|RD_s|RD_t,        0, I32R6},
+{"bgtzc",   "s,t,p",    0x5c000000, 0xffe00000, CBD|RD_s|RD_t,        0, I32R6},
+{"bltzc",   "s,t,p",    0x5c000000, 0xfc000000, CBD|RD_s|RD_t,        0, I32R6},
+{"bltc",    "s,t,p",    0x5c000000, 0xfc000000, CBD|RD_s|RD_t,        0, I32R6},
+{"blezalc", "s,t,p",    0x18000000, 0xffe00000, CBD|RD_s|RD_t,        0, I32R6},
+{"bgezalc", "s,t,p",    0x18000000, 0xfc000000, CBD|RD_s|RD_t,        0, I32R6},
+{"bgeuc",   "s,t,p",    0x18000000, 0xfc000000, CBD|RD_s|RD_t,        0, I32R6},
+{"bgtzalc", "s,t,p",    0x1c000000, 0xffe00000, CBD|RD_s|RD_t,        0, I32R6},
+{"bltzalc", "s,t,p",    0x1c000000, 0xfc000000, CBD|RD_s|RD_t,        0, I32R6},
+{"bltuc",   "s,t,p",    0x1c000000, 0xfc000000, CBD|RD_s|RD_t,        0, I32R6},
+{"nal",     "p",        0x04100000, 0xffff0000, WR_31,                0, I32R6},
+{"bal",     "p",        0x04110000, 0xffff0000, UBD|WR_31,            0, I32R6},
+{"bc1eqz",  "T,p",      0x45200000, 0xffe00000, CBD|RD_T|FP_S|FP_D,   0, I32R6},
+{"bc1nez",  "T,p",      0x45a00000, 0xffe00000, CBD|RD_T|FP_S|FP_D,   0, I32R6},
+{"bc2eqz",  "E,p",      0x49200000, 0xffe00000, CBD|RD_C2,            0, I32R6},
+{"bc2nez",  "E,p",      0x49a00000, 0xffe00000, CBD|RD_C2,            0, I32R6},
+{"cmp.af.s",   "D,S,T", 0x46800000, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.un.s",   "D,S,T", 0x46800001, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.eq.s",   "D,S,T", 0x46800002, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.ueq.s",  "D,S,T", 0x46800003, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.lt.s",   "D,S,T", 0x46800004, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.ult.s",  "D,S,T", 0x46800005, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.le.s",   "D,S,T", 0x46800006, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.ule.s",  "D,S,T", 0x46800007, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.saf.s",  "D,S,T", 0x46800008, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.sun.s",  "D,S,T", 0x46800009, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.seq.s",  "D,S,T", 0x4680000a, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.sueq.s", "D,S,T", 0x4680000b, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.slt.s",  "D,S,T", 0x4680000c, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.sult.s", "D,S,T", 0x4680000d, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.sle.s",  "D,S,T", 0x4680000e, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.sule.s", "D,S,T", 0x4680000f, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.or.s",   "D,S,T", 0x46800011, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.une.s",  "D,S,T", 0x46800012, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.ne.s",   "D,S,T", 0x46800013, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.sor.s",  "D,S,T", 0x46800019, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.sune.s", "D,S,T", 0x4680001a, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.sne.s",  "D,S,T", 0x4680001b, 0xffe0003f, RD_S|RD_T|WR_D|FP_S,  0, I32R6},
+{"cmp.af.d",   "D,S,T", 0x46a00000, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.un.d",   "D,S,T", 0x46a00001, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.eq.d",   "D,S,T", 0x46a00002, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.ueq.d",  "D,S,T", 0x46a00003, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.lt.d",   "D,S,T", 0x46a00004, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.ult.d",  "D,S,T", 0x46a00005, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.le.d",   "D,S,T", 0x46a00006, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.ule.d",  "D,S,T", 0x46a00007, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.saf.d",  "D,S,T", 0x46a00008, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.sun.d",  "D,S,T", 0x46a00009, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.seq.d",  "D,S,T", 0x46a0000a, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.sueq.d", "D,S,T", 0x46a0000b, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.slt.d",  "D,S,T", 0x46a0000c, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.sult.d", "D,S,T", 0x46a0000d, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.sle.d",  "D,S,T", 0x46a0000e, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.sule.d", "D,S,T", 0x46a0000f, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.or.d",   "D,S,T", 0x46a00011, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.une.d",  "D,S,T", 0x46a00012, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.ne.d",   "D,S,T", 0x46a00013, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.sor.d",  "D,S,T", 0x46a00019, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.sune.d", "D,S,T", 0x46a0001a, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"cmp.sne.d",  "D,S,T", 0x46a0001b, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+
+/* MSA */
+{"sll.b",   "+d,+e,+f", 0x7800000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"sll.h",   "+d,+e,+f", 0x7820000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"sll.w",   "+d,+e,+f", 0x7840000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"sll.d",   "+d,+e,+f", 0x7860000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"slli.b",  "+d,+e,+7", 0x78700009, 0xfff8003f, WR_VD|RD_VS,        0, MSA},
+{"slli.h",  "+d,+e,+8", 0x78600009, 0xfff0003f, WR_VD|RD_VS,        0, MSA},
+{"slli.w",  "+d,+e,+9", 0x78400009, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"slli.d",  "+d,+e,'",  0x78000009, 0xffc0003f, WR_VD|RD_VS,        0, MSA},
+{"sra.b",   "+d,+e,+f", 0x7880000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"sra.h",   "+d,+e,+f", 0x78a0000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"sra.w",   "+d,+e,+f", 0x78c0000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"sra.d",   "+d,+e,+f", 0x78e0000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"srai.b",  "+d,+e,+7", 0x78f00009, 0xfff8003f, WR_VD|RD_VS,        0, MSA},
+{"srai.h",  "+d,+e,+8", 0x78e00009, 0xfff0003f, WR_VD|RD_VS,        0, MSA},
+{"srai.w",  "+d,+e,+9", 0x78c00009, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"srai.d",  "+d,+e,'",  0x78800009, 0xffc0003f, WR_VD|RD_VS,        0, MSA},
+{"srl.b",   "+d,+e,+f", 0x7900000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"srl.h",   "+d,+e,+f", 0x7920000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"srl.w",   "+d,+e,+f", 0x7940000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"srl.d",   "+d,+e,+f", 0x7960000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"srli.b",  "+d,+e,+7", 0x79700009, 0xfff8003f, WR_VD|RD_VS,        0, MSA},
+{"srli.h",  "+d,+e,+8", 0x79600009, 0xfff0003f, WR_VD|RD_VS,        0, MSA},
+{"srli.w",  "+d,+e,+9", 0x79400009, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"srli.d",  "+d,+e,'",  0x79000009, 0xffc0003f, WR_VD|RD_VS,        0, MSA},
+{"bclr.b",  "+d,+e,+f", 0x7980000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"bclr.h",  "+d,+e,+f", 0x79a0000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"bclr.w",  "+d,+e,+f", 0x79c0000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"bclr.d",  "+d,+e,+f", 0x79e0000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"bclri.b", "+d,+e,+7", 0x79f00009, 0xfff8003f, WR_VD|RD_VS,        0, MSA},
+{"bclri.h", "+d,+e,+8", 0x79e00009, 0xfff0003f, WR_VD|RD_VS,        0, MSA},
+{"bclri.w", "+d,+e,+9", 0x79c00009, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"bclri.d", "+d,+e,'",  0x79800009, 0xffc0003f, WR_VD|RD_VS,        0, MSA},
+{"bset.b",  "+d,+e,+f", 0x7a00000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"bset.h",  "+d,+e,+f", 0x7a20000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"bset.w",  "+d,+e,+f", 0x7a40000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"bset.d",  "+d,+e,+f", 0x7a60000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"bseti.b", "+d,+e,+7", 0x7a700009, 0xfff8003f, WR_VD|RD_VS,        0, MSA},
+{"bseti.h", "+d,+e,+8", 0x7a600009, 0xfff0003f, WR_VD|RD_VS,        0, MSA},
+{"bseti.w", "+d,+e,+9", 0x7a400009, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"bseti.d", "+d,+e,'",  0x7a000009, 0xffc0003f, WR_VD|RD_VS,        0, MSA},
+{"bneg.b",  "+d,+e,+f", 0x7a80000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"bneg.h",  "+d,+e,+f", 0x7aa0000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"bneg.w",  "+d,+e,+f", 0x7ac0000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"bneg.d",  "+d,+e,+f", 0x7ae0000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"bnegi.b", "+d,+e,+7", 0x7af00009, 0xfff8003f, WR_VD|RD_VS,        0, MSA},
+{"bnegi.h", "+d,+e,+8", 0x7ae00009, 0xfff0003f, WR_VD|RD_VS,        0, MSA},
+{"bnegi.w", "+d,+e,+9", 0x7ac00009, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"bnegi.d", "+d,+e,'",  0x7a800009, 0xffc0003f, WR_VD|RD_VS,        0, MSA},
+{"binsl.b", "+d,+e,+f", 0x7b00000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"binsl.h", "+d,+e,+f", 0x7b20000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"binsl.w", "+d,+e,+f", 0x7b40000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"binsl.d", "+d,+e,+f", 0x7b60000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"binsli.b", "+d,+e,+7", 0x7b700009, 0xfff8003f, WR_VD|RD_VS,       0, MSA},
+{"binsli.h", "+d,+e,+8", 0x7b600009, 0xfff0003f, WR_VD|RD_VS,       0, MSA},
+{"binsli.w", "+d,+e,+9", 0x7b400009, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"binsli.d", "+d,+e,'",  0x7b000009, 0xffc0003f, WR_VD|RD_VS,       0, MSA},
+{"binsr.b", "+d,+e,+f", 0x7b80000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"binsr.h", "+d,+e,+f", 0x7ba0000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"binsr.w", "+d,+e,+f", 0x7bc0000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"binsr.d", "+d,+e,+f", 0x7be0000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"binsri.b", "+d,+e,+7", 0x7bf00009, 0xfff8003f, WR_VD|RD_VS,       0, MSA},
+{"binsri.h", "+d,+e,+8", 0x7be00009, 0xfff0003f, WR_VD|RD_VS,       0, MSA},
+{"binsri.w", "+d,+e,+9", 0x7bc00009, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"binsri.d", "+d,+e,'",  0x7b800009, 0xffc0003f, WR_VD|RD_VS,       0, MSA},
+{"addv.b",  "+d,+e,+f", 0x7800000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"addv.h",  "+d,+e,+f", 0x7820000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"addv.w",  "+d,+e,+f", 0x7840000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"addv.d",  "+d,+e,+f", 0x7860000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"addvi.b", "+d,+e,k",  0x78000006, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"addvi.h", "+d,+e,k",  0x78200006, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"addvi.w", "+d,+e,k",  0x78400006, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"addvi.d", "+d,+e,k",  0x78600006, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"subv.b",  "+d,+e,+f", 0x7880000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"subv.h",  "+d,+e,+f", 0x78a0000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"subv.w",  "+d,+e,+f", 0x78c0000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"subv.d",  "+d,+e,+f", 0x78e0000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"subvi.b", "+d,+e,k",  0x78800006, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"subvi.h", "+d,+e,k",  0x78a00006, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"subvi.w", "+d,+e,k",  0x78c00006, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"subvi.d", "+d,+e,k",  0x78e00006, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"max_s.b", "+d,+e,+f", 0x7900000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"max_s.h", "+d,+e,+f", 0x7920000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"max_s.w", "+d,+e,+f", 0x7940000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"max_s.d", "+d,+e,+f", 0x7960000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"maxi_s.b", "+d,+e,+5", 0x79000006, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"maxi_s.h", "+d,+e,+5", 0x79200006, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"maxi_s.w", "+d,+e,+5", 0x79400006, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"maxi_s.d", "+d,+e,+5", 0x79600006, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"max_u.b", "+d,+e,+f", 0x7980000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"max_u.h", "+d,+e,+f", 0x79a0000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"max_u.w", "+d,+e,+f", 0x79c0000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"max_u.d", "+d,+e,+f", 0x79e0000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"maxi_u.b", "+d,+e,k",  0x79800006, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"maxi_u.h", "+d,+e,k",  0x79a00006, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"maxi_u.w", "+d,+e,k",  0x79c00006, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"maxi_u.d", "+d,+e,k",  0x79e00006, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"min_s.b", "+d,+e,+f", 0x7a00000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"min_s.h", "+d,+e,+f", 0x7a20000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"min_s.w", "+d,+e,+f", 0x7a40000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"min_s.d", "+d,+e,+f", 0x7a60000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"mini_s.b", "+d,+e,+5", 0x7a000006, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"mini_s.h", "+d,+e,+5", 0x7a200006, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"mini_s.w", "+d,+e,+5", 0x7a400006, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"mini_s.d", "+d,+e,+5", 0x7a600006, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"min_u.b", "+d,+e,+f", 0x7a80000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"min_u.h", "+d,+e,+f", 0x7aa0000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"min_u.w", "+d,+e,+f", 0x7ac0000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"min_u.d", "+d,+e,+f", 0x7ae0000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"mini_u.b", "+d,+e,k",  0x7a800006, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"mini_u.h", "+d,+e,k",  0x7aa00006, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"mini_u.w", "+d,+e,k",  0x7ac00006, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"mini_u.d", "+d,+e,k",  0x7ae00006, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"max_a.b", "+d,+e,+f", 0x7b00000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"max_a.h", "+d,+e,+f", 0x7b20000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"max_a.w", "+d,+e,+f", 0x7b40000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"max_a.d", "+d,+e,+f", 0x7b60000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"min_a.b", "+d,+e,+f", 0x7b80000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"min_a.h", "+d,+e,+f", 0x7ba0000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"min_a.w", "+d,+e,+f", 0x7bc0000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"min_a.d", "+d,+e,+f", 0x7be0000e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ceq.b",   "+d,+e,+f", 0x7800000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ceq.h",   "+d,+e,+f", 0x7820000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ceq.w",   "+d,+e,+f", 0x7840000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ceq.d",   "+d,+e,+f", 0x7860000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ceqi.b",  "+d,+e,+5", 0x78000007, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"ceqi.h",  "+d,+e,+5", 0x78200007, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"ceqi.w",  "+d,+e,+5", 0x78400007, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"ceqi.d",  "+d,+e,+5", 0x78600007, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"clt_s.b", "+d,+e,+f", 0x7900000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"clt_s.h", "+d,+e,+f", 0x7920000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"clt_s.w", "+d,+e,+f", 0x7940000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"clt_s.d", "+d,+e,+f", 0x7960000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"clti_s.b", "+d,+e,+5", 0x79000007, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"clti_s.h", "+d,+e,+5", 0x79200007, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"clti_s.w", "+d,+e,+5", 0x79400007, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"clti_s.d", "+d,+e,+5", 0x79600007, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"clt_u.b", "+d,+e,+f", 0x7980000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"clt_u.h", "+d,+e,+f", 0x79a0000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"clt_u.w", "+d,+e,+f", 0x79c0000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"clt_u.d", "+d,+e,+f", 0x79e0000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"clti_u.b", "+d,+e,k",  0x79800007, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"clti_u.h", "+d,+e,k",  0x79a00007, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"clti_u.w", "+d,+e,k",  0x79c00007, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"clti_u.d", "+d,+e,k",  0x79e00007, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"cle_s.b", "+d,+e,+f", 0x7a00000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"cle_s.h", "+d,+e,+f", 0x7a20000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"cle_s.w", "+d,+e,+f", 0x7a40000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"cle_s.d", "+d,+e,+f", 0x7a60000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"clei_s.b", "+d,+e,+5", 0x7a000007, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"clei_s.h", "+d,+e,+5", 0x7a200007, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"clei_s.w", "+d,+e,+5", 0x7a400007, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"clei_s.d", "+d,+e,+5", 0x7a600007, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"cle_u.b", "+d,+e,+f", 0x7a80000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"cle_u.h", "+d,+e,+f", 0x7aa0000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"cle_u.w", "+d,+e,+f", 0x7ac0000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"cle_u.d", "+d,+e,+f", 0x7ae0000f, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"clei_u.b", "+d,+e,k",  0x7a800007, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"clei_u.h", "+d,+e,k",  0x7aa00007, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"clei_u.w", "+d,+e,k",  0x7ac00007, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"clei_u.d", "+d,+e,k",  0x7ae00007, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"ld.b",    "+d,+^(d)", 0x78000020, 0xfc00003f, WR_VD|LDD,       RD_d, MSA},
+{"ld.h",    "+d,+#(d)", 0x78000021, 0xfc00003f, WR_VD|LDD,       RD_d, MSA},
+{"ld.w",    "+d,+$(d)", 0x78000022, 0xfc00003f, WR_VD|LDD,       RD_d, MSA},
+{"ld.d",    "+d,+%(d)", 0x78000023, 0xfc00003f, WR_VD|LDD,       RD_d, MSA},
+{"st.b",    "+d,+^(d)", 0x78000024, 0xfc00003f, RD_VD|SM,        RD_d, MSA},
+{"st.h",    "+d,+#(d)", 0x78000025, 0xfc00003f, RD_VD|SM,        RD_d, MSA},
+{"st.w",    "+d,+$(d)", 0x78000026, 0xfc00003f, RD_VD|SM,        RD_d, MSA},
+{"st.d",    "+d,+%(d)", 0x78000027, 0xfc00003f, RD_VD|SM,        RD_d, MSA},
+{"sat_s.b", "+d,+e,+7", 0x7870000a, 0xfff8003f, WR_VD|RD_VS,        0, MSA},
+{"sat_s.h", "+d,+e,+8", 0x7860000a, 0xfff0003f, WR_VD|RD_VS,        0, MSA},
+{"sat_s.w", "+d,+e,+9", 0x7840000a, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"sat_s.d", "+d,+e,'",  0x7800000a, 0xffc0003f, WR_VD|RD_VS,        0, MSA},
+{"sat_u.b", "+d,+e,+7", 0x78f0000a, 0xfff8003f, WR_VD|RD_VS,        0, MSA},
+{"sat_u.h", "+d,+e,+8", 0x78e0000a, 0xfff0003f, WR_VD|RD_VS,        0, MSA},
+{"sat_u.w", "+d,+e,+9", 0x78c0000a, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"sat_u.d", "+d,+e,'",  0x7880000a, 0xffc0003f, WR_VD|RD_VS,        0, MSA},
+{"add_a.b", "+d,+e,+f", 0x78000010, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"add_a.h", "+d,+e,+f", 0x78200010, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"add_a.w", "+d,+e,+f", 0x78400010, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"add_a.d", "+d,+e,+f", 0x78600010, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"adds_a.b", "+d,+e,+f", 0x78800010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"adds_a.h", "+d,+e,+f", 0x78a00010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"adds_a.w", "+d,+e,+f", 0x78c00010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"adds_a.d", "+d,+e,+f", 0x78e00010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"adds_s.b", "+d,+e,+f", 0x79000010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"adds_s.h", "+d,+e,+f", 0x79200010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"adds_s.w", "+d,+e,+f", 0x79400010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"adds_s.d", "+d,+e,+f", 0x79600010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"adds_u.b", "+d,+e,+f", 0x79800010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"adds_u.h", "+d,+e,+f", 0x79a00010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"adds_u.w", "+d,+e,+f", 0x79c00010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"adds_u.d", "+d,+e,+f", 0x79e00010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"ave_s.b", "+d,+e,+f", 0x7a000010, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ave_s.h", "+d,+e,+f", 0x7a200010, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ave_s.w", "+d,+e,+f", 0x7a400010, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ave_s.d", "+d,+e,+f", 0x7a600010, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ave_u.b", "+d,+e,+f", 0x7a800010, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ave_u.h", "+d,+e,+f", 0x7aa00010, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ave_u.w", "+d,+e,+f", 0x7ac00010, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ave_u.d", "+d,+e,+f", 0x7ae00010, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"aver_s.b", "+d,+e,+f", 0x7b000010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"aver_s.h", "+d,+e,+f", 0x7b200010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"aver_s.w", "+d,+e,+f", 0x7b400010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"aver_s.d", "+d,+e,+f", 0x7b600010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"aver_u.b", "+d,+e,+f", 0x7b800010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"aver_u.h", "+d,+e,+f", 0x7ba00010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"aver_u.w", "+d,+e,+f", 0x7bc00010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"aver_u.d", "+d,+e,+f", 0x7be00010, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"subs_s.b", "+d,+e,+f", 0x78000011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"subs_s.h", "+d,+e,+f", 0x78200011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"subs_s.w", "+d,+e,+f", 0x78400011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"subs_s.d", "+d,+e,+f", 0x78600011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"subs_u.b", "+d,+e,+f", 0x78800011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"subs_u.h", "+d,+e,+f", 0x78a00011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"subs_u.w", "+d,+e,+f", 0x78c00011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"subs_u.d", "+d,+e,+f", 0x78e00011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"subsus_u.b", "+d,+e,+f", 0x79000011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"subsus_u.h", "+d,+e,+f", 0x79200011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"subsus_u.w", "+d,+e,+f", 0x79400011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"subsus_u.d", "+d,+e,+f", 0x79600011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"subsuu_s.b", "+d,+e,+f", 0x79800011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"subsuu_s.h", "+d,+e,+f", 0x79a00011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"subsuu_s.w", "+d,+e,+f", 0x79c00011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"subsuu_s.d", "+d,+e,+f", 0x79e00011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"asub_s.b", "+d,+e,+f", 0x7a000011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"asub_s.h", "+d,+e,+f", 0x7a200011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"asub_s.w", "+d,+e,+f", 0x7a400011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"asub_s.d", "+d,+e,+f", 0x7a600011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"asub_u.b", "+d,+e,+f", 0x7a800011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"asub_u.h", "+d,+e,+f", 0x7aa00011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"asub_u.w", "+d,+e,+f", 0x7ac00011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"asub_u.d", "+d,+e,+f", 0x7ae00011, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"mulv.b",  "+d,+e,+f", 0x78000012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"mulv.h",  "+d,+e,+f", 0x78200012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"mulv.w",  "+d,+e,+f", 0x78400012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"mulv.d",  "+d,+e,+f", 0x78600012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"maddv.b", "+d,+e,+f", 0x78800012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"maddv.h", "+d,+e,+f", 0x78a00012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"maddv.w", "+d,+e,+f", 0x78c00012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"maddv.d", "+d,+e,+f", 0x78e00012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"msubv.b", "+d,+e,+f", 0x79000012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"msubv.h", "+d,+e,+f", 0x79200012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"msubv.w", "+d,+e,+f", 0x79400012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"msubv.d", "+d,+e,+f", 0x79600012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"div_s.b", "+d,+e,+f", 0x7a000012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"div_s.h", "+d,+e,+f", 0x7a200012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"div_s.w", "+d,+e,+f", 0x7a400012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"div_s.d", "+d,+e,+f", 0x7a600012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"div_u.b", "+d,+e,+f", 0x7a800012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"div_u.h", "+d,+e,+f", 0x7aa00012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"div_u.w", "+d,+e,+f", 0x7ac00012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"div_u.d", "+d,+e,+f", 0x7ae00012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"mod_s.b", "+d,+e,+f", 0x7b000012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"mod_s.h", "+d,+e,+f", 0x7b200012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"mod_s.w", "+d,+e,+f", 0x7b400012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"mod_s.d", "+d,+e,+f", 0x7b600012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"mod_u.b", "+d,+e,+f", 0x7b800012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"mod_u.h", "+d,+e,+f", 0x7ba00012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"mod_u.w", "+d,+e,+f", 0x7bc00012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"mod_u.d", "+d,+e,+f", 0x7be00012, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"dotp_s.h", "+d,+e,+f", 0x78200013, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"dotp_s.w", "+d,+e,+f", 0x78400013, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"dotp_s.d", "+d,+e,+f", 0x78600013, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"dotp_u.h", "+d,+e,+f", 0x78a00013, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"dotp_u.w", "+d,+e,+f", 0x78c00013, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"dotp_u.d", "+d,+e,+f", 0x78e00013, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"dpadd_s.h", "+d,+e,+f", 0x79200013, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"dpadd_s.w", "+d,+e,+f", 0x79400013, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"dpadd_s.d", "+d,+e,+f", 0x79600013, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"dpadd_u.h", "+d,+e,+f", 0x79a00013, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"dpadd_u.w", "+d,+e,+f", 0x79c00013, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"dpadd_u.d", "+d,+e,+f", 0x79e00013, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"dpsub_s.h", "+d,+e,+f", 0x7a200013, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"dpsub_s.w", "+d,+e,+f", 0x7a400013, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"dpsub_s.d", "+d,+e,+f", 0x7a600013, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"dpsub_u.h", "+d,+e,+f", 0x7aa00013, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"dpsub_u.w", "+d,+e,+f", 0x7ac00013, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"dpsub_u.d", "+d,+e,+f", 0x7ae00013, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"sld.b",   "+d,+e[t]", 0x78000014, 0xffe0003f, WR_VD|RD_VS|RD_t,   0, MSA},
+{"sld.h",   "+d,+e[t]", 0x78200014, 0xffe0003f, WR_VD|RD_VS|RD_t,   0, MSA},
+{"sld.w",   "+d,+e[t]", 0x78400014, 0xffe0003f, WR_VD|RD_VS|RD_t,   0, MSA},
+{"sld.d",   "+d,+e[t]", 0x78600014, 0xffe0003f, WR_VD|RD_VS|RD_t,   0, MSA},
+{"sldi.b",  "+d,+e[+9]", 0x78000019, 0xffe0003f, WR_VD|RD_VS,       0, MSA},
+{"sldi.h",  "+d,+e[+8]", 0x78200019, 0xfff0003f, WR_VD|RD_VS,       0, MSA},
+{"sldi.w",  "+d,+e[+7]", 0x78300019, 0xfff8003f, WR_VD|RD_VS,       0, MSA},
+{"sldi.d",  "+d,+e[+6]", 0x78380019, 0xfffc003f, WR_VD|RD_VS,       0, MSA},
+{"splat.b", "+d,+e[t]", 0x78800014, 0xffe0003f, WR_VD|RD_VS|RD_t,   0, MSA},
+{"splat.h", "+d,+e[t]", 0x78a00014, 0xffe0003f, WR_VD|RD_VS|RD_t,   0, MSA},
+{"splat.w", "+d,+e[t]", 0x78c00014, 0xffe0003f, WR_VD|RD_VS|RD_t,   0, MSA},
+{"splat.d", "+d,+e[t]", 0x78e00014, 0xffe0003f, WR_VD|RD_VS|RD_t,   0, MSA},
+{"splati.b", "+d,+e[+9]", 0x78400019, 0xffe0003f, WR_VD|RD_VS,      0, MSA},
+{"splati.h", "+d,+e[+8]", 0x78600019, 0xfff0003f, WR_VD|RD_VS,      0, MSA},
+{"splati.w", "+d,+e[+7]", 0x78700019, 0xfff8003f, WR_VD|RD_VS,      0, MSA},
+{"splati.d", "+d,+e[+6]", 0x78780019, 0xfffc003f, WR_VD|RD_VS,      0, MSA},
+{"pckev.b", "+d,+e,+f", 0x79000014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"pckev.h", "+d,+e,+f", 0x79200014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"pckev.w", "+d,+e,+f", 0x79400014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"pckev.d", "+d,+e,+f", 0x79600014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"pckod.b", "+d,+e,+f", 0x79800014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"pckod.h", "+d,+e,+f", 0x79a00014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"pckod.w", "+d,+e,+f", 0x79c00014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"pckod.d", "+d,+e,+f", 0x79e00014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ilvl.b",  "+d,+e,+f", 0x7a000014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ilvl.h",  "+d,+e,+f", 0x7a200014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ilvl.w",  "+d,+e,+f", 0x7a400014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ilvl.d",  "+d,+e,+f", 0x7a600014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ilvr.b",  "+d,+e,+f", 0x7a800014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ilvr.h",  "+d,+e,+f", 0x7aa00014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ilvr.w",  "+d,+e,+f", 0x7ac00014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ilvr.d",  "+d,+e,+f", 0x7ae00014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ilvev.b", "+d,+e,+f", 0x7b000014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ilvev.h", "+d,+e,+f", 0x7b200014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ilvev.w", "+d,+e,+f", 0x7b400014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ilvev.d", "+d,+e,+f", 0x7b600014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ilvod.b", "+d,+e,+f", 0x7b800014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ilvod.h", "+d,+e,+f", 0x7ba00014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ilvod.w", "+d,+e,+f", 0x7bc00014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ilvod.d", "+d,+e,+f", 0x7be00014, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"vshf.b",  "+d,+e,+f", 0x78000015, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"vshf.h",  "+d,+e,+f", 0x78200015, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"vshf.w",  "+d,+e,+f", 0x78400015, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"vshf.d",  "+d,+e,+f", 0x78600015, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"srar.b",  "+d,+e,+f", 0x78800015, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"srar.h",  "+d,+e,+f", 0x78a00015, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"srar.w",  "+d,+e,+f", 0x78c00015, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"srar.d",  "+d,+e,+f", 0x78e00015, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"srari.b", "+d,+e,+7", 0x7970000a, 0xfff8003f, WR_VD|RD_VS,        0, MSA},
+{"srari.h", "+d,+e,+8", 0x7960000a, 0xfff0003f, WR_VD|RD_VS,        0, MSA},
+{"srari.w", "+d,+e,+9", 0x7940000a, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"srari.d", "+d,+e,'",  0x7900000a, 0xffc0003f, WR_VD|RD_VS,        0, MSA},
+{"srlr.b",  "+d,+e,+f", 0x79000015, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"srlr.h",  "+d,+e,+f", 0x79200015, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"srlr.w",  "+d,+e,+f", 0x79400015, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"srlr.d",  "+d,+e,+f", 0x79600015, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"srlri.b", "+d,+e,+7", 0x79f0000a, 0xfff8003f, WR_VD|RD_VS,        0, MSA},
+{"srlri.h", "+d,+e,+8", 0x79e0000a, 0xfff0003f, WR_VD|RD_VS,        0, MSA},
+{"srlri.w", "+d,+e,+9", 0x79c0000a, 0xffe0003f, WR_VD|RD_VS,        0, MSA},
+{"srlri.d", "+d,+e,'",  0x7980000a, 0xffc0003f, WR_VD|RD_VS,        0, MSA},
+{"hadd_s.h", "+d,+e,+f", 0x7a200015, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"hadd_s.w", "+d,+e,+f", 0x7a400015, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"hadd_s.d", "+d,+e,+f", 0x7a600015, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"hadd_u.h", "+d,+e,+f", 0x7aa00015, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"hadd_u.w", "+d,+e,+f", 0x7ac00015, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"hadd_u.d", "+d,+e,+f", 0x7ae00015, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"hsub_s.h", "+d,+e,+f", 0x7b200015, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"hsub_s.w", "+d,+e,+f", 0x7b400015, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"hsub_s.d", "+d,+e,+f", 0x7b600015, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"hsub_u.h", "+d,+e,+f", 0x7ba00015, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"hsub_u.w", "+d,+e,+f", 0x7bc00015, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"hsub_u.d", "+d,+e,+f", 0x7be00015, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"and.v",   "+d,+e,+f", 0x7800001e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"andi.b",  "+d,+e,5",  0x78000000, 0xff00003f, WR_VD|RD_VS,        0, MSA},
+{"or.v",    "+d,+e,+f", 0x7820001e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ori.b",   "+d,+e,5",  0x79000000, 0xff00003f, WR_VD|RD_VS,        0, MSA},
+{"nor.v",   "+d,+e,+f", 0x7840001e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"nori.b",  "+d,+e,5",  0x7a000000, 0xff00003f, WR_VD|RD_VS,        0, MSA},
+{"xor.v",   "+d,+e,+f", 0x7860001e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"xori.b",  "+d,+e,5",  0x7b000000, 0xff00003f, WR_VD|RD_VS,        0, MSA},
+{"bmnz.v",  "+d,+e,+f", 0x7880001e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"bmnzi.b", "+d,+e,5",  0x78000001, 0xff00003f, WR_VD|RD_VS,        0, MSA},
+{"bmz.v",   "+d,+e,+f", 0x78a0001e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"bmzi.b",  "+d,+e,5",  0x79000001, 0xff00003f, WR_VD|RD_VS,        0, MSA},
+{"bsel.v",  "+d,+e,+f", 0x78c0001e, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"bseli.b", "+d,+e,5",  0x7a000001, 0xff00003f, WR_VD|RD_VS,        0, MSA},
+{"shf.b",   "+d,+e,5",  0x78000002, 0xff00003f, WR_VD|RD_VS,        0, MSA},
+{"shf.h",   "+d,+e,5",  0x79000002, 0xff00003f, WR_VD|RD_VS,        0, MSA},
+{"shf.w",   "+d,+e,5",  0x7a000002, 0xff00003f, WR_VD|RD_VS,        0, MSA},
+{"bnz.v",    "+f,p",    0x45e00000, 0xffe00000, CBD|RD_VT,          0, MSA},
+{"bz.v",    "+f,p",     0x45600000, 0xffe00000, CBD|RD_VT,          0, MSA},
+{"fill.b",  "+d,d",     0x7b00001e, 0xffff003f, WR_VD,           RD_d, MSA},
+{"fill.h",  "+d,d",     0x7b01001e, 0xffff003f, WR_VD,           RD_d, MSA},
+{"fill.w",  "+d,d",     0x7b02001e, 0xffff003f, WR_VD,           RD_d, MSA},
+{"fill.d",  "+d,d",     0x7b03001e, 0xffff003f, WR_VD,           RD_d, MSA64},
+{"pcnt.b",  "+d,+e",    0x7b04001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"pcnt.h",  "+d,+e",    0x7b05001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"pcnt.w",  "+d,+e",    0x7b06001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"pcnt.d",  "+d,+e",    0x7b07001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"nloc.b",  "+d,+e",    0x7b08001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"nloc.h",  "+d,+e",    0x7b09001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"nloc.w",  "+d,+e",    0x7b0a001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"nloc.d",  "+d,+e",    0x7b0b001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"nlzc.b",  "+d,+e",    0x7b0c001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"nlzc.h",  "+d,+e",    0x7b0d001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"nlzc.w",  "+d,+e",    0x7b0e001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"nlzc.d",  "+d,+e",    0x7b0f001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"copy_s.b", "+i,+e[+9]", 0x78800019, 0xffe0003f, RD_VS,       RD_rd6, MSA},
+{"copy_s.h", "+i,+e[+8]", 0x78a00019, 0xfff0003f, RD_VS,       RD_rd6, MSA},
+{"copy_s.w", "+i,+e[+7]", 0x78b00019, 0xfff8003f, RD_VS,       RD_rd6, MSA},
+{"copy_s.d", "+i,+e[+6]", 0x78b80019, 0xfffc003f, RD_VS,       RD_rd6, MSA64},
+{"copy_u.b", "+i,+e[+9]", 0x78c00019, 0xffe0003f, RD_VS,       RD_rd6, MSA},
+{"copy_u.h", "+i,+e[+8]", 0x78e00019, 0xfff0003f, RD_VS,       RD_rd6, MSA},
+{"copy_u.w", "+i,+e[+7]", 0x78f00019, 0xfff8003f, RD_VS,       RD_rd6, MSA},
+{"copy_u.d", "+i,+e[+6]", 0x78f80019, 0xfffc003f, RD_VS,       RD_rd6, MSA64},
+{"insert.b", "+d[+9],d", 0x79000019, 0xffe0003f, WR_VD|RD_VD,    RD_d, MSA},
+{"insert.h", "+d[+8],d", 0x79200019, 0xfff0003f, WR_VD|RD_VD,    RD_d, MSA},
+{"insert.w", "+d[+7],d", 0x79300019, 0xfff8003f, WR_VD|RD_VD,    RD_d, MSA},
+{"insert.d", "+d[+6],d", 0x79380019, 0xfffc003f, WR_VD|RD_VD,    RD_d, MSA64},
+{"insve.b", "+d[+9],+e[+~]", 0x79400019, 0xffe0003f, WR_VD|RD_VD|RD_VS, 0, MSA},
+{"insve.h", "+d[+8],+e[+~]", 0x79600019, 0xfff0003f, WR_VD|RD_VD|RD_VS, 0, MSA},
+{"insve.w", "+d[+7],+e[+~]", 0x79700019, 0xfff8003f, WR_VD|RD_VD|RD_VS, 0, MSA},
+{"insve.d", "+d[+6],+e[+~]", 0x79780019, 0xfffc003f, WR_VD|RD_VD|RD_VS, 0, MSA},
+{"bnz.b",    "+f,p",    0x47800000, 0xffe00000, CBD|RD_VT,          0, MSA},
+{"bnz.h",    "+f,p",    0x47a00000, 0xffe00000, CBD|RD_VT,          0, MSA},
+{"bnz.w",    "+f,p",    0x47c00000, 0xffe00000, CBD|RD_VT,          0, MSA},
+{"bnz.d",    "+f,p",    0x47e00000, 0xffe00000, CBD|RD_VT,          0, MSA},
+{"bz.b",    "+f,p",     0x47000000, 0xffe00000, CBD|RD_VT,          0, MSA},
+{"bz.h",    "+f,p",     0x47200000, 0xffe00000, CBD|RD_VT,          0, MSA},
+{"bz.w",    "+f,p",     0x47400000, 0xffe00000, CBD|RD_VT,          0, MSA},
+{"bz.d",    "+f,p",     0x47600000, 0xffe00000, CBD|RD_VT,          0, MSA},
+{"ldi.b",   "+d,+0",    0x7b000007, 0xffe0003f, WR_VD,              0, MSA},
+{"ldi.h",   "+d,+0",    0x7b200007, 0xffe0003f, WR_VD,              0, MSA},
+{"ldi.w",   "+d,+0",    0x7b400007, 0xffe0003f, WR_VD,              0, MSA},
+{"ldi.d",   "+d,+0",    0x7b600007, 0xffe0003f, WR_VD,              0, MSA},
+{"fcaf.w",  "+d,+e,+f", 0x7800001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fcaf.d",  "+d,+e,+f", 0x7820001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fcun.w",  "+d,+e,+f", 0x7840001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fcun.d",  "+d,+e,+f", 0x7860001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fceq.w",  "+d,+e,+f", 0x7880001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fceq.d",  "+d,+e,+f", 0x78a0001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fcueq.w", "+d,+e,+f", 0x78c0001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fcueq.d", "+d,+e,+f", 0x78e0001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fclt.w",  "+d,+e,+f", 0x7900001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fclt.d",  "+d,+e,+f", 0x7920001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fcult.w", "+d,+e,+f", 0x7940001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fcult.d", "+d,+e,+f", 0x7960001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fcle.w",  "+d,+e,+f", 0x7980001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fcle.d",  "+d,+e,+f", 0x79a0001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fcule.w", "+d,+e,+f", 0x79c0001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fcule.d", "+d,+e,+f", 0x79e0001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsaf.w",  "+d,+e,+f", 0x7a00001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsaf.d",  "+d,+e,+f", 0x7a20001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsun.w",  "+d,+e,+f", 0x7a40001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsun.d",  "+d,+e,+f", 0x7a60001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fseq.w",  "+d,+e,+f", 0x7a80001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fseq.d",  "+d,+e,+f", 0x7aa0001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsueq.w", "+d,+e,+f", 0x7ac0001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsueq.d", "+d,+e,+f", 0x7ae0001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fslt.w",  "+d,+e,+f", 0x7b00001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fslt.d",  "+d,+e,+f", 0x7b20001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsult.w", "+d,+e,+f", 0x7b40001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsult.d", "+d,+e,+f", 0x7b60001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsle.w",  "+d,+e,+f", 0x7b80001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsle.d",  "+d,+e,+f", 0x7ba0001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsule.w", "+d,+e,+f", 0x7bc0001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsule.d", "+d,+e,+f", 0x7be0001a, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fadd.w",  "+d,+e,+f", 0x7800001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fadd.d",  "+d,+e,+f", 0x7820001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsub.w",  "+d,+e,+f", 0x7840001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsub.d",  "+d,+e,+f", 0x7860001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fmul.w",  "+d,+e,+f", 0x7880001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fmul.d",  "+d,+e,+f", 0x78a0001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fdiv.w",  "+d,+e,+f", 0x78c0001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fdiv.d",  "+d,+e,+f", 0x78e0001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fmadd.w", "+d,+e,+f", 0x7900001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fmadd.d", "+d,+e,+f", 0x7920001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fmsub.w", "+d,+e,+f", 0x7940001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fmsub.d", "+d,+e,+f", 0x7960001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fexp2.w", "+d,+e,+f", 0x79c0001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fexp2.d", "+d,+e,+f", 0x79e0001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fexdo.h", "+d,+e,+f", 0x7a00001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fexdo.w", "+d,+e,+f", 0x7a20001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ftq.h",   "+d,+e,+f", 0x7a80001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"ftq.w",   "+d,+e,+f", 0x7aa0001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fmin.w",  "+d,+e,+f", 0x7b00001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fmin.d",  "+d,+e,+f", 0x7b20001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fmin_a.w", "+d,+e,+f", 0x7b40001b, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"fmin_a.d", "+d,+e,+f", 0x7b60001b, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"fmax.w",  "+d,+e,+f", 0x7b80001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fmax.d",  "+d,+e,+f", 0x7ba0001b, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fmax_a.w", "+d,+e,+f", 0x7bc0001b, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"fmax_a.d", "+d,+e,+f", 0x7be0001b, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"fcor.w",  "+d,+e,+f", 0x7840001c, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fcor.d",  "+d,+e,+f", 0x7860001c, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fcune.w", "+d,+e,+f", 0x7880001c, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fcune.d", "+d,+e,+f", 0x78a0001c, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fcne.w",  "+d,+e,+f", 0x78c0001c, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fcne.d",  "+d,+e,+f", 0x78e0001c, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"mul_q.h", "+d,+e,+f", 0x7900001c, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"mul_q.w", "+d,+e,+f", 0x7920001c, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"madd_q.h", "+d,+e,+f", 0x7940001c, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"madd_q.w", "+d,+e,+f", 0x7960001c, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"msub_q.h", "+d,+e,+f", 0x7980001c, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"msub_q.w", "+d,+e,+f", 0x79a0001c, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"fsor.w",  "+d,+e,+f", 0x7a40001c, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsor.d",  "+d,+e,+f", 0x7a60001c, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsune.w", "+d,+e,+f", 0x7a80001c, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsune.d", "+d,+e,+f", 0x7aa0001c, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsne.w",  "+d,+e,+f", 0x7ac0001c, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"fsne.d",  "+d,+e,+f", 0x7ae0001c, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
+{"mulr_q.h", "+d,+e,+f", 0x7b00001c, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"mulr_q.w", "+d,+e,+f", 0x7b20001c, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"maddr_q.h", "+d,+e,+f", 0x7b40001c, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"maddr_q.w", "+d,+e,+f", 0x7b60001c, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"msubr_q.h", "+d,+e,+f", 0x7b80001c, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"msubr_q.w", "+d,+e,+f", 0x7ba0001c, 0xffe0003f, WR_VD|RD_VS|RD_VT, 0, MSA},
+{"fclass.w", "+d,+e",    0x7b20001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"fclass.d", "+d,+e",    0x7b21001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"fsqrt.w", "+d,+e",    0x7b26001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"fsqrt.d", "+d,+e",    0x7b27001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"frsqrt.w", "+d,+e",    0x7b28001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"frsqrt.d", "+d,+e",    0x7b29001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"frcp.w",  "+d,+e",    0x7b2a001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"frcp.d",  "+d,+e",    0x7b2b001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"frint.w", "+d,+e",    0x7b2c001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"frint.d", "+d,+e",    0x7b2d001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"flog2.w", "+d,+e",    0x7b2e001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"flog2.d", "+d,+e",    0x7b2f001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"fexupl.w", "+d,+e",    0x7b30001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"fexupl.d", "+d,+e",    0x7b31001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"fexupr.w", "+d,+e",    0x7b32001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"fexupr.d", "+d,+e",    0x7b33001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"ffql.w",  "+d,+e",    0x7b34001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"ffql.d",  "+d,+e",    0x7b35001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"ffqr.w",  "+d,+e",    0x7b36001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"ffqr.d",  "+d,+e",    0x7b37001e, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"ftint_s.w", "+d,+e",   0x7b38001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"ftint_s.d", "+d,+e",   0x7b39001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"ftint_u.w", "+d,+e",   0x7b3a001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"ftint_u.d", "+d,+e",   0x7b3b001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"ffint_s.w", "+d,+e",   0x7b3c001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"ffint_s.d", "+d,+e",   0x7b3d001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"ffint_u.w", "+d,+e",   0x7b3e001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"ffint_u.d", "+d,+e",   0x7b3f001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"ftrunc_s.w", "+d,+e",  0x7b40001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"ftrunc_s.d", "+d,+e",  0x7b41001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"ftrunc_u.w", "+d,+e",  0x7b42001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"ftrunc_u.d", "+d,+e",  0x7b43001e, 0xffff003f, WR_VD|RD_VS,       0, MSA},
+{"ctcmsa",  "+h,d",     0x783e0019, 0xffff003f, COD,             RD_d, MSA},
+{"cfcmsa",  "+i,+g",    0x787e0019, 0xffff003f, COD,                0, MSA},
+{"move.v",  "+d,+e",    0x78be0019, 0xffff003f, WR_VD|RD_VS,        0, MSA},
+{"lsa",     "d,v,t,+@", 0x00000005, 0xfc00073f, WR_d|RD_s|RD_t,     0, MSA},
+{"dlsa",    "d,v,t,+@", 0x00000015, 0xfc00073f, WR_d|RD_s|RD_t,     0, MSA64},
+
 {"pref",    "k,o(b)",   0xcc000000, 0xfc000000, RD_b,           	0,		I4|I32|G3	},
 {"prefx",   "h,t(b)",	0x4c00000f, 0xfc0007ff, RD_b|RD_t,		0,		I4|I33	},
 {"nop",     "",         0x00000000, 0xffffffff, 0,              	INSN2_ALIAS,	I1      }, /* sll */
@@ -1753,6 +2483,7 @@ const struct mips_opcode mips_builtin_opcodes[] =
 {"lld",	    "t,o(b)",	0xd0000000, 0xfc000000, LDD|RD_b|WR_t,		0,		I3	},
 {"lld",     "t,A(b)",	0,    (int) M_LLD_AB,	INSN_MACRO,		0,		I3	},
 {"lui",     "t,u",	0x3c000000, 0xffe00000,	WR_t,			0,		I1	},
+{"aui",     "s,t,u",    0x3c000000, 0xfc000000, RD_s|WR_t,            0, I32R6},
 {"luxc1",   "D,t(b)",	0x4c000005, 0xfc00f83f, LDD|WR_D|RD_t|RD_b|FP_D, 0,		I5|I33|N55},
 {"lw",      "t,o(b)",	0x8c000000, 0xfc000000,	LDD|RD_b|WR_t,		0,		I1	},
 {"lw",      "t,A(b)",	0,    (int) M_LW_AB,	INSN_MACRO,		0,		I1	},
@@ -2255,6 +2986,8 @@ const struct mips_opcode mips_builtin_opcodes[] =
 {"tlbp",    "",         0x42000008, 0xffffffff, INSN_TLB,       	0,		I1   	},
 {"tlbr",    "",         0x42000001, 0xffffffff, INSN_TLB,       	0,		I1   	},
 {"tlbwi",   "",         0x42000002, 0xffffffff, INSN_TLB,       	0,		I1   	},
+{"tlbinv",  "",         0x42000003, 0xffffffff, INSN_TLB,             0, I32  },
+{"tlbinvf", "",         0x42000004, 0xffffffff, INSN_TLB,             0, I32  },
 {"tlbwr",   "",         0x42000006, 0xffffffff, INSN_TLB,       	0,		I1   	},
 {"tlti",    "s,j",	0x040a0000, 0xfc1f0000,	RD_s|TRAP,		0,		I2	},
 {"tlt",     "s,t",	0x00000032, 0xfc00ffff, RD_s|RD_t|TRAP,		0,		I2	},
@@ -2843,6 +3576,13 @@ static const char * const mips_fpr_names_64[32] =
   "fs0",  "fs1",  "fs2",  "fs3",  "fs4",  "fs5",  "fs6",  "fs7"
 };
 
+static const char * const mips_wr_names[32] = {
+  "w0",  "w1",  "w2",  "w3",  "w4",  "w5",  "w6",  "w7",
+  "w8",  "w9",  "w10", "w11", "w12", "w13", "w14", "w15",
+  "w16", "w17", "w18", "w19", "w20", "w21", "w22", "w23",
+  "w24", "w25", "w26", "w27", "w28", "w29", "w30", "w31"
+};
+
 static const char * const mips_cp0_names_numeric[32] =
 {
   "$0",   "$1",   "$2",   "$3",   "$4",   "$5",   "$6",   "$7",
@@ -3061,6 +3801,20 @@ static const char * const mips_hwr_names_mips3264r2[32] =
   "$24",  "$25",  "$26",  "$27",  "$28",  "$29",  "$30",  "$31"
 };
 
+static const char * const mips_msa_control_names_numeric[32] = {
+  "$0",   "$1",   "$2",   "$3",   "$4",   "$5",   "$6",   "$7",
+  "$8",   "$9",   "$10",  "$11",  "$12",  "$13",  "$14",  "$15",
+  "$16",  "$17",  "$18",  "$19",  "$20",  "$21",  "$22",  "$23",
+  "$24",  "$25",  "$26",  "$27",  "$28",  "$29",  "$30",  "$31"
+};
+
+static const char * const mips_msa_control_names_mips3264r2[32] = {
+  "MSAIR", "MSACSR", "$2", "$3",  "$4",   "$5",   "$6",   "$7",
+  "$8",   "$9",   "$10",  "$11",  "$12",  "$13",  "$14",  "$15",
+  "$16",  "$17",  "$18",  "$19",  "$20",  "$21",  "$22",  "$23",
+  "$24",  "$25",  "$26",  "$27",  "$28",  "$29",  "$30",  "$31"
+};
+
 struct mips_abi_choice
 {
   const char *name;
@@ -3178,7 +3932,7 @@ static const struct mips_arch_choice mips_arch_choices[] =
 
   { "mips32r2",	1, bfd_mach_mipsisa32r2, CPU_MIPS32R2,
     (ISA_MIPS32R2 | INSN_MIPS16 | INSN_SMARTMIPS | INSN_DSP | INSN_DSPR2
-     | INSN_MIPS3D | INSN_MT),
+     | INSN_MIPS3D | INSN_MT | INSN_MSA),
     mips_cp0_names_mips3264r2,
     mips_cp0sel_names_mips3264r2, ARRAY_SIZE (mips_cp0sel_names_mips3264r2),
     mips_hwr_names_mips3264r2 },
@@ -3532,6 +4286,89 @@ print_insn_args (const char *d,
 				     (l >> OP_SH_UDI4) & OP_MASK_UDI4);
 	      break;
 
+        case '5': /* 5-bit signed immediate in bit 16 */
+            delta = ((l >> OP_SH_RT) & OP_MASK_RT);
+            if (delta & 0x10) { /* test sign bit */
+                delta |= ~OP_MASK_RT;
+            }
+            (*info->fprintf_func) (info->stream, "%d", delta);
+            break;
+
+        case '6':
+            (*info->fprintf_func) (info->stream, "0x%lx",
+                    (l >> OP_SH_2BIT) & OP_MASK_2BIT);
+            break;
+
+        case '7':
+            (*info->fprintf_func) (info->stream, "0x%lx",
+                    (l >> OP_SH_3BIT) & OP_MASK_3BIT);
+            break;
+
+        case '8':
+            (*info->fprintf_func) (info->stream, "0x%lx",
+                    (l >> OP_SH_4BIT) & OP_MASK_4BIT);
+            break;
+
+        case '9':
+            (*info->fprintf_func) (info->stream, "0x%lx",
+                    (l >> OP_SH_5BIT) & OP_MASK_5BIT);
+            break;
+
+        case ':':
+            (*info->fprintf_func) (info->stream, "0x%lx",
+                    (l >> OP_SH_1BIT) & OP_MASK_1BIT);
+            break;
+
+        case '!': /* 10-bit pc-relative target in bit 11 */
+            delta = ((l >> OP_SH_10BIT) & OP_MASK_10BIT);
+            if (delta & 0x200) { /* test sign bit */
+                delta |= ~OP_MASK_10BIT;
+            }
+            info->target = (delta << 2) + pc + INSNLEN;
+            (*info->print_address_func) (info->target, info);
+            break;
+
+        case '~':
+            (*info->fprintf_func) (info->stream, "0");
+            break;
+
+        case '@':
+            (*info->fprintf_func) (info->stream, "0x%lx",
+                    ((l >> OP_SH_1_TO_4) & OP_MASK_1_TO_4)+1);
+            break;
+
+        case '^': /* 10-bit signed immediate << 0 in bit 16 */
+            delta = ((l >> OP_SH_IMM10) & OP_MASK_IMM10);
+            if (delta & 0x200) { /* test sign bit */
+                delta |= ~OP_MASK_IMM10;
+            }
+            (*info->fprintf_func) (info->stream, "%d", delta);
+            break;
+
+        case '#': /* 10-bit signed immediate << 1 in bit 16 */
+            delta = ((l >> OP_SH_IMM10) & OP_MASK_IMM10);
+            if (delta & 0x200) { /* test sign bit */
+                delta |= ~OP_MASK_IMM10;
+            }
+            (*info->fprintf_func) (info->stream, "%d", delta << 1);
+            break;
+
+        case '$': /* 10-bit signed immediate << 2 in bit 16 */
+            delta = ((l >> OP_SH_IMM10) & OP_MASK_IMM10);
+            if (delta & 0x200) { /* test sign bit */
+                delta |= ~OP_MASK_IMM10;
+            }
+            (*info->fprintf_func) (info->stream, "%d", delta << 2);
+            break;
+
+        case '%': /* 10-bit signed immediate << 3 in bit 16 */
+            delta = ((l >> OP_SH_IMM10) & OP_MASK_IMM10);
+            if (delta & 0x200) { /* test sign bit */
+                delta |= ~OP_MASK_IMM10;
+            }
+            (*info->fprintf_func) (info->stream, "%d", delta << 3);
+            break;
+
 	    case 'C':
 	    case 'H':
 	      msbd = (l >> OP_SH_EXTMSBD) & OP_MASK_EXTMSBD;
@@ -3575,6 +4412,42 @@ print_insn_args (const char *d,
 	      (*info->fprintf_func) (info->stream, "0x%x", msbd + 1);
 	      break;
 
+            case 'o':
+                switch (*(d+1)) {
+                case '1':
+                    d++;
+                    delta = l & ((1 << 18) - 1);
+                    if (delta & 0x20000) {
+                        delta |= ~0x1ffff;
+                    }
+                    break;
+                case '2':
+                    d++;
+                    delta = l & ((1 << 19) - 1);
+                    if (delta & 0x40000) {
+                        delta |= ~0x3ffff;
+                    }
+                    break;
+                default:
+                    delta = (l >> OP_SH_DELTA_R6) & OP_MASK_DELTA_R6;
+                    if (delta & 0x8000) {
+                        delta |= ~0xffff;
+                    }
+                }
+
+                (*info->fprintf_func) (info->stream, "%d", delta);
+                break;
+
+            case 'p':
+                /* Sign extend the displacement with 26 bits.  */
+                delta = (l >> OP_SH_DELTA) & OP_MASK_TARGET;
+                if (delta & 0x2000000) {
+                    delta |= ~0x3FFFFFF;
+                }
+                info->target = (delta << 2) + pc + INSNLEN;
+                (*info->print_address_func) (info->target, info);
+                break;
+
 	    case 't': /* Coprocessor 0 reg name */
 	      (*info->fprintf_func) (info->stream, "%s",
 				     mips_cp0_names[(l >> OP_SH_RT) &
@@ -3603,6 +4476,38 @@ print_insn_args (const char *d,
 		break;
 	      }
 
+        case 'd':
+            (*info->fprintf_func) (info->stream, "%s",
+                    mips_wr_names[(l >> OP_SH_FD) & OP_MASK_FD]);
+            break;
+
+        case 'e':
+            (*info->fprintf_func) (info->stream, "%s",
+                    mips_wr_names[(l >> OP_SH_FS) & OP_MASK_FS]);
+            break;
+
+        case 'f':
+            (*info->fprintf_func) (info->stream, "%s",
+                    mips_wr_names[(l >> OP_SH_FT) & OP_MASK_FT]);
+            break;
+
+        case 'g':
+            (*info->fprintf_func) (info->stream, "%s",
+                    mips_msa_control_names_mips3264r2[(l >> OP_SH_MSACR11)
+                                                      & OP_MASK_MSACR11]);
+            break;
+
+        case 'h':
+            (*info->fprintf_func) (info->stream, "%s",
+                    mips_msa_control_names_mips3264r2[(l >> OP_SH_MSACR6)
+                                                      & OP_MASK_MSACR6]);
+            break;
+
+        case 'i':
+            (*info->fprintf_func) (info->stream, "%s",
+                    mips_gpr_names[(l >> OP_SH_GPR) & OP_MASK_GPR]);
+            break;
+
 	    default:
 	      /* xgettext:c-format */
 	      (*info->fprintf_func) (info->stream,
@@ -3726,7 +4631,8 @@ print_insn_args (const char *d,
 
 	case 'j': /* Same as i, but sign-extended.  */
 	case 'o':
-	  delta = (l >> OP_SH_DELTA) & OP_MASK_DELTA;
+            delta = (l >> OP_SH_DELTA) & OP_MASK_DELTA;
+
 	  if (delta & 0x8000)
 	    delta |= ~0xffff;
 	  (*info->fprintf_func) (info->stream, "%d",
@@ -4052,6 +4958,23 @@ print_insn_mips (bfd_vma memaddr,
 		  && strcmp (op->name, "jalx"))
 		continue;
 
+              if (strcmp(op->name, "bovc") == 0
+                  || strcmp(op->name, "bnvc") == 0) {
+                  if (((word >> OP_SH_RS) & OP_MASK_RS) <
+                      ((word >> OP_SH_RT) & OP_MASK_RT)) {
+                      continue;
+                  }
+              }
+              if (strcmp(op->name, "bgezc") == 0
+                  || strcmp(op->name, "bltzc") == 0
+                  || strcmp(op->name, "bgezalc") == 0
+                  || strcmp(op->name, "bltzalc") == 0) {
+                  if (((word >> OP_SH_RS) & OP_MASK_RS) !=
+                      ((word >> OP_SH_RT) & OP_MASK_RT)) {
+                      continue;
+                  }
+              }
+
 	      /* Figure out instruction type and branch delay information.  */
 	      if ((op->pinfo & INSN_UNCOND_BRANCH_DELAY) != 0)
 	        {
diff --git a/disas/sparc.c b/disas/sparc.c
index 8eb22e6fc..8e755d1ba 100644
--- a/disas/sparc.c
+++ b/disas/sparc.c
@@ -1175,15 +1175,11 @@ static const struct sparc_opcode sparc_opcodes[] = {
 { "subcc",      F3(2, 0x14, 0), F3(~2, ~0x14, ~0)|ASI(~0),      "1,2,d", 0, v6 },
 { "subcc",      F3(2, 0x14, 1), F3(~2, ~0x14, ~1),              "1,i,d", 0, v6 },
 
-{ "subx",       F3(2, 0x0c, 0), F3(~2, ~0x0c, ~0)|ASI(~0),      "1,2,d", 0, v6notv9 },
-{ "subx",       F3(2, 0x0c, 1), F3(~2, ~0x0c, ~1),              "1,i,d", 0, v6notv9 },
-{ "subc",       F3(2, 0x0c, 0), F3(~2, ~0x0c, ~0)|ASI(~0),      "1,2,d", 0, v9 },
-{ "subc",       F3(2, 0x0c, 1), F3(~2, ~0x0c, ~1),              "1,i,d", 0, v9 },
+{ "subc",       F3(2, 0x0c, 0), F3(~2, ~0x0c, ~0)|ASI(~0),      "1,2,d", 0, v6 },
+{ "subc",       F3(2, 0x0c, 1), F3(~2, ~0x0c, ~1),              "1,i,d", 0, v6 },
 
-{ "subxcc",     F3(2, 0x1c, 0), F3(~2, ~0x1c, ~0)|ASI(~0),      "1,2,d", 0, v6notv9 },
-{ "subxcc",     F3(2, 0x1c, 1), F3(~2, ~0x1c, ~1),              "1,i,d", 0, v6notv9 },
-{ "subccc",     F3(2, 0x1c, 0), F3(~2, ~0x1c, ~0)|ASI(~0),      "1,2,d", 0, v9 },
-{ "subccc",     F3(2, 0x1c, 1), F3(~2, ~0x1c, ~1),              "1,i,d", 0, v9 },
+{ "subccc",     F3(2, 0x1c, 0), F3(~2, ~0x1c, ~0)|ASI(~0),      "1,2,d", 0, v6 },
+{ "subccc",     F3(2, 0x1c, 1), F3(~2, ~0x1c, ~1),              "1,i,d", 0, v6 },
 
 { "and",        F3(2, 0x01, 0), F3(~2, ~0x01, ~0)|ASI(~0),      "1,2,d", 0, v6 },
 { "and",        F3(2, 0x01, 1), F3(~2, ~0x01, ~1),              "1,i,d", 0, v6 },
@@ -1215,19 +1211,13 @@ static const struct sparc_opcode sparc_opcodes[] = {
 { "addcc",      F3(2, 0x10, 1), F3(~2, ~0x10, ~1),              "1,i,d", 0, v6 },
 { "addcc",      F3(2, 0x10, 1), F3(~2, ~0x10, ~1),              "i,1,d", 0, v6 },
 
-{ "addx",       F3(2, 0x08, 0), F3(~2, ~0x08, ~0)|ASI(~0),      "1,2,d", 0, v6notv9 },
-{ "addx",       F3(2, 0x08, 1), F3(~2, ~0x08, ~1),              "1,i,d", 0, v6notv9 },
-{ "addx",       F3(2, 0x08, 1), F3(~2, ~0x08, ~1),              "i,1,d", 0, v6notv9 },
-{ "addc",       F3(2, 0x08, 0), F3(~2, ~0x08, ~0)|ASI(~0),      "1,2,d", 0, v9 },
-{ "addc",       F3(2, 0x08, 1), F3(~2, ~0x08, ~1),              "1,i,d", 0, v9 },
-{ "addc",       F3(2, 0x08, 1), F3(~2, ~0x08, ~1),              "i,1,d", 0, v9 },
+{ "addc",       F3(2, 0x08, 0), F3(~2, ~0x08, ~0)|ASI(~0),      "1,2,d", 0, v6 },
+{ "addc",       F3(2, 0x08, 1), F3(~2, ~0x08, ~1),              "1,i,d", 0, v6 },
+{ "addc",       F3(2, 0x08, 1), F3(~2, ~0x08, ~1),              "i,1,d", 0, v6 },
 
-{ "addxcc",     F3(2, 0x18, 0), F3(~2, ~0x18, ~0)|ASI(~0),      "1,2,d", 0, v6notv9 },
-{ "addxcc",     F3(2, 0x18, 1), F3(~2, ~0x18, ~1),              "1,i,d", 0, v6notv9 },
-{ "addxcc",     F3(2, 0x18, 1), F3(~2, ~0x18, ~1),              "i,1,d", 0, v6notv9 },
-{ "addccc",     F3(2, 0x18, 0), F3(~2, ~0x18, ~0)|ASI(~0),      "1,2,d", 0, v9 },
-{ "addccc",     F3(2, 0x18, 1), F3(~2, ~0x18, ~1),              "1,i,d", 0, v9 },
-{ "addccc",     F3(2, 0x18, 1), F3(~2, ~0x18, ~1),              "i,1,d", 0, v9 },
+{ "addccc",     F3(2, 0x18, 0), F3(~2, ~0x18, ~0)|ASI(~0),      "1,2,d", 0, v6 },
+{ "addccc",     F3(2, 0x18, 1), F3(~2, ~0x18, ~1),              "1,i,d", 0, v6 },
+{ "addccc",     F3(2, 0x18, 1), F3(~2, ~0x18, ~1),              "i,1,d", 0, v6 },
 
 { "smul",       F3(2, 0x0b, 0), F3(~2, ~0x0b, ~0)|ASI(~0),      "1,2,d", 0, v8 },
 { "smul",       F3(2, 0x0b, 1), F3(~2, ~0x0b, ~1),              "1,i,d", 0, v8 },
@@ -2042,6 +2032,10 @@ IMPDEP ("impdep2", 0x37),
 
 #undef IMPDEP
 
+{ "addxc", F3F(2, 0x36, 0x011), F3F(~2, ~0x36, ~0x011), "1,2,d", 0, v9b },
+{ "addxccc", F3F(2, 0x36, 0x013), F3F(~2, ~0x36, ~0x013), "1,2,d", 0, v9b },
+{ "umulxhi", F3F(2, 0x36, 0x016), F3F(~2, ~0x36, ~0x016), "1,2,d", 0, v9b },
+
 };
 
 static const int sparc_num_opcodes = ((sizeof sparc_opcodes)/(sizeof sparc_opcodes[0]));
diff --git a/dma-helpers.c b/dma-helpers.c
index 499b52bc5..6918572e1 100644
--- a/dma-helpers.c
+++ b/dma-helpers.c
@@ -7,6 +7,7 @@
  * (GNU GPL), version 2 or later.
  */
 
+#include "sysemu/block-backend.h"
 #include "sysemu/dma.h"
 #include "trace.h"
 #include "qemu/range.h"
@@ -67,13 +68,12 @@ void qemu_sglist_destroy(QEMUSGList *qsg)
 }
 
 typedef struct {
-    BlockDriverAIOCB common;
-    BlockDriverState *bs;
-    BlockDriverAIOCB *acb;
+    BlockAIOCB common;
+    BlockBackend *blk;
+    BlockAIOCB *acb;
     QEMUSGList *sg;
     uint64_t sector_num;
     DMADirection dir;
-    bool in_cancel;
     int sg_cur_index;
     dma_addr_t sg_cur_byte;
     QEMUIOVector iov;
@@ -81,7 +81,7 @@ typedef struct {
     DMAIOFunc *io_func;
 } DMAAIOCB;
 
-static void dma_bdrv_cb(void *opaque, int ret);
+static void dma_blk_cb(void *opaque, int ret);
 
 static void reschedule_dma(void *opaque)
 {
@@ -89,7 +89,7 @@ static void reschedule_dma(void *opaque)
 
     qemu_bh_delete(dbs->bh);
     dbs->bh = NULL;
-    dma_bdrv_cb(dbs, 0);
+    dma_blk_cb(dbs, 0);
 }
 
 static void continue_after_map_failure(void *opaque)
@@ -100,7 +100,7 @@ static void continue_after_map_failure(void *opaque)
     qemu_bh_schedule(dbs->bh);
 }
 
-static void dma_bdrv_unmap(DMAAIOCB *dbs)
+static void dma_blk_unmap(DMAAIOCB *dbs)
 {
     int i;
 
@@ -116,7 +116,7 @@ static void dma_complete(DMAAIOCB *dbs, int ret)
 {
     trace_dma_complete(dbs, ret, dbs->common.cb);
 
-    dma_bdrv_unmap(dbs);
+    dma_blk_unmap(dbs);
     if (dbs->common.cb) {
         dbs->common.cb(dbs->common.opaque, ret);
     }
@@ -125,21 +125,16 @@ static void dma_complete(DMAAIOCB *dbs, int ret)
         qemu_bh_delete(dbs->bh);
         dbs->bh = NULL;
     }
-    if (!dbs->in_cancel) {
-        /* Requests may complete while dma_aio_cancel is in progress.  In
-         * this case, the AIOCB should not be released because it is still
-         * referenced by dma_aio_cancel.  */
-        qemu_aio_release(dbs);
-    }
+    qemu_aio_unref(dbs);
 }
 
-static void dma_bdrv_cb(void *opaque, int ret)
+static void dma_blk_cb(void *opaque, int ret)
 {
     DMAAIOCB *dbs = (DMAAIOCB *)opaque;
     dma_addr_t cur_addr, cur_len;
     void *mem;
 
-    trace_dma_bdrv_cb(dbs, ret);
+    trace_dma_blk_cb(dbs, ret);
 
     dbs->acb = NULL;
     dbs->sector_num += dbs->iov.size / 512;
@@ -148,7 +143,7 @@ static void dma_bdrv_cb(void *opaque, int ret)
         dma_complete(dbs, ret);
         return;
     }
-    dma_bdrv_unmap(dbs);
+    dma_blk_unmap(dbs);
 
     while (dbs->sg_cur_index < dbs->sg->nsg) {
         cur_addr = dbs->sg->sg[dbs->sg_cur_index].base + dbs->sg_cur_byte;
@@ -174,72 +169,66 @@ static void dma_bdrv_cb(void *opaque, int ret)
         qemu_iovec_discard_back(&dbs->iov, dbs->iov.size & ~BDRV_SECTOR_MASK);
     }
 
-    dbs->acb = dbs->io_func(dbs->bs, dbs->sector_num, &dbs->iov,
-                            dbs->iov.size / 512, dma_bdrv_cb, dbs);
+    dbs->acb = dbs->io_func(dbs->blk, dbs->sector_num, &dbs->iov,
+                            dbs->iov.size / 512, dma_blk_cb, dbs);
     assert(dbs->acb);
 }
 
-static void dma_aio_cancel(BlockDriverAIOCB *acb)
+static void dma_aio_cancel(BlockAIOCB *acb)
 {
     DMAAIOCB *dbs = container_of(acb, DMAAIOCB, common);
 
     trace_dma_aio_cancel(dbs);
 
     if (dbs->acb) {
-        BlockDriverAIOCB *acb = dbs->acb;
-        dbs->acb = NULL;
-        dbs->in_cancel = true;
-        bdrv_aio_cancel(acb);
-        dbs->in_cancel = false;
+        blk_aio_cancel_async(dbs->acb);
     }
-    dbs->common.cb = NULL;
-    dma_complete(dbs, 0);
 }
 
+
 static const AIOCBInfo dma_aiocb_info = {
     .aiocb_size         = sizeof(DMAAIOCB),
-    .cancel             = dma_aio_cancel,
+    .cancel_async       = dma_aio_cancel,
 };
 
-BlockDriverAIOCB *dma_bdrv_io(
-    BlockDriverState *bs, QEMUSGList *sg, uint64_t sector_num,
-    DMAIOFunc *io_func, BlockDriverCompletionFunc *cb,
+BlockAIOCB *dma_blk_io(
+    BlockBackend *blk, QEMUSGList *sg, uint64_t sector_num,
+    DMAIOFunc *io_func, BlockCompletionFunc *cb,
     void *opaque, DMADirection dir)
 {
-    DMAAIOCB *dbs = qemu_aio_get(&dma_aiocb_info, bs, cb, opaque);
+    DMAAIOCB *dbs = blk_aio_get(&dma_aiocb_info, blk, cb, opaque);
 
-    trace_dma_bdrv_io(dbs, bs, sector_num, (dir == DMA_DIRECTION_TO_DEVICE));
+    trace_dma_blk_io(dbs, blk, sector_num, (dir == DMA_DIRECTION_TO_DEVICE));
 
     dbs->acb = NULL;
-    dbs->bs = bs;
+    dbs->blk = blk;
     dbs->sg = sg;
     dbs->sector_num = sector_num;
     dbs->sg_cur_index = 0;
     dbs->sg_cur_byte = 0;
     dbs->dir = dir;
-    dbs->in_cancel = false;
     dbs->io_func = io_func;
     dbs->bh = NULL;
     qemu_iovec_init(&dbs->iov, sg->nsg);
-    dma_bdrv_cb(dbs, 0);
+    dma_blk_cb(dbs, 0);
     return &dbs->common;
 }
 
 
-BlockDriverAIOCB *dma_bdrv_read(BlockDriverState *bs,
-                                QEMUSGList *sg, uint64_t sector,
-                                void (*cb)(void *opaque, int ret), void *opaque)
+BlockAIOCB *dma_blk_read(BlockBackend *blk,
+                         QEMUSGList *sg, uint64_t sector,
+                         void (*cb)(void *opaque, int ret), void *opaque)
 {
-    return dma_bdrv_io(bs, sg, sector, bdrv_aio_readv, cb, opaque,
-                       DMA_DIRECTION_FROM_DEVICE);
+    return dma_blk_io(blk, sg, sector, blk_aio_readv, cb, opaque,
+                      DMA_DIRECTION_FROM_DEVICE);
 }
 
-BlockDriverAIOCB *dma_bdrv_write(BlockDriverState *bs,
-                                 QEMUSGList *sg, uint64_t sector,
-                                 void (*cb)(void *opaque, int ret), void *opaque)
+BlockAIOCB *dma_blk_write(BlockBackend *blk,
+                          QEMUSGList *sg, uint64_t sector,
+                          void (*cb)(void *opaque, int ret), void *opaque)
 {
-    return dma_bdrv_io(bs, sg, sector, bdrv_aio_writev, cb, opaque,
-                       DMA_DIRECTION_TO_DEVICE);
+    return dma_blk_io(blk, sg, sector, blk_aio_writev, cb, opaque,
+                      DMA_DIRECTION_TO_DEVICE);
 }
 
 
@@ -274,8 +263,8 @@ uint64_t dma_buf_write(uint8_t *ptr, int32_t len, QEMUSGList *sg)
     return dma_buf_rw(ptr, len, sg, DMA_DIRECTION_TO_DEVICE);
 }
 
-void dma_acct_start(BlockDriverState *bs, BlockAcctCookie *cookie,
+void dma_acct_start(BlockBackend *blk, BlockAcctCookie *cookie,
                     QEMUSGList *sg, enum BlockAcctType type)
 {
-    bdrv_acct_start(bs, cookie, sg->size, type);
+    block_acct_start(blk_get_stats(blk), cookie, sg->size, type);
 }
diff --git a/docs/blkdebug.txt b/docs/blkdebug.txt
new file mode 100644
index 000000000..b67a36d5c
--- /dev/null
+++ b/docs/blkdebug.txt
@@ -0,0 +1,161 @@
+Block I/O error injection using blkdebug
+----------------------------------------
+Copyright (C) 2014 Red Hat Inc
+
+This work is licensed under the terms of the GNU GPL, version 2 or later.  See
+the COPYING file in the top-level directory.
+
+The blkdebug block driver is a rule-based error injection engine.  It can be
+used to exercise error code paths in block drivers including ENOSPC (out of
+space) and EIO.
+
+This document gives an overview of the features available in blkdebug.
+
+Background
+----------
+Block drivers have many error code paths that handle I/O errors.  Image formats
+are especially complex since metadata I/O errors during cluster allocation or
+while updating tables happen halfway through request processing and require
+discipline to keep image files consistent.
+
+Error injection allows test cases to trigger I/O errors at specific points.
+This way, all error paths can be tested to make sure they are correct.
+
+Rules
+-----
+The blkdebug block driver takes a list of "rules" that tell the error injection
+engine when to fail an I/O request.
+
+Each I/O request is evaluated against the rules.  If a rule matches the request
+then its "action" is executed.
+
+Rules can be placed in a configuration file; the configuration file
+follows the same .ini-like format used by QEMU's -readconfig option, and
+each section of the file represents a rule.
+
+The following configuration file defines a single rule:
+
+  $ cat blkdebug.conf
+  [inject-error]
+  event = "read_aio"
+  errno = "28"
+
+This rule fails all aio read requests with ENOSPC (28).  Note that the errno
+value depends on the host.  On Linux, see
+/usr/include/asm-generic/errno-base.h for errno values.
+
+Invoke QEMU as follows:
+
+  $ qemu-system-x86_64
+        -drive if=none,cache=none,file=blkdebug:blkdebug.conf:test.img,id=drive0 \
+        -device virtio-blk-pci,drive=drive0,id=virtio-blk-pci0
+
+Rules support the following attributes:
+
+  event - which type of operation to match (e.g. read_aio, write_aio,
+          flush_to_os, flush_to_disk).  See the "Events" section for
+          information on events.
+
+  state - (optional) the engine must be in this state number in order for this
+          rule to match.  See the "State transitions" section for information
+          on states.
+
+  errno - the numeric errno value to return when a request matches this rule.
+          The errno values depend on the host since the numeric values are not
+          standarized in the POSIX specification.
+
+  sector - (optional) a sector number that the request must overlap in order to
+           match this rule
+
+  once - (optional, default "off") only execute this action on the first
+         matching request
+
+  immediately - (optional, default "off") return a NULL BlockAIOCB
+                pointer and fail without an errno instead.  This
+                exercises the code path where BlockAIOCB fails and the
+                caller's BlockCompletionFunc is not invoked.
+
+Events
+------
+Block drivers provide information about the type of I/O request they are about
+to make so rules can match specific types of requests.  For example, the qcow2
+block driver tells blkdebug when it accesses the L1 table so rules can match
+only L1 table accesses and not other metadata or guest data requests.
+
+The core events are:
+
+  read_aio - guest data read
+
+  write_aio - guest data write
+
+  flush_to_os - write out unwritten block driver state (e.g. cached metadata)
+
+  flush_to_disk - flush the host block device's disk cache
+
+See block/blkdebug.c:event_names[] for the full list of events.  You may need
+to grep block driver source code to understand the meaning of specific events.
+
+State transitions
+-----------------
+There are cases where more power is needed to match a particular I/O request in
+a longer sequence of requests.  For example:
+
+  write_aio
+  flush_to_disk
+  write_aio
+
+How do we match the 2nd write_aio but not the first?  This is where state
+transitions come in.
+
+The error injection engine has an integer called the "state" that always starts
+initialized to 1.  The state integer is internal to blkdebug and cannot be
+observed from outside but rules can interact with it for powerful matching
+behavior.
+
+Rules can be conditional on the current state and they can transition to a new
+state.
+
+When a rule's "state" attribute is non-zero then the current state must equal
+the attribute in order for the rule to match.
+
+For example, to match the 2nd write_aio:
+
+  [set-state]
+  event = "write_aio"
+  state = "1"
+  new_state = "2"
+
+  [inject-error]
+  event = "write_aio"
+  state = "2"
+  errno = "5"
+
+The first write_aio request matches the set-state rule and transitions from
+state 1 to state 2.  Once state 2 has been entered, the set-state rule no
+longer matches since it requires state 1.  But the inject-error rule now
+matches the next write_aio request and injects EIO (5).
+
+State transition rules support the following attributes:
+
+  event - which type of operation to match (e.g. read_aio, write_aio,
+          flush_to_os, flush_to_disk).  See the "Events" section for
+          information on events.
+
+  state - (optional) the engine must be in this state number in order for this
+          rule to match
+
+  new_state - transition to this state number
+
+Suspend and resume
+------------------
+Exercising code paths in block drivers may require specific ordering amongst
+concurrent requests.  The "breakpoint" feature allows requests to be halted on
+a blkdebug event and resumed later.  This makes it possible to achieve
+deterministic ordering when multiple requests are in flight.
+
+Breakpoints on blkdebug events are associated with a user-defined "tag" string.
+This tag serves as an identifier by which the request can be resumed at a later
+point.
+
+See the qemu-io(1) break, resume, remove_break, and wait_break commands for
+details.
diff --git a/docs/image-fuzzer.txt b/docs/image-fuzzer.txt
new file mode 100644
index 000000000..3e23ebec3
--- /dev/null
+++ b/docs/image-fuzzer.txt
@@ -0,0 +1,239 @@
+# Specification for the fuzz testing tool
+#
+# Copyright (C) 2014 Maria Kustova <maria.k@catit.be>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+Image fuzzer
+============
+
+Description
+-----------
+
+The goal of the image fuzzer is to catch crashes of qemu-io/qemu-img
+by providing to them randomly corrupted images.
+Test images are generated from scratch and have valid inner structure with some
+elements, e.g. L1/L2 tables, having random invalid values.
+
+
+Test runner
+-----------
+
+The test runner generates test images, executes tests utilizing generated
+images, indicates their results and collects all test related artifacts (logs,
+core dumps, test images, backing files).
+The test means execution of all available commands under test with the same
+generated test image.
+By default, the test runner generates new tests and executes them until
+keyboard interruption. But if a test seed is specified via the '--seed' runner
+parameter, then only one test with this seed will be executed, after its finish
+the runner will exit.
+
+The runner uses an external image fuzzer to generate test images. An image
+generator should be specified as a mandatory parameter of the test runner.
+Details about interactions between the runner and fuzzers see "Module
+interfaces".
+
+The runner activates generation of core dumps during test executions, but it
+assumes that core dumps will be generated in the current working directory.
+For comprehensive test results, please, set up your test environment
+properly.
+
+Paths to binaries under test (SUTs) qemu-img and qemu-io are retrieved from
+environment variables. If the environment check fails the runner will
+use SUTs installed in system paths.
+qemu-img is required for creation of backing files, so it's mandatory to set
+the related environment variable if it's not installed in the system path.
+For details about environment variables see qemu-iotests/check.
+
+The runner accepts a JSON array of fields expected to be fuzzed via the
+'--config' argument, e.g.
+
+       '[["feature_name_table"], ["header", "l1_table_offset"]]'
+
+Each sublist can have one or two strings defining image structure elements.
+In the latter case a parent element should be placed on the first position,
+and a field name on the second one.
+
+The runner accepts a list of commands under test as a JSON array via
+the '--command' argument. Each command is a list containing a SUT and all its
+arguments, e.g.
+
+       runner.py -c '[["qemu-io", "$test_img", "-c", "write $off $len"]]'
+     /tmp/test ../qcow2
+
+For variable arguments next aliases can be used:
+    - $test_img for a fuzzed img
+    - $off for an offset in the fuzzed image
+    - $len for a data size
+
+Values for last two aliases will be generated based on a size of a virtual
+disk of the generated image.
+In case when no commands are specified the runner will execute commands from
+the default list:
+    - qemu-img check
+    - qemu-img info
+    - qemu-img convert
+    - qemu-io -c read
+    - qemu-io -c write
+    - qemu-io -c aio_read
+    - qemu-io -c aio_write
+    - qemu-io -c flush
+    - qemu-io -c discard
+    - qemu-io -c truncate
+
+
+Qcow2 image generator
+---------------------
+
+The 'qcow2' generator is a Python package providing 'create_image' method as
+a single public API. See details in 'Test runner/image fuzzer' chapter of
+'Module interfaces'.
+
+Qcow2 contains two submodules: fuzz.py and layout.py.
+
+'fuzz.py' contains all fuzzing functions, one per image field. It's assumed
+that after code analysis every field will have own constraints for its value.
+For now only universal potentially dangerous values are used, e.g. type limits
+for integers or unsafe symbols as '%s' for strings. For bitmasks random amount
+of bits are set to ones. All fuzzed values are checked on non-equality to the
+current valid value of the field. In case of equality the value will be
+regenerated.
+
+'layout.py' creates a random valid image, fuzzes a random subset of the image
+fields by 'fuzz.py' module and writes a fuzzed image to the file specified.
+If a fuzzer configuration is specified, then it has the next interpretation:
+
+    1. If a list contains a parent image element only, then some random portion
+    of fields of this element will be fuzzed every test.
+    The same behavior is applied for the entire image if no configuration is
+    used. This case is useful for the test specialization.
+
+    2. If a list contains a parent element and a field name, then a field
+    will be always fuzzed for every test. This case is useful for regression
+    testing.
+
+The generator can create header fields, header extensions, L1/L2 tables and
+refcount table and blocks.
+
+Module interfaces
+-----------------
+
+* Test runner/image fuzzer
+
+The runner calls an image generator specifying the path to a test image file,
+path to a backing file and its format and a fuzzer configuration.
+An image generator is expected to provide a
+
+   'create_image(test_img_path, backing_file_path=None,
+                 backing_file_format=None, fuzz_config=None)'
+
+method that creates a test image, writes it to the specified file and returns
+the size of the virtual disk.
+The file should be created if it doesn't exist or overwritten otherwise.
+fuzz_config has a form of a list of lists. Every sublist can have one
+or two elements: first element is a name of a parent image element, second one
+if exists is a name of a field in this element.
+Example,
+        [['header', 'l1_table_offset'],
+         ['header', 'nb_snapshots'],
+         ['feature_name_table']]
+
+Random seed is set by the runner at every test execution for the regression
+purpose, so an image generator is not recommended to modify it internally.
+
+
+Overall fuzzer requirements
+===========================
+
+Input data:
+----------
+
+ - image template (generator)
+ - work directory
+ - action vector (optional)
+ - seed (optional)
+ - SUT and its arguments (optional)
+
+
+Fuzzer requirements:
+-------------------
+
+1.  Should be able to inject random data
+2.  Should be able to select a random value from the manually pregenerated
+    vector (boundary values, e.g. max/min cluster size)
+3.  Image template should describe a general structure invariant for all
+    test images (image format description)
+4.  Image template should be autonomous and other fuzzer parts should not
+    rely on it
+5.  Image template should contain reference rules (not only block+size
+    description)
+6.  Should generate the test image with the correct structure based on an image
+    template
+7.  Should accept a seed as an argument (for regression purpose)
+8.  Should generate a seed if it is not specified as an input parameter.
+9.  The same seed should generate the same image for the same action vector,
+    specified or generated.
+10. Should accept a vector of actions as an argument (for test reproducing and
+    for test case specification, e.g. group of tests for header structure,
+    group of test for snapshots, etc)
+11. Action vector should be randomly generated from the pool of available
+    actions, if it is not specified as an input parameter
+12. Pool of actions should be defined automatically based on an image template
+13. Should accept a SUT and its call parameters as an argument or select them
+    randomly otherwise. As far as it's expected to be rarely changed, the list
+    of all possible test commands can be available in the test runner
+    internally.
+14. Should support an external cancellation of a test run
+15. Seed should be logged (for regression purpose)
+16. All files related to a test result should be collected: a test image,
+    SUT logs, fuzzer logs and crash dumps
+17. Should be compatible with python version 2.4-2.7
+18. Usage of external libraries should be limited as much as possible.
+
+
+Image formats:
+-------------
+
+Main target image format is qcow2, but support of image templates should
+provide an ability to add any other image format.
+
+
+Effectiveness:
+-------------
+
+The fuzzer can be controlled via template, seed and action vector;
+it makes the fuzzer itself invariant to an image format and test logic.
+It should be able to perform rather complex and precise tests, that can be
+specified via an action vector. Otherwise, knowledge about an image structure
+allows the fuzzer to generate the pool of all available areas can be fuzzed
+and randomly select some of them and so compose its own action vector.
+Also complexity of a template defines complexity of the fuzzer, so its
+functionality can be varied from simple model-independent fuzzing to smart
+model-based one.
+
+
+Glossary:
+--------
+
+Action vector is a sequence of structure elements retrieved from an image
+format, each of them will be fuzzed for the test image. It's a subset of
+elements of the action pool. Example: header, refcount table, etc.
+Action pool is all available elements of an image structure that generated
+automatically from an image template.
+Image template is a formal description of an image structure and relations
+between image blocks.
+Test image is an output image of the fuzzer defined by the current seed and
+action vector.
diff --git a/docs/memory.txt b/docs/memory.txt
index 5bdbdb369..b12f1f049 100644
--- a/docs/memory.txt
+++ b/docs/memory.txt
@@ -74,11 +74,16 @@ Region lifecycle
 ----------------
 
 A region is created by one of the constructor functions (memory_region_init*())
-and destroyed by the destructor (memory_region_destroy()).  In between,
-a region can be added to an address space by using memory_region_add_subregion()
-and removed using memory_region_del_subregion().  Region attributes may be
-changed at any point; they take effect once the region becomes exposed to the
-guest.
+and attached to an object.  It is then destroyed by object_unparent() or simply
+when the parent object dies.
+
+In between, a region can be added to an address space
+by using memory_region_add_subregion() and removed using
+memory_region_del_subregion().  Destroying the region implicitly
+removes the region from the address space.
+
+Region attributes may be changed at any point; they take effect once
+the region becomes exposed to the guest.
 
 Overlapping regions and priority
 --------------------------------
diff --git a/docs/multiple-iothreads.txt b/docs/multiple-iothreads.txt
new file mode 100644
index 000000000..40b841991
--- /dev/null
+++ b/docs/multiple-iothreads.txt
@@ -0,0 +1,134 @@
+Copyright (c) 2014 Red Hat Inc.
+
+This work is licensed under the terms of the GNU GPL, version 2 or later.  See
+the COPYING file in the top-level directory.
+
+
+This document explains the IOThread feature and how to write code that runs
+outside the QEMU global mutex.
+
+The main loop and IOThreads
+---------------------------
+QEMU is an event-driven program that can do several things at once using an
+event loop.  The VNC server and the QMP monitor are both processed from the
+same event loop, which monitors their file descriptors until they become
+readable and then invokes a callback.
+
+The default event loop is called the main loop (see main-loop.c).  It is
+possible to create additional event loop threads using -object
+iothread,id=my-iothread.
+
+Side note: The main loop and IOThread are both event loops but their code is
+not shared completely.  Sometimes it is useful to remember that although they
+are conceptually similar they are currently not interchangeable.
+
+Why IOThreads are useful
+------------------------
+IOThreads allow the user to control the placement of work.  The main loop is a
+scalability bottleneck on hosts with many CPUs.  Work can be spread across
+several IOThreads instead of just one main loop.  When set up correctly this
+can improve I/O latency and reduce jitter seen by the guest.
+
+The main loop is also deeply associated with the QEMU global mutex, which is a
+scalability bottleneck in itself.  vCPU threads and the main loop use the QEMU
+global mutex to serialize execution of QEMU code.  This mutex is necessary
+because a lot of QEMU's code historically was not thread-safe.
+
+The fact that all I/O processing is done in a single main loop and that the
+QEMU global mutex is contended by all vCPU threads and the main loop explain
+why it is desirable to place work into IOThreads.
+
+The experimental virtio-blk data-plane implementation has been benchmarked and
+shows these effects:
+ftp://public.dhe.ibm.com/linux/pdfs/KVM_Virtualized_IO_Performance_Paper.pdf
+
+How to program for IOThreads
+----------------------------
+The main difference between legacy code and new code that can run in an
+IOThread is dealing explicitly with the event loop object, AioContext
+(see include/block/aio.h).  Code that only works in the main loop
+implicitly uses the main loop's AioContext.  Code that supports running
+in IOThreads must be aware of its AioContext.
+
+AioContext supports the following services:
+ * File descriptor monitoring (read/write/error on POSIX hosts)
+ * Event notifiers (inter-thread signalling)
+ * Timers
+ * Bottom Halves (BH) deferred callbacks
+
+There are several old APIs that use the main loop AioContext:
+ * LEGACY qemu_aio_set_fd_handler() - monitor a file descriptor
+ * LEGACY qemu_aio_set_event_notifier() - monitor an event notifier
+ * LEGACY timer_new_ms() - create a timer
+ * LEGACY qemu_bh_new() - create a BH
+ * LEGACY qemu_aio_wait() - run an event loop iteration
+
+Since they implicitly work on the main loop they cannot be used in code that
+runs in an IOThread.  They might cause a crash or deadlock if called from an
+IOThread since the QEMU global mutex is not held.
+
+Instead, use the AioContext functions directly (see include/block/aio.h):
+ * aio_set_fd_handler() - monitor a file descriptor
+ * aio_set_event_notifier() - monitor an event notifier
+ * aio_timer_new() - create a timer
+ * aio_bh_new() - create a BH
+ * aio_poll() - run an event loop iteration
+
+The AioContext can be obtained from the IOThread using
+iothread_get_aio_context() or for the main loop using qemu_get_aio_context().
+Code that takes an AioContext argument works both in IOThreads or the main
+loop, depending on which AioContext instance the caller passes in.
+
+How to synchronize with an IOThread
+-----------------------------------
+AioContext is not thread-safe so some rules must be followed when using file
+descriptors, event notifiers, timers, or BHs across threads:
+
+1. AioContext functions can be called safely from file descriptor, event
+notifier, timer, or BH callbacks invoked by the AioContext.  No locking is
+necessary.
+
+2. Other threads wishing to access the AioContext must use
+aio_context_acquire()/aio_context_release() for mutual exclusion.  Once the
+context is acquired no other thread can access it or run event loop iterations
+in this AioContext.
+
+aio_context_acquire()/aio_context_release() calls may be nested.  This
+means you can call them if you're not sure whether #1 applies.
+
+There is currently no lock ordering rule if a thread needs to acquire multiple
+AioContexts simultaneously.  Therefore, it is only safe for code holding the
+QEMU global mutex to acquire other AioContexts.
+
+Side note: the best way to schedule a function call across threads is to create
+a BH in the target AioContext beforehand and then call qemu_bh_schedule().  No
+acquire/release or locking is needed for the qemu_bh_schedule() call.  But be
+sure to acquire the AioContext for aio_bh_new() if necessary.
+
+The relationship between AioContext and the block layer
+-------------------------------------------------------
+The AioContext originates from the QEMU block layer because it provides a
+scoped way of running event loop iterations until all work is done.  This
+feature is used to complete all in-flight block I/O requests (see
+bdrv_drain_all()).  Nowadays AioContext is a generic event loop that can be
+used by any QEMU subsystem.
+
+The block layer has support for AioContext integrated.  Each BlockDriverState
+is associated with an AioContext using bdrv_set_aio_context() and
+bdrv_get_aio_context().  This allows block layer code to process I/O inside the
+right AioContext.  Other subsystems may wish to follow a similar approach.
+
+Block layer code must therefore expect to run in an IOThread and avoid using
+old APIs that implicitly use the main loop.  See the "How to program for
+IOThreads" above for information on how to do that.
+
+If main loop code such as a QMP function wishes to access a BlockDriverState it
+must first call aio_context_acquire(bdrv_get_aio_context(bs)) to ensure the
+IOThread does not run in parallel.
+
+Long-running jobs (usually in the form of coroutines) are best scheduled in the
+BlockDriverState's AioContext to avoid the need to acquire/release around each
+bdrv_*() call.  Be aware that there is currently no mechanism to get notified
+when bdrv_set_aio_context() moves this BlockDriverState to a different
+AioContext (see bdrv_detach_aio_context()/bdrv_attach_aio_context()), so you
+may need to add this if you want to support long-running jobs.
diff --git a/docs/qapi-code-gen.txt b/docs/qapi-code-gen.txt
index a6197a913..8313ba6af 100644
--- a/docs/qapi-code-gen.txt
+++ b/docs/qapi-code-gen.txt
@@ -1,10 +1,5 @@
 = How to use the QAPI code generator =
 
-* Note: as of this writing, QMP does not use QAPI. Eventually QMP
-commands will be converted to use QAPI internally. The following
-information describes QMP/QAPI as it will exist after the
-conversion.
-
 QAPI is a native C API within QEMU which provides management-level
 functionality to internal/external users. For external
 users/processes, this interface is made available by a JSON-based
@@ -19,7 +14,7 @@ marshaling/dispatch code for the guest agent server running in the
 guest.
 
 This document will describe how the schemas, scripts, and resulting
-code is used.
+code are used.
 
 
 == QMP/Guest agent schema ==
@@ -234,6 +229,7 @@ Resulting in this JSON object:
   "data": { "b": "test string" },
   "timestamp": { "seconds": 1267020223, "microseconds": 435656 } }
 
+
 == Code generation ==
 
 Schemas are fed into 3 scripts to generate all the code/files that, paired
@@ -256,6 +252,8 @@ command which takes that type as a parameter and returns the same type:
       'data':    {'arg1': 'UserDefOne'},
       'returns': 'UserDefOne' }
 
+    { 'event': 'MY_EVENT' }
+
 === scripts/qapi-types.py ===
 
 Used to generate the C types defined by a schema. The following files are
@@ -277,7 +275,7 @@ Example:
     $ cat qapi-generated/example-qapi-types.c
 [Uninteresting stuff omitted...]
 
-    void qapi_free_UserDefOneList(UserDefOneList * obj)
+    void qapi_free_UserDefOneList(UserDefOneList *obj)
     {
         QapiDeallocVisitor *md;
         Visitor *v;
@@ -292,7 +290,7 @@ Example:
         qapi_dealloc_visitor_cleanup(md);
     }
 
-    void qapi_free_UserDefOne(UserDefOne * obj)
+    void qapi_free_UserDefOne(UserDefOne *obj)
     {
         QapiDeallocVisitor *md;
         Visitor *v;
@@ -331,11 +329,11 @@ Example:
     struct UserDefOne
     {
         int64_t integer;
-        char * string;
+        char *string;
     };
 
-    void qapi_free_UserDefOneList(UserDefOneList * obj);
-    void qapi_free_UserDefOne(UserDefOne * obj);
+    void qapi_free_UserDefOneList(UserDefOneList *obj);
+    void qapi_free_UserDefOne(UserDefOne *obj);
 
     #endif
 
@@ -364,7 +362,7 @@ Example:
     $ cat qapi-generated/example-qapi-visit.c
 [Uninteresting stuff omitted...]
 
-    static void visit_type_UserDefOne_fields(Visitor *m, UserDefOne ** obj, Error **errp)
+    static void visit_type_UserDefOne_fields(Visitor *m, UserDefOne **obj, Error **errp)
     {
         Error *err = NULL;
         visit_type_int(m, &(*obj)->integer, "integer", &err);
@@ -380,7 +378,7 @@ Example:
         error_propagate(errp, err);
     }
 
-    void visit_type_UserDefOne(Visitor *m, UserDefOne ** obj, const char *name, Error **errp)
+    void visit_type_UserDefOne(Visitor *m, UserDefOne **obj, const char *name, Error **errp)
     {
         Error *err = NULL;
 
@@ -394,7 +392,7 @@ Example:
         error_propagate(errp, err);
     }
 
-    void visit_type_UserDefOneList(Visitor *m, UserDefOneList ** obj, const char *name, Error **errp)
+    void visit_type_UserDefOneList(Visitor *m, UserDefOneList **obj, const char *name, Error **errp)
     {
         Error *err = NULL;
         GenericList *i, **prev;
@@ -427,8 +425,8 @@ Example:
 
 [Visitors for builtin types omitted...]
 
-    void visit_type_UserDefOne(Visitor *m, UserDefOne ** obj, const char *name, Error **errp);
-    void visit_type_UserDefOneList(Visitor *m, UserDefOneList ** obj, const char *name, Error **errp);
+    void visit_type_UserDefOne(Visitor *m, UserDefOne **obj, const char *name, Error **errp);
+    void visit_type_UserDefOneList(Visitor *m, UserDefOneList **obj, const char *name, Error **errp);
 
     #endif
 
@@ -451,10 +449,12 @@ $(prefix)qmp-commands.h: Function prototypes for the QMP commands
 
 Example:
 
+    $ python scripts/qapi-commands.py --output-dir="qapi-generated"
+    --prefix="example-" --input-file=example-schema.json
     $ cat qapi-generated/example-qmp-marshal.c
 [Uninteresting stuff omitted...]
 
-    static void qmp_marshal_output_my_command(UserDefOne * ret_in, QObject **ret_out, Error **errp)
+    static void qmp_marshal_output_my_command(UserDefOne *ret_in, QObject **ret_out, Error **errp)
     {
         Error *local_err = NULL;
         QmpOutputVisitor *mo = qmp_output_visitor_new();
@@ -480,11 +480,11 @@ Example:
     static void qmp_marshal_input_my_command(QDict *args, QObject **ret, Error **errp)
     {
         Error *local_err = NULL;
-        UserDefOne * retval = NULL;
+        UserDefOne *retval = NULL;
         QmpInputVisitor *mi = qmp_input_visitor_new_strict(QOBJECT(args));
         QapiDeallocVisitor *md;
         Visitor *v;
-        UserDefOne * arg1 = NULL;
+        UserDefOne *arg1 = NULL;
 
         v = qmp_input_get_visitor(mi);
         visit_type_UserDefOne(v, &arg1, "arg1", &local_err);
@@ -525,6 +525,66 @@ Example:
     #include "qapi/qmp/qdict.h"
     #include "qapi/error.h"
 
-    UserDefOne * qmp_my_command(UserDefOne * arg1, Error **errp);
+    UserDefOne *qmp_my_command(UserDefOne *arg1, Error **errp);
+
+    #endif
+
+=== scripts/qapi-event.py ===
+
+Used to generate the event-related C code defined by a schema. The
+following files are created:
+
+$(prefix)qapi-event.h - Function prototypes for each event type, plus an
+                        enumeration of all event names
+$(prefix)qapi-event.c - Implementation of functions to send an event
+
+Example:
+
+    $ python scripts/qapi-event.py --output-dir="qapi-generated"
+    --prefix="example-" --input-file=example-schema.json
+    $ cat qapi-generated/example-qapi-event.c
+[Uninteresting stuff omitted...]
+
+    void qapi_event_send_my_event(Error **errp)
+    {
+        QDict *qmp;
+        Error *local_err = NULL;
+        QMPEventFuncEmit emit;
+        emit = qmp_event_get_func_emit();
+        if (!emit) {
+            return;
+        }
+
+        qmp = qmp_event_build_dict("MY_EVENT");
+
+        emit(EXAMPLE_QAPI_EVENT_MY_EVENT, qmp, &local_err);
+
+        error_propagate(errp, local_err);
+        QDECREF(qmp);
+    }
+
+    const char *EXAMPLE_QAPIEvent_lookup[] = {
+        "MY_EVENT",
+        NULL,
+    };
+    $ cat qapi-generated/example-qapi-event.h
+[Uninteresting stuff omitted...]
+
+    #ifndef EXAMPLE_QAPI_EVENT_H
+    #define EXAMPLE_QAPI_EVENT_H
+
+    #include "qapi/error.h"
+    #include "qapi/qmp/qdict.h"
+    #include "example-qapi-types.h"
+
+
+    void qapi_event_send_my_event(Error **errp);
+
+    extern const char *EXAMPLE_QAPIEvent_lookup[];
+    typedef enum EXAMPLE_QAPIEvent
+    {
+        EXAMPLE_QAPI_EVENT_MY_EVENT = 0,
+        EXAMPLE_QAPI_EVENT_MAX = 1,
+    } EXAMPLE_QAPIEvent;
 
     #endif
diff --git a/docs/rdma.txt b/docs/rdma.txt
index 1f5d9e9fe..2bdd0a5be 100644
--- a/docs/rdma.txt
+++ b/docs/rdma.txt
@@ -18,7 +18,7 @@ Contents:
 * RDMA Migration Protocol Description
 * Versioning and Capabilities
 * QEMUFileRDMA Interface
-* Migration of pc.ram
+* Migration of VM's ram
 * Error handling
 * TODO
 
@@ -149,7 +149,7 @@ The only difference between a SEND message and an RDMA
 message is that SEND messages cause notifications
 to be posted to the completion queue (CQ) on the
 infiniband receiver side, whereas RDMA messages (used
-for pc.ram) do not (to behave like an actual DMA).
+for VM's ram) do not (to behave like an actual DMA).
 
 Messages in infiniband require two things:
 
@@ -355,7 +355,7 @@ If the buffer is empty, then we follow the same steps
 listed above and issue another "QEMU File" protocol command,
 asking for a new SEND message to re-fill the buffer.
 
-Migration of pc.ram:
+Migration of VM's ram:
 ====================
 
 At the beginning of the migration, (migration-rdma.c),
diff --git a/docs/specs/qcow2.txt b/docs/specs/qcow2.txt
index 3f713a644..121dfc8cc 100644
--- a/docs/specs/qcow2.txt
+++ b/docs/specs/qcow2.txt
@@ -110,6 +110,7 @@ in the description of a field.
                     in bits: refcount_bits = 1 << refcount_order). For version 2
                     images, the order is always assumed to be 4
                     (i.e. refcount_bits = 16).
+                    This value may not exceed 6 (i.e. refcount_bits = 64).
 
         100 - 103:  header_length
                     Length of the header structure in bytes. For version 2
@@ -135,12 +136,12 @@ be stored. Each extension has a structure like the following:
 Unless stated otherwise, each header extension type shall appear at most once
 in the same image.
 
-The remaining space between the end of the header extension area and the end of
-the first cluster can be used for the backing file name. It is not allowed to
-store other data here, so that an implementation can safely modify the header
-and add extensions without harming data of compatible features that it
-doesn't support. Compatible features that need space for additional data can
-use a header extension.
+If the image has a backing file then the backing file name should be stored in
+the remaining space between the end of the header extension area and the end of
+the first cluster. It is not allowed to store other data here, so that an
+implementation can safely modify the header and add extensions without harming
+data of compatible features that it doesn't support. Compatible features that
+need space for additional data can use a header extension.
 
 
 == Feature name table ==
@@ -183,7 +184,7 @@ blocks and are exactly one cluster in size.
 Given a offset into the image file, the refcount of its cluster can be obtained
 as follows:
 
-    refcount_block_entries = (cluster_size / sizeof(uint16_t))
+    refcount_block_entries = (cluster_size * 8 / refcount_bits)
 
     refcount_block_index = (offset / cluster_size) % refcount_block_entries
     refcount_table_index = (offset / cluster_size) / refcount_block_entries
diff --git a/docs/specs/standard-vga.txt b/docs/specs/standard-vga.txt
index f82773e67..19d2a7450 100644
--- a/docs/specs/standard-vga.txt
+++ b/docs/specs/standard-vga.txt
@@ -70,3 +70,12 @@ Likewise applies to the pci variant only for obvious reasons.
 0500 - 0515 : bochs dispi interface registers, mapped flat
               without index/data ports.  Use (index << 1)
               as offset for (16bit) register access.
+
+0600 - 0607 : qemu extended registers.  qemu 2.2+ only.
+              The pci revision is 2 (or greater) when
+              these registers are present.  The registers
+              are 32bit.
+  0600      : qemu extended register region size, in bytes.
+  0604      : framebuffer endianness register.
+              - 0xbebebebe indicates big endian.
+              - 0x1e1e1e1e indicates little endian.
diff --git a/docs/tracing.txt b/docs/tracing.txt
index c6ab1c11c..7117c5e7d 100644
--- a/docs/tracing.txt
+++ b/docs/tracing.txt
@@ -23,7 +23,7 @@ for debugging, profiling, and observing execution.
 
 4. Pretty-print the binary trace file:
 
-    ./scripts/simpletrace.py trace-events trace-*
+    ./scripts/simpletrace.py trace-events trace-* # Override * with QEMU <pid>
 
 == Trace events ==
 
@@ -139,12 +139,12 @@ events are not tightly coupled to a specific trace backend, such as LTTng or
 SystemTap.  Support for trace backends can be added by extending the "tracetool"
 script.
 
-The trace backend is chosen at configure time and only one trace backend can
-be built into the binary:
+The trace backends are chosen at configure time:
 
-    ./configure --trace-backends=simple
+    ./configure --enable-trace-backends=simple
 
 For a list of supported trace backends, try ./configure --help or see below.
+If multiple backends are enabled, the trace is sent to them all.
 
 The following subsections describe the supported trace backends.
 
@@ -307,3 +307,43 @@ guard such computations and avoid its compilation when the event is disabled:
 You can check both if the event has been disabled and is dynamically enabled at
 the same time using the 'trace_event_get_state' routine (see header
 "trace/control.h" for more information).
+
+=== "tcg" ===
+
+Guest code generated by TCG can be traced by defining an event with the "tcg"
+event property. Internally, this property generates two events:
+"<eventname>_trans" to trace the event at translation time, and
+"<eventname>_exec" to trace the event at execution time.
+
+Instead of using these two events, you should instead use the function
+"trace_<eventname>_tcg" during translation (TCG code generation). This function
+will automatically call "trace_<eventname>_trans", and will generate the
+necessary TCG code to call "trace_<eventname>_exec" during guest code execution.
+
+Events with the "tcg" property can be declared in the "trace-events" file with a
+mix of native and TCG types, and "trace_<eventname>_tcg" will gracefully forward
+them to the "<eventname>_trans" and "<eventname>_exec" events. Since TCG values
+are not known at translation time, these are ignored by the "<eventname>_trans"
+event. Because of this, the entry in the "trace-events" file needs two printing
+formats (separated by a comma):
+
+    tcg foo(uint8_t a1, TCGv_i32 a2) "a1=%d", "a1=%d a2=%d"
+
+For example:
+
+    #include "trace-tcg.h"
+    
+    void some_disassembly_func (...)
+    {
+        uint8_t a1 = ...;
+        TCGv_i32 a2 = ...;
+        trace_foo_tcg(a1, a2);
+    }
+
+This will immediately call:
+
+    void trace_foo_trans(uint8_t a1);
+
+and will generate the TCG code to call:
+
+    void trace_foo(uint8_t a1, uint32_t a2);
diff --git a/docs/writing-qmp-commands.txt b/docs/writing-qmp-commands.txt
index 4d86c2477..f3df2066a 100644
--- a/docs/writing-qmp-commands.txt
+++ b/docs/writing-qmp-commands.txt
@@ -365,6 +365,8 @@ documentation for information about the other types.
 
 === User Defined Types ===
 
+FIXME This example needs to be redone after commit 6d32717
+
 For this example we will write the query-alarm-clock command, which returns
 information about QEMU's timer alarm. For more information about it, please
 check the "-clock" command-line option.
diff --git a/dump.c b/dump.c
index ce646bcc5..9c7dad8f8 100644
--- a/dump.c
+++ b/dump.c
@@ -71,23 +71,20 @@ uint64_t cpu_to_dump64(DumpState *s, uint64_t val)
 
 static int dump_cleanup(DumpState *s)
 {
-    int ret = 0;
-
     guest_phys_blocks_free(&s->guest_phys_blocks);
     memory_mapping_list_free(&s->list);
-    if (s->fd != -1) {
-        close(s->fd);
-    }
+    close(s->fd);
     if (s->resume) {
         vm_start();
     }
 
-    return ret;
+    return 0;
 }
 
-static void dump_error(DumpState *s, const char *reason)
+static void dump_error(DumpState *s, const char *reason, Error **errp)
 {
     dump_cleanup(s);
+    error_setg(errp, "%s", reason);
 }
 
 static int fd_write_vmcore(const void *buf, size_t size, void *opaque)
@@ -103,7 +100,7 @@ static int fd_write_vmcore(const void *buf, size_t size, void *opaque)
     return 0;
 }
 
-static int write_elf64_header(DumpState *s)
+static void write_elf64_header(DumpState *s, Error **errp)
 {
     Elf64_Ehdr elf_header;
     int ret;
@@ -130,14 +127,11 @@ static int write_elf64_header(DumpState *s)
 
     ret = fd_write_vmcore(&elf_header, sizeof(elf_header), s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write elf header.\n");
-        return -1;
+        dump_error(s, "dump: failed to write elf header", errp);
     }
-
-    return 0;
 }
 
-static int write_elf32_header(DumpState *s)
+static void write_elf32_header(DumpState *s, Error **errp)
 {
     Elf32_Ehdr elf_header;
     int ret;
@@ -164,16 +158,13 @@ static int write_elf32_header(DumpState *s)
 
     ret = fd_write_vmcore(&elf_header, sizeof(elf_header), s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write elf header.\n");
-        return -1;
+        dump_error(s, "dump: failed to write elf header", errp);
     }
-
-    return 0;
 }
 
-static int write_elf64_load(DumpState *s, MemoryMapping *memory_mapping,
-                            int phdr_index, hwaddr offset,
-                            hwaddr filesz)
+static void write_elf64_load(DumpState *s, MemoryMapping *memory_mapping,
+                             int phdr_index, hwaddr offset,
+                             hwaddr filesz, Error **errp)
 {
     Elf64_Phdr phdr;
     int ret;
@@ -190,16 +181,13 @@ static int write_elf64_load(DumpState *s, MemoryMapping *memory_mapping,
 
     ret = fd_write_vmcore(&phdr, sizeof(Elf64_Phdr), s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write program header table.\n");
-        return -1;
+        dump_error(s, "dump: failed to write program header table", errp);
     }
-
-    return 0;
 }
 
-static int write_elf32_load(DumpState *s, MemoryMapping *memory_mapping,
-                            int phdr_index, hwaddr offset,
-                            hwaddr filesz)
+static void write_elf32_load(DumpState *s, MemoryMapping *memory_mapping,
+                             int phdr_index, hwaddr offset,
+                             hwaddr filesz, Error **errp)
 {
     Elf32_Phdr phdr;
     int ret;
@@ -216,14 +204,11 @@ static int write_elf32_load(DumpState *s, MemoryMapping *memory_mapping,
 
     ret = fd_write_vmcore(&phdr, sizeof(Elf32_Phdr), s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write program header table.\n");
-        return -1;
+        dump_error(s, "dump: failed to write program header table", errp);
     }
-
-    return 0;
 }
 
-static int write_elf64_note(DumpState *s)
+static void write_elf64_note(DumpState *s, Error **errp)
 {
     Elf64_Phdr phdr;
     hwaddr begin = s->memory_offset - s->note_size;
@@ -239,11 +224,8 @@ static int write_elf64_note(DumpState *s)
 
     ret = fd_write_vmcore(&phdr, sizeof(Elf64_Phdr), s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write program header table.\n");
-        return -1;
+        dump_error(s, "dump: failed to write program header table", errp);
     }
-
-    return 0;
 }
 
 static inline int cpu_index(CPUState *cpu)
@@ -251,7 +233,8 @@ static inline int cpu_index(CPUState *cpu)
     return cpu->cpu_index + 1;
 }
 
-static int write_elf64_notes(WriteCoreDumpFunction f, DumpState *s)
+static void write_elf64_notes(WriteCoreDumpFunction f, DumpState *s,
+                              Error **errp)
 {
     CPUState *cpu;
     int ret;
@@ -261,23 +244,21 @@ static int write_elf64_notes(WriteCoreDumpFunction f, DumpState *s)
         id = cpu_index(cpu);
         ret = cpu_write_elf64_note(f, cpu, id, s);
         if (ret < 0) {
-            dump_error(s, "dump: failed to write elf notes.\n");
-            return -1;
+            dump_error(s, "dump: failed to write elf notes", errp);
+            return;
         }
     }
 
     CPU_FOREACH(cpu) {
         ret = cpu_write_elf64_qemunote(f, cpu, s);
         if (ret < 0) {
-            dump_error(s, "dump: failed to write CPU status.\n");
-            return -1;
+            dump_error(s, "dump: failed to write CPU status", errp);
+            return;
         }
     }
-
-    return 0;
 }
 
-static int write_elf32_note(DumpState *s)
+static void write_elf32_note(DumpState *s, Error **errp)
 {
     hwaddr begin = s->memory_offset - s->note_size;
     Elf32_Phdr phdr;
@@ -293,14 +274,12 @@ static int write_elf32_note(DumpState *s)
 
     ret = fd_write_vmcore(&phdr, sizeof(Elf32_Phdr), s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write program header table.\n");
-        return -1;
+        dump_error(s, "dump: failed to write program header table", errp);
     }
-
-    return 0;
 }
 
-static int write_elf32_notes(WriteCoreDumpFunction f, DumpState *s)
+static void write_elf32_notes(WriteCoreDumpFunction f, DumpState *s,
+                              Error **errp)
 {
     CPUState *cpu;
     int ret;
@@ -310,23 +289,21 @@ static int write_elf32_notes(WriteCoreDumpFunction f, DumpState *s)
         id = cpu_index(cpu);
         ret = cpu_write_elf32_note(f, cpu, id, s);
         if (ret < 0) {
-            dump_error(s, "dump: failed to write elf notes.\n");
-            return -1;
+            dump_error(s, "dump: failed to write elf notes", errp);
+            return;
         }
     }
 
     CPU_FOREACH(cpu) {
         ret = cpu_write_elf32_qemunote(f, cpu, s);
         if (ret < 0) {
-            dump_error(s, "dump: failed to write CPU status.\n");
-            return -1;
+            dump_error(s, "dump: failed to write CPU status", errp);
+            return;
         }
     }
-
-    return 0;
 }
 
-static int write_elf_section(DumpState *s, int type)
+static void write_elf_section(DumpState *s, int type, Error **errp)
 {
     Elf32_Shdr shdr32;
     Elf64_Shdr shdr64;
@@ -348,50 +325,44 @@ static int write_elf_section(DumpState *s, int type)
 
     ret = fd_write_vmcore(&shdr, shdr_size, s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write section header table.\n");
-        return -1;
+        dump_error(s, "dump: failed to write section header table", errp);
     }
-
-    return 0;
 }
 
-static int write_data(DumpState *s, void *buf, int length)
+static void write_data(DumpState *s, void *buf, int length, Error **errp)
 {
     int ret;
 
     ret = fd_write_vmcore(buf, length, s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to save memory.\n");
-        return -1;
+        dump_error(s, "dump: failed to save memory", errp);
     }
-
-    return 0;
 }
 
-/* write the memroy to vmcore. 1 page per I/O. */
-static int write_memory(DumpState *s, GuestPhysBlock *block, ram_addr_t start,
-                        int64_t size)
+/* write the memory to vmcore. 1 page per I/O. */
+static void write_memory(DumpState *s, GuestPhysBlock *block, ram_addr_t start,
+                         int64_t size, Error **errp)
 {
     int64_t i;
-    int ret;
+    Error *local_err = NULL;
 
     for (i = 0; i < size / TARGET_PAGE_SIZE; i++) {
-        ret = write_data(s, block->host_addr + start + i * TARGET_PAGE_SIZE,
-                         TARGET_PAGE_SIZE);
-        if (ret < 0) {
-            return ret;
+        write_data(s, block->host_addr + start + i * TARGET_PAGE_SIZE,
+                   TARGET_PAGE_SIZE, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
         }
     }
 
     if ((size % TARGET_PAGE_SIZE) != 0) {
-        ret = write_data(s, block->host_addr + start + i * TARGET_PAGE_SIZE,
-                         size % TARGET_PAGE_SIZE);
-        if (ret < 0) {
-            return ret;
+        write_data(s, block->host_addr + start + i * TARGET_PAGE_SIZE,
+                   size % TARGET_PAGE_SIZE, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
         }
     }
-
-    return 0;
 }
 
 /* get the memory's offset and size in the vmcore */
@@ -456,13 +427,13 @@ static void get_offset_range(hwaddr phys_addr,
     }
 }
 
-static int write_elf_loads(DumpState *s)
+static void write_elf_loads(DumpState *s, Error **errp)
 {
     hwaddr offset, filesz;
     MemoryMapping *memory_mapping;
     uint32_t phdr_index = 1;
-    int ret;
     uint32_t max_index;
+    Error *local_err = NULL;
 
     if (s->have_section) {
         max_index = s->sh_info;
@@ -475,29 +446,28 @@ static int write_elf_loads(DumpState *s)
                          memory_mapping->length,
                          s, &offset, &filesz);
         if (s->dump_info.d_class == ELFCLASS64) {
-            ret = write_elf64_load(s, memory_mapping, phdr_index++, offset,
-                                   filesz);
+            write_elf64_load(s, memory_mapping, phdr_index++, offset,
+                             filesz, &local_err);
         } else {
-            ret = write_elf32_load(s, memory_mapping, phdr_index++, offset,
-                                   filesz);
+            write_elf32_load(s, memory_mapping, phdr_index++, offset,
+                             filesz, &local_err);
         }
 
-        if (ret < 0) {
-            return -1;
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
         }
 
         if (phdr_index >= max_index) {
             break;
         }
     }
-
-    return 0;
 }
 
 /* write elf header, PT_NOTE and elf note to vmcore. */
-static int dump_begin(DumpState *s)
+static void dump_begin(DumpState *s, Error **errp)
 {
-    int ret;
+    Error *local_err = NULL;
 
     /*
      * the vmcore's format is:
@@ -525,69 +495,81 @@ static int dump_begin(DumpState *s)
 
     /* write elf header to vmcore */
     if (s->dump_info.d_class == ELFCLASS64) {
-        ret = write_elf64_header(s);
+        write_elf64_header(s, &local_err);
     } else {
-        ret = write_elf32_header(s);
+        write_elf32_header(s, &local_err);
     }
-    if (ret < 0) {
-        return -1;
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
     }
 
     if (s->dump_info.d_class == ELFCLASS64) {
         /* write PT_NOTE to vmcore */
-        if (write_elf64_note(s) < 0) {
-            return -1;
+        write_elf64_note(s, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
         }
 
         /* write all PT_LOAD to vmcore */
-        if (write_elf_loads(s) < 0) {
-            return -1;
+        write_elf_loads(s, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
         }
 
         /* write section to vmcore */
         if (s->have_section) {
-            if (write_elf_section(s, 1) < 0) {
-                return -1;
+            write_elf_section(s, 1, &local_err);
+            if (local_err) {
+                error_propagate(errp, local_err);
+                return;
             }
         }
 
         /* write notes to vmcore */
-        if (write_elf64_notes(fd_write_vmcore, s) < 0) {
-            return -1;
+        write_elf64_notes(fd_write_vmcore, s, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
         }
-
     } else {
         /* write PT_NOTE to vmcore */
-        if (write_elf32_note(s) < 0) {
-            return -1;
+        write_elf32_note(s, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
         }
 
         /* write all PT_LOAD to vmcore */
-        if (write_elf_loads(s) < 0) {
-            return -1;
+        write_elf_loads(s, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
         }
 
         /* write section to vmcore */
         if (s->have_section) {
-            if (write_elf_section(s, 0) < 0) {
-                return -1;
+            write_elf_section(s, 0, &local_err);
+            if (local_err) {
+                error_propagate(errp, local_err);
+                return;
             }
         }
 
         /* write notes to vmcore */
-        if (write_elf32_notes(fd_write_vmcore, s) < 0) {
-            return -1;
+        write_elf32_notes(fd_write_vmcore, s, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
         }
     }
-
-    return 0;
 }
 
-/* write PT_LOAD to vmcore */
-static int dump_completed(DumpState *s)
+static void dump_completed(DumpState *s)
 {
     dump_cleanup(s);
-    return 0;
 }
 
 static int get_next_block(DumpState *s, GuestPhysBlock *block)
@@ -618,13 +600,13 @@ static int get_next_block(DumpState *s, GuestPhysBlock *block)
 }
 
 /* write all memory to vmcore */
-static int dump_iterate(DumpState *s)
+static void dump_iterate(DumpState *s, Error **errp)
 {
     GuestPhysBlock *block;
     int64_t size;
-    int ret;
+    Error *local_err = NULL;
 
-    while (1) {
+    do {
         block = s->next_block;
 
         size = block->target_end - block->target_start;
@@ -634,34 +616,28 @@ static int dump_iterate(DumpState *s)
                 size -= block->target_end - (s->begin + s->length);
             }
         }
-        ret = write_memory(s, block, s->start, size);
-        if (ret == -1) {
-            return ret;
+        write_memory(s, block, s->start, size, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
         }
 
-        ret = get_next_block(s, block);
-        if (ret == 1) {
-            dump_completed(s);
-            return 0;
-        }
-    }
+    } while (!get_next_block(s, block));
+
+    dump_completed(s);
 }
 
-static int create_vmcore(DumpState *s)
+static void create_vmcore(DumpState *s, Error **errp)
 {
-    int ret;
-
-    ret = dump_begin(s);
-    if (ret < 0) {
-        return -1;
-    }
+    Error *local_err = NULL;
 
-    ret = dump_iterate(s);
-    if (ret < 0) {
-        return -1;
+    dump_begin(s, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
     }
 
-    return 0;
+    dump_iterate(s, errp);
 }
 
 static int write_start_flat_header(int fd)
@@ -742,9 +718,8 @@ static int buf_write_note(const void *buf, size_t size, void *opaque)
 }
 
 /* write common header, sub header and elf note to vmcore */
-static int create_header32(DumpState *s)
+static void create_header32(DumpState *s, Error **errp)
 {
-    int ret = 0;
     DiskDumpHeader32 *dh = NULL;
     KdumpSubHeader32 *kh = NULL;
     size_t size;
@@ -753,6 +728,7 @@ static int create_header32(DumpState *s)
     uint32_t bitmap_blocks;
     uint32_t status = 0;
     uint64_t offset_note;
+    Error *local_err = NULL;
 
     /* write common header, the version of kdump-compressed format is 6th */
     size = sizeof(DiskDumpHeader32);
@@ -788,8 +764,7 @@ static int create_header32(DumpState *s)
     dh->status = cpu_to_dump32(s, status);
 
     if (write_buffer(s->fd, 0, dh, size) < 0) {
-        dump_error(s, "dump: failed to write disk dump header.\n");
-        ret = -1;
+        dump_error(s, "dump: failed to write disk dump header", errp);
         goto out;
     }
 
@@ -808,8 +783,7 @@ static int create_header32(DumpState *s)
 
     if (write_buffer(s->fd, DISKDUMP_HEADER_BLOCKS *
                      block_size, kh, size) < 0) {
-        dump_error(s, "dump: failed to write kdump sub header.\n");
-        ret = -1;
+        dump_error(s, "dump: failed to write kdump sub header", errp);
         goto out;
     }
 
@@ -818,15 +792,14 @@ static int create_header32(DumpState *s)
     s->note_buf_offset = 0;
 
     /* use s->note_buf to store notes temporarily */
-    if (write_elf32_notes(buf_write_note, s) < 0) {
-        ret = -1;
+    write_elf32_notes(buf_write_note, s, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
         goto out;
     }
-
     if (write_buffer(s->fd, offset_note, s->note_buf,
                      s->note_size) < 0) {
-        dump_error(s, "dump: failed to write notes");
-        ret = -1;
+        dump_error(s, "dump: failed to write notes", errp);
         goto out;
     }
 
@@ -842,14 +815,11 @@ out:
     g_free(dh);
     g_free(kh);
     g_free(s->note_buf);
-
-    return ret;
 }
 
 /* write common header, sub header and elf note to vmcore */
-static int create_header64(DumpState *s)
+static void create_header64(DumpState *s, Error **errp)
 {
-    int ret = 0;
     DiskDumpHeader64 *dh = NULL;
     KdumpSubHeader64 *kh = NULL;
     size_t size;
@@ -858,6 +828,7 @@ static int create_header64(DumpState *s)
     uint32_t bitmap_blocks;
     uint32_t status = 0;
     uint64_t offset_note;
+    Error *local_err = NULL;
 
     /* write common header, the version of kdump-compressed format is 6th */
     size = sizeof(DiskDumpHeader64);
@@ -893,8 +864,7 @@ static int create_header64(DumpState *s)
     dh->status = cpu_to_dump32(s, status);
 
     if (write_buffer(s->fd, 0, dh, size) < 0) {
-        dump_error(s, "dump: failed to write disk dump header.\n");
-        ret = -1;
+        dump_error(s, "dump: failed to write disk dump header", errp);
         goto out;
     }
 
@@ -913,8 +883,7 @@ static int create_header64(DumpState *s)
 
     if (write_buffer(s->fd, DISKDUMP_HEADER_BLOCKS *
                      block_size, kh, size) < 0) {
-        dump_error(s, "dump: failed to write kdump sub header.\n");
-        ret = -1;
+        dump_error(s, "dump: failed to write kdump sub header", errp);
         goto out;
     }
 
@@ -923,15 +892,15 @@ static int create_header64(DumpState *s)
     s->note_buf_offset = 0;
 
     /* use s->note_buf to store notes temporarily */
-    if (write_elf64_notes(buf_write_note, s) < 0) {
-        ret = -1;
+    write_elf64_notes(buf_write_note, s, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
         goto out;
     }
 
     if (write_buffer(s->fd, offset_note, s->note_buf,
                      s->note_size) < 0) {
-        dump_error(s, "dump: failed to write notes");
-        ret = -1;
+        dump_error(s, "dump: failed to write notes", errp);
         goto out;
     }
 
@@ -947,16 +916,19 @@ out:
     g_free(dh);
     g_free(kh);
     g_free(s->note_buf);
-
-    return ret;
 }
 
-static int write_dump_header(DumpState *s)
+static void write_dump_header(DumpState *s, Error **errp)
 {
+     Error *local_err = NULL;
+
     if (s->dump_info.d_class == ELFCLASS32) {
-        return create_header32(s);
+        create_header32(s, &local_err);
     } else {
-        return create_header64(s);
+        create_header64(s, &local_err);
+    }
+    if (local_err) {
+        error_propagate(errp, local_err);
     }
 }
 
@@ -1070,7 +1042,7 @@ static bool get_next_page(GuestPhysBlock **blockptr, uint64_t *pfnptr,
     return true;
 }
 
-static int write_dump_bitmap(DumpState *s)
+static void write_dump_bitmap(DumpState *s, Error **errp)
 {
     int ret = 0;
     uint64_t last_pfn, pfn;
@@ -1091,8 +1063,7 @@ static int write_dump_bitmap(DumpState *s)
     while (get_next_page(&block_iter, &pfn, NULL, s)) {
         ret = set_dump_bitmap(last_pfn, pfn, true, dump_bitmap_buf, s);
         if (ret < 0) {
-            dump_error(s, "dump: failed to set dump_bitmap.\n");
-            ret = -1;
+            dump_error(s, "dump: failed to set dump_bitmap", errp);
             goto out;
         }
 
@@ -1109,8 +1080,7 @@ static int write_dump_bitmap(DumpState *s)
         ret = set_dump_bitmap(last_pfn, last_pfn + PFN_BUFBITMAP, false,
                               dump_bitmap_buf, s);
         if (ret < 0) {
-            dump_error(s, "dump: failed to sync dump_bitmap.\n");
-            ret = -1;
+            dump_error(s, "dump: failed to sync dump_bitmap", errp);
             goto out;
         }
     }
@@ -1120,8 +1090,6 @@ static int write_dump_bitmap(DumpState *s)
 
 out:
     g_free(dump_bitmap_buf);
-
-    return ret;
 }
 
 static void prepare_data_cache(DataCache *data_cache, DumpState *s,
@@ -1201,7 +1169,7 @@ static inline bool is_zero_page(const uint8_t *buf, size_t page_size)
     return buffer_is_zero(buf, page_size);
 }
 
-static int write_dump_pages(DumpState *s)
+static void write_dump_pages(DumpState *s, Error **errp)
 {
     int ret = 0;
     DataCache page_desc, page_data;
@@ -1245,7 +1213,7 @@ static int write_dump_pages(DumpState *s)
     ret = write_cache(&page_data, buf, TARGET_PAGE_SIZE, false);
     g_free(buf);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write page data(zero page).\n");
+        dump_error(s, "dump: failed to write page data (zero page)", errp);
         goto out;
     }
 
@@ -1261,7 +1229,7 @@ static int write_dump_pages(DumpState *s)
             ret = write_cache(&page_desc, &pd_zero, sizeof(PageDescriptor),
                               false);
             if (ret < 0) {
-                dump_error(s, "dump: failed to write page desc.\n");
+                dump_error(s, "dump: failed to write page desc", errp);
                 goto out;
             }
         } else {
@@ -1286,7 +1254,7 @@ static int write_dump_pages(DumpState *s)
 
                 ret = write_cache(&page_data, buf_out, size_out, false);
                 if (ret < 0) {
-                    dump_error(s, "dump: failed to write page data.\n");
+                    dump_error(s, "dump: failed to write page data", errp);
                     goto out;
                 }
 #ifdef CONFIG_LZO
@@ -1299,7 +1267,7 @@ static int write_dump_pages(DumpState *s)
 
                 ret = write_cache(&page_data, buf_out, size_out, false);
                 if (ret < 0) {
-                    dump_error(s, "dump: failed to write page data.\n");
+                    dump_error(s, "dump: failed to write page data", errp);
                     goto out;
                 }
 #endif
@@ -1313,7 +1281,7 @@ static int write_dump_pages(DumpState *s)
 
                 ret = write_cache(&page_data, buf_out, size_out, false);
                 if (ret < 0) {
-                    dump_error(s, "dump: failed to write page data.\n");
+                    dump_error(s, "dump: failed to write page data", errp);
                     goto out;
                 }
 #endif
@@ -1328,7 +1296,7 @@ static int write_dump_pages(DumpState *s)
 
                 ret = write_cache(&page_data, buf, TARGET_PAGE_SIZE, false);
                 if (ret < 0) {
-                    dump_error(s, "dump: failed to write page data.\n");
+                    dump_error(s, "dump: failed to write page data", errp);
                     goto out;
                 }
             }
@@ -1340,7 +1308,7 @@ static int write_dump_pages(DumpState *s)
 
             ret = write_cache(&page_desc, &pd, sizeof(PageDescriptor), false);
             if (ret < 0) {
-                dump_error(s, "dump: failed to write page desc.\n");
+                dump_error(s, "dump: failed to write page desc", errp);
                 goto out;
             }
         }
@@ -1348,12 +1316,12 @@ static int write_dump_pages(DumpState *s)
 
     ret = write_cache(&page_desc, NULL, 0, true);
     if (ret < 0) {
-        dump_error(s, "dump: failed to sync cache for page_desc.\n");
+        dump_error(s, "dump: failed to sync cache for page_desc", errp);
         goto out;
     }
     ret = write_cache(&page_data, NULL, 0, true);
     if (ret < 0) {
-        dump_error(s, "dump: failed to sync cache for page_data.\n");
+        dump_error(s, "dump: failed to sync cache for page_data", errp);
         goto out;
     }
 
@@ -1366,13 +1334,12 @@ out:
 #endif
 
     g_free(buf_out);
-
-    return ret;
 }
 
-static int create_kdump_vmcore(DumpState *s)
+static void create_kdump_vmcore(DumpState *s, Error **errp)
 {
     int ret;
+    Error *local_err = NULL;
 
     /*
      * the kdump-compressed format is:
@@ -1398,34 +1365,35 @@ static int create_kdump_vmcore(DumpState *s)
 
     ret = write_start_flat_header(s->fd);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write start flat header.\n");
-        return -1;
+        dump_error(s, "dump: failed to write start flat header", errp);
+        return;
     }
 
-    ret = write_dump_header(s);
-    if (ret < 0) {
-        return -1;
+    write_dump_header(s, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
     }
 
-    ret = write_dump_bitmap(s);
-    if (ret < 0) {
-        return -1;
+    write_dump_bitmap(s, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
     }
 
-    ret = write_dump_pages(s);
-    if (ret < 0) {
-        return -1;
+    write_dump_pages(s, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
     }
 
     ret = write_end_flat_header(s->fd);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write end flat header.\n");
-        return -1;
+        dump_error(s, "dump: failed to write end flat header", errp);
+        return;
     }
 
     dump_completed(s);
-
-    return 0;
 }
 
 static ram_addr_t get_start_block(DumpState *s)
@@ -1464,9 +1432,9 @@ static void get_max_mapnr(DumpState *s)
     s->max_mapnr = paddr_to_pfn(last_block->target_end);
 }
 
-static int dump_init(DumpState *s, int fd, bool has_format,
-                     DumpGuestMemoryFormat format, bool paging, bool has_filter,
-                     int64_t begin, int64_t length, Error **errp)
+static void dump_init(DumpState *s, int fd, bool has_format,
+                      DumpGuestMemoryFormat format, bool paging, bool has_filter,
+                      int64_t begin, int64_t length, Error **errp)
 {
     CPUState *cpu;
     int nr_cpus;
@@ -1499,6 +1467,8 @@ static int dump_init(DumpState *s, int fd, bool has_format,
     s->begin = begin;
     s->length = length;
 
+    memory_mapping_list_init(&s->list);
+
     guest_phys_blocks_init(&s->guest_phys_blocks);
     guest_phys_blocks_append(&s->guest_phys_blocks);
 
@@ -1526,7 +1496,6 @@ static int dump_init(DumpState *s, int fd, bool has_format,
     }
 
     /* get memory mapping */
-    memory_mapping_list_init(&s->list);
     if (paging) {
         qemu_get_guest_memory_mapping(&s->list, &s->guest_phys_blocks, &err);
         if (err != NULL) {
@@ -1570,7 +1539,7 @@ static int dump_init(DumpState *s, int fd, bool has_format,
             s->flag_compress = 0;
         }
 
-        return 0;
+        return;
     }
 
     if (s->has_filter) {
@@ -1619,16 +1588,10 @@ static int dump_init(DumpState *s, int fd, bool has_format,
         }
     }
 
-    return 0;
+    return;
 
 cleanup:
-    guest_phys_blocks_free(&s->guest_phys_blocks);
-
-    if (s->resume) {
-        vm_start();
-    }
-
-    return -1;
+    dump_cleanup(s);
 }
 
 void qmp_dump_guest_memory(bool paging, const char *file, bool has_begin,
@@ -1639,7 +1602,7 @@ void qmp_dump_guest_memory(bool paging, const char *file, bool has_begin,
     const char *p;
     int fd = -1;
     DumpState *s;
-    int ret;
+    Error *local_err = NULL;
 
     /*
      * kdump-compressed format need the whole memory dumped, so paging or
@@ -1699,21 +1662,18 @@ void qmp_dump_guest_memory(bool paging, const char *file, bool has_begin,
 
     s = g_malloc0(sizeof(DumpState));
 
-    ret = dump_init(s, fd, has_format, format, paging, has_begin,
-                    begin, length, errp);
-    if (ret < 0) {
+    dump_init(s, fd, has_format, format, paging, has_begin,
+              begin, length, &local_err);
+    if (local_err) {
         g_free(s);
+        error_propagate(errp, local_err);
         return;
     }
 
     if (has_format && format != DUMP_GUEST_MEMORY_FORMAT_ELF) {
-        if (create_kdump_vmcore(s) < 0) {
-            error_set(errp, QERR_IO_ERROR);
-        }
+        create_kdump_vmcore(s, errp);
     } else {
-        if (create_vmcore(s) < 0) {
-            error_set(errp, QERR_IO_ERROR);
-        }
+        create_vmcore(s, errp);
     }
 
     g_free(s);
diff --git a/exec.c b/exec.c
index 765bd942e..71ac104b3 100644
--- a/exec.c
+++ b/exec.c
@@ -373,7 +373,7 @@ MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
             break;
         }
 
-        iotlb = mr->iommu_ops->translate(mr, addr);
+        iotlb = mr->iommu_ops->translate(mr, addr, is_write);
         addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
                 | (addr & iotlb.addr_mask));
         len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
@@ -430,15 +430,50 @@ static int cpu_common_post_load(void *opaque, int version_id)
     return 0;
 }
 
+static int cpu_common_pre_load(void *opaque)
+{
+    CPUState *cpu = opaque;
+
+    cpu->exception_index = 0;
+
+    return 0;
+}
+
+static bool cpu_common_exception_index_needed(void *opaque)
+{
+    CPUState *cpu = opaque;
+
+    return cpu->exception_index != 0;
+}
+
+static const VMStateDescription vmstate_cpu_common_exception_index = {
+    .name = "cpu_common/exception_index",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT32(exception_index, CPUState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 const VMStateDescription vmstate_cpu_common = {
     .name = "cpu_common",
     .version_id = 1,
     .minimum_version_id = 1,
+    .pre_load = cpu_common_pre_load,
     .post_load = cpu_common_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_UINT32(halted, CPUState),
         VMSTATE_UINT32(interrupt_request, CPUState),
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (VMStateSubsection[]) {
+        {
+            .vmsd = &vmstate_cpu_common_exception_index,
+            .needed = cpu_common_exception_index_needed,
+        } , {
+            /* empty */
+        }
     }
 };
 
@@ -537,6 +572,16 @@ void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
 {
 }
 
+int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
+                          int flags)
+{
+    return -ENOSYS;
+}
+
+void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
+{
+}
+
 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
                           int flags, CPUWatchpoint **watchpoint)
 {
@@ -547,12 +592,10 @@ int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
                           int flags, CPUWatchpoint **watchpoint)
 {
-    vaddr len_mask = ~(len - 1);
     CPUWatchpoint *wp;
 
-    /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
-    if ((len & (len - 1)) || (addr & ~len_mask) ||
-            len == 0 || len > TARGET_PAGE_SIZE) {
+    /* forbid ranges which are empty or run off the end of the address space */
+    if (len == 0 || (addr + len - 1) < addr) {
         error_report("tried to set invalid watchpoint at %"
                      VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
         return -EINVAL;
@@ -560,7 +603,7 @@ int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
     wp = g_malloc(sizeof(*wp));
 
     wp->vaddr = addr;
-    wp->len_mask = len_mask;
+    wp->len = len;
     wp->flags = flags;
 
     /* keep all GDB-injected watchpoints in front */
@@ -581,11 +624,10 @@ int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
                           int flags)
 {
-    vaddr len_mask = ~(len - 1);
     CPUWatchpoint *wp;
 
     QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
-        if (addr == wp->vaddr && len_mask == wp->len_mask
+        if (addr == wp->vaddr && len == wp->len
                 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
             cpu_watchpoint_remove_by_ref(cpu, wp);
             return 0;
@@ -615,6 +657,27 @@ void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
         }
     }
 }
+
+/* Return true if this watchpoint address matches the specified
+ * access (ie the address range covered by the watchpoint overlaps
+ * partially or completely with the address range covered by the
+ * access).
+ */
+static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
+                                                  vaddr addr,
+                                                  vaddr len)
+{
+    /* We know the lengths are non-zero, but a little caution is
+     * required to avoid errors in the case where the range ends
+     * exactly at the top of the address space and so addr + len
+     * wraps round to zero.
+     */
+    vaddr wpend = wp->vaddr + wp->len - 1;
+    vaddr addrend = addr + len - 1;
+
+    return !(addr > wpend || wp->vaddr > addrend);
+}
+
 #endif
 
 /* Add a breakpoint.  */
@@ -826,7 +889,7 @@ hwaddr memory_region_section_get_iotlb(CPUState *cpu,
     /* Make accesses to pages with watchpoints go via the
        watchpoint trap routines.  */
     QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
-        if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
+        if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
             /* Avoid trapping reads of pages with a write breakpoint. */
             if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
                 iotlb = PHYS_SECTION_WATCH + paddr;
@@ -846,14 +909,15 @@ static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
                              uint16_t section);
 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
 
-static void *(*phys_mem_alloc)(size_t size) = qemu_anon_ram_alloc;
+static void *(*phys_mem_alloc)(size_t size, uint64_t *align) =
+                               qemu_anon_ram_alloc;
 
 /*
  * Set a custom physical guest memory alloator.
  * Accelerators with unusual needs may need this.  Hopefully, we can
  * get rid of it eventually.
  */
-void phys_mem_set_alloc(void *(*alloc)(size_t))
+void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align))
 {
     phys_mem_alloc = alloc;
 }
@@ -996,7 +1060,7 @@ void qemu_mutex_unlock_ramlist(void)
 
 #define HUGETLBFS_MAGIC       0x958458f6
 
-static long gethugepagesize(const char *path)
+static long gethugepagesize(const char *path, Error **errp)
 {
     struct statfs fs;
     int ret;
@@ -1006,7 +1070,8 @@ static long gethugepagesize(const char *path)
     } while (ret != 0 && errno == EINTR);
 
     if (ret != 0) {
-        perror(path);
+        error_setg_errno(errp, errno, "failed to get page size of file %s",
+                         path);
         return 0;
     }
 
@@ -1024,17 +1089,23 @@ static void *file_ram_alloc(RAMBlock *block,
     char *filename;
     char *sanitized_name;
     char *c;
-    void *area;
+    void *area = NULL;
     int fd;
-    unsigned long hpagesize;
+    uint64_t hpagesize;
+    Error *local_err = NULL;
 
-    hpagesize = gethugepagesize(path);
-    if (!hpagesize) {
+    hpagesize = gethugepagesize(path, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
         goto error;
     }
+    block->mr->align = hpagesize;
 
     if (memory < hpagesize) {
-        return NULL;
+        error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
+                   "or larger than huge page size 0x%" PRIx64,
+                   memory, hpagesize);
+        goto error;
     }
 
     if (kvm_enabled() && !kvm_has_sync_mmu()) {
@@ -1044,7 +1115,7 @@ static void *file_ram_alloc(RAMBlock *block,
     }
 
     /* Make name safe to use with mkstemp by replacing '/' with '_'. */
-    sanitized_name = g_strdup(block->mr->name);
+    sanitized_name = g_strdup(memory_region_name(block->mr));
     for (c = sanitized_name; *c != '\0'; c++) {
         if (*c == '/')
             *c = '_';
@@ -1095,6 +1166,7 @@ static void *file_ram_alloc(RAMBlock *block,
 
 error:
     if (mem_prealloc) {
+        error_report("%s\n", error_get_pretty(*errp));
         exit(1);
     }
     return NULL;
@@ -1224,7 +1296,7 @@ static int memory_try_enable_merging(void *addr, size_t len)
     return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
 }
 
-static ram_addr_t ram_block_add(RAMBlock *new_block)
+static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
 {
     RAMBlock *block;
     ram_addr_t old_ram_size, new_ram_size;
@@ -1239,11 +1311,14 @@ static ram_addr_t ram_block_add(RAMBlock *new_block)
         if (xen_enabled()) {
             xen_ram_alloc(new_block->offset, new_block->length, new_block->mr);
         } else {
-            new_block->host = phys_mem_alloc(new_block->length);
+            new_block->host = phys_mem_alloc(new_block->length,
+                                             &new_block->mr->align);
             if (!new_block->host) {
-                fprintf(stderr, "Cannot set up guest memory '%s': %s\n",
-                        new_block->mr->name, strerror(errno));
-                exit(1);
+                error_setg_errno(errp, errno,
+                                 "cannot set up guest memory '%s'",
+                                 memory_region_name(new_block->mr));
+                qemu_mutex_unlock_ramlist();
+                return -1;
             }
             memory_try_enable_merging(new_block->host, new_block->length);
         }
@@ -1294,6 +1369,8 @@ ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
                                     Error **errp)
 {
     RAMBlock *new_block;
+    ram_addr_t addr;
+    Error *local_err = NULL;
 
     if (xen_enabled()) {
         error_setg(errp, "-mem-path not supported with Xen");
@@ -1323,14 +1400,22 @@ ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
         return -1;
     }
 
-    return ram_block_add(new_block);
+    addr = ram_block_add(new_block, &local_err);
+    if (local_err) {
+        g_free(new_block);
+        error_propagate(errp, local_err);
+        return -1;
+    }
+    return addr;
 }
 #endif
 
 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
-                                   MemoryRegion *mr)
+                                   MemoryRegion *mr, Error **errp)
 {
     RAMBlock *new_block;
+    ram_addr_t addr;
+    Error *local_err = NULL;
 
     size = TARGET_PAGE_ALIGN(size);
     new_block = g_malloc0(sizeof(*new_block));
@@ -1341,12 +1426,18 @@ ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
     if (host) {
         new_block->flags |= RAM_PREALLOC;
     }
-    return ram_block_add(new_block);
+    addr = ram_block_add(new_block, &local_err);
+    if (local_err) {
+        g_free(new_block);
+        error_propagate(errp, local_err);
+        return -1;
+    }
+    return addr;
 }
 
-ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
+ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
 {
-    return qemu_ram_alloc_from_ptr(size, NULL, mr);
+    return qemu_ram_alloc_from_ptr(size, NULL, mr, errp);
 }
 
 void qemu_ram_free_from_ptr(ram_addr_t addr)
@@ -1590,7 +1681,7 @@ static const MemoryRegionOps notdirty_mem_ops = {
 };
 
 /* Generate a debug exception if a watchpoint has been hit.  */
-static void check_watchpoint(int offset, int len_mask, int flags)
+static void check_watchpoint(int offset, int len, int flags)
 {
     CPUState *cpu = current_cpu;
     CPUArchState *env = cpu->env_ptr;
@@ -1608,9 +1699,14 @@ static void check_watchpoint(int offset, int len_mask, int flags)
     }
     vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
     QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
-        if ((vaddr == (wp->vaddr & len_mask) ||
-             (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
-            wp->flags |= BP_WATCHPOINT_HIT;
+        if (cpu_watchpoint_address_matches(wp, vaddr, len)
+            && (wp->flags & flags)) {
+            if (flags == BP_MEM_READ) {
+                wp->flags |= BP_WATCHPOINT_HIT_READ;
+            } else {
+                wp->flags |= BP_WATCHPOINT_HIT_WRITE;
+            }
+            wp->hitaddr = vaddr;
             if (!cpu->watchpoint_hit) {
                 cpu->watchpoint_hit = wp;
                 tb_check_watchpoint(cpu);
@@ -1635,7 +1731,7 @@ static void check_watchpoint(int offset, int len_mask, int flags)
 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
-    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
+    check_watchpoint(addr & ~TARGET_PAGE_MASK, size, BP_MEM_READ);
     switch (size) {
     case 1: return ldub_phys(&address_space_memory, addr);
     case 2: return lduw_phys(&address_space_memory, addr);
@@ -1647,7 +1743,7 @@ static uint64_t watch_mem_read(void *opaque, hwaddr addr,
 static void watch_mem_write(void *opaque, hwaddr addr,
                             uint64_t val, unsigned size)
 {
-    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
+    check_watchpoint(addr & ~TARGET_PAGE_MASK, size, BP_MEM_WRITE);
     switch (size) {
     case 1:
         stb_phys(&address_space_memory, addr, val);
@@ -1973,10 +2069,8 @@ int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
 static void invalidate_and_set_dirty(hwaddr addr,
                                      hwaddr length)
 {
-    if (cpu_physical_memory_is_clean(addr)) {
-        /* invalidate code */
-        tb_invalidate_phys_page_range(addr, addr + length, 0);
-        /* set dirty bit */
+    if (cpu_physical_memory_range_includes_clean(addr, length)) {
+        tb_invalidate_phys_range(addr, addr + length, 0);
         cpu_physical_memory_set_dirty_range_nocode(addr, length);
     }
     xen_modified_memory(addr, length);
diff --git a/fpu/softfloat.c b/fpu/softfloat.c
index 9274ebf10..16b21ebe6 100644
--- a/fpu/softfloat.c
+++ b/fpu/softfloat.c
@@ -7240,13 +7240,17 @@ int float128_compare_quiet( float128 a, float128 b STATUS_PARAM )
  * minnum() and maxnum correspond to the IEEE 754-2008 minNum()
  * and maxNum() operations. min() and max() are the typical min/max
  * semantics provided by many CPUs which predate that specification.
+ *
+ * minnummag() and maxnummag() functions correspond to minNumMag()
+ * and minNumMag() from the IEEE-754 2008.
  */
 #define MINMAX(s)                                                       \
 static inline float ## s float ## s ## _minmax(float ## s a, float ## s b,     \
-                                        int ismin, int isieee STATUS_PARAM) \
+                                               int ismin, int isieee,   \
+                                               int ismag STATUS_PARAM)  \
 {                                                                       \
     flag aSign, bSign;                                                  \
-    uint ## s ## _t av, bv;                                             \
+    uint ## s ## _t av, bv, aav, abv;                                   \
     a = float ## s ## _squash_input_denormal(a STATUS_VAR);             \
     b = float ## s ## _squash_input_denormal(b STATUS_VAR);             \
     if (float ## s ## _is_any_nan(a) ||                                 \
@@ -7266,6 +7270,17 @@ static inline float ## s float ## s ## _minmax(float ## s a, float ## s b,     \
     bSign = extractFloat ## s ## Sign(b);                               \
     av = float ## s ## _val(a);                                         \
     bv = float ## s ## _val(b);                                         \
+    if (ismag) {                                                        \
+        aav = float ## s ## _abs(av);                                   \
+        abv = float ## s ## _abs(bv);                                   \
+        if (aav != abv) {                                               \
+            if (ismin) {                                                \
+                return (aav < abv) ? a : b;                             \
+            } else {                                                    \
+                return (aav < abv) ? b : a;                             \
+            }                                                           \
+        }                                                               \
+    }                                                                   \
     if (aSign != bSign) {                                               \
         if (ismin) {                                                    \
             return aSign ? a : b;                                       \
@@ -7283,22 +7298,32 @@ static inline float ## s float ## s ## _minmax(float ## s a, float ## s b,     \
                                                                         \
 float ## s float ## s ## _min(float ## s a, float ## s b STATUS_PARAM)  \
 {                                                                       \
-    return float ## s ## _minmax(a, b, 1, 0 STATUS_VAR);                \
+    return float ## s ## _minmax(a, b, 1, 0, 0 STATUS_VAR);             \
 }                                                                       \
                                                                         \
 float ## s float ## s ## _max(float ## s a, float ## s b STATUS_PARAM)  \
 {                                                                       \
-    return float ## s ## _minmax(a, b, 0, 0 STATUS_VAR);                \
+    return float ## s ## _minmax(a, b, 0, 0, 0 STATUS_VAR);             \
 }                                                                       \
                                                                         \
 float ## s float ## s ## _minnum(float ## s a, float ## s b STATUS_PARAM) \
 {                                                                       \
-    return float ## s ## _minmax(a, b, 1, 1 STATUS_VAR);                \
+    return float ## s ## _minmax(a, b, 1, 1, 0 STATUS_VAR);             \
 }                                                                       \
                                                                         \
 float ## s float ## s ## _maxnum(float ## s a, float ## s b STATUS_PARAM) \
 {                                                                       \
-    return float ## s ## _minmax(a, b, 0, 1 STATUS_VAR);                \
+    return float ## s ## _minmax(a, b, 0, 1, 0 STATUS_VAR);             \
+}                                                                       \
+                                                                        \
+float ## s float ## s ## _minnummag(float ## s a, float ## s b STATUS_PARAM) \
+{                                                                       \
+    return float ## s ## _minmax(a, b, 1, 1, 1 STATUS_VAR);             \
+}                                                                       \
+                                                                        \
+float ## s float ## s ## _maxnummag(float ## s a, float ## s b STATUS_PARAM) \
+{                                                                       \
+    return float ## s ## _minmax(a, b, 0, 1, 1 STATUS_VAR);             \
 }
 
 MINMAX(32)
diff --git a/gdb-xml/s390-acr.xml b/gdb-xml/s390-acr.xml
new file mode 100644
index 000000000..71dfb2052
--- /dev/null
+++ b/gdb-xml/s390-acr.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!-- Copyright (C) 2010-2014 Free Software Foundation, Inc.
+
+     Copying and distribution of this file, with or without modification,
+     are permitted in any medium without royalty provided the copyright
+     notice and this notice are preserved.  -->
+
+<!DOCTYPE feature SYSTEM "gdb-target.dtd">
+<feature name="org.gnu.gdb.s390.acr">
+  <reg name="acr0" bitsize="32" type="uint32" group="access"/>
+  <reg name="acr1" bitsize="32" type="uint32" group="access"/>
+  <reg name="acr2" bitsize="32" type="uint32" group="access"/>
+  <reg name="acr3" bitsize="32" type="uint32" group="access"/>
+  <reg name="acr4" bitsize="32" type="uint32" group="access"/>
+  <reg name="acr5" bitsize="32" type="uint32" group="access"/>
+  <reg name="acr6" bitsize="32" type="uint32" group="access"/>
+  <reg name="acr7" bitsize="32" type="uint32" group="access"/>
+  <reg name="acr8" bitsize="32" type="uint32" group="access"/>
+  <reg name="acr9" bitsize="32" type="uint32" group="access"/>
+  <reg name="acr10" bitsize="32" type="uint32" group="access"/>
+  <reg name="acr11" bitsize="32" type="uint32" group="access"/>
+  <reg name="acr12" bitsize="32" type="uint32" group="access"/>
+  <reg name="acr13" bitsize="32" type="uint32" group="access"/>
+  <reg name="acr14" bitsize="32" type="uint32" group="access"/>
+  <reg name="acr15" bitsize="32" type="uint32" group="access"/>
+</feature>
diff --git a/gdb-xml/s390-fpr.xml b/gdb-xml/s390-fpr.xml
new file mode 100644
index 000000000..7de0c136a
--- /dev/null
+++ b/gdb-xml/s390-fpr.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<!-- Copyright (C) 2010-2014 Free Software Foundation, Inc.
+
+     Copying and distribution of this file, with or without modification,
+     are permitted in any medium without royalty provided the copyright
+     notice and this notice are preserved.  -->
+
+<!DOCTYPE feature SYSTEM "gdb-target.dtd">
+<feature name="org.gnu.gdb.s390.fpr">
+  <reg name="fpc" bitsize="32" type="uint32" group="float"/>
+  <reg name="f0" bitsize="64" type="ieee_double" group="float"/>
+  <reg name="f1" bitsize="64" type="ieee_double" group="float"/>
+  <reg name="f2" bitsize="64" type="ieee_double" group="float"/>
+  <reg name="f3" bitsize="64" type="ieee_double" group="float"/>
+  <reg name="f4" bitsize="64" type="ieee_double" group="float"/>
+  <reg name="f5" bitsize="64" type="ieee_double" group="float"/>
+  <reg name="f6" bitsize="64" type="ieee_double" group="float"/>
+  <reg name="f7" bitsize="64" type="ieee_double" group="float"/>
+  <reg name="f8" bitsize="64" type="ieee_double" group="float"/>
+  <reg name="f9" bitsize="64" type="ieee_double" group="float"/>
+  <reg name="f10" bitsize="64" type="ieee_double" group="float"/>
+  <reg name="f11" bitsize="64" type="ieee_double" group="float"/>
+  <reg name="f12" bitsize="64" type="ieee_double" group="float"/>
+  <reg name="f13" bitsize="64" type="ieee_double" group="float"/>
+  <reg name="f14" bitsize="64" type="ieee_double" group="float"/>
+  <reg name="f15" bitsize="64" type="ieee_double" group="float"/>
+</feature>
diff --git a/gdb-xml/s390x-core64.xml b/gdb-xml/s390x-core64.xml
new file mode 100644
index 000000000..15234378e
--- /dev/null
+++ b/gdb-xml/s390x-core64.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0"?>
+<!-- Copyright (C) 2010-2014 Free Software Foundation, Inc.
+
+     Copying and distribution of this file, with or without modification,
+     are permitted in any medium without royalty provided the copyright
+     notice and this notice are preserved.  -->
+
+<!DOCTYPE feature SYSTEM "gdb-target.dtd">
+<feature name="org.gnu.gdb.s390.core">
+  <reg name="pswm" bitsize="64" type="uint64" group="psw"/>
+  <reg name="pswa" bitsize="64" type="uint64" group="psw"/>
+  <reg name="r0" bitsize="64" type="uint64" group="general"/>
+  <reg name="r1" bitsize="64" type="uint64" group="general"/>
+  <reg name="r2" bitsize="64" type="uint64" group="general"/>
+  <reg name="r3" bitsize="64" type="uint64" group="general"/>
+  <reg name="r4" bitsize="64" type="uint64" group="general"/>
+  <reg name="r5" bitsize="64" type="uint64" group="general"/>
+  <reg name="r6" bitsize="64" type="uint64" group="general"/>
+  <reg name="r7" bitsize="64" type="uint64" group="general"/>
+  <reg name="r8" bitsize="64" type="uint64" group="general"/>
+  <reg name="r9" bitsize="64" type="uint64" group="general"/>
+  <reg name="r10" bitsize="64" type="uint64" group="general"/>
+  <reg name="r11" bitsize="64" type="uint64" group="general"/>
+  <reg name="r12" bitsize="64" type="uint64" group="general"/>
+  <reg name="r13" bitsize="64" type="uint64" group="general"/>
+  <reg name="r14" bitsize="64" type="uint64" group="general"/>
+  <reg name="r15" bitsize="64" type="uint64" group="general"/>
+</feature>
diff --git a/gdbstub.c b/gdbstub.c
index 8afe0b701..0faca568d 100644
--- a/gdbstub.c
+++ b/gdbstub.c
@@ -625,11 +625,23 @@ void gdb_register_coprocessor(CPUState *cpu,
 }
 
 #ifndef CONFIG_USER_ONLY
-static const int xlat_gdb_type[] = {
-    [GDB_WATCHPOINT_WRITE]  = BP_GDB | BP_MEM_WRITE,
-    [GDB_WATCHPOINT_READ]   = BP_GDB | BP_MEM_READ,
-    [GDB_WATCHPOINT_ACCESS] = BP_GDB | BP_MEM_ACCESS,
-};
+/* Translate GDB watchpoint type to a flags value for cpu_watchpoint_* */
+static inline int xlat_gdb_type(CPUState *cpu, int gdbtype)
+{
+    static const int xlat[] = {
+        [GDB_WATCHPOINT_WRITE]  = BP_GDB | BP_MEM_WRITE,
+        [GDB_WATCHPOINT_READ]   = BP_GDB | BP_MEM_READ,
+        [GDB_WATCHPOINT_ACCESS] = BP_GDB | BP_MEM_ACCESS,
+    };
+
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+    int cputype = xlat[gdbtype];
+
+    if (cc->gdb_stop_before_watchpoint) {
+        cputype |= BP_STOP_BEFORE_ACCESS;
+    }
+    return cputype;
+}
 #endif
 
 static int gdb_breakpoint_insert(target_ulong addr, target_ulong len, int type)
@@ -656,10 +668,11 @@ static int gdb_breakpoint_insert(target_ulong addr, target_ulong len, int type)
     case GDB_WATCHPOINT_READ:
     case GDB_WATCHPOINT_ACCESS:
         CPU_FOREACH(cpu) {
-            err = cpu_watchpoint_insert(cpu, addr, len, xlat_gdb_type[type],
-                                        NULL);
-            if (err)
+            err = cpu_watchpoint_insert(cpu, addr, len,
+                                        xlat_gdb_type(cpu, type), NULL);
+            if (err) {
                 break;
+            }
         }
         return err;
 #endif
@@ -692,7 +705,8 @@ static int gdb_breakpoint_remove(target_ulong addr, target_ulong len, int type)
     case GDB_WATCHPOINT_READ:
     case GDB_WATCHPOINT_ACCESS:
         CPU_FOREACH(cpu) {
-            err = cpu_watchpoint_remove(cpu, addr, len, xlat_gdb_type[type]);
+            err = cpu_watchpoint_remove(cpu, addr, len,
+                                        xlat_gdb_type(cpu, type));
             if (err)
                 break;
         }
@@ -809,7 +823,10 @@ static int gdb_handle_packet(GDBState *s, const char *line_buf)
                 action = *p++;
                 signal = 0;
                 if (action == 'C' || action == 'S') {
-                    signal = strtoul(p, (char **)&p, 16);
+                    signal = gdb_signal_to_target(strtoul(p, (char **)&p, 16));
+                    if (signal == -1) {
+                        signal = 0;
+                    }
                 } else if (action != 'c' && action != 's') {
                     res = 0;
                     break;
@@ -1707,7 +1724,7 @@ int gdbserver_start(const char *device)
         qemu_add_vm_change_state_handler(gdb_vm_state_change, NULL);
 
         /* Initialize a monitor terminal for gdb */
-        mon_chr = g_malloc0(sizeof(*mon_chr));
+        mon_chr = qemu_chr_alloc();
         mon_chr->chr_write = gdb_monitor_write;
         monitor_init(mon_chr, 0);
     } else {
diff --git a/hmp-commands.hx b/hmp-commands.hx
index d0943b1ff..e37bc8b01 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -832,19 +832,17 @@ The values that can be specified here depend on the machine type, but are
 the same that can be specified in the @code{-boot} command line option.
 ETEXI
 
-#if defined(TARGET_I386) || defined(TARGET_S390X)
     {
         .name       = "nmi",
         .args_type  = "",
         .params     = "",
-        .help       = "inject an NMI on all guest's CPUs",
+        .help       = "inject an NMI",
         .mhandler.cmd = hmp_inject_nmi,
     },
-#endif
 STEXI
 @item nmi @var{cpu}
 @findex nmi
-Inject an NMI (x86) or RESTART (s390x) on the given CPU.
+Inject an NMI on the default CPU (x86/s390) or all CPUs (ppc64).
 
 ETEXI
 
@@ -1750,8 +1748,6 @@ show information about active capturing
 show list of VM snapshots
 @item info status
 show the current VM status (running|paused)
-@item info pcmcia
-show guest PCMCIA status
 @item info mice
 show which guest mouse is receiving events
 @item info vnc
@@ -1780,6 +1776,8 @@ show qdev device model list
 show roms
 @item info tpm
 show the TPM device
+@item info memory-devices
+show the memory devices
 @end table
 ETEXI
 
diff --git a/hmp.c b/hmp.c
index 4d1838e9e..63d76868b 100644
--- a/hmp.c
+++ b/hmp.c
@@ -679,6 +679,8 @@ void hmp_info_block_jobs(Monitor *mon, const QDict *qdict)
         }
         list = list->next;
     }
+
+    qapi_free_BlockJobInfoList(list);
 }
 
 void hmp_info_tpm(Monitor *mon, const QDict *qdict)
@@ -1687,6 +1689,7 @@ void hmp_info_memdev(Monitor *mon, const QDict *qdict)
     MemdevList *memdev_list = qmp_query_memdev(&err);
     MemdevList *m = memdev_list;
     StringOutputVisitor *ov;
+    char *str;
     int i = 0;
 
 
@@ -1704,13 +1707,54 @@ void hmp_info_memdev(Monitor *mon, const QDict *qdict)
                        m->value->prealloc ? "true" : "false");
         monitor_printf(mon, "  policy: %s\n",
                        HostMemPolicy_lookup[m->value->policy]);
-        monitor_printf(mon, "  host nodes: %s\n",
-                       string_output_get_string(ov));
+        str = string_output_get_string(ov);
+        monitor_printf(mon, "  host nodes: %s\n", str);
 
+        g_free(str);
         string_output_visitor_cleanup(ov);
         m = m->next;
         i++;
     }
 
     monitor_printf(mon, "\n");
+
+    qapi_free_MemdevList(memdev_list);
+}
+
+void hmp_info_memory_devices(Monitor *mon, const QDict *qdict)
+{
+    Error *err = NULL;
+    MemoryDeviceInfoList *info_list = qmp_query_memory_devices(&err);
+    MemoryDeviceInfoList *info;
+    MemoryDeviceInfo *value;
+    PCDIMMDeviceInfo *di;
+
+    for (info = info_list; info; info = info->next) {
+        value = info->value;
+
+        if (value) {
+            switch (value->kind) {
+            case MEMORY_DEVICE_INFO_KIND_DIMM:
+                di = value->dimm;
+
+                monitor_printf(mon, "Memory device [%s]: \"%s\"\n",
+                               MemoryDeviceInfoKind_lookup[value->kind],
+                               di->id ? di->id : "");
+                monitor_printf(mon, "  addr: 0x%" PRIx64 "\n", di->addr);
+                monitor_printf(mon, "  slot: %" PRId64 "\n", di->slot);
+                monitor_printf(mon, "  node: %" PRId64 "\n", di->node);
+                monitor_printf(mon, "  size: %" PRIu64 "\n", di->size);
+                monitor_printf(mon, "  memdev: %s\n", di->memdev);
+                monitor_printf(mon, "  hotplugged: %s\n",
+                               di->hotplugged ? "true" : "false");
+                monitor_printf(mon, "  hotpluggable: %s\n",
+                               di->hotpluggable ? "true" : "false");
+                break;
+            default:
+                break;
+            }
+        }
+    }
+
+    qapi_free_MemoryDeviceInfoList(info_list);
 }
diff --git a/hmp.h b/hmp.h
index 4fd3c4a90..4bb5dca45 100644
--- a/hmp.h
+++ b/hmp.h
@@ -94,6 +94,7 @@ void hmp_cpu_add(Monitor *mon, const QDict *qdict);
 void hmp_object_add(Monitor *mon, const QDict *qdict);
 void hmp_object_del(Monitor *mon, const QDict *qdict);
 void hmp_info_memdev(Monitor *mon, const QDict *qdict);
+void hmp_info_memory_devices(Monitor *mon, const QDict *qdict);
 void object_add_completion(ReadLineState *rs, int nb_args, const char *str);
 void object_del_completion(ReadLineState *rs, int nb_args, const char *str);
 void device_add_completion(ReadLineState *rs, int nb_args, const char *str);
diff --git a/hw/9pfs/virtio-9p-local.c b/hw/9pfs/virtio-9p-local.c
index 3b0b6a9b1..a183eee66 100644
--- a/hw/9pfs/virtio-9p-local.c
+++ b/hw/9pfs/virtio-9p-local.c
@@ -135,17 +135,17 @@ static int local_lstat(FsContext *fs_ctx, V9fsPath *fs_path, struct stat *stbuf)
         mode_t tmp_mode;
         dev_t tmp_dev;
         if (getxattr(buffer, "user.virtfs.uid", &tmp_uid, sizeof(uid_t)) > 0) {
-            stbuf->st_uid = tmp_uid;
+            stbuf->st_uid = le32_to_cpu(tmp_uid);
         }
         if (getxattr(buffer, "user.virtfs.gid", &tmp_gid, sizeof(gid_t)) > 0) {
-            stbuf->st_gid = tmp_gid;
+            stbuf->st_gid = le32_to_cpu(tmp_gid);
         }
         if (getxattr(buffer, "user.virtfs.mode",
                     &tmp_mode, sizeof(mode_t)) > 0) {
-            stbuf->st_mode = tmp_mode;
+            stbuf->st_mode = le32_to_cpu(tmp_mode);
         }
         if (getxattr(buffer, "user.virtfs.rdev", &tmp_dev, sizeof(dev_t)) > 0) {
-                stbuf->st_rdev = tmp_dev;
+            stbuf->st_rdev = le64_to_cpu(tmp_dev);
         }
     } else if (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE) {
         local_mapped_file_attr(fs_ctx, path, stbuf);
@@ -255,29 +255,29 @@ static int local_set_xattr(const char *path, FsCred *credp)
     int err;
 
     if (credp->fc_uid != -1) {
-        err = setxattr(path, "user.virtfs.uid", &credp->fc_uid, sizeof(uid_t),
-                0);
+        uint32_t tmp_uid = cpu_to_le32(credp->fc_uid);
+        err = setxattr(path, "user.virtfs.uid", &tmp_uid, sizeof(uid_t), 0);
         if (err) {
             return err;
         }
     }
     if (credp->fc_gid != -1) {
-        err = setxattr(path, "user.virtfs.gid", &credp->fc_gid, sizeof(gid_t),
-                0);
+        uint32_t tmp_gid = cpu_to_le32(credp->fc_gid);
+        err = setxattr(path, "user.virtfs.gid", &tmp_gid, sizeof(gid_t), 0);
         if (err) {
             return err;
         }
     }
     if (credp->fc_mode != -1) {
-        err = setxattr(path, "user.virtfs.mode", &credp->fc_mode,
-                sizeof(mode_t), 0);
+        uint32_t tmp_mode = cpu_to_le32(credp->fc_mode);
+        err = setxattr(path, "user.virtfs.mode", &tmp_mode, sizeof(mode_t), 0);
         if (err) {
             return err;
         }
     }
     if (credp->fc_rdev != -1) {
-        err = setxattr(path, "user.virtfs.rdev", &credp->fc_rdev,
-                sizeof(dev_t), 0);
+        uint64_t tmp_rdev = cpu_to_le64(credp->fc_rdev);
+        err = setxattr(path, "user.virtfs.rdev", &tmp_rdev, sizeof(dev_t), 0);
         if (err) {
             return err;
         }
@@ -397,12 +397,15 @@ static int local_readdir_r(FsContext *ctx, V9fsFidOpenState *fs,
 
 again:
     ret = readdir_r(fs->dir, entry, result);
-    if (ctx->export_flags & V9FS_SM_MAPPED_FILE) {
+    if (ctx->export_flags & V9FS_SM_MAPPED) {
+        entry->d_type = DT_UNKNOWN;
+    } else if (ctx->export_flags & V9FS_SM_MAPPED_FILE) {
         if (!ret && *result != NULL &&
             !strcmp(entry->d_name, VIRTFS_META_DIR)) {
             /* skp the meta data directory */
             goto again;
         }
+        entry->d_type = DT_UNKNOWN;
     }
     return ret;
 }
@@ -630,21 +633,17 @@ static int local_fstat(FsContext *fs_ctx, int fid_type,
         mode_t tmp_mode;
         dev_t tmp_dev;
 
-        if (fgetxattr(fd, "user.virtfs.uid",
-                      &tmp_uid, sizeof(uid_t)) > 0) {
-            stbuf->st_uid = tmp_uid;
+        if (fgetxattr(fd, "user.virtfs.uid", &tmp_uid, sizeof(uid_t)) > 0) {
+            stbuf->st_uid = le32_to_cpu(tmp_uid);
         }
-        if (fgetxattr(fd, "user.virtfs.gid",
-                      &tmp_gid, sizeof(gid_t)) > 0) {
-            stbuf->st_gid = tmp_gid;
+        if (fgetxattr(fd, "user.virtfs.gid", &tmp_gid, sizeof(gid_t)) > 0) {
+            stbuf->st_gid = le32_to_cpu(tmp_gid);
         }
-        if (fgetxattr(fd, "user.virtfs.mode",
-                      &tmp_mode, sizeof(mode_t)) > 0) {
-            stbuf->st_mode = tmp_mode;
+        if (fgetxattr(fd, "user.virtfs.mode", &tmp_mode, sizeof(mode_t)) > 0) {
+            stbuf->st_mode = le32_to_cpu(tmp_mode);
         }
-        if (fgetxattr(fd, "user.virtfs.rdev",
-                      &tmp_dev, sizeof(dev_t)) > 0) {
-                stbuf->st_rdev = tmp_dev;
+        if (fgetxattr(fd, "user.virtfs.rdev", &tmp_dev, sizeof(dev_t)) > 0) {
+            stbuf->st_rdev = le64_to_cpu(tmp_dev);
         }
     } else if (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE) {
         errno = EOPNOTSUPP;
diff --git a/hw/9pfs/virtio-9p-proxy.c b/hw/9pfs/virtio-9p-proxy.c
index b57966d9d..59c7445de 100644
--- a/hw/9pfs/virtio-9p-proxy.c
+++ b/hw/9pfs/virtio-9p-proxy.c
@@ -1104,14 +1104,15 @@ static int connect_namedsocket(const char *path)
 
     sockfd = socket(AF_UNIX, SOCK_STREAM, 0);
     if (sockfd < 0) {
-        fprintf(stderr, "socket %s\n", strerror(errno));
+        fprintf(stderr, "failed to create socket: %s\n", strerror(errno));
         return -1;
     }
     strcpy(helper.sun_path, path);
     helper.sun_family = AF_UNIX;
     size = strlen(helper.sun_path) + sizeof(helper.sun_family);
     if (connect(sockfd, (struct sockaddr *)&helper, size) < 0) {
-        fprintf(stderr, "socket error\n");
+        fprintf(stderr, "failed to connect to %s: %s\n", path, strerror(errno));
+        close(sockfd);
         return -1;
     }
 
@@ -1154,10 +1155,12 @@ static int proxy_init(FsContext *ctx)
         sock_id = atoi(ctx->fs_root);
         if (sock_id < 0) {
             fprintf(stderr, "socket descriptor not initialized\n");
-            g_free(proxy);
-            return -1;
         }
     }
+    if (sock_id < 0) {
+        g_free(proxy);
+        return -1;
+    }
     g_free(ctx->fs_root);
     ctx->fs_root = NULL;
 
diff --git a/hw/acpi/core.c b/hw/acpi/core.c
index a7368fb24..51913d693 100644
--- a/hw/acpi/core.c
+++ b/hw/acpi/core.c
@@ -376,8 +376,11 @@ static void acpi_notify_wakeup(Notifier *notifier, void *data)
 /* ACPI PM1a EVT */
 uint16_t acpi_pm1_evt_get_sts(ACPIREGS *ar)
 {
-    int64_t d = acpi_pm_tmr_get_clock();
-    if (d >= ar->tmr.overflow_time) {
+    /* Compare ns-clock, not PM timer ticks, because
+       acpi_pm_tmr_update function uses ns for setting the timer. */
+    int64_t d = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+    if (d >= muldiv64(ar->tmr.overflow_time,
+                      get_ticks_per_sec(), PM_TIMER_FREQUENCY)) {
         ar->pm1.evt.sts |= ACPI_BITMASK_TIMER_STATUS;
     }
     return ar->pm1.evt.sts;
diff --git a/hw/acpi/cpu_hotplug.c b/hw/acpi/cpu_hotplug.c
index 2ad83a0ed..b8ebfadc3 100644
--- a/hw/acpi/cpu_hotplug.c
+++ b/hw/acpi/cpu_hotplug.c
@@ -36,28 +36,40 @@ static const MemoryRegionOps AcpiCpuHotplug_ops = {
     },
 };
 
-void AcpiCpuHotplug_add(ACPIGPE *gpe, AcpiCpuHotplug *g, CPUState *cpu)
+static void acpi_set_cpu_present_bit(AcpiCpuHotplug *g, CPUState *cpu,
+                                     Error **errp)
 {
     CPUClass *k = CPU_GET_CLASS(cpu);
     int64_t cpu_id;
 
-    *gpe->sts = *gpe->sts | ACPI_CPU_HOTPLUG_STATUS;
-    cpu_id = k->get_arch_id(CPU(cpu));
-    g_assert((cpu_id / 8) < ACPI_GPE_PROC_LEN);
+    cpu_id = k->get_arch_id(cpu);
+    if ((cpu_id / 8) >= ACPI_GPE_PROC_LEN) {
+        error_setg(errp, "acpi: invalid cpu id: %" PRIi64, cpu_id);
+        return;
+    }
+
     g->sts[cpu_id / 8] |= (1 << (cpu_id % 8));
 }
 
-void AcpiCpuHotplug_init(MemoryRegion *parent, Object *owner,
-                         AcpiCpuHotplug *gpe_cpu, uint16_t base)
+void acpi_cpu_plug_cb(ACPIREGS *ar, qemu_irq irq,
+                      AcpiCpuHotplug *g, DeviceState *dev, Error **errp)
+{
+    acpi_set_cpu_present_bit(g, CPU(dev), errp);
+    if (*errp != NULL) {
+        return;
+    }
+
+    ar->gpe.sts[0] |= ACPI_CPU_HOTPLUG_STATUS;
+    acpi_update_sci(ar, irq);
+}
+
+void acpi_cpu_hotplug_init(MemoryRegion *parent, Object *owner,
+                           AcpiCpuHotplug *gpe_cpu, uint16_t base)
 {
     CPUState *cpu;
 
     CPU_FOREACH(cpu) {
-        CPUClass *cc = CPU_GET_CLASS(cpu);
-        int64_t id = cc->get_arch_id(cpu);
-
-        g_assert((id / 8) < ACPI_GPE_PROC_LEN);
-        gpe_cpu->sts[id / 8] |= (1 << (id % 8));
+        acpi_set_cpu_present_bit(gpe_cpu, cpu, &error_abort);
     }
     memory_region_init_io(&gpe_cpu->io, owner, &AcpiCpuHotplug_ops,
                           gpe_cpu, "acpi-cpu-hotplug", ACPI_GPE_PROC_LEN);
diff --git a/hw/acpi/ich9.c b/hw/acpi/ich9.c
index 7b14bbbee..ea991a3c6 100644
--- a/hw/acpi/ich9.c
+++ b/hw/acpi/ich9.c
@@ -209,15 +209,6 @@ static void pm_powerdown_req(Notifier *n, void *opaque)
     acpi_pm1_evt_power_down(&pm->acpi_regs);
 }
 
-static void ich9_cpu_added_req(Notifier *n, void *opaque)
-{
-    ICH9LPCPMRegs *pm = container_of(n, ICH9LPCPMRegs, cpu_added_notifier);
-
-    assert(pm != NULL);
-    AcpiCpuHotplug_add(&pm->acpi_regs.gpe, &pm->gpe_cpu, CPU(opaque));
-    acpi_update_sci(&pm->acpi_regs, pm->irq);
-}
-
 void ich9_pm_init(PCIDevice *lpc_pci, ICH9LPCPMRegs *pm,
                   qemu_irq sci_irq)
 {
@@ -244,10 +235,8 @@ void ich9_pm_init(PCIDevice *lpc_pci, ICH9LPCPMRegs *pm,
     pm->powerdown_notifier.notify = pm_powerdown_req;
     qemu_register_powerdown_notifier(&pm->powerdown_notifier);
 
-    AcpiCpuHotplug_init(pci_address_space_io(lpc_pci), OBJECT(lpc_pci),
-                        &pm->gpe_cpu, ICH9_CPU_HOTPLUG_IO_BASE);
-    pm->cpu_added_notifier.notify = ich9_cpu_added_req;
-    qemu_register_cpu_added_notifier(&pm->cpu_added_notifier);
+    acpi_cpu_hotplug_init(pci_address_space_io(lpc_pci), OBJECT(lpc_pci),
+                          &pm->gpe_cpu, ICH9_CPU_HOTPLUG_IO_BASE);
 
     if (pm->acpi_memory_hotplug.is_enabled) {
         acpi_memory_hotplug_init(pci_address_space_io(lpc_pci), OBJECT(lpc_pci),
@@ -304,6 +293,8 @@ void ich9_pm_device_plug_cb(ICH9LPCPMRegs *pm, DeviceState *dev, Error **errp)
         object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
         acpi_memory_plug_cb(&pm->acpi_regs, pm->irq, &pm->acpi_memory_hotplug,
                             dev, errp);
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
+        acpi_cpu_plug_cb(&pm->acpi_regs, pm->irq, &pm->gpe_cpu, dev, errp);
     } else {
         error_setg(errp, "acpi: device plug request for not supported device"
                    " type: %s", object_get_typename(OBJECT(dev)));
diff --git a/hw/acpi/pcihp.c b/hw/acpi/pcihp.c
index fae663af1..34dedf1e8 100644
--- a/hw/acpi/pcihp.c
+++ b/hw/acpi/pcihp.c
@@ -231,7 +231,7 @@ static uint64_t pci_read(void *opaque, hwaddr addr, unsigned int size)
     uint32_t val = 0;
     int bsel = s->hotplug_select;
 
-    if (bsel < 0 || bsel > ACPI_PCIHP_MAX_HOTPLUG_BUS) {
+    if (bsel < 0 || bsel >= ACPI_PCIHP_MAX_HOTPLUG_BUS) {
         return 0;
     }
 
diff --git a/hw/acpi/piix4.c b/hw/acpi/piix4.c
index b72b34e5c..481a16c60 100644
--- a/hw/acpi/piix4.c
+++ b/hw/acpi/piix4.c
@@ -36,6 +36,7 @@
 #include "hw/mem/pc-dimm.h"
 #include "hw/acpi/memory_hotplug.h"
 #include "hw/acpi/acpi_dev_interface.h"
+#include "hw/xen/xen.h"
 
 //#define DEBUG
 
@@ -83,7 +84,6 @@ typedef struct PIIX4PMState {
     uint8_t s4_val;
 
     AcpiCpuHotplug gpe_cpu;
-    Notifier cpu_added_notifier;
 
     MemHotplugState acpi_memory_hotplug;
 } PIIX4PMState;
@@ -348,14 +348,16 @@ static void piix4_device_plug_cb(HotplugHandler *hotplug_dev,
     } else if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE)) {
         acpi_pcihp_device_plug_cb(&s->ar, s->irq, &s->acpi_pci_hotplug, dev,
                                   errp);
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
+        acpi_cpu_plug_cb(&s->ar, s->irq, &s->gpe_cpu, dev, errp);
     } else {
         error_setg(errp, "acpi: device plug request for not supported device"
                    " type: %s", object_get_typename(OBJECT(dev)));
     }
 }
 
-static void piix4_device_unplug_cb(HotplugHandler *hotplug_dev,
-                                   DeviceState *dev, Error **errp)
+static void piix4_device_unplug_request_cb(HotplugHandler *hotplug_dev,
+                                           DeviceState *dev, Error **errp)
 {
     PIIX4PMState *s = PIIX4_PM(hotplug_dev);
 
@@ -500,6 +502,9 @@ I2CBus *piix4_pm_init(PCIBus *bus, int devfn, uint32_t smb_io_base,
     s->irq = sci_irq;
     s->smi_irq = smi_irq;
     s->kvm_enabled = kvm_enabled;
+    if (xen_enabled()) {
+        s->use_acpi_pci_hotplug = false;
+    }
 
     qdev_init_nofail(dev);
 
@@ -544,15 +549,6 @@ static const MemoryRegionOps piix4_gpe_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static void piix4_cpu_added_req(Notifier *n, void *opaque)
-{
-    PIIX4PMState *s = container_of(n, PIIX4PMState, cpu_added_notifier);
-
-    assert(s != NULL);
-    AcpiCpuHotplug_add(&s->ar.gpe, &s->gpe_cpu, CPU(opaque));
-    acpi_update_sci(&s->ar, s->irq);
-}
-
 static void piix4_acpi_system_hot_add_init(MemoryRegion *parent,
                                            PCIBus *bus, PIIX4PMState *s)
 {
@@ -563,10 +559,8 @@ static void piix4_acpi_system_hot_add_init(MemoryRegion *parent,
     acpi_pcihp_init(&s->acpi_pci_hotplug, bus, parent,
                     s->use_acpi_pci_hotplug);
 
-    AcpiCpuHotplug_init(parent, OBJECT(s), &s->gpe_cpu,
-                        PIIX4_CPU_HOTPLUG_IO_BASE);
-    s->cpu_added_notifier.notify = piix4_cpu_added_req;
-    qemu_register_cpu_added_notifier(&s->cpu_added_notifier);
+    acpi_cpu_hotplug_init(parent, OBJECT(s), &s->gpe_cpu,
+                          PIIX4_CPU_HOTPLUG_IO_BASE);
 
     if (s->acpi_memory_hotplug.is_enabled) {
         acpi_memory_hotplug_init(parent, OBJECT(s), &s->acpi_memory_hotplug);
@@ -615,7 +609,7 @@ static void piix4_pm_class_init(ObjectClass *klass, void *data)
     dc->cannot_instantiate_with_device_add_yet = true;
     dc->hotpluggable = false;
     hc->plug = piix4_device_plug_cb;
-    hc->unplug = piix4_device_unplug_cb;
+    hc->unplug_request = piix4_device_unplug_request_cb;
     adevc->ospm_status = piix4_ospm_status;
 }
 
diff --git a/hw/alpha/dp264.c b/hw/alpha/dp264.c
index b178a0360..84a55e41a 100644
--- a/hw/alpha/dp264.c
+++ b/hw/alpha/dp264.c
@@ -97,7 +97,7 @@ static void clipper_init(MachineState *machine)
     /* IDE disk setup.  */
     {
         DriveInfo *hd[MAX_IDE_BUS * MAX_IDE_DEVS];
-        ide_drive_get(hd, MAX_IDE_BUS);
+        ide_drive_get(hd, ARRAY_SIZE(hd));
 
         pci_cmd646_ide_init(pci_bus, hd, 0);
     }
diff --git a/hw/alpha/typhoon.c b/hw/alpha/typhoon.c
index 67a107028..53100061d 100644
--- a/hw/alpha/typhoon.c
+++ b/hw/alpha/typhoon.c
@@ -660,7 +660,8 @@ static bool window_translate(TyphoonWindow *win, hwaddr addr,
 /* Handle PCI-to-system address translation.  */
 /* TODO: A translation failure here ought to set PCI error codes on the
    Pchip and generate a machine check interrupt.  */
-static IOMMUTLBEntry typhoon_translate_iommu(MemoryRegion *iommu, hwaddr addr)
+static IOMMUTLBEntry typhoon_translate_iommu(MemoryRegion *iommu, hwaddr addr,
+                                             bool is_write)
 {
     TyphoonPchip *pchip = container_of(iommu, TyphoonPchip, iommu);
     IOMMUTLBEntry ret;
@@ -843,7 +844,8 @@ PCIBus *typhoon_init(ram_addr_t ram_size, ISABus **isa_bus,
 
     /* Main memory region, 0x00.0000.0000.  Real hardware supports 32GB,
        but the address space hole reserved at this point is 8TB.  */
-    memory_region_init_ram(&s->ram_region, OBJECT(s), "ram", ram_size);
+    memory_region_init_ram(&s->ram_region, OBJECT(s), "ram", ram_size,
+                           &error_abort);
     vmstate_register_ram_global(&s->ram_region);
     memory_region_add_subregion(addr_space, 0, &s->ram_region);
 
diff --git a/hw/arm/armv7m.c b/hw/arm/armv7m.c
index 397e8dfb3..ef24ca40f 100644
--- a/hw/arm/armv7m.c
+++ b/hw/arm/armv7m.c
@@ -166,7 +166,7 @@ static void armv7m_reset(void *opaque)
    flash_size and sram_size are in kb.
    Returns the NVIC array.  */
 
-qemu_irq *armv7m_init(MemoryRegion *address_space_mem,
+qemu_irq *armv7m_init(MemoryRegion *system_memory,
                       int flash_size, int sram_size,
                       const char *kernel_filename, const char *cpu_model)
 {
@@ -210,13 +210,14 @@ qemu_irq *armv7m_init(MemoryRegion *address_space_mem,
 #endif
 
     /* Flash programming is done via the SCU, so pretend it is ROM.  */
-    memory_region_init_ram(flash, NULL, "armv7m.flash", flash_size);
+    memory_region_init_ram(flash, NULL, "armv7m.flash", flash_size,
+                           &error_abort);
     vmstate_register_ram_global(flash);
     memory_region_set_readonly(flash, true);
-    memory_region_add_subregion(address_space_mem, 0, flash);
-    memory_region_init_ram(sram, NULL, "armv7m.sram", sram_size);
+    memory_region_add_subregion(system_memory, 0, flash);
+    memory_region_init_ram(sram, NULL, "armv7m.sram", sram_size, &error_abort);
     vmstate_register_ram_global(sram);
-    memory_region_add_subregion(address_space_mem, 0x20000000, sram);
+    memory_region_add_subregion(system_memory, 0x20000000, sram);
     armv7m_bitband_init();
 
     nvic = qdev_create(NULL, "armv7m_nvic");
@@ -255,9 +256,9 @@ qemu_irq *armv7m_init(MemoryRegion *address_space_mem,
     /* Hack to map an additional page of ram at the top of the address
        space.  This stops qemu complaining about executing code outside RAM
        when returning from an exception.  */
-    memory_region_init_ram(hack, NULL, "armv7m.hack", 0x1000);
+    memory_region_init_ram(hack, NULL, "armv7m.hack", 0x1000, &error_abort);
     vmstate_register_ram_global(hack);
-    memory_region_add_subregion(address_space_mem, 0xfffff000, hack);
+    memory_region_add_subregion(system_memory, 0xfffff000, hack);
 
     qemu_register_reset(armv7m_reset, cpu);
     return pic;
diff --git a/hw/arm/boot.c b/hw/arm/boot.c
index 3d1f4a255..0014c34dd 100644
--- a/hw/arm/boot.c
+++ b/hw/arm/boot.c
@@ -312,7 +312,26 @@ static void set_kernel_args_old(const struct arm_boot_info *info)
     }
 }
 
-static int load_dtb(hwaddr addr, const struct arm_boot_info *binfo)
+/**
+ * load_dtb() - load a device tree binary image into memory
+ * @addr:       the address to load the image at
+ * @binfo:      struct describing the boot environment
+ * @addr_limit: upper limit of the available memory area at @addr
+ *
+ * Load a device tree supplied by the machine or by the user  with the
+ * '-dtb' command line option, and put it at offset @addr in target
+ * memory.
+ *
+ * If @addr_limit contains a meaningful value (i.e., it is strictly greater
+ * than @addr), the device tree is only loaded if its size does not exceed
+ * the limit.
+ *
+ * Returns: the size of the device tree image on success,
+ *          0 if the image size exceeds the limit,
+ *          -1 on errors.
+ */
+static int load_dtb(hwaddr addr, const struct arm_boot_info *binfo,
+                    hwaddr addr_limit)
 {
     void *fdt = NULL;
     int size, rc;
@@ -341,6 +360,15 @@ static int load_dtb(hwaddr addr, const struct arm_boot_info *binfo)
         }
     }
 
+    if (addr_limit > addr && size > (addr_limit - addr)) {
+        /* Installing the device tree blob at addr would exceed addr_limit.
+         * Whether this constitutes failure is up to the caller to decide,
+         * so just return 0 as size, i.e., no error.
+         */
+        g_free(fdt);
+        return 0;
+    }
+
     acells = qemu_fdt_getprop_cell(fdt, "/", "#address-cells");
     scells = qemu_fdt_getprop_cell(fdt, "/", "#size-cells");
     if (acells == 0 || scells == 0) {
@@ -396,11 +424,14 @@ static int load_dtb(hwaddr addr, const struct arm_boot_info *binfo)
 
     qemu_fdt_dumpdtb(fdt, size);
 
-    cpu_physical_memory_write(addr, fdt, size);
+    /* Put the DTB into the memory map as a ROM image: this will ensure
+     * the DTB is copied again upon reset, even if addr points into RAM.
+     */
+    rom_add_blob_fixed("dtb", fdt, size, addr);
 
     g_free(fdt);
 
-    return 0;
+    return size;
 
 fail:
     g_free(fdt);
@@ -417,8 +448,12 @@ static void do_cpu_reset(void *opaque)
     if (info) {
         if (!info->is_linux) {
             /* Jump to the entry point.  */
-            env->regs[15] = info->entry & 0xfffffffe;
-            env->thumb = info->entry & 1;
+            if (env->aarch64) {
+                env->pc = info->entry;
+            } else {
+                env->regs[15] = info->entry & 0xfffffffe;
+                env->thumb = info->entry & 1;
+            }
         } else {
             if (CPU(cpu) == first_cpu) {
                 if (env->aarch64) {
@@ -443,18 +478,37 @@ static void do_cpu_reset(void *opaque)
 
 void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
 {
-    CPUState *cs = CPU(cpu);
+    CPUState *cs;
     int kernel_size;
     int initrd_size;
     int is_linux = 0;
-    uint64_t elf_entry;
+    uint64_t elf_entry, elf_low_addr, elf_high_addr;
     int elf_machine;
     hwaddr entry, kernel_load_offset;
     int big_endian;
     static const ARMInsnFixup *primary_loader;
 
+    /* CPU objects (unlike devices) are not automatically reset on system
+     * reset, so we must always register a handler to do so. If we're
+     * actually loading a kernel, the handler is also responsible for
+     * arranging that we start it correctly.
+     */
+    for (cs = CPU(cpu); cs; cs = CPU_NEXT(cs)) {
+        qemu_register_reset(do_cpu_reset, ARM_CPU(cs));
+    }
+
     /* Load the kernel.  */
     if (!info->kernel_filename) {
+
+        if (have_dtb(info)) {
+            /* If we have a device tree blob, but no kernel to supply it to,
+             * copy it to the base of RAM for a bootloader to pick up.
+             */
+            if (load_dtb(info->loader_start, info, 0) < 0) {
+                exit(1);
+            }
+        }
+
         /* If no kernel specified, do nothing; we will start from address 0
          * (typically a boot ROM image) in the same way as hardware.
          */
@@ -504,11 +558,36 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
 
     /* Assume that raw images are linux kernels, and ELF images are not.  */
     kernel_size = load_elf(info->kernel_filename, NULL, NULL, &elf_entry,
-                           NULL, NULL, big_endian, elf_machine, 1);
+                           &elf_low_addr, &elf_high_addr, big_endian,
+                           elf_machine, 1);
+    if (kernel_size > 0 && have_dtb(info)) {
+        /* If there is still some room left at the base of RAM, try and put
+         * the DTB there like we do for images loaded with -bios or -pflash.
+         */
+        if (elf_low_addr > info->loader_start
+            || elf_high_addr < info->loader_start) {
+            /* Pass elf_low_addr as address limit to load_dtb if it may be
+             * pointing into RAM, otherwise pass '0' (no limit)
+             */
+            if (elf_low_addr < info->loader_start) {
+                elf_low_addr = 0;
+            }
+            if (load_dtb(info->loader_start, info, elf_low_addr) < 0) {
+                exit(1);
+            }
+        }
+    }
     entry = elf_entry;
     if (kernel_size < 0) {
         kernel_size = load_uimage(info->kernel_filename, &entry, NULL,
-                                  &is_linux);
+                                  &is_linux, NULL, NULL);
+    }
+    /* On aarch64, it's the bootloader's job to uncompress the kernel. */
+    if (arm_feature(&cpu->env, ARM_FEATURE_AARCH64) && kernel_size < 0) {
+        entry = info->loader_start + kernel_load_offset;
+        kernel_size = load_image_gzipped(info->kernel_filename, entry,
+                                         info->ram_size - kernel_load_offset);
+        is_linux = 1;
     }
     if (kernel_size < 0) {
         entry = info->loader_start + kernel_load_offset;
@@ -558,7 +637,7 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
              */
             hwaddr dtb_start = QEMU_ALIGN_UP(info->initrd_start + initrd_size,
                                              4096);
-            if (load_dtb(dtb_start, info)) {
+            if (load_dtb(dtb_start, info, 0) < 0) {
                 exit(1);
             }
             fixupcontext[FIXUP_ARGPTR] = dtb_start;
@@ -582,9 +661,7 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
     }
     info->is_linux = is_linux;
 
-    for (; cs; cs = CPU_NEXT(cs)) {
-        cpu = ARM_CPU(cs);
-        cpu->env.boot_info = info;
-        qemu_register_reset(do_cpu_reset, cpu);
+    for (cs = CPU(cpu); cs; cs = CPU_NEXT(cs)) {
+        ARM_CPU(cs)->env.boot_info = info;
     }
 }
diff --git a/hw/arm/collie.c b/hw/arm/collie.c
index ed7851fe0..6c9b82fc5 100644
--- a/hw/arm/collie.c
+++ b/hw/arm/collie.c
@@ -15,7 +15,7 @@
 #include "strongarm.h"
 #include "hw/arm/arm.h"
 #include "hw/block/flash.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 
 static struct arm_boot_info collie_binfo = {
@@ -41,13 +41,13 @@ static void collie_init(MachineState *machine)
 
     dinfo = drive_get(IF_PFLASH, 0, 0);
     pflash_cfi01_register(SA_CS0, NULL, "collie.fl1", 0x02000000,
-                    dinfo ? dinfo->bdrv : NULL, (64 * 1024),
-                    512, 4, 0x00, 0x00, 0x00, 0x00, 0);
+                    dinfo ? blk_by_legacy_dinfo(dinfo) : NULL,
+                    (64 * 1024), 512, 4, 0x00, 0x00, 0x00, 0x00, 0);
 
     dinfo = drive_get(IF_PFLASH, 0, 1);
     pflash_cfi01_register(SA_CS1, NULL, "collie.fl2", 0x02000000,
-                    dinfo ? dinfo->bdrv : NULL, (64 * 1024),
-                    512, 4, 0x00, 0x00, 0x00, 0x00, 0);
+                    dinfo ? blk_by_legacy_dinfo(dinfo) : NULL,
+                    (64 * 1024), 512, 4, 0x00, 0x00, 0x00, 0x00, 0);
 
     sysbus_create_simple("scoop", 0x40800000, NULL);
 
diff --git a/hw/arm/cubieboard.c b/hw/arm/cubieboard.c
index e2260e379..d1e53be92 100644
--- a/hw/arm/cubieboard.c
+++ b/hw/arm/cubieboard.c
@@ -64,7 +64,7 @@ static void cubieboard_init(MachineState *machine)
     }
 
     memory_region_init_ram(&s->sdram, NULL, "cubieboard.ram",
-                           machine->ram_size);
+                           machine->ram_size, &error_abort);
     vmstate_register_ram_global(&s->sdram);
     memory_region_add_subregion(get_system_memory(), AW_A10_SDRAM_BASE,
                                 &s->sdram);
diff --git a/hw/arm/digic_boards.c b/hw/arm/digic_boards.c
index d1424eee2..2a4b8720a 100644
--- a/hw/arm/digic_boards.c
+++ b/hw/arm/digic_boards.c
@@ -51,7 +51,7 @@ typedef struct DigicBoard {
 
 static void digic4_board_setup_ram(DigicBoardState *s, hwaddr ram_size)
 {
-    memory_region_init_ram(&s->ram, NULL, "ram", ram_size);
+    memory_region_init_ram(&s->ram, NULL, "ram", ram_size, &error_abort);
     memory_region_add_subregion(get_system_memory(), 0, &s->ram);
     vmstate_register_ram_global(&s->ram);
 }
diff --git a/hw/arm/exynos4210.c b/hw/arm/exynos4210.c
index 6426d168d..582794c19 100644
--- a/hw/arm/exynos4210.c
+++ b/hw/arm/exynos4210.c
@@ -248,7 +248,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
 
     /* Internal ROM */
     memory_region_init_ram(&s->irom_mem, NULL, "exynos4210.irom",
-                           EXYNOS4210_IROM_SIZE);
+                           EXYNOS4210_IROM_SIZE, &error_abort);
     vmstate_register_ram_global(&s->irom_mem);
     memory_region_set_readonly(&s->irom_mem, true);
     memory_region_add_subregion(system_mem, EXYNOS4210_IROM_BASE_ADDR,
@@ -264,7 +264,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
 
     /* Internal RAM */
     memory_region_init_ram(&s->iram_mem, NULL, "exynos4210.iram",
-                           EXYNOS4210_IRAM_SIZE);
+                           EXYNOS4210_IRAM_SIZE, &error_abort);
     vmstate_register_ram_global(&s->iram_mem);
     memory_region_add_subregion(system_mem, EXYNOS4210_IRAM_BASE_ADDR,
                                 &s->iram_mem);
@@ -273,13 +273,14 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
     mem_size = ram_size;
     if (mem_size > EXYNOS4210_DRAM_MAX_SIZE) {
         memory_region_init_ram(&s->dram1_mem, NULL, "exynos4210.dram1",
-                mem_size - EXYNOS4210_DRAM_MAX_SIZE);
+                mem_size - EXYNOS4210_DRAM_MAX_SIZE, &error_abort);
         vmstate_register_ram_global(&s->dram1_mem);
         memory_region_add_subregion(system_mem, EXYNOS4210_DRAM1_BASE_ADDR,
                 &s->dram1_mem);
         mem_size = EXYNOS4210_DRAM_MAX_SIZE;
     }
-    memory_region_init_ram(&s->dram0_mem, NULL, "exynos4210.dram0", mem_size);
+    memory_region_init_ram(&s->dram0_mem, NULL, "exynos4210.dram0", mem_size,
+                           &error_abort);
     vmstate_register_ram_global(&s->dram0_mem);
     memory_region_add_subregion(system_mem, EXYNOS4210_DRAM0_BASE_ADDR,
             &s->dram0_mem);
diff --git a/hw/arm/gumstix.c b/hw/arm/gumstix.c
index 3f8465ebc..8103278b1 100644
--- a/hw/arm/gumstix.c
+++ b/hw/arm/gumstix.c
@@ -40,7 +40,7 @@
 #include "hw/block/flash.h"
 #include "hw/devices.h"
 #include "hw/boards.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 #include "sysemu/qtest.h"
 
@@ -71,7 +71,7 @@ static void connex_init(MachineState *machine)
     be = 0;
 #endif
     if (!pflash_cfi01_register(0x00000000, NULL, "connext.rom", connex_rom,
-                               dinfo ? dinfo->bdrv : NULL,
+                               dinfo ? blk_by_legacy_dinfo(dinfo) : NULL,
                                sector_len, connex_rom / sector_len,
                                2, 0, 0, 0, 0, be)) {
         fprintf(stderr, "qemu: Error registering flash memory.\n");
@@ -109,7 +109,7 @@ static void verdex_init(MachineState *machine)
     be = 0;
 #endif
     if (!pflash_cfi01_register(0x00000000, NULL, "verdex.rom", verdex_rom,
-                               dinfo ? dinfo->bdrv : NULL,
+                               dinfo ? blk_by_legacy_dinfo(dinfo) : NULL,
                                sector_len, verdex_rom / sector_len,
                                2, 0, 0, 0, 0, be)) {
         fprintf(stderr, "qemu: Error registering flash memory.\n");
diff --git a/hw/arm/highbank.c b/hw/arm/highbank.c
index 834043421..30f744a1b 100644
--- a/hw/arm/highbank.c
+++ b/hw/arm/highbank.c
@@ -24,7 +24,7 @@
 #include "net/net.h"
 #include "sysemu/sysemu.h"
 #include "hw/boards.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 #include "qemu/error-report.h"
 
@@ -255,12 +255,13 @@ static void calxeda_init(MachineState *machine, enum cxmachines machine_id)
 
     sysmem = get_system_memory();
     dram = g_new(MemoryRegion, 1);
-    memory_region_init_ram(dram, NULL, "highbank.dram", ram_size);
+    memory_region_init_ram(dram, NULL, "highbank.dram", ram_size, &error_abort);
     /* SDRAM at address zero.  */
     memory_region_add_subregion(sysmem, 0, dram);
 
     sysram = g_new(MemoryRegion, 1);
-    memory_region_init_ram(sysram, NULL, "highbank.sysram", 0x8000);
+    memory_region_init_ram(sysram, NULL, "highbank.sysram", 0x8000,
+                           &error_abort);
     memory_region_add_subregion(sysmem, 0xfff88000, sysram);
     if (bios_name != NULL) {
         sysboot_filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
diff --git a/hw/arm/integratorcp.c b/hw/arm/integratorcp.c
index 0e476c3db..266ec18fb 100644
--- a/hw/arm/integratorcp.c
+++ b/hw/arm/integratorcp.c
@@ -264,7 +264,8 @@ static int integratorcm_init(SysBusDevice *dev)
     s->cm_init = 0x00000112;
     s->cm_refcnt_offset = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL), 24,
                                    1000);
-    memory_region_init_ram(&s->flash, OBJECT(s), "integrator.flash", 0x100000);
+    memory_region_init_ram(&s->flash, OBJECT(s), "integrator.flash", 0x100000,
+                           &error_abort);
     vmstate_register_ram_global(&s->flash);
 
     memory_region_init_io(&s->iomem, OBJECT(s), &integratorcm_ops, s,
@@ -485,7 +486,7 @@ static void integratorcp_init(MachineState *machine)
         exit(1);
     }
 
-    memory_region_init_ram(ram, NULL, "integrator.ram", ram_size);
+    memory_region_init_ram(ram, NULL, "integrator.ram", ram_size, &error_abort);
     vmstate_register_ram_global(ram);
     /* ??? On a real system the first 1Mb is mapped as SSRAM or boot flash.  */
     /* ??? RAM should repeat to fill physical memory space.  */
diff --git a/hw/arm/kzm.c b/hw/arm/kzm.c
index 0555d1265..94ceab6c8 100644
--- a/hw/arm/kzm.c
+++ b/hw/arm/kzm.c
@@ -97,14 +97,14 @@ static void kzm_init(MachineState *machine)
 
     /* On a real system, the first 16k is a `secure boot rom' */
 
-    memory_region_init_ram(ram, NULL, "kzm.ram", ram_size);
+    memory_region_init_ram(ram, NULL, "kzm.ram", ram_size, &error_abort);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(address_space_mem, KZM_RAMADDRESS, ram);
 
     memory_region_init_alias(ram_alias, NULL, "ram.alias", ram, 0, ram_size);
     memory_region_add_subregion(address_space_mem, 0x88000000, ram_alias);
 
-    memory_region_init_ram(sram, NULL, "kzm.sram", 0x4000);
+    memory_region_init_ram(sram, NULL, "kzm.sram", 0x4000, &error_abort);
     memory_region_add_subregion(address_space_mem, 0x1FFFC000, sram);
 
     dev = sysbus_create_varargs("imx_avic", 0x68000000,
diff --git a/hw/arm/mainstone.c b/hw/arm/mainstone.c
index 44f187310..0da02a67e 100644
--- a/hw/arm/mainstone.c
+++ b/hw/arm/mainstone.c
@@ -18,7 +18,7 @@
 #include "hw/devices.h"
 #include "hw/boards.h"
 #include "hw/block/flash.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "hw/sysbus.h"
 #include "exec/address-spaces.h"
 #include "sysemu/qtest.h"
@@ -123,7 +123,8 @@ static void mainstone_common_init(MemoryRegion *address_space_mem,
 
     /* Setup CPU & memory */
     mpu = pxa270_init(address_space_mem, mainstone_binfo.ram_size, cpu_model);
-    memory_region_init_ram(rom, NULL, "mainstone.rom", MAINSTONE_ROM);
+    memory_region_init_ram(rom, NULL, "mainstone.rom", MAINSTONE_ROM,
+                           &error_abort);
     vmstate_register_ram_global(rom);
     memory_region_set_readonly(rom, true);
     memory_region_add_subregion(address_space_mem, 0, rom);
@@ -148,9 +149,9 @@ static void mainstone_common_init(MemoryRegion *address_space_mem,
         if (!pflash_cfi01_register(mainstone_flash_base[i], NULL,
                                    i ? "mainstone.flash1" : "mainstone.flash0",
                                    MAINSTONE_FLASH,
-                                   dinfo->bdrv, sector_len,
-                                   MAINSTONE_FLASH / sector_len, 4, 0, 0, 0, 0,
-                                   be)) {
+                                   blk_by_legacy_dinfo(dinfo),
+                                   sector_len, MAINSTONE_FLASH / sector_len,
+                                   4, 0, 0, 0, 0, be)) {
             fprintf(stderr, "qemu: Error registering flash memory.\n");
             exit(1);
         }
diff --git a/hw/arm/musicpal.c b/hw/arm/musicpal.c
index 6a134f23d..3712de6cb 100644
--- a/hw/arm/musicpal.c
+++ b/hw/arm/musicpal.c
@@ -18,11 +18,10 @@
 #include "hw/char/serial.h"
 #include "qemu/timer.h"
 #include "hw/ptimer.h"
-#include "block/block.h"
 #include "hw/block/flash.h"
 #include "ui/console.h"
 #include "hw/i2c/i2c.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 #include "ui/pixel_ops.h"
 
@@ -1601,11 +1600,13 @@ static void musicpal_init(MachineState *machine)
     }
 
     /* For now we use a fixed - the original - RAM size */
-    memory_region_init_ram(ram, NULL, "musicpal.ram", MP_RAM_DEFAULT_SIZE);
+    memory_region_init_ram(ram, NULL, "musicpal.ram", MP_RAM_DEFAULT_SIZE,
+                           &error_abort);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(address_space_mem, 0, ram);
 
-    memory_region_init_ram(sram, NULL, "musicpal.sram", MP_SRAM_SIZE);
+    memory_region_init_ram(sram, NULL, "musicpal.sram", MP_SRAM_SIZE,
+                           &error_abort);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(address_space_mem, MP_SRAM_BASE, sram);
 
@@ -1630,7 +1631,9 @@ static void musicpal_init(MachineState *machine)
     /* Register flash */
     dinfo = drive_get(IF_PFLASH, 0, 0);
     if (dinfo) {
-        flash_size = bdrv_getlength(dinfo->bdrv);
+        BlockBackend *blk = blk_by_legacy_dinfo(dinfo);
+
+        flash_size = blk_getlength(blk);
         if (flash_size != 8*1024*1024 && flash_size != 16*1024*1024 &&
             flash_size != 32*1024*1024) {
             fprintf(stderr, "Invalid flash image size\n");
@@ -1645,16 +1648,14 @@ static void musicpal_init(MachineState *machine)
 #ifdef TARGET_WORDS_BIGENDIAN
         pflash_cfi02_register(0x100000000ULL-MP_FLASH_SIZE_MAX, NULL,
                               "musicpal.flash", flash_size,
-                              dinfo->bdrv, 0x10000,
-                              (flash_size + 0xffff) >> 16,
+                              blk, 0x10000, (flash_size + 0xffff) >> 16,
                               MP_FLASH_SIZE_MAX / flash_size,
                               2, 0x00BF, 0x236D, 0x0000, 0x0000,
                               0x5555, 0x2AAA, 1);
 #else
         pflash_cfi02_register(0x100000000ULL-MP_FLASH_SIZE_MAX, NULL,
                               "musicpal.flash", flash_size,
-                              dinfo->bdrv, 0x10000,
-                              (flash_size + 0xffff) >> 16,
+                              blk, 0x10000, (flash_size + 0xffff) >> 16,
                               MP_FLASH_SIZE_MAX / flash_size,
                               2, 0x00BF, 0x236D, 0x0000, 0x0000,
                               0x5555, 0x2AAA, 0);
diff --git a/hw/arm/nseries.c b/hw/arm/nseries.c
index 4f092d644..c7ebaa6ab 100644
--- a/hw/arm/nseries.c
+++ b/hw/arm/nseries.c
@@ -31,7 +31,7 @@
 #include "hw/hw.h"
 #include "hw/bt.h"
 #include "hw/loader.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "hw/sysbus.h"
 #include "exec/address-spaces.h"
 
@@ -172,8 +172,9 @@ static void n8x0_nand_setup(struct n800_s *s)
     qdev_prop_set_uint16(s->nand, "version_id", 0);
     qdev_prop_set_int32(s->nand, "shift", 1);
     dinfo = drive_get(IF_MTD, 0, 0);
-    if (dinfo && dinfo->bdrv) {
-        qdev_prop_set_drive_nofail(s->nand, "drive", dinfo->bdrv);
+    if (dinfo) {
+        qdev_prop_set_drive_nofail(s->nand, "drive",
+                                   blk_by_legacy_dinfo(dinfo));
     }
     qdev_init_nofail(s->nand);
     sysbus_connect_irq(SYS_BUS_DEVICE(s->nand), 0,
diff --git a/hw/arm/omap1.c b/hw/arm/omap1.c
index e7cc5d757..abb183c2d 100644
--- a/hw/arm/omap1.c
+++ b/hw/arm/omap1.c
@@ -21,6 +21,7 @@
 #include "hw/arm/omap.h"
 #include "sysemu/sysemu.h"
 #include "hw/arm/soc_dma.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "qemu/range.h"
 #include "hw/sysbus.h"
@@ -3854,10 +3855,12 @@ struct omap_mpu_state_s *omap310_mpu_init(MemoryRegion *system_memory,
     omap_clk_init(s);
 
     /* Memory-mapped stuff */
-    memory_region_init_ram(&s->emiff_ram, NULL, "omap1.dram", s->sdram_size);
+    memory_region_init_ram(&s->emiff_ram, NULL, "omap1.dram", s->sdram_size,
+                           &error_abort);
     vmstate_register_ram_global(&s->emiff_ram);
     memory_region_add_subregion(system_memory, OMAP_EMIFF_BASE, &s->emiff_ram);
-    memory_region_init_ram(&s->imif_ram, NULL, "omap1.sram", s->sram_size);
+    memory_region_init_ram(&s->imif_ram, NULL, "omap1.sram", s->sram_size,
+                           &error_abort);
     vmstate_register_ram_global(&s->imif_ram);
     memory_region_add_subregion(system_memory, OMAP_IMIF_BASE, &s->imif_ram);
 
@@ -3976,7 +3979,8 @@ struct omap_mpu_state_s *omap310_mpu_init(MemoryRegion *system_memory,
         fprintf(stderr, "qemu: missing SecureDigital device\n");
         exit(1);
     }
-    s->mmc = omap_mmc_init(0xfffb7800, system_memory, dinfo->bdrv,
+    s->mmc = omap_mmc_init(0xfffb7800, system_memory,
+                           blk_by_legacy_dinfo(dinfo),
                            qdev_get_gpio_in(s->ih[1], OMAP_INT_OQN),
                            &s->drq[OMAP_DMA_MMC_TX],
                     omap_findclk(s, "mmc_ck"));
diff --git a/hw/arm/omap2.c b/hw/arm/omap2.c
index dc53a7abb..b083ebebc 100644
--- a/hw/arm/omap2.c
+++ b/hw/arm/omap2.c
@@ -18,6 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "hw/hw.h"
 #include "hw/arm/arm.h"
@@ -2266,10 +2267,12 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
     omap_clk_init(s);
 
     /* Memory-mapped stuff */
-    memory_region_init_ram(&s->sdram, NULL, "omap2.dram", s->sdram_size);
+    memory_region_init_ram(&s->sdram, NULL, "omap2.dram", s->sdram_size,
+                           &error_abort);
     vmstate_register_ram_global(&s->sdram);
     memory_region_add_subregion(sysmem, OMAP2_Q2_BASE, &s->sdram);
-    memory_region_init_ram(&s->sram, NULL, "omap2.sram", s->sram_size);
+    memory_region_init_ram(&s->sram, NULL, "omap2.sram", s->sram_size,
+                           &error_abort);
     vmstate_register_ram_global(&s->sram);
     memory_region_add_subregion(sysmem, OMAP2_SRAM_BASE, &s->sram);
 
@@ -2459,7 +2462,8 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
         fprintf(stderr, "qemu: missing SecureDigital device\n");
         exit(1);
     }
-    s->mmc = omap2_mmc_init(omap_l4tao(s->l4, 9), dinfo->bdrv,
+    s->mmc = omap2_mmc_init(omap_l4tao(s->l4, 9),
+                    blk_by_legacy_dinfo(dinfo),
                     qdev_get_gpio_in(s->ih[0], OMAP_INT_24XX_MMC_IRQ),
                     &s->drq[OMAP24XX_DMA_MMC1_TX],
                     omap_findclk(s, "mmc_fclk"), omap_findclk(s, "mmc_iclk"));
diff --git a/hw/arm/omap_sx1.c b/hw/arm/omap_sx1.c
index b4f6da606..671e02c4e 100644
--- a/hw/arm/omap_sx1.c
+++ b/hw/arm/omap_sx1.c
@@ -31,7 +31,7 @@
 #include "hw/boards.h"
 #include "hw/arm/arm.h"
 #include "hw/block/flash.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/qtest.h"
 #include "exec/address-spaces.h"
 
@@ -122,7 +122,8 @@ static void sx1_init(MachineState *machine, const int version)
                            machine->cpu_model);
 
     /* External Flash (EMIFS) */
-    memory_region_init_ram(flash, NULL, "omap_sx1.flash0-0", flash_size);
+    memory_region_init_ram(flash, NULL, "omap_sx1.flash0-0", flash_size,
+                           &error_abort);
     vmstate_register_ram_global(flash);
     memory_region_set_readonly(flash, true);
     memory_region_add_subregion(address_space, OMAP_CS0_BASE, flash);
@@ -153,8 +154,8 @@ static void sx1_init(MachineState *machine, const int version)
     if ((dinfo = drive_get(IF_PFLASH, 0, fl_idx)) != NULL) {
         if (!pflash_cfi01_register(OMAP_CS0_BASE, NULL,
                                    "omap_sx1.flash0-1", flash_size,
-                                   dinfo->bdrv, sector_size,
-                                   flash_size / sector_size,
+                                   blk_by_legacy_dinfo(dinfo),
+                                   sector_size, flash_size / sector_size,
                                    4, 0, 0, 0, 0, be)) {
             fprintf(stderr, "qemu: Error registering flash memory %d.\n",
                            fl_idx);
@@ -164,7 +165,8 @@ static void sx1_init(MachineState *machine, const int version)
 
     if ((version == 1) &&
             (dinfo = drive_get(IF_PFLASH, 0, fl_idx)) != NULL) {
-        memory_region_init_ram(flash_1, NULL, "omap_sx1.flash1-0", flash1_size);
+        memory_region_init_ram(flash_1, NULL, "omap_sx1.flash1-0", flash1_size,
+                               &error_abort);
         vmstate_register_ram_global(flash_1);
         memory_region_set_readonly(flash_1, true);
         memory_region_add_subregion(address_space, OMAP_CS1_BASE, flash_1);
@@ -176,8 +178,8 @@ static void sx1_init(MachineState *machine, const int version)
 
         if (!pflash_cfi01_register(OMAP_CS1_BASE, NULL,
                                    "omap_sx1.flash1-1", flash1_size,
-                                   dinfo->bdrv, sector_size,
-                                   flash1_size / sector_size,
+                                   blk_by_legacy_dinfo(dinfo),
+                                   sector_size, flash1_size / sector_size,
                                    4, 0, 0, 0, 0, be)) {
             fprintf(stderr, "qemu: Error registering flash memory %d.\n",
                            fl_idx);
diff --git a/hw/arm/palm.c b/hw/arm/palm.c
index e61995f96..7f1cfb8f6 100644
--- a/hw/arm/palm.c
+++ b/hw/arm/palm.c
@@ -212,7 +212,8 @@ static void palmte_init(MachineState *machine)
     mpu = omap310_mpu_init(address_space_mem, sdram_size, cpu_model);
 
     /* External Flash (EMIFS) */
-    memory_region_init_ram(flash, NULL, "palmte.flash", flash_size);
+    memory_region_init_ram(flash, NULL, "palmte.flash", flash_size,
+                           &error_abort);
     vmstate_register_ram_global(flash);
     memory_region_set_readonly(flash, true);
     memory_region_add_subregion(address_space_mem, OMAP_CS0_BASE, flash);
diff --git a/hw/arm/pxa2xx.c b/hw/arm/pxa2xx.c
index 557e0f127..693dfec9f 100644
--- a/hw/arm/pxa2xx.c
+++ b/hw/arm/pxa2xx.c
@@ -14,6 +14,7 @@
 #include "hw/i2c/i2c.h"
 #include "hw/ssi.h"
 #include "sysemu/char.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 
 static struct {
@@ -2055,10 +2056,12 @@ PXA2xxState *pxa270_init(MemoryRegion *address_space,
     s->reset = qemu_allocate_irq(pxa2xx_reset, s, 0);
 
     /* SDRAM & Internal Memory Storage */
-    memory_region_init_ram(&s->sdram, NULL, "pxa270.sdram", sdram_size);
+    memory_region_init_ram(&s->sdram, NULL, "pxa270.sdram", sdram_size,
+                           &error_abort);
     vmstate_register_ram_global(&s->sdram);
     memory_region_add_subregion(address_space, PXA2XX_SDRAM_BASE, &s->sdram);
-    memory_region_init_ram(&s->internal, NULL, "pxa270.internal", 0x40000);
+    memory_region_init_ram(&s->internal, NULL, "pxa270.internal", 0x40000,
+                           &error_abort);
     vmstate_register_ram_global(&s->internal);
     memory_region_add_subregion(address_space, PXA2XX_INTERNAL_BASE,
                                 &s->internal);
@@ -2083,7 +2086,8 @@ PXA2xxState *pxa270_init(MemoryRegion *address_space,
         fprintf(stderr, "qemu: missing SecureDigital device\n");
         exit(1);
     }
-    s->mmc = pxa2xx_mmci_init(address_space, 0x41100000, dinfo->bdrv,
+    s->mmc = pxa2xx_mmci_init(address_space, 0x41100000,
+                    blk_by_legacy_dinfo(dinfo),
                     qdev_get_gpio_in(s->pic, PXA2XX_PIC_MMC),
                     qdev_get_gpio_in(s->dma, PXA2XX_RX_RQ_MMCI),
                     qdev_get_gpio_in(s->dma, PXA2XX_TX_RQ_MMCI));
@@ -2186,11 +2190,12 @@ PXA2xxState *pxa255_init(MemoryRegion *address_space, unsigned int sdram_size)
     s->reset = qemu_allocate_irq(pxa2xx_reset, s, 0);
 
     /* SDRAM & Internal Memory Storage */
-    memory_region_init_ram(&s->sdram, NULL, "pxa255.sdram", sdram_size);
+    memory_region_init_ram(&s->sdram, NULL, "pxa255.sdram", sdram_size,
+                           &error_abort);
     vmstate_register_ram_global(&s->sdram);
     memory_region_add_subregion(address_space, PXA2XX_SDRAM_BASE, &s->sdram);
     memory_region_init_ram(&s->internal, NULL, "pxa255.internal",
-                           PXA2XX_INTERNAL_SIZE);
+                           PXA2XX_INTERNAL_SIZE, &error_abort);
     vmstate_register_ram_global(&s->internal);
     memory_region_add_subregion(address_space, PXA2XX_INTERNAL_BASE,
                                 &s->internal);
@@ -2214,7 +2219,8 @@ PXA2xxState *pxa255_init(MemoryRegion *address_space, unsigned int sdram_size)
         fprintf(stderr, "qemu: missing SecureDigital device\n");
         exit(1);
     }
-    s->mmc = pxa2xx_mmci_init(address_space, 0x41100000, dinfo->bdrv,
+    s->mmc = pxa2xx_mmci_init(address_space, 0x41100000,
+                    blk_by_legacy_dinfo(dinfo),
                     qdev_get_gpio_in(s->pic, PXA2XX_PIC_MMC),
                     qdev_get_gpio_in(s->dma, PXA2XX_RX_RQ_MMCI),
                     qdev_get_gpio_in(s->dma, PXA2XX_TX_RQ_MMCI));
diff --git a/hw/arm/realview.c b/hw/arm/realview.c
index 64b92518d..af65aa408 100644
--- a/hw/arm/realview.c
+++ b/hw/arm/realview.c
@@ -16,7 +16,7 @@
 #include "sysemu/sysemu.h"
 #include "hw/boards.h"
 #include "hw/i2c/i2c.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 #include "qemu/error-report.h"
 
@@ -137,12 +137,14 @@ static void realview_init(MachineState *machine,
         /* Core tile RAM.  */
         low_ram_size = ram_size - 0x20000000;
         ram_size = 0x20000000;
-        memory_region_init_ram(ram_lo, NULL, "realview.lowmem", low_ram_size);
+        memory_region_init_ram(ram_lo, NULL, "realview.lowmem", low_ram_size,
+                               &error_abort);
         vmstate_register_ram_global(ram_lo);
         memory_region_add_subregion(sysmem, 0x20000000, ram_lo);
     }
 
-    memory_region_init_ram(ram_hi, NULL, "realview.highmem", ram_size);
+    memory_region_init_ram(ram_hi, NULL, "realview.highmem", ram_size,
+                           &error_abort);
     vmstate_register_ram_global(ram_hi);
     low_ram_size = ram_size;
     if (low_ram_size > 0x10000000)
@@ -337,7 +339,8 @@ static void realview_init(MachineState *machine,
        startup code.  I guess this works on real hardware because the
        BootROM happens to be in ROM/flash or in memory that isn't clobbered
        until after Linux boots the secondary CPUs.  */
-    memory_region_init_ram(ram_hack, NULL, "realview.hack", 0x1000);
+    memory_region_init_ram(ram_hack, NULL, "realview.hack", 0x1000,
+                           &error_abort);
     vmstate_register_ram_global(ram_hack);
     memory_region_add_subregion(sysmem, SMP_BOOT_ADDR, ram_hack);
 
diff --git a/hw/arm/spitz.c b/hw/arm/spitz.c
index 03cc6ce2f..a16831c2e 100644
--- a/hw/arm/spitz.c
+++ b/hw/arm/spitz.c
@@ -22,10 +22,9 @@
 #include "hw/devices.h"
 #include "hw/arm/sharpsl.h"
 #include "ui/console.h"
-#include "block/block.h"
 #include "audio/audio.h"
 #include "hw/boards.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "hw/sysbus.h"
 #include "exec/address-spaces.h"
 
@@ -170,7 +169,8 @@ static int sl_nand_init(SysBusDevice *dev)
 
     s->ctl = 0;
     nand = drive_get(IF_MTD, 0, 0);
-    s->nand = nand_init(nand ? nand->bdrv : NULL, s->manf_id, s->chip_id);
+    s->nand = nand_init(nand ? blk_by_legacy_dinfo(nand) : NULL,
+                        s->manf_id, s->chip_id);
 
     memory_region_init_io(&s->iomem, OBJECT(s), &sl_ops, s, "sl", 0x40);
     sysbus_init_mmio(dev, &s->iomem);
@@ -912,7 +912,7 @@ static void spitz_common_init(MachineState *machine,
 
     sl_flash_register(mpu, (model == spitz) ? FLASH_128M : FLASH_1024M);
 
-    memory_region_init_ram(rom, NULL, "spitz.rom", SPITZ_ROM);
+    memory_region_init_ram(rom, NULL, "spitz.rom", SPITZ_ROM, &error_abort);
     vmstate_register_ram_global(rom);
     memory_region_set_readonly(rom, true);
     memory_region_add_subregion(address_space_mem, 0, rom);
diff --git a/hw/arm/stellaris.c b/hw/arm/stellaris.c
index 80028e80c..64bd4b4c4 100644
--- a/hw/arm/stellaris.c
+++ b/hw/arm/stellaris.c
@@ -1208,7 +1208,6 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
         0x40024000, 0x40025000, 0x40026000};
     static const int gpio_irq[7] = {0, 1, 2, 3, 4, 30, 31};
 
-    MemoryRegion *address_space_mem = get_system_memory();
     qemu_irq *pic;
     DeviceState *gpio_dev[7];
     qemu_irq gpio_in[7][8];
@@ -1223,7 +1222,7 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
 
     flash_size = ((board->dc0 & 0xffff) + 1) << 1;
     sram_size = (board->dc0 >> 18) + 1;
-    pic = armv7m_init(address_space_mem,
+    pic = armv7m_init(get_system_memory(),
                       flash_size, sram_size, kernel_filename, cpu_model);
 
     if (board->dc1 & (1 << 16)) {
diff --git a/hw/arm/strongarm.c b/hw/arm/strongarm.c
index 9e2a0d48a..32063459d 100644
--- a/hw/arm/strongarm.c
+++ b/hw/arm/strongarm.c
@@ -1604,7 +1604,8 @@ StrongARMState *sa1110_init(MemoryRegion *sysmem,
         exit(1);
     }
 
-    memory_region_init_ram(&s->sdram, NULL, "strongarm.sdram", sdram_size);
+    memory_region_init_ram(&s->sdram, NULL, "strongarm.sdram", sdram_size,
+                           &error_abort);
     vmstate_register_ram_global(&s->sdram);
     memory_region_add_subregion(sysmem, SA_SDCS0, &s->sdram);
 
diff --git a/hw/arm/tosa.c b/hw/arm/tosa.c
index abc0f2a96..73572ebe0 100644
--- a/hw/arm/tosa.c
+++ b/hw/arm/tosa.c
@@ -17,11 +17,10 @@
 #include "hw/devices.h"
 #include "hw/arm/sharpsl.h"
 #include "hw/pcmcia.h"
-#include "block/block.h"
 #include "hw/boards.h"
 #include "hw/i2c/i2c.h"
 #include "hw/ssi.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "hw/sysbus.h"
 #include "exec/address-spaces.h"
 
@@ -228,7 +227,7 @@ static void tosa_init(MachineState *machine)
 
     mpu = pxa255_init(address_space_mem, tosa_binfo.ram_size);
 
-    memory_region_init_ram(rom, NULL, "tosa.rom", TOSA_ROM);
+    memory_region_init_ram(rom, NULL, "tosa.rom", TOSA_ROM, &error_abort);
     vmstate_register_ram_global(rom);
     memory_region_set_readonly(rom, true);
     memory_region_add_subregion(address_space_mem, 0, rom);
diff --git a/hw/arm/versatilepb.c b/hw/arm/versatilepb.c
index dea5fc7a9..e6ef0a2e7 100644
--- a/hw/arm/versatilepb.c
+++ b/hw/arm/versatilepb.c
@@ -15,7 +15,7 @@
 #include "hw/pci/pci.h"
 #include "hw/i2c/i2c.h"
 #include "hw/boards.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 #include "hw/block/flash.h"
 
@@ -198,7 +198,8 @@ static void versatile_init(MachineState *machine, int board_id)
         fprintf(stderr, "Unable to find CPU definition\n");
         exit(1);
     }
-    memory_region_init_ram(ram, NULL, "versatile.ram", machine->ram_size);
+    memory_region_init_ram(ram, NULL, "versatile.ram", machine->ram_size,
+                           &error_abort);
     vmstate_register_ram_global(ram);
     /* ??? RAM should repeat to fill physical memory space.  */
     /* SDRAM at address zero.  */
@@ -337,7 +338,8 @@ static void versatile_init(MachineState *machine, int board_id)
 
     dinfo = drive_get(IF_PFLASH, 0, 0);
     if (!pflash_cfi01_register(VERSATILE_FLASH_ADDR, NULL, "versatile.flash",
-                          VERSATILE_FLASH_SIZE, dinfo ? dinfo->bdrv : NULL,
+                          VERSATILE_FLASH_SIZE,
+                          dinfo ? blk_by_legacy_dinfo(dinfo) : NULL,
                           VERSATILE_FLASH_SECT_SIZE,
                           VERSATILE_FLASH_SIZE / VERSATILE_FLASH_SECT_SIZE,
                           4, 0x0089, 0x0018, 0x0000, 0x0, 0)) {
diff --git a/hw/arm/vexpress.c b/hw/arm/vexpress.c
index a88732c7e..7cbd13f18 100644
--- a/hw/arm/vexpress.c
+++ b/hw/arm/vexpress.c
@@ -30,7 +30,7 @@
 #include "hw/boards.h"
 #include "hw/loader.h"
 #include "exec/address-spaces.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "hw/block/flash.h"
 #include "sysemu/device_tree.h"
 #include "qemu/error-report.h"
@@ -252,7 +252,8 @@ static void a9_daughterboard_init(const VEDBoardInfo *daughterboard,
         exit(1);
     }
 
-    memory_region_init_ram(ram, NULL, "vexpress.highmem", ram_size);
+    memory_region_init_ram(ram, NULL, "vexpress.highmem", ram_size,
+                           &error_abort);
     vmstate_register_ram_global(ram);
     low_ram_size = ram_size;
     if (low_ram_size > 0x4000000) {
@@ -346,7 +347,8 @@ static void a15_daughterboard_init(const VEDBoardInfo *daughterboard,
         }
     }
 
-    memory_region_init_ram(ram, NULL, "vexpress.highmem", ram_size);
+    memory_region_init_ram(ram, NULL, "vexpress.highmem", ram_size,
+                           &error_abort);
     vmstate_register_ram_global(ram);
     /* RAM is from 0x80000000 upwards; there is no low-memory alias for it. */
     memory_region_add_subregion(sysmem, 0x80000000, ram);
@@ -364,7 +366,8 @@ static void a15_daughterboard_init(const VEDBoardInfo *daughterboard,
     /* 0x2b060000: SP805 watchdog: not modelled */
     /* 0x2b0a0000: PL341 dynamic memory controller: not modelled */
     /* 0x2e000000: system SRAM */
-    memory_region_init_ram(sram, NULL, "vexpress.a15sram", 0x10000);
+    memory_region_init_ram(sram, NULL, "vexpress.a15sram", 0x10000,
+                           &error_abort);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(sysmem, 0x2e000000, sram);
 
@@ -488,7 +491,8 @@ static pflash_t *ve_pflash_cfi01_register(hwaddr base, const char *name,
 {
     DeviceState *dev = qdev_create(NULL, "cfi.pflash01");
 
-    if (di && qdev_prop_set_drive(dev, "drive", di->bdrv)) {
+    if (di && qdev_prop_set_drive(dev, "drive",
+                                  blk_by_legacy_dinfo(di))) {
         abort();
     }
 
@@ -634,12 +638,14 @@ static void vexpress_common_init(VEDBoardInfo *daughterboard,
     }
 
     sram_size = 0x2000000;
-    memory_region_init_ram(sram, NULL, "vexpress.sram", sram_size);
+    memory_region_init_ram(sram, NULL, "vexpress.sram", sram_size,
+                           &error_abort);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(sysmem, map[VE_SRAM], sram);
 
     vram_size = 0x800000;
-    memory_region_init_ram(vram, NULL, "vexpress.vram", vram_size);
+    memory_region_init_ram(vram, NULL, "vexpress.vram", vram_size,
+                           &error_abort);
     vmstate_register_ram_global(vram);
     memory_region_add_subregion(sysmem, map[VE_VIDEORAM], vram);
 
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 89532bd78..314e55b56 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -33,10 +33,12 @@
 #include "hw/arm/primecell.h"
 #include "hw/devices.h"
 #include "net/net.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/device_tree.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
 #include "hw/boards.h"
+#include "hw/loader.h"
 #include "exec/address-spaces.h"
 #include "qemu/bitops.h"
 #include "qemu/error-report.h"
@@ -98,17 +100,17 @@ typedef struct VirtBoardInfo {
  */
 static const MemMapEntry a15memmap[] = {
     /* Space up to 0x8000000 is reserved for a boot ROM */
-    [VIRT_FLASH] = { 0, 0x8000000 },
-    [VIRT_CPUPERIPHS] = { 0x8000000, 0x20000 },
+    [VIRT_FLASH] =      {          0, 0x08000000 },
+    [VIRT_CPUPERIPHS] = { 0x08000000, 0x00020000 },
     /* GIC distributor and CPU interfaces sit inside the CPU peripheral space */
-    [VIRT_GIC_DIST] = { 0x8000000, 0x10000 },
-    [VIRT_GIC_CPU] = { 0x8010000, 0x10000 },
-    [VIRT_UART] = { 0x9000000, 0x1000 },
-    [VIRT_RTC] = { 0x9010000, 0x1000 },
-    [VIRT_MMIO] = { 0xa000000, 0x200 },
+    [VIRT_GIC_DIST] =   { 0x08000000, 0x00010000 },
+    [VIRT_GIC_CPU] =    { 0x08010000, 0x00010000 },
+    [VIRT_UART] =       { 0x09000000, 0x00001000 },
+    [VIRT_RTC] =        { 0x09010000, 0x00001000 },
+    [VIRT_MMIO] =       { 0x0a000000, 0x00000200 },
     /* ...repeating for a total of NUM_VIRTIO_TRANSPORTS, each of that size */
     /* 0x10000000 .. 0x40000000 reserved for PCI */
-    [VIRT_MEM] = { 0x40000000, 30ULL * 1024 * 1024 * 1024 },
+    [VIRT_MEM] =        { 0x40000000, 30ULL * 1024 * 1024 * 1024 },
 };
 
 static const int a15irqmap[] = {
@@ -189,26 +191,48 @@ static void create_fdt(VirtBoardInfo *vbi)
 
 static void fdt_add_psci_node(const VirtBoardInfo *vbi)
 {
+    uint32_t cpu_suspend_fn;
+    uint32_t cpu_off_fn;
+    uint32_t cpu_on_fn;
+    uint32_t migrate_fn;
     void *fdt = vbi->fdt;
     ARMCPU *armcpu = ARM_CPU(qemu_get_cpu(0));
 
-    /* No PSCI for TCG yet */
-    if (kvm_enabled()) {
-        qemu_fdt_add_subnode(fdt, "/psci");
-        if (armcpu->psci_version == 2) {
-            const char comp[] = "arm,psci-0.2\0arm,psci";
-            qemu_fdt_setprop(fdt, "/psci", "compatible", comp, sizeof(comp));
+    qemu_fdt_add_subnode(fdt, "/psci");
+    if (armcpu->psci_version == 2) {
+        const char comp[] = "arm,psci-0.2\0arm,psci";
+        qemu_fdt_setprop(fdt, "/psci", "compatible", comp, sizeof(comp));
+
+        cpu_off_fn = QEMU_PSCI_0_2_FN_CPU_OFF;
+        if (arm_feature(&armcpu->env, ARM_FEATURE_AARCH64)) {
+            cpu_suspend_fn = QEMU_PSCI_0_2_FN64_CPU_SUSPEND;
+            cpu_on_fn = QEMU_PSCI_0_2_FN64_CPU_ON;
+            migrate_fn = QEMU_PSCI_0_2_FN64_MIGRATE;
         } else {
-            qemu_fdt_setprop_string(fdt, "/psci", "compatible", "arm,psci");
+            cpu_suspend_fn = QEMU_PSCI_0_2_FN_CPU_SUSPEND;
+            cpu_on_fn = QEMU_PSCI_0_2_FN_CPU_ON;
+            migrate_fn = QEMU_PSCI_0_2_FN_MIGRATE;
         }
+    } else {
+        qemu_fdt_setprop_string(fdt, "/psci", "compatible", "arm,psci");
 
-        qemu_fdt_setprop_string(fdt, "/psci", "method", "hvc");
-        qemu_fdt_setprop_cell(fdt, "/psci", "cpu_suspend",
-                                  PSCI_FN_CPU_SUSPEND);
-        qemu_fdt_setprop_cell(fdt, "/psci", "cpu_off", PSCI_FN_CPU_OFF);
-        qemu_fdt_setprop_cell(fdt, "/psci", "cpu_on", PSCI_FN_CPU_ON);
-        qemu_fdt_setprop_cell(fdt, "/psci", "migrate", PSCI_FN_MIGRATE);
+        cpu_suspend_fn = QEMU_PSCI_0_1_FN_CPU_SUSPEND;
+        cpu_off_fn = QEMU_PSCI_0_1_FN_CPU_OFF;
+        cpu_on_fn = QEMU_PSCI_0_1_FN_CPU_ON;
+        migrate_fn = QEMU_PSCI_0_1_FN_MIGRATE;
     }
+
+    /* We adopt the PSCI spec's nomenclature, and use 'conduit' to refer
+     * to the instruction that should be used to invoke PSCI functions.
+     * However, the device tree binding uses 'method' instead, so that is
+     * what we should use here.
+     */
+    qemu_fdt_setprop_string(fdt, "/psci", "method", "hvc");
+
+    qemu_fdt_setprop_cell(fdt, "/psci", "cpu_suspend", cpu_suspend_fn);
+    qemu_fdt_setprop_cell(fdt, "/psci", "cpu_off", cpu_off_fn);
+    qemu_fdt_setprop_cell(fdt, "/psci", "cpu_on", cpu_on_fn);
+    qemu_fdt_setprop_cell(fdt, "/psci", "migrate", migrate_fn);
 }
 
 static void fdt_add_timer_nodes(const VirtBoardInfo *vbi)
@@ -217,14 +241,23 @@ static void fdt_add_timer_nodes(const VirtBoardInfo *vbi)
      * but for the GIC implementation provided by both QEMU and KVM
      * they are edge-triggered.
      */
+    ARMCPU *armcpu;
     uint32_t irqflags = GIC_FDT_IRQ_FLAGS_EDGE_LO_HI;
 
     irqflags = deposit32(irqflags, GIC_FDT_IRQ_PPI_CPU_START,
                          GIC_FDT_IRQ_PPI_CPU_WIDTH, (1 << vbi->smp_cpus) - 1);
 
     qemu_fdt_add_subnode(vbi->fdt, "/timer");
-    qemu_fdt_setprop_string(vbi->fdt, "/timer",
-                                "compatible", "arm,armv7-timer");
+
+    armcpu = ARM_CPU(qemu_get_cpu(0));
+    if (arm_feature(&armcpu->env, ARM_FEATURE_V8)) {
+        const char compat[] = "arm,armv8-timer\0arm,armv7-timer";
+        qemu_fdt_setprop(vbi->fdt, "/timer", "compatible",
+                         compat, sizeof(compat));
+    } else {
+        qemu_fdt_setprop_string(vbi->fdt, "/timer", "compatible",
+                                "arm,armv7-timer");
+    }
     qemu_fdt_setprop_cells(vbi->fdt, "/timer", "interrupts",
                                GIC_FDT_IRQ_TYPE_PPI, 13, irqflags,
                                GIC_FDT_IRQ_TYPE_PPI, 14, irqflags,
@@ -350,11 +383,13 @@ static void create_uart(const VirtBoardInfo *vbi, qemu_irq *pic)
                                      2, base, 2, size);
     qemu_fdt_setprop_cells(vbi->fdt, nodename, "interrupts",
                                GIC_FDT_IRQ_TYPE_SPI, irq,
-                               GIC_FDT_IRQ_FLAGS_EDGE_LO_HI);
+                               GIC_FDT_IRQ_FLAGS_LEVEL_HI);
     qemu_fdt_setprop_cells(vbi->fdt, nodename, "clocks",
                                vbi->clock_phandle, vbi->clock_phandle);
     qemu_fdt_setprop(vbi->fdt, nodename, "clock-names",
                          clocknames, sizeof(clocknames));
+
+    qemu_fdt_setprop_string(vbi->fdt, "/chosen", "stdout-path", nodename);
     g_free(nodename);
 }
 
@@ -375,7 +410,7 @@ static void create_rtc(const VirtBoardInfo *vbi, qemu_irq *pic)
                                  2, base, 2, size);
     qemu_fdt_setprop_cells(vbi->fdt, nodename, "interrupts",
                            GIC_FDT_IRQ_TYPE_SPI, irq,
-                           GIC_FDT_IRQ_FLAGS_EDGE_LO_HI);
+                           GIC_FDT_IRQ_FLAGS_LEVEL_HI);
     qemu_fdt_setprop_cell(vbi->fdt, nodename, "clocks", vbi->clock_phandle);
     qemu_fdt_setprop_string(vbi->fdt, nodename, "clock-names", "apb_pclk");
     g_free(nodename);
@@ -416,6 +451,74 @@ static void create_virtio_devices(const VirtBoardInfo *vbi, qemu_irq *pic)
     }
 }
 
+static void create_one_flash(const char *name, hwaddr flashbase,
+                             hwaddr flashsize)
+{
+    /* Create and map a single flash device. We use the same
+     * parameters as the flash devices on the Versatile Express board.
+     */
+    DriveInfo *dinfo = drive_get_next(IF_PFLASH);
+    DeviceState *dev = qdev_create(NULL, "cfi.pflash01");
+    const uint64_t sectorlength = 256 * 1024;
+
+    if (dinfo && qdev_prop_set_drive(dev, "drive",
+                                     blk_by_legacy_dinfo(dinfo))) {
+        abort();
+    }
+
+    qdev_prop_set_uint32(dev, "num-blocks", flashsize / sectorlength);
+    qdev_prop_set_uint64(dev, "sector-length", sectorlength);
+    qdev_prop_set_uint8(dev, "width", 4);
+    qdev_prop_set_uint8(dev, "device-width", 2);
+    qdev_prop_set_uint8(dev, "big-endian", 0);
+    qdev_prop_set_uint16(dev, "id0", 0x89);
+    qdev_prop_set_uint16(dev, "id1", 0x18);
+    qdev_prop_set_uint16(dev, "id2", 0x00);
+    qdev_prop_set_uint16(dev, "id3", 0x00);
+    qdev_prop_set_string(dev, "name", name);
+    qdev_init_nofail(dev);
+
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, flashbase);
+}
+
+static void create_flash(const VirtBoardInfo *vbi)
+{
+    /* Create two flash devices to fill the VIRT_FLASH space in the memmap.
+     * Any file passed via -bios goes in the first of these.
+     */
+    hwaddr flashsize = vbi->memmap[VIRT_FLASH].size / 2;
+    hwaddr flashbase = vbi->memmap[VIRT_FLASH].base;
+    char *nodename;
+
+    if (bios_name) {
+        const char *fn;
+
+        if (drive_get(IF_PFLASH, 0, 0)) {
+            error_report("The contents of the first flash device may be "
+                         "specified with -bios or with -drive if=pflash... "
+                         "but you cannot use both options at once");
+            exit(1);
+        }
+        fn = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
+        if (!fn || load_image_targphys(fn, flashbase, flashsize) < 0) {
+            error_report("Could not load ROM image '%s'", bios_name);
+            exit(1);
+        }
+    }
+
+    create_one_flash("virt.flash0", flashbase, flashsize);
+    create_one_flash("virt.flash1", flashbase + flashsize, flashsize);
+
+    nodename = g_strdup_printf("/flash@%" PRIx64, flashbase);
+    qemu_fdt_add_subnode(vbi->fdt, nodename);
+    qemu_fdt_setprop_string(vbi->fdt, nodename, "compatible", "cfi-flash");
+    qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "reg",
+                                 2, flashbase, 2, flashsize,
+                                 2, flashbase + flashsize, 2, flashsize);
+    qemu_fdt_setprop_cell(vbi->fdt, nodename, "bank-width", 4);
+    g_free(nodename);
+}
+
 static void *machvirt_dtb(const struct arm_boot_info *binfo, int *fdt_size)
 {
     const VirtBoardInfo *board = (const VirtBoardInfo *)binfo;
@@ -446,23 +549,12 @@ static void machvirt_init(MachineState *machine)
 
     vbi->smp_cpus = smp_cpus;
 
-    /*
-     * Only supported method of starting secondary CPUs is PSCI and
-     * PSCI is not yet supported with TCG, so limit smp_cpus to 1
-     * if we're not using KVM.
-     */
-    if (!kvm_enabled() && smp_cpus > 1) {
-        error_report("mach-virt: must enable KVM to use multiple CPUs");
-        exit(1);
-    }
-
     if (machine->ram_size > vbi->memmap[VIRT_MEM].size) {
         error_report("mach-virt: cannot model more than 30GB RAM");
         exit(1);
     }
 
     create_fdt(vbi);
-    fdt_add_timer_nodes(vbi);
 
     for (n = 0; n < smp_cpus; n++) {
         ObjectClass *oc = cpu_class_by_name(TYPE_ARM_CPU, cpu_model);
@@ -474,6 +566,9 @@ static void machvirt_init(MachineState *machine)
         }
         cpuobj = object_new(object_class_get_name(oc));
 
+        object_property_set_int(cpuobj, QEMU_PSCI_CONDUIT_HVC, "psci-conduit",
+                                NULL);
+
         /* Secondary CPUs start in PSCI powered-down state */
         if (n > 0) {
             object_property_set_bool(cpuobj, true, "start-powered-off", NULL);
@@ -486,13 +581,17 @@ static void machvirt_init(MachineState *machine)
 
         object_property_set_bool(cpuobj, true, "realized", NULL);
     }
+    fdt_add_timer_nodes(vbi);
     fdt_add_cpu_nodes(vbi);
     fdt_add_psci_node(vbi);
 
-    memory_region_init_ram(ram, NULL, "mach-virt.ram", machine->ram_size);
+    memory_region_init_ram(ram, NULL, "mach-virt.ram", machine->ram_size,
+                           &error_abort);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(sysmem, vbi->memmap[VIRT_MEM].base, ram);
 
+    create_flash(vbi);
+
     create_gic(vbi, pic);
 
     create_uart(vbi, pic);
@@ -520,7 +619,7 @@ static QEMUMachine machvirt_a15_machine = {
     .name = "virt",
     .desc = "ARM Virtual Machine",
     .init = machvirt_init,
-    .max_cpus = 4,
+    .max_cpus = 8,
 };
 
 static void machvirt_machine_init(void)
diff --git a/hw/arm/xilinx_zynq.c b/hw/arm/xilinx_zynq.c
index ba5aa82cd..b59039297 100644
--- a/hw/arm/xilinx_zynq.c
+++ b/hw/arm/xilinx_zynq.c
@@ -22,7 +22,7 @@
 #include "sysemu/sysemu.h"
 #include "hw/boards.h"
 #include "hw/block/flash.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "hw/loader.h"
 #include "hw/ssi.h"
 #include "qemu/error-report.h"
@@ -149,12 +149,14 @@ static void zynq_init(MachineState *machine)
     }
 
     /* DDR remapped to address zero.  */
-    memory_region_init_ram(ext_ram, NULL, "zynq.ext_ram", ram_size);
+    memory_region_init_ram(ext_ram, NULL, "zynq.ext_ram", ram_size,
+                           &error_abort);
     vmstate_register_ram_global(ext_ram);
     memory_region_add_subregion(address_space_mem, 0, ext_ram);
 
     /* 256K of on-chip memory */
-    memory_region_init_ram(ocm_ram, NULL, "zynq.ocm_ram", 256 << 10);
+    memory_region_init_ram(ocm_ram, NULL, "zynq.ocm_ram", 256 << 10,
+                           &error_abort);
     vmstate_register_ram_global(ocm_ram);
     memory_region_add_subregion(address_space_mem, 0xFFFC0000, ocm_ram);
 
@@ -162,7 +164,8 @@ static void zynq_init(MachineState *machine)
 
     /* AMD */
     pflash_cfi02_register(0xe2000000, NULL, "zynq.pflash", FLASH_SIZE,
-                          dinfo ? dinfo->bdrv : NULL, FLASH_SECTOR_SIZE,
+                          dinfo ? blk_by_legacy_dinfo(dinfo) : NULL,
+                          FLASH_SECTOR_SIZE,
                           FLASH_SIZE/FLASH_SECTOR_SIZE, 1,
                           1, 0x0066, 0x0022, 0x0000, 0x0000, 0x0555, 0x2aa,
                               0);
diff --git a/hw/arm/z2.c b/hw/arm/z2.c
index 36b3b504f..17355479a 100644
--- a/hw/arm/z2.c
+++ b/hw/arm/z2.c
@@ -20,7 +20,7 @@
 #include "hw/boards.h"
 #include "sysemu/sysemu.h"
 #include "hw/block/flash.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "ui/console.h"
 #include "audio/audio.h"
 #include "exec/address-spaces.h"
@@ -336,9 +336,9 @@ static void z2_init(MachineState *machine)
 
     if (!pflash_cfi01_register(Z2_FLASH_BASE,
                                NULL, "z2.flash0", Z2_FLASH_SIZE,
-                               dinfo ? dinfo->bdrv : NULL, sector_len,
-                               Z2_FLASH_SIZE / sector_len, 4, 0, 0, 0, 0,
-                               be)) {
+                               dinfo ? blk_by_legacy_dinfo(dinfo) : NULL,
+                               sector_len, Z2_FLASH_SIZE / sector_len,
+                               4, 0, 0, 0, 0, be)) {
         fprintf(stderr, "qemu: Error registering flash memory.\n");
         exit(1);
     }
diff --git a/hw/audio/ac97.c b/hw/audio/ac97.c
index 45cb1185c..111ec0e84 100644
--- a/hw/audio/ac97.c
+++ b/hw/audio/ac97.c
@@ -1321,9 +1321,9 @@ static const MemoryRegionOps ac97_io_nabm_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static void ac97_on_reset (void *opaque)
+static void ac97_on_reset (DeviceState *dev)
 {
-    AC97LinkState *s = opaque;
+    AC97LinkState *s = container_of(dev, AC97LinkState, dev.qdev);
 
     reset_bm_regs (s, &s->bm_regs[0]);
     reset_bm_regs (s, &s->bm_regs[1]);
@@ -1382,20 +1382,11 @@ static int ac97_initfn (PCIDevice *dev)
                            "ac97-nabm", 256);
     pci_register_bar (&s->dev, 0, PCI_BASE_ADDRESS_SPACE_IO, &s->io_nam);
     pci_register_bar (&s->dev, 1, PCI_BASE_ADDRESS_SPACE_IO, &s->io_nabm);
-    qemu_register_reset (ac97_on_reset, s);
     AUD_register_card ("ac97", &s->card);
-    ac97_on_reset (s);
+    ac97_on_reset (&s->dev.qdev);
     return 0;
 }
 
-static void ac97_exitfn (PCIDevice *dev)
-{
-    AC97LinkState *s = DO_UPCAST (AC97LinkState, dev, dev);
-
-    memory_region_destroy (&s->io_nam);
-    memory_region_destroy (&s->io_nabm);
-}
-
 static int ac97_init (PCIBus *bus)
 {
     pci_create_simple (bus, -1, "AC97");
@@ -1413,7 +1404,6 @@ static void ac97_class_init (ObjectClass *klass, void *data)
     PCIDeviceClass *k = PCI_DEVICE_CLASS (klass);
 
     k->init = ac97_initfn;
-    k->exit = ac97_exitfn;
     k->vendor_id = PCI_VENDOR_ID_INTEL;
     k->device_id = PCI_DEVICE_ID_INTEL_82801AA_5;
     k->revision = 0x01;
@@ -1422,6 +1412,7 @@ static void ac97_class_init (ObjectClass *klass, void *data)
     dc->desc = "Intel 82801AA AC97 Audio";
     dc->vmsd = &vmstate_ac97;
     dc->props = ac97_properties;
+    dc->reset = ac97_on_reset;
 }
 
 static const TypeInfo ac97_info = {
diff --git a/hw/audio/es1370.c b/hw/audio/es1370.c
index 5dbf80381..e67d1ea16 100644
--- a/hw/audio/es1370.c
+++ b/hw/audio/es1370.c
@@ -1042,13 +1042,6 @@ static int es1370_initfn (PCIDevice *dev)
     return 0;
 }
 
-static void es1370_exitfn (PCIDevice *dev)
-{
-    ES1370State *s = DO_UPCAST (ES1370State, dev, dev);
-
-    memory_region_destroy (&s->io);
-}
-
 static int es1370_init (PCIBus *bus)
 {
     pci_create_simple (bus, -1, "ES1370");
@@ -1061,7 +1054,6 @@ static void es1370_class_init (ObjectClass *klass, void *data)
     PCIDeviceClass *k = PCI_DEVICE_CLASS (klass);
 
     k->init = es1370_initfn;
-    k->exit = es1370_exitfn;
     k->vendor_id = PCI_VENDOR_ID_ENSONIQ;
     k->device_id = PCI_DEVICE_ID_ENSONIQ_ES1370;
     k->class_id = PCI_CLASS_MULTIMEDIA_AUDIO;
diff --git a/hw/audio/gus.c b/hw/audio/gus.c
index bba684047..4a43ce7ad 100644
--- a/hw/audio/gus.c
+++ b/hw/audio/gus.c
@@ -212,7 +212,7 @@ static int GUS_read_DMA (void *opaque, int nchan, int dma_pos, int dma_len)
         pos += copied;
     }
 
-    if (0 == ((mode >> 4) & 1)) {
+    if (((mode >> 4) & 1) == 0) {
         DMA_release_DREQ (s->emu.gusdma);
     }
     return dma_len;
diff --git a/hw/audio/hda-codec.c b/hw/audio/hda-codec.c
index cbcf521c5..3c03ff566 100644
--- a/hw/audio/hda-codec.c
+++ b/hw/audio/hda-codec.c
@@ -489,8 +489,9 @@ static int hda_audio_init(HDACodecDevice *hda, const struct desc_codec *desc)
     for (i = 0; i < a->desc->nnodes; i++) {
         node = a->desc->nodes + i;
         param = hda_codec_find_param(node, AC_PAR_AUDIO_WIDGET_CAP);
-        if (NULL == param)
+        if (param == NULL) {
             continue;
+        }
         type = (param->val & AC_WCAP_TYPE) >> AC_WCAP_TYPE_SHIFT;
         switch (type) {
         case AC_WID_AUD_OUT:
diff --git a/hw/audio/intel-hda.c b/hw/audio/intel-hda.c
index aa49b4748..2885231a6 100644
--- a/hw/audio/intel-hda.c
+++ b/hw/audio/intel-hda.c
@@ -187,6 +187,7 @@ struct IntelHDAState {
     /* properties */
     uint32_t debug;
     uint32_t msi;
+    bool old_msi_addr;
 };
 
 #define TYPE_INTEL_HDA_GENERIC "intel-hda-generic"
@@ -1141,7 +1142,7 @@ static int intel_hda_init(PCIDevice *pci)
                           "intel-hda", 0x4000);
     pci_register_bar(&d->pci, 0, 0, &d->mmio);
     if (d->msi) {
-        msi_init(&d->pci, 0x50, 1, true, false);
+        msi_init(&d->pci, d->old_msi_addr ? 0x50 : 0x60, 1, true, false);
     }
 
     hda_codec_bus_init(DEVICE(pci), &d->codecs, sizeof(d->codecs),
@@ -1155,7 +1156,6 @@ static void intel_hda_exit(PCIDevice *pci)
     IntelHDAState *d = INTEL_HDA(pci);
 
     msi_uninit(&d->pci);
-    memory_region_destroy(&d->mmio);
 }
 
 static int intel_hda_post_load(void *opaque, int version)
@@ -1236,6 +1236,7 @@ static const VMStateDescription vmstate_intel_hda = {
 static Property intel_hda_properties[] = {
     DEFINE_PROP_UINT32("debug", IntelHDAState, debug, 0),
     DEFINE_PROP_UINT32("msi", IntelHDAState, msi, 1),
+    DEFINE_PROP_BOOL("old_msi_addr", IntelHDAState, old_msi_addr, false),
     DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/hw/audio/sb16.c b/hw/audio/sb16.c
index 60c4b3b49..bda26d012 100644
--- a/hw/audio/sb16.c
+++ b/hw/audio/sb16.c
@@ -928,7 +928,7 @@ static IO_WRITE_PROTO (dsp_write)
 /*         if (s->highspeed) */
 /*             break; */
 
-        if (0 == s->needed_bytes) {
+        if (s->needed_bytes == 0) {
             command (s, val);
 #if 0
             if (0 == s->needed_bytes) {
@@ -1212,7 +1212,7 @@ static int SB_read_DMA (void *opaque, int nchan, int dma_pos, int dma_len)
 #endif
 
     if (till <= copy) {
-        if (0 == s->dma_auto) {
+        if (s->dma_auto == 0) {
             copy = till;
         }
     }
@@ -1224,7 +1224,7 @@ static int SB_read_DMA (void *opaque, int nchan, int dma_pos, int dma_len)
     if (s->left_till_irq <= 0) {
         s->mixer_regs[0x82] |= (nchan & 4) ? 2 : 1;
         qemu_irq_raise (s->pic);
-        if (0 == s->dma_auto) {
+        if (s->dma_auto == 0) {
             control (s, 0);
             speaker (s, 0);
         }
diff --git a/hw/block/Makefile.objs b/hw/block/Makefile.objs
index bf46f03b7..d4c3ab758 100644
--- a/hw/block/Makefile.objs
+++ b/hw/block/Makefile.objs
@@ -12,4 +12,4 @@ common-obj-$(CONFIG_NVME_PCI) += nvme.o
 obj-$(CONFIG_SH4) += tc58128.o
 
 obj-$(CONFIG_VIRTIO) += virtio-blk.o
-obj-$(CONFIG_VIRTIO_BLK_DATA_PLANE) += dataplane/
+obj-$(CONFIG_VIRTIO) += dataplane/
diff --git a/hw/block/block.c b/hw/block/block.c
index 33dd3f33b..a625773d4 100644
--- a/hw/block/block.c
+++ b/hw/block/block.c
@@ -8,6 +8,7 @@
  */
 
 #include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "hw/block/block.h"
 #include "qemu/error-report.h"
 
@@ -17,28 +18,33 @@ void blkconf_serial(BlockConf *conf, char **serial)
 
     if (!*serial) {
         /* try to fall back to value set with legacy -drive serial=... */
-        dinfo = drive_get_by_blockdev(conf->bs);
-        *serial = g_strdup(dinfo->serial);
+        dinfo = blk_legacy_dinfo(conf->blk);
+        if (dinfo) {
+            *serial = g_strdup(dinfo->serial);
+        }
     }
 }
 
-int blkconf_geometry(BlockConf *conf, int *ptrans,
-                     unsigned cyls_max, unsigned heads_max, unsigned secs_max)
+void blkconf_geometry(BlockConf *conf, int *ptrans,
+                      unsigned cyls_max, unsigned heads_max, unsigned secs_max,
+                      Error **errp)
 {
     DriveInfo *dinfo;
 
     if (!conf->cyls && !conf->heads && !conf->secs) {
         /* try to fall back to value set with legacy -drive cyls=... */
-        dinfo = drive_get_by_blockdev(conf->bs);
-        conf->cyls  = dinfo->cyls;
-        conf->heads = dinfo->heads;
-        conf->secs  = dinfo->secs;
-        if (ptrans) {
-            *ptrans = dinfo->trans;
+        dinfo = blk_legacy_dinfo(conf->blk);
+        if (dinfo) {
+            conf->cyls  = dinfo->cyls;
+            conf->heads = dinfo->heads;
+            conf->secs  = dinfo->secs;
+            if (ptrans) {
+                *ptrans = dinfo->trans;
+            }
         }
     }
     if (!conf->cyls && !conf->heads && !conf->secs) {
-        hd_geometry_guess(conf->bs,
+        hd_geometry_guess(conf->blk,
                           &conf->cyls, &conf->heads, &conf->secs,
                           ptrans);
     } else if (ptrans && *ptrans == BIOS_ATA_TRANSLATION_AUTO) {
@@ -46,17 +52,16 @@ int blkconf_geometry(BlockConf *conf, int *ptrans,
     }
     if (conf->cyls || conf->heads || conf->secs) {
         if (conf->cyls < 1 || conf->cyls > cyls_max) {
-            error_report("cyls must be between 1 and %u", cyls_max);
-            return -1;
+            error_setg(errp, "cyls must be between 1 and %u", cyls_max);
+            return;
         }
         if (conf->heads < 1 || conf->heads > heads_max) {
-            error_report("heads must be between 1 and %u", heads_max);
-            return -1;
+            error_setg(errp, "heads must be between 1 and %u", heads_max);
+            return;
         }
         if (conf->secs < 1 || conf->secs > secs_max) {
-            error_report("secs must be between 1 and %u", secs_max);
-            return -1;
+            error_setg(errp, "secs must be between 1 and %u", secs_max);
+            return;
         }
     }
-    return 0;
 }
diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c
index d6ba65ca2..1222a37f4 100644
--- a/hw/block/dataplane/virtio-blk.c
+++ b/hw/block/dataplane/virtio-blk.c
@@ -17,7 +17,7 @@
 #include "qemu/thread.h"
 #include "qemu/error-report.h"
 #include "hw/virtio/dataplane/vring.h"
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "hw/virtio/virtio-blk.h"
 #include "virtio-blk.h"
 #include "block/aio.h"
@@ -28,8 +28,9 @@ struct VirtIOBlockDataPlane {
     bool started;
     bool starting;
     bool stopping;
+    bool disabled;
 
-    VirtIOBlkConf *blk;
+    VirtIOBlkConf *conf;
 
     VirtIODevice *vdev;
     Vring vring;                    /* virtqueue vring */
@@ -93,7 +94,7 @@ static void handle_notify(EventNotifier *e)
     VirtIOBlock *vblk = VIRTIO_BLK(s->vdev);
 
     event_notifier_test_and_clear(&s->host_notifier);
-    bdrv_io_plug(s->blk->conf.bs);
+    blk_io_plug(s->conf->conf.blk);
     for (;;) {
         MultiReqBuffer mrb = {
             .num_writes = 0,
@@ -119,7 +120,7 @@ static void handle_notify(EventNotifier *e)
             virtio_blk_handle_request(req, &mrb);
         }
 
-        virtio_submit_multiwrite(s->blk->conf.bs, &mrb);
+        virtio_submit_multiwrite(s->conf->conf.blk, &mrb);
 
         if (likely(ret == -EAGAIN)) { /* vring emptied */
             /* Re-enable guest->host notifies and stop processing the vring.
@@ -132,11 +133,11 @@ static void handle_notify(EventNotifier *e)
             break;
         }
     }
-    bdrv_io_unplug(s->blk->conf.bs);
+    blk_io_unplug(s->conf->conf.blk);
 }
 
 /* Context: QEMU global mutex held */
-void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *blk,
+void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
                                   VirtIOBlockDataPlane **dataplane,
                                   Error **errp)
 {
@@ -147,7 +148,7 @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *blk,
 
     *dataplane = NULL;
 
-    if (!blk->data_plane && !blk->iothread) {
+    if (!conf->data_plane && !conf->iothread) {
         return;
     }
 
@@ -162,19 +163,20 @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *blk,
     /* If dataplane is (re-)enabled while the guest is running there could be
      * block jobs that can conflict.
      */
-    if (bdrv_op_is_blocked(blk->conf.bs, BLOCK_OP_TYPE_DATAPLANE, &local_err)) {
-        error_report("cannot start dataplane thread: %s",
-                      error_get_pretty(local_err));
+    if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE,
+                          &local_err)) {
+        error_setg(errp, "cannot start dataplane thread: %s",
+                   error_get_pretty(local_err));
         error_free(local_err);
         return;
     }
 
     s = g_new0(VirtIOBlockDataPlane, 1);
     s->vdev = vdev;
-    s->blk = blk;
+    s->conf = conf;
 
-    if (blk->iothread) {
-        s->iothread = blk->iothread;
+    if (conf->iothread) {
+        s->iothread = conf->iothread;
         object_ref(OBJECT(s->iothread));
     } else {
         /* Create per-device IOThread if none specified.  This is for
@@ -191,7 +193,14 @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *blk,
     s->bh = aio_bh_new(s->ctx, notify_guest_bh, s);
 
     error_setg(&s->blocker, "block device is in use by data plane");
-    bdrv_op_block_all(blk->conf.bs, s->blocker);
+    blk_op_block_all(conf->conf.blk, s->blocker);
+    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_RESIZE, s->blocker);
+    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_DRIVE_DEL, s->blocker);
+    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_BACKUP_SOURCE, s->blocker);
+    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_COMMIT, s->blocker);
+    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_MIRROR, s->blocker);
+    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_STREAM, s->blocker);
+    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_REPLACE, s->blocker);
 
     *dataplane = s;
 }
@@ -204,7 +213,7 @@ void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s)
     }
 
     virtio_blk_data_plane_stop(s);
-    bdrv_op_unblock_all(s->blk->conf.bs, s->blocker);
+    blk_op_unblock_all(s->conf->conf.blk, s->blocker);
     error_free(s->blocker);
     object_unref(OBJECT(s->iothread));
     qemu_bh_delete(s->bh);
@@ -218,8 +227,9 @@ void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s)
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
     VirtIOBlock *vblk = VIRTIO_BLK(s->vdev);
     VirtQueue *vq;
+    int r;
 
-    if (s->started) {
+    if (s->started || s->disabled) {
         return;
     }
 
@@ -231,22 +241,23 @@ void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s)
 
     vq = virtio_get_queue(s->vdev, 0);
     if (!vring_setup(&s->vring, s->vdev, 0)) {
-        s->starting = false;
-        return;
+        goto fail_vring;
     }
 
     /* Set up guest notifier (irq) */
-    if (k->set_guest_notifiers(qbus->parent, 1, true) != 0) {
-        fprintf(stderr, "virtio-blk failed to set guest notifier, "
-                "ensure -enable-kvm is set\n");
-        exit(1);
+    r = k->set_guest_notifiers(qbus->parent, 1, true);
+    if (r != 0) {
+        fprintf(stderr, "virtio-blk failed to set guest notifier (%d), "
+                "ensure -enable-kvm is set\n", r);
+        goto fail_guest_notifiers;
     }
     s->guest_notifier = virtio_queue_get_guest_notifier(vq);
 
     /* Set up virtqueue notify */
-    if (k->set_host_notifier(qbus->parent, 0, true) != 0) {
-        fprintf(stderr, "virtio-blk failed to set host notifier\n");
-        exit(1);
+    r = k->set_host_notifier(qbus->parent, 0, true);
+    if (r != 0) {
+        fprintf(stderr, "virtio-blk failed to set host notifier (%d)\n", r);
+        goto fail_host_notifier;
     }
     s->host_notifier = *virtio_queue_get_host_notifier(vq);
 
@@ -257,7 +268,7 @@ void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s)
     s->started = true;
     trace_virtio_blk_data_plane_start(s);
 
-    bdrv_set_aio_context(s->blk->conf.bs, s->ctx);
+    blk_set_aio_context(s->conf->conf.blk, s->ctx);
 
     /* Kick right away to begin processing requests already in vring */
     event_notifier_set(virtio_queue_get_host_notifier(vq));
@@ -266,6 +277,15 @@ void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s)
     aio_context_acquire(s->ctx);
     aio_set_event_notifier(s->ctx, &s->host_notifier, handle_notify);
     aio_context_release(s->ctx);
+    return;
+
+  fail_host_notifier:
+    k->set_guest_notifiers(qbus->parent, 1, false);
+  fail_guest_notifiers:
+    vring_teardown(&s->vring, s->vdev, 0);
+    s->disabled = true;
+  fail_vring:
+    s->starting = false;
 }
 
 /* Context: QEMU global mutex held */
@@ -274,6 +294,13 @@ void virtio_blk_data_plane_stop(VirtIOBlockDataPlane *s)
     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s->vdev)));
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
     VirtIOBlock *vblk = VIRTIO_BLK(s->vdev);
+
+
+    /* Better luck next time. */
+    if (s->disabled) {
+        s->disabled = false;
+        return;
+    }
     if (!s->started || s->stopping) {
         return;
     }
@@ -287,7 +314,7 @@ void virtio_blk_data_plane_stop(VirtIOBlockDataPlane *s)
     aio_set_event_notifier(s->ctx, &s->host_notifier, NULL);
 
     /* Drain and switch bs back to the QEMU main loop */
-    bdrv_set_aio_context(s->blk->conf.bs, qemu_get_aio_context());
+    blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context());
 
     aio_context_release(s->ctx);
 
diff --git a/hw/block/dataplane/virtio-blk.h b/hw/block/dataplane/virtio-blk.h
index 1750c9905..c88d40e72 100644
--- a/hw/block/dataplane/virtio-blk.h
+++ b/hw/block/dataplane/virtio-blk.h
@@ -19,7 +19,7 @@
 
 typedef struct VirtIOBlockDataPlane VirtIOBlockDataPlane;
 
-void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *blk,
+void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
                                   VirtIOBlockDataPlane **dataplane,
                                   Error **errp);
 void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s);
diff --git a/hw/block/fdc.c b/hw/block/fdc.c
index 490d127df..739a03ed5 100644
--- a/hw/block/fdc.c
+++ b/hw/block/fdc.c
@@ -33,6 +33,7 @@
 #include "qemu/timer.h"
 #include "hw/isa/isa.h"
 #include "hw/sysbus.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/sysemu.h"
 #include "qemu/log.h"
@@ -113,7 +114,7 @@ static const FDFormat fd_formats[] = {
     { FDRIVE_DRV_NONE, -1, -1, 0, 0, },
 };
 
-static void pick_geometry(BlockDriverState *bs, int *nb_heads,
+static void pick_geometry(BlockBackend *blk, int *nb_heads,
                           int *max_track, int *last_sect,
                           FDriveType drive_in, FDriveType *drive,
                           FDriveRate *rate)
@@ -122,7 +123,7 @@ static void pick_geometry(BlockDriverState *bs, int *nb_heads,
     uint64_t nb_sectors, size;
     int i, first_match, match;
 
-    bdrv_get_geometry(bs, &nb_sectors);
+    blk_get_geometry(blk, &nb_sectors);
     match = -1;
     first_match = -1;
     for (i = 0; ; i++) {
@@ -175,7 +176,7 @@ typedef enum FDiskFlags {
 
 typedef struct FDrive {
     FDCtrl *fdctrl;
-    BlockDriverState *bs;
+    BlockBackend *blk;
     /* Drive status */
     FDriveType drive;
     uint8_t perpendicular;    /* 2.88 MB access mode    */
@@ -260,7 +261,7 @@ static int fd_seek(FDrive *drv, uint8_t head, uint8_t track, uint8_t sect,
 #endif
         drv->head = head;
         if (drv->track != track) {
-            if (drv->bs != NULL && bdrv_is_inserted(drv->bs)) {
+            if (drv->blk != NULL && blk_is_inserted(drv->blk)) {
                 drv->media_changed = 0;
             }
             ret = 1;
@@ -269,7 +270,7 @@ static int fd_seek(FDrive *drv, uint8_t head, uint8_t track, uint8_t sect,
         drv->sect = sect;
     }
 
-    if (drv->bs == NULL || !bdrv_is_inserted(drv->bs)) {
+    if (drv->blk == NULL || !blk_is_inserted(drv->blk)) {
         ret = 2;
     }
 
@@ -291,11 +292,11 @@ static void fd_revalidate(FDrive *drv)
     FDriveRate rate;
 
     FLOPPY_DPRINTF("revalidate\n");
-    if (drv->bs != NULL) {
-        ro = bdrv_is_read_only(drv->bs);
-        pick_geometry(drv->bs, &nb_heads, &max_track,
+    if (drv->blk != NULL) {
+        ro = blk_is_read_only(drv->blk);
+        pick_geometry(drv->blk, &nb_heads, &max_track,
                       &last_sect, drv->drive, &drive, &rate);
-        if (!bdrv_is_inserted(drv->bs)) {
+        if (!blk_is_inserted(drv->blk)) {
             FLOPPY_DPRINTF("No disk in drive\n");
         } else {
             FLOPPY_DPRINTF("Floppy disk (%d h %d t %d s) %s\n", nb_heads,
@@ -665,7 +666,7 @@ static bool fdrive_media_changed_needed(void *opaque)
 {
     FDrive *drive = opaque;
 
-    return (drive->bs != NULL && drive->media_changed != 1);
+    return (drive->blk != NULL && drive->media_changed != 1);
 }
 
 static const VMStateDescription vmstate_fdrive_media_changed = {
@@ -695,10 +696,34 @@ static const VMStateDescription vmstate_fdrive_media_rate = {
     }
 };
 
+static bool fdrive_perpendicular_needed(void *opaque)
+{
+    FDrive *drive = opaque;
+
+    return drive->perpendicular != 0;
+}
+
+static const VMStateDescription vmstate_fdrive_perpendicular = {
+    .name = "fdrive/perpendicular",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(perpendicular, FDrive),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static int fdrive_post_load(void *opaque, int version_id)
+{
+    fd_revalidate(opaque);
+    return 0;
+}
+
 static const VMStateDescription vmstate_fdrive = {
     .name = "fdrive",
     .version_id = 1,
     .minimum_version_id = 1,
+    .post_load = fdrive_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_UINT8(head, FDrive),
         VMSTATE_UINT8(track, FDrive),
@@ -713,6 +738,9 @@ static const VMStateDescription vmstate_fdrive = {
             .vmsd = &vmstate_fdrive_media_rate,
             .needed = &fdrive_media_rate_needed,
         } , {
+            .vmsd = &vmstate_fdrive_perpendicular,
+            .needed = &fdrive_perpendicular_needed,
+        } , {
             /* empty */
         }
     }
@@ -734,6 +762,40 @@ static int fdc_post_load(void *opaque, int version_id)
     return 0;
 }
 
+static bool fdc_reset_sensei_needed(void *opaque)
+{
+    FDCtrl *s = opaque;
+
+    return s->reset_sensei != 0;
+}
+
+static const VMStateDescription vmstate_fdc_reset_sensei = {
+    .name = "fdc/reset_sensei",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT32(reset_sensei, FDCtrl),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool fdc_result_timer_needed(void *opaque)
+{
+    FDCtrl *s = opaque;
+
+    return timer_pending(s->result_timer);
+}
+
+static const VMStateDescription vmstate_fdc_result_timer = {
+    .name = "fdc/result_timer",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_TIMER(result_timer, FDCtrl),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_fdc = {
     .name = "fdc",
     .version_id = 2,
@@ -770,6 +832,17 @@ static const VMStateDescription vmstate_fdc = {
         VMSTATE_STRUCT_ARRAY(drives, FDCtrl, MAX_FD, 1,
                              vmstate_fdrive, FDrive),
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (VMStateSubsection[]) {
+        {
+            .vmsd = &vmstate_fdc_reset_sensei,
+            .needed = fdc_reset_sensei_needed,
+        } , {
+            .vmsd = &vmstate_fdc_result_timer,
+            .needed = fdc_result_timer_needed,
+        } , {
+            /* empty */
+        }
     }
 };
 
@@ -838,12 +911,15 @@ static void fdctrl_reset(FDCtrl *fdctrl, int do_irq)
     /* Initialise controller */
     fdctrl->sra = 0;
     fdctrl->srb = 0xc0;
-    if (!fdctrl->drives[1].bs)
+    if (!fdctrl->drives[1].blk) {
         fdctrl->sra |= FD_SRA_nDRV2;
+    }
     fdctrl->cur_drv = 0;
     fdctrl->dor = FD_DOR_nRESET;
     fdctrl->dor |= (fdctrl->dma_chann != -1) ? FD_DOR_DMAEN : 0;
     fdctrl->msr = FD_MSR_RQM;
+    fdctrl->reset_sensei = 0;
+    timer_del(fdctrl->result_timer);
     /* FIFO state */
     fdctrl->data_pos = 0;
     fdctrl->data_len = 0;
@@ -1329,7 +1405,7 @@ static int fdctrl_transfer_handler (void *opaque, int nchan,
         status2 = FD_SR2_SNS;
     if (dma_len > fdctrl->data_len)
         dma_len = fdctrl->data_len;
-    if (cur_drv->bs == NULL) {
+    if (cur_drv->blk == NULL) {
         if (fdctrl->data_dir == FD_DIR_WRITE)
             fdctrl_stop_transfer(fdctrl, FD_SR0_ABNTERM | FD_SR0_SEEK, 0x00, 0x00);
         else
@@ -1350,8 +1426,8 @@ static int fdctrl_transfer_handler (void *opaque, int nchan,
         if (fdctrl->data_dir != FD_DIR_WRITE ||
             len < FD_SECTOR_LEN || rel_pos != 0) {
             /* READ & SCAN commands and realign to a sector for WRITE */
-            if (bdrv_read(cur_drv->bs, fd_sector(cur_drv),
-                          fdctrl->fifo, 1) < 0) {
+            if (blk_read(cur_drv->blk, fd_sector(cur_drv),
+                         fdctrl->fifo, 1) < 0) {
                 FLOPPY_DPRINTF("Floppy: error getting sector %d\n",
                                fd_sector(cur_drv));
                 /* Sure, image size is too small... */
@@ -1378,8 +1454,8 @@ static int fdctrl_transfer_handler (void *opaque, int nchan,
 
             DMA_read_memory (nchan, fdctrl->fifo + rel_pos,
                              fdctrl->data_pos, len);
-            if (bdrv_write(cur_drv->bs, fd_sector(cur_drv),
-                           fdctrl->fifo, 1) < 0) {
+            if (blk_write(cur_drv->blk, fd_sector(cur_drv),
+                          fdctrl->fifo, 1) < 0) {
                 FLOPPY_DPRINTF("error writing sector %d\n",
                                fd_sector(cur_drv));
                 fdctrl_stop_transfer(fdctrl, FD_SR0_ABNTERM | FD_SR0_SEEK, 0x00, 0x00);
@@ -1454,7 +1530,8 @@ static uint32_t fdctrl_read_data(FDCtrl *fdctrl)
                                    fd_sector(cur_drv));
                     return 0;
                 }
-            if (bdrv_read(cur_drv->bs, fd_sector(cur_drv), fdctrl->fifo, 1) < 0) {
+            if (blk_read(cur_drv->blk, fd_sector(cur_drv), fdctrl->fifo, 1)
+                < 0) {
                 FLOPPY_DPRINTF("error getting sector %d\n",
                                fd_sector(cur_drv));
                 /* Sure, image size is too small... */
@@ -1523,8 +1600,8 @@ static void fdctrl_format_sector(FDCtrl *fdctrl)
         break;
     }
     memset(fdctrl->fifo, 0, FD_SECTOR_LEN);
-    if (cur_drv->bs == NULL ||
-        bdrv_write(cur_drv->bs, fd_sector(cur_drv), fdctrl->fifo, 1) < 0) {
+    if (cur_drv->blk == NULL ||
+        blk_write(cur_drv->blk, fd_sector(cur_drv), fdctrl->fifo, 1) < 0) {
         FLOPPY_DPRINTF("error formatting sector %d\n", fd_sector(cur_drv));
         fdctrl_stop_transfer(fdctrl, FD_SR0_ABNTERM | FD_SR0_SEEK, 0x00, 0x00);
     } else {
@@ -1914,7 +1991,8 @@ static void fdctrl_write_data(FDCtrl *fdctrl, uint32_t value)
         if (pos == FD_SECTOR_LEN - 1 ||
             fdctrl->data_pos == fdctrl->data_len) {
             cur_drv = get_cur_drv(fdctrl);
-            if (bdrv_write(cur_drv->bs, fd_sector(cur_drv), fdctrl->fifo, 1) < 0) {
+            if (blk_write(cur_drv->blk, fd_sector(cur_drv), fdctrl->fifo, 1)
+                < 0) {
                 FLOPPY_DPRINTF("error writing sector %d\n",
                                fd_sector(cur_drv));
                 return;
@@ -2002,12 +2080,12 @@ static void fdctrl_connect_drives(FDCtrl *fdctrl, Error **errp)
         drive = &fdctrl->drives[i];
         drive->fdctrl = fdctrl;
 
-        if (drive->bs) {
-            if (bdrv_get_on_error(drive->bs, 0) != BLOCKDEV_ON_ERROR_ENOSPC) {
+        if (drive->blk) {
+            if (blk_get_on_error(drive->blk, 0) != BLOCKDEV_ON_ERROR_ENOSPC) {
                 error_setg(errp, "fdc doesn't support drive option werror");
                 return;
             }
-            if (bdrv_get_on_error(drive->bs, 1) != BLOCKDEV_ON_ERROR_REPORT) {
+            if (blk_get_on_error(drive->blk, 1) != BLOCKDEV_ON_ERROR_REPORT) {
                 error_setg(errp, "fdc doesn't support drive option rerror");
                 return;
             }
@@ -2015,8 +2093,8 @@ static void fdctrl_connect_drives(FDCtrl *fdctrl, Error **errp)
 
         fd_init(drive);
         fdctrl_change_cb(drive, 0);
-        if (drive->bs) {
-            bdrv_set_dev_ops(drive->bs, &fdctrl_block_ops, drive);
+        if (drive->blk) {
+            blk_set_dev_ops(drive->blk, &fdctrl_block_ops, drive);
         }
     }
 }
@@ -2033,10 +2111,10 @@ ISADevice *fdctrl_init_isa(ISABus *bus, DriveInfo **fds)
     dev = DEVICE(isadev);
 
     if (fds[0]) {
-        qdev_prop_set_drive_nofail(dev, "driveA", fds[0]->bdrv);
+        qdev_prop_set_drive_nofail(dev, "driveA", blk_by_legacy_dinfo(fds[0]));
     }
     if (fds[1]) {
-        qdev_prop_set_drive_nofail(dev, "driveB", fds[1]->bdrv);
+        qdev_prop_set_drive_nofail(dev, "driveB", blk_by_legacy_dinfo(fds[1]));
     }
     qdev_init_nofail(dev);
 
@@ -2056,10 +2134,10 @@ void fdctrl_init_sysbus(qemu_irq irq, int dma_chann,
     fdctrl = &sys->state;
     fdctrl->dma_chann = dma_chann; /* FIXME */
     if (fds[0]) {
-        qdev_prop_set_drive_nofail(dev, "driveA", fds[0]->bdrv);
+        qdev_prop_set_drive_nofail(dev, "driveA", blk_by_legacy_dinfo(fds[0]));
     }
     if (fds[1]) {
-        qdev_prop_set_drive_nofail(dev, "driveB", fds[1]->bdrv);
+        qdev_prop_set_drive_nofail(dev, "driveB", blk_by_legacy_dinfo(fds[1]));
     }
     qdev_init_nofail(dev);
     sbd = SYS_BUS_DEVICE(dev);
@@ -2075,7 +2153,7 @@ void sun4m_fdctrl_init(qemu_irq irq, hwaddr io_base,
 
     dev = qdev_create(NULL, "SUNW,fdtwo");
     if (fds[0]) {
-        qdev_prop_set_drive_nofail(dev, "drive", fds[0]->bdrv);
+        qdev_prop_set_drive_nofail(dev, "drive", blk_by_legacy_dinfo(fds[0]));
     }
     qdev_init_nofail(dev);
     sys = SYSBUS_FDC(dev);
@@ -2142,9 +2220,6 @@ static void isabus_fdc_realize(DeviceState *dev, Error **errp)
         error_propagate(errp, err);
         return;
     }
-
-    add_boot_device_path(isa->bootindexA, dev, "/floppy@0");
-    add_boot_device_path(isa->bootindexB, dev, "/floppy@1");
 }
 
 static void sysbus_fdc_initfn(Object *obj)
@@ -2215,10 +2290,8 @@ static Property isa_fdc_properties[] = {
     DEFINE_PROP_UINT32("iobase", FDCtrlISABus, iobase, 0x3f0),
     DEFINE_PROP_UINT32("irq", FDCtrlISABus, irq, 6),
     DEFINE_PROP_UINT32("dma", FDCtrlISABus, dma, 2),
-    DEFINE_PROP_DRIVE("driveA", FDCtrlISABus, state.drives[0].bs),
-    DEFINE_PROP_DRIVE("driveB", FDCtrlISABus, state.drives[1].bs),
-    DEFINE_PROP_INT32("bootindexA", FDCtrlISABus, bootindexA, -1),
-    DEFINE_PROP_INT32("bootindexB", FDCtrlISABus, bootindexB, -1),
+    DEFINE_PROP_DRIVE("driveA", FDCtrlISABus, state.drives[0].blk),
+    DEFINE_PROP_DRIVE("driveB", FDCtrlISABus, state.drives[1].blk),
     DEFINE_PROP_BIT("check_media_rate", FDCtrlISABus, state.check_media_rate,
                     0, true),
     DEFINE_PROP_END_OF_LIST(),
@@ -2236,11 +2309,24 @@ static void isabus_fdc_class_init(ObjectClass *klass, void *data)
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
 }
 
+static void isabus_fdc_instance_init(Object *obj)
+{
+    FDCtrlISABus *isa = ISA_FDC(obj);
+
+    device_add_bootindex_property(obj, &isa->bootindexA,
+                                  "bootindexA", "/floppy@0",
+                                  DEVICE(obj), NULL);
+    device_add_bootindex_property(obj, &isa->bootindexB,
+                                  "bootindexB", "/floppy@1",
+                                  DEVICE(obj), NULL);
+}
+
 static const TypeInfo isa_fdc_info = {
     .name          = TYPE_ISA_FDC,
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(FDCtrlISABus),
     .class_init    = isabus_fdc_class_init,
+    .instance_init = isabus_fdc_instance_init,
 };
 
 static const VMStateDescription vmstate_sysbus_fdc ={
@@ -2254,8 +2340,8 @@ static const VMStateDescription vmstate_sysbus_fdc ={
 };
 
 static Property sysbus_fdc_properties[] = {
-    DEFINE_PROP_DRIVE("driveA", FDCtrlSysBus, state.drives[0].bs),
-    DEFINE_PROP_DRIVE("driveB", FDCtrlSysBus, state.drives[1].bs),
+    DEFINE_PROP_DRIVE("driveA", FDCtrlSysBus, state.drives[0].blk),
+    DEFINE_PROP_DRIVE("driveB", FDCtrlSysBus, state.drives[1].blk),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -2275,7 +2361,7 @@ static const TypeInfo sysbus_fdc_info = {
 };
 
 static Property sun4m_fdc_properties[] = {
-    DEFINE_PROP_DRIVE("drive", FDCtrlSysBus, state.drives[0].bs),
+    DEFINE_PROP_DRIVE("drive", FDCtrlSysBus, state.drives[0].blk),
     DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/hw/block/hd-geometry.c b/hw/block/hd-geometry.c
index 6feb4f817..6fcf74df4 100644
--- a/hw/block/hd-geometry.c
+++ b/hw/block/hd-geometry.c
@@ -30,7 +30,7 @@
  * THE SOFTWARE.
  */
 
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "hw/block/block.h"
 #include "trace.h"
 
@@ -49,7 +49,7 @@ struct partition {
 
 /* try to guess the disk logical geometry from the MSDOS partition table.
    Return 0 if OK, -1 if could not guess */
-static int guess_disk_lchs(BlockDriverState *bs,
+static int guess_disk_lchs(BlockBackend *blk,
                            int *pcylinders, int *pheads, int *psectors)
 {
     uint8_t buf[BDRV_SECTOR_SIZE];
@@ -58,14 +58,14 @@ static int guess_disk_lchs(BlockDriverState *bs,
     uint32_t nr_sects;
     uint64_t nb_sectors;
 
-    bdrv_get_geometry(bs, &nb_sectors);
+    blk_get_geometry(blk, &nb_sectors);
 
     /**
      * The function will be invoked during startup not only in sync I/O mode,
      * but also in async I/O mode. So the I/O throttling function has to
      * be disabled temporarily here, not permanently.
      */
-    if (bdrv_read_unthrottled(bs, 0, buf, 1) < 0) {
+    if (blk_read_unthrottled(blk, 0, buf, 1) < 0) {
         return -1;
     }
     /* test msdos magic */
@@ -90,20 +90,20 @@ static int guess_disk_lchs(BlockDriverState *bs,
             *pheads = heads;
             *psectors = sectors;
             *pcylinders = cylinders;
-            trace_hd_geometry_lchs_guess(bs, cylinders, heads, sectors);
+            trace_hd_geometry_lchs_guess(blk, cylinders, heads, sectors);
             return 0;
         }
     }
     return -1;
 }
 
-static void guess_chs_for_size(BlockDriverState *bs,
+static void guess_chs_for_size(BlockBackend *blk,
                 uint32_t *pcyls, uint32_t *pheads, uint32_t *psecs)
 {
     uint64_t nb_sectors;
     int cylinders;
 
-    bdrv_get_geometry(bs, &nb_sectors);
+    blk_get_geometry(blk, &nb_sectors);
 
     cylinders = nb_sectors / (16 * 63);
     if (cylinders > 16383) {
@@ -116,21 +116,21 @@ static void guess_chs_for_size(BlockDriverState *bs,
     *psecs = 63;
 }
 
-void hd_geometry_guess(BlockDriverState *bs,
+void hd_geometry_guess(BlockBackend *blk,
                        uint32_t *pcyls, uint32_t *pheads, uint32_t *psecs,
                        int *ptrans)
 {
     int cylinders, heads, secs, translation;
 
-    if (guess_disk_lchs(bs, &cylinders, &heads, &secs) < 0) {
+    if (guess_disk_lchs(blk, &cylinders, &heads, &secs) < 0) {
         /* no LCHS guess: use a standard physical disk geometry  */
-        guess_chs_for_size(bs, pcyls, pheads, psecs);
+        guess_chs_for_size(blk, pcyls, pheads, psecs);
         translation = hd_bios_chs_auto_trans(*pcyls, *pheads, *psecs);
     } else if (heads > 16) {
         /* LCHS guess with heads > 16 means that a BIOS LBA
            translation was active, so a standard physical disk
            geometry is OK */
-        guess_chs_for_size(bs, pcyls, pheads, psecs);
+        guess_chs_for_size(blk, pcyls, pheads, psecs);
         translation = *pcyls * *pheads <= 131072
             ? BIOS_ATA_TRANSLATION_LARGE
             : BIOS_ATA_TRANSLATION_LBA;
@@ -146,7 +146,7 @@ void hd_geometry_guess(BlockDriverState *bs,
     if (ptrans) {
         *ptrans = translation;
     }
-    trace_hd_geometry_guess(bs, *pcyls, *pheads, *psecs, translation);
+    trace_hd_geometry_guess(blk, *pcyls, *pheads, *psecs, translation);
 }
 
 int hd_bios_chs_auto_trans(uint32_t cyls, uint32_t heads, uint32_t secs)
diff --git a/hw/block/m25p80.c b/hw/block/m25p80.c
index 5893773f0..ff1106b6c 100644
--- a/hw/block/m25p80.c
+++ b/hw/block/m25p80.c
@@ -22,6 +22,7 @@
  */
 
 #include "hw/hw.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "hw/ssi.h"
 
@@ -245,7 +246,7 @@ typedef struct Flash {
 
     uint32_t r;
 
-    BlockDriverState *bdrv;
+    BlockBackend *blk;
 
     uint8_t *storage;
     uint32_t size;
@@ -279,7 +280,7 @@ typedef struct M25P80Class {
 #define M25P80_GET_CLASS(obj) \
      OBJECT_GET_CLASS(M25P80Class, (obj), TYPE_M25P80)
 
-static void bdrv_sync_complete(void *opaque, int ret)
+static void blk_sync_complete(void *opaque, int ret)
 {
     /* do nothing. Masters do not directly interact with the backing store,
      * only the working copy so no mutexing required.
@@ -288,20 +289,20 @@ static void bdrv_sync_complete(void *opaque, int ret)
 
 static void flash_sync_page(Flash *s, int page)
 {
-    int bdrv_sector, nb_sectors;
+    int blk_sector, nb_sectors;
     QEMUIOVector iov;
 
-    if (!s->bdrv || bdrv_is_read_only(s->bdrv)) {
+    if (!s->blk || blk_is_read_only(s->blk)) {
         return;
     }
 
-    bdrv_sector = (page * s->pi->page_size) / BDRV_SECTOR_SIZE;
+    blk_sector = (page * s->pi->page_size) / BDRV_SECTOR_SIZE;
     nb_sectors = DIV_ROUND_UP(s->pi->page_size, BDRV_SECTOR_SIZE);
     qemu_iovec_init(&iov, 1);
-    qemu_iovec_add(&iov, s->storage + bdrv_sector * BDRV_SECTOR_SIZE,
+    qemu_iovec_add(&iov, s->storage + blk_sector * BDRV_SECTOR_SIZE,
                    nb_sectors * BDRV_SECTOR_SIZE);
-    bdrv_aio_writev(s->bdrv, bdrv_sector, &iov, nb_sectors, bdrv_sync_complete,
-                    NULL);
+    blk_aio_writev(s->blk, blk_sector, &iov, nb_sectors, blk_sync_complete,
+                   NULL);
 }
 
 static inline void flash_sync_area(Flash *s, int64_t off, int64_t len)
@@ -309,7 +310,7 @@ static inline void flash_sync_area(Flash *s, int64_t off, int64_t len)
     int64_t start, end, nb_sectors;
     QEMUIOVector iov;
 
-    if (!s->bdrv || bdrv_is_read_only(s->bdrv)) {
+    if (!s->blk || blk_is_read_only(s->blk)) {
         return;
     }
 
@@ -320,7 +321,7 @@ static inline void flash_sync_area(Flash *s, int64_t off, int64_t len)
     qemu_iovec_init(&iov, 1);
     qemu_iovec_add(&iov, s->storage + (start * BDRV_SECTOR_SIZE),
                                         nb_sectors * BDRV_SECTOR_SIZE);
-    bdrv_aio_writev(s->bdrv, start, &iov, nb_sectors, bdrv_sync_complete, NULL);
+    blk_aio_writev(s->blk, start, &iov, nb_sectors, blk_sync_complete, NULL);
 }
 
 static void flash_erase(Flash *s, int offset, FlashCMD cmd)
@@ -620,17 +621,17 @@ static int m25p80_init(SSISlave *ss)
 
     s->size = s->pi->sector_size * s->pi->n_sectors;
     s->dirty_page = -1;
-    s->storage = qemu_blockalign(s->bdrv, s->size);
+    s->storage = blk_blockalign(s->blk, s->size);
 
     dinfo = drive_get_next(IF_MTD);
 
-    if (dinfo && dinfo->bdrv) {
+    if (dinfo) {
         DB_PRINT_L(0, "Binding to IF_MTD drive\n");
-        s->bdrv = dinfo->bdrv;
+        s->blk = blk_by_legacy_dinfo(dinfo);
 
         /* FIXME: Move to late init */
-        if (bdrv_read(s->bdrv, 0, s->storage, DIV_ROUND_UP(s->size,
-                                                    BDRV_SECTOR_SIZE))) {
+        if (blk_read(s->blk, 0, s->storage,
+                     DIV_ROUND_UP(s->size, BDRV_SECTOR_SIZE))) {
             fprintf(stderr, "Failed to initialize SPI flash!\n");
             return 1;
         }
diff --git a/hw/block/nand.c b/hw/block/nand.c
index 38eefd436..1882a0cbe 100644
--- a/hw/block/nand.c
+++ b/hw/block/nand.c
@@ -20,7 +20,7 @@
 
 # include "hw/hw.h"
 # include "hw/block/flash.h"
-# include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "hw/qdev.h"
 #include "qemu/error-report.h"
 
@@ -61,7 +61,7 @@ struct NANDFlashState {
     int size, pages;
     int page_shift, oob_shift, erase_shift, addr_shift;
     uint8_t *storage;
-    BlockDriverState *bdrv;
+    BlockBackend *blk;
     int mem_oob;
 
     uint8_t cle, ale, ce, wp, gnd;
@@ -400,12 +400,12 @@ static void nand_realize(DeviceState *dev, Error **errp)
 
     pagesize = 1 << s->oob_shift;
     s->mem_oob = 1;
-    if (s->bdrv) {
-        if (bdrv_is_read_only(s->bdrv)) {
+    if (s->blk) {
+        if (blk_is_read_only(s->blk)) {
             error_setg(errp, "Can't use a read-only drive");
             return;
         }
-        if (bdrv_getlength(s->bdrv) >=
+        if (blk_getlength(s->blk) >=
                 (s->pages << s->page_shift) + (s->pages << s->oob_shift)) {
             pagesize = 0;
             s->mem_oob = 0;
@@ -424,7 +424,7 @@ static void nand_realize(DeviceState *dev, Error **errp)
 static Property nand_properties[] = {
     DEFINE_PROP_UINT8("manufacturer_id", NANDFlashState, manf_id, 0),
     DEFINE_PROP_UINT8("chip_id", NANDFlashState, chip_id, 0),
-    DEFINE_PROP_DRIVE("drive", NANDFlashState, bdrv),
+    DEFINE_PROP_DRIVE("drive", NANDFlashState, blk),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -624,7 +624,7 @@ uint32_t nand_getbuswidth(DeviceState *dev)
     return s->buswidth << 3;
 }
 
-DeviceState *nand_init(BlockDriverState *bdrv, int manf_id, int chip_id)
+DeviceState *nand_init(BlockBackend *blk, int manf_id, int chip_id)
 {
     DeviceState *dev;
 
@@ -634,8 +634,8 @@ DeviceState *nand_init(BlockDriverState *bdrv, int manf_id, int chip_id)
     dev = DEVICE(object_new(TYPE_NAND));
     qdev_prop_set_uint8(dev, "manufacturer_id", manf_id);
     qdev_prop_set_uint8(dev, "chip_id", chip_id);
-    if (bdrv) {
-        qdev_prop_set_drive_nofail(dev, "drive", bdrv);
+    if (blk) {
+        qdev_prop_set_drive_nofail(dev, "drive", blk);
     }
 
     qdev_init_nofail(dev);
@@ -654,14 +654,14 @@ static void glue(nand_blk_write_, PAGE_SIZE)(NANDFlashState *s)
     if (PAGE(s->addr) >= s->pages)
         return;
 
-    if (!s->bdrv) {
+    if (!s->blk) {
         mem_and(s->storage + PAGE_START(s->addr) + (s->addr & PAGE_MASK) +
                         s->offset, s->io, s->iolen);
     } else if (s->mem_oob) {
         sector = SECTOR(s->addr);
         off = (s->addr & PAGE_MASK) + s->offset;
         soff = SECTOR_OFFSET(s->addr);
-        if (bdrv_read(s->bdrv, sector, iobuf, PAGE_SECTORS) < 0) {
+        if (blk_read(s->blk, sector, iobuf, PAGE_SECTORS) < 0) {
             printf("%s: read error in sector %" PRIu64 "\n", __func__, sector);
             return;
         }
@@ -673,21 +673,21 @@ static void glue(nand_blk_write_, PAGE_SIZE)(NANDFlashState *s)
                             MIN(OOB_SIZE, off + s->iolen - PAGE_SIZE));
         }
 
-        if (bdrv_write(s->bdrv, sector, iobuf, PAGE_SECTORS) < 0) {
+        if (blk_write(s->blk, sector, iobuf, PAGE_SECTORS) < 0) {
             printf("%s: write error in sector %" PRIu64 "\n", __func__, sector);
         }
     } else {
         off = PAGE_START(s->addr) + (s->addr & PAGE_MASK) + s->offset;
         sector = off >> 9;
         soff = off & 0x1ff;
-        if (bdrv_read(s->bdrv, sector, iobuf, PAGE_SECTORS + 2) < 0) {
+        if (blk_read(s->blk, sector, iobuf, PAGE_SECTORS + 2) < 0) {
             printf("%s: read error in sector %" PRIu64 "\n", __func__, sector);
             return;
         }
 
         mem_and(iobuf + soff, s->io, s->iolen);
 
-        if (bdrv_write(s->bdrv, sector, iobuf, PAGE_SECTORS + 2) < 0) {
+        if (blk_write(s->blk, sector, iobuf, PAGE_SECTORS + 2) < 0) {
             printf("%s: write error in sector %" PRIu64 "\n", __func__, sector);
         }
     }
@@ -705,7 +705,7 @@ static void glue(nand_blk_erase_, PAGE_SIZE)(NANDFlashState *s)
         return;
     }
 
-    if (!s->bdrv) {
+    if (!s->blk) {
         memset(s->storage + PAGE_START(addr),
                         0xff, (PAGE_SIZE + OOB_SIZE) << s->erase_shift);
     } else if (s->mem_oob) {
@@ -714,17 +714,17 @@ static void glue(nand_blk_erase_, PAGE_SIZE)(NANDFlashState *s)
         i = SECTOR(addr);
         page = SECTOR(addr + (ADDR_SHIFT + s->erase_shift));
         for (; i < page; i ++)
-            if (bdrv_write(s->bdrv, i, iobuf, 1) < 0) {
+            if (blk_write(s->blk, i, iobuf, 1) < 0) {
                 printf("%s: write error in sector %" PRIu64 "\n", __func__, i);
             }
     } else {
         addr = PAGE_START(addr);
         page = addr >> 9;
-        if (bdrv_read(s->bdrv, page, iobuf, 1) < 0) {
+        if (blk_read(s->blk, page, iobuf, 1) < 0) {
             printf("%s: read error in sector %" PRIu64 "\n", __func__, page);
         }
         memset(iobuf + (addr & 0x1ff), 0xff, (~addr & 0x1ff) + 1);
-        if (bdrv_write(s->bdrv, page, iobuf, 1) < 0) {
+        if (blk_write(s->blk, page, iobuf, 1) < 0) {
             printf("%s: write error in sector %" PRIu64 "\n", __func__, page);
         }
 
@@ -732,18 +732,18 @@ static void glue(nand_blk_erase_, PAGE_SIZE)(NANDFlashState *s)
         i = (addr & ~0x1ff) + 0x200;
         for (addr += ((PAGE_SIZE + OOB_SIZE) << s->erase_shift) - 0x200;
                         i < addr; i += 0x200) {
-            if (bdrv_write(s->bdrv, i >> 9, iobuf, 1) < 0) {
+            if (blk_write(s->blk, i >> 9, iobuf, 1) < 0) {
                 printf("%s: write error in sector %" PRIu64 "\n",
                        __func__, i >> 9);
             }
         }
 
         page = i >> 9;
-        if (bdrv_read(s->bdrv, page, iobuf, 1) < 0) {
+        if (blk_read(s->blk, page, iobuf, 1) < 0) {
             printf("%s: read error in sector %" PRIu64 "\n", __func__, page);
         }
         memset(iobuf, 0xff, ((addr - 1) & 0x1ff) + 1);
-        if (bdrv_write(s->bdrv, page, iobuf, 1) < 0) {
+        if (blk_write(s->blk, page, iobuf, 1) < 0) {
             printf("%s: write error in sector %" PRIu64 "\n", __func__, page);
         }
     }
@@ -756,9 +756,9 @@ static void glue(nand_blk_load_, PAGE_SIZE)(NANDFlashState *s,
         return;
     }
 
-    if (s->bdrv) {
+    if (s->blk) {
         if (s->mem_oob) {
-            if (bdrv_read(s->bdrv, SECTOR(addr), s->io, PAGE_SECTORS) < 0) {
+            if (blk_read(s->blk, SECTOR(addr), s->io, PAGE_SECTORS) < 0) {
                 printf("%s: read error in sector %" PRIu64 "\n",
                                 __func__, SECTOR(addr));
             }
@@ -767,8 +767,8 @@ static void glue(nand_blk_load_, PAGE_SIZE)(NANDFlashState *s,
                             OOB_SIZE);
             s->ioaddr = s->io + SECTOR_OFFSET(s->addr) + offset;
         } else {
-            if (bdrv_read(s->bdrv, PAGE_START(addr) >> 9,
-                                    s->io, (PAGE_SECTORS + 2)) < 0) {
+            if (blk_read(s->blk, PAGE_START(addr) >> 9,
+                         s->io, (PAGE_SECTORS + 2)) < 0) {
                 printf("%s: read error in sector %" PRIu64 "\n",
                                 __func__, PAGE_START(addr) >> 9);
             }
diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index 5fd8f8982..13276589e 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -24,6 +24,9 @@
 #include <hw/hw.h>
 #include <hw/pci/msix.h>
 #include <hw/pci/pci.h>
+#include "sysemu/sysemu.h"
+#include "qapi/visitor.h"
+#include "sysemu/block-backend.h"
 
 #include "nvme.h"
 
@@ -197,7 +200,7 @@ static void nvme_rw_cb(void *opaque, int ret)
     NvmeCtrl *n = sq->ctrl;
     NvmeCQueue *cq = n->cq[sq->cqid];
 
-    bdrv_acct_done(n->conf.bs, &req->acct);
+    block_acct_done(blk_get_stats(n->conf.blk), &req->acct);
     if (!ret) {
         req->status = NVME_SUCCESS;
     } else {
@@ -231,11 +234,11 @@ static uint16_t nvme_rw(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
     }
     assert((nlb << data_shift) == req->qsg.size);
 
-    dma_acct_start(n->conf.bs, &req->acct, &req->qsg, is_write ?
-        BDRV_ACCT_WRITE : BDRV_ACCT_READ);
+    dma_acct_start(n->conf.blk, &req->acct, &req->qsg,
+                   is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ);
     req->aiocb = is_write ?
-        dma_bdrv_write(n->conf.bs, &req->qsg, aio_slba, nvme_rw_cb, req) :
-        dma_bdrv_read(n->conf.bs, &req->qsg, aio_slba, nvme_rw_cb, req);
+        dma_blk_write(n->conf.blk, &req->qsg, aio_slba, nvme_rw_cb, req) :
+        dma_blk_read(n->conf.blk, &req->qsg, aio_slba, nvme_rw_cb, req);
 
     return NVME_NO_COMPLETE;
 }
@@ -288,7 +291,7 @@ static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeCmd *cmd)
     while (!QTAILQ_EMPTY(&sq->out_req_list)) {
         req = QTAILQ_FIRST(&sq->out_req_list);
         assert(req->aiocb);
-        bdrv_aio_cancel(req->aiocb);
+        blk_aio_cancel(req->aiocb);
     }
     if (!nvme_check_cqid(n, sq->cqid)) {
         cq = n->cq[sq->cqid];
@@ -319,7 +322,7 @@ static void nvme_init_sq(NvmeSQueue *sq, NvmeCtrl *n, uint64_t dma_addr,
     sq->size = size;
     sq->cqid = cqid;
     sq->head = sq->tail = 0;
-    sq->io_req = g_malloc(sq->size * sizeof(*sq->io_req));
+    sq->io_req = g_new(NvmeRequest, sq->size);
 
     QTAILQ_INIT(&sq->req_list);
     QTAILQ_INIT(&sq->out_req_list);
@@ -563,7 +566,7 @@ static void nvme_clear_ctrl(NvmeCtrl *n)
         }
     }
 
-    bdrv_flush(n->conf.bs);
+    blk_flush(n->conf.blk);
     n->bar.cc = 0;
 }
 
@@ -580,8 +583,7 @@ static int nvme_start_ctrl(NvmeCtrl *n)
             NVME_CC_IOCQES(n->bar.cc) > NVME_CTRL_CQES_MAX(n->id_ctrl.cqes) ||
             NVME_CC_IOSQES(n->bar.cc) < NVME_CTRL_SQES_MIN(n->id_ctrl.sqes) ||
             NVME_CC_IOSQES(n->bar.cc) > NVME_CTRL_SQES_MAX(n->id_ctrl.sqes) ||
-            !NVME_AQA_ASQS(n->bar.aqa) || NVME_AQA_ASQS(n->bar.aqa) > 4095 ||
-            !NVME_AQA_ACQS(n->bar.aqa) || NVME_AQA_ACQS(n->bar.aqa) > 4095) {
+            !NVME_AQA_ASQS(n->bar.aqa) || !NVME_AQA_ACQS(n->bar.aqa)) {
         return -1;
     }
 
@@ -748,11 +750,11 @@ static int nvme_init(PCIDevice *pci_dev)
     int64_t bs_size;
     uint8_t *pci_conf;
 
-    if (!(n->conf.bs)) {
+    if (!n->conf.blk) {
         return -1;
     }
 
-    bs_size = bdrv_getlength(n->conf.bs);
+    bs_size = blk_getlength(n->conf.blk);
     if (bs_size < 0) {
         return -1;
     }
@@ -773,9 +775,9 @@ static int nvme_init(PCIDevice *pci_dev)
     n->reg_size = 1 << qemu_fls(0x1004 + 2 * (n->num_queues + 1) * 4);
     n->ns_size = bs_size / (uint64_t)n->num_namespaces;
 
-    n->namespaces = g_malloc0(sizeof(*n->namespaces)*n->num_namespaces);
-    n->sq = g_malloc0(sizeof(*n->sq)*n->num_queues);
-    n->cq = g_malloc0(sizeof(*n->cq)*n->num_queues);
+    n->namespaces = g_new0(NvmeNamespace, n->num_namespaces);
+    n->sq = g_new0(NvmeSQueue *, n->num_queues);
+    n->cq = g_new0(NvmeCQueue *, n->num_queues);
 
     memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n,
                           "nvme", n->reg_size);
@@ -839,7 +841,6 @@ static void nvme_exit(PCIDevice *pci_dev)
     g_free(n->cq);
     g_free(n->sq);
     msix_uninit_exclusive_bar(pci_dev);
-    memory_region_destroy(&n->iomem);
 }
 
 static Property nvme_props[] = {
@@ -872,11 +873,53 @@ static void nvme_class_init(ObjectClass *oc, void *data)
     dc->vmsd = &nvme_vmstate;
 }
 
+static void nvme_get_bootindex(Object *obj, Visitor *v, void *opaque,
+                                  const char *name, Error **errp)
+{
+    NvmeCtrl *s = NVME(obj);
+
+    visit_type_int32(v, &s->conf.bootindex, name, errp);
+}
+
+static void nvme_set_bootindex(Object *obj, Visitor *v, void *opaque,
+                                  const char *name, Error **errp)
+{
+    NvmeCtrl *s = NVME(obj);
+    int32_t boot_index;
+    Error *local_err = NULL;
+
+    visit_type_int32(v, &boot_index, name, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    /* check whether bootindex is present in fw_boot_order list  */
+    check_boot_index(boot_index, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    /* change bootindex to a new one */
+    s->conf.bootindex = boot_index;
+
+out:
+    if (local_err) {
+        error_propagate(errp, local_err);
+    }
+}
+
+static void nvme_instance_init(Object *obj)
+{
+    object_property_add(obj, "bootindex", "int32",
+                        nvme_get_bootindex,
+                        nvme_set_bootindex, NULL, NULL, NULL);
+    object_property_set_int(obj, -1, "bootindex", NULL);
+}
+
 static const TypeInfo nvme_info = {
     .name          = "nvme",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(NvmeCtrl),
     .class_init    = nvme_class_init,
+    .instance_init = nvme_instance_init,
 };
 
 static void nvme_register_types(void)
diff --git a/hw/block/nvme.h b/hw/block/nvme.h
index bd8fc3e4b..993c51131 100644
--- a/hw/block/nvme.h
+++ b/hw/block/nvme.h
@@ -636,7 +636,7 @@ typedef struct NvmeAsyncEvent {
 
 typedef struct NvmeRequest {
     struct NvmeSQueue       *sq;
-    BlockDriverAIOCB        *aiocb;
+    BlockAIOCB              *aiocb;
     uint16_t                status;
     NvmeCqe                 cqe;
     BlockAcctCookie         acct;
diff --git a/hw/block/onenand.c b/hw/block/onenand.c
index 5388122eb..348630d90 100644
--- a/hw/block/onenand.c
+++ b/hw/block/onenand.c
@@ -22,6 +22,7 @@
 #include "hw/hw.h"
 #include "hw/block/flash.h"
 #include "hw/irq.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "exec/memory.h"
 #include "exec/address-spaces.h"
@@ -49,8 +50,8 @@ typedef struct OneNANDState {
     hwaddr base;
     qemu_irq intr;
     qemu_irq rdy;
-    BlockDriverState *bdrv;
-    BlockDriverState *bdrv_cur;
+    BlockBackend *blk;
+    BlockBackend *blk_cur;
     uint8_t *image;
     uint8_t *otp;
     uint8_t *current;
@@ -213,7 +214,7 @@ static void onenand_reset(OneNANDState *s, int cold)
     s->wpstatus = 0x0002;
     s->cycle = 0;
     s->otpmode = 0;
-    s->bdrv_cur = s->bdrv;
+    s->blk_cur = s->blk;
     s->current = s->image;
     s->secs_cur = s->secs;
 
@@ -221,7 +222,7 @@ static void onenand_reset(OneNANDState *s, int cold)
         /* Lock the whole flash */
         memset(s->blockwp, ONEN_LOCK_LOCKED, s->blocks);
 
-        if (s->bdrv_cur && bdrv_read(s->bdrv_cur, 0, s->boot[0], 8) < 0) {
+        if (s->blk_cur && blk_read(s->blk_cur, 0, s->boot[0], 8) < 0) {
             hw_error("%s: Loading the BootRAM failed.\n", __func__);
         }
     }
@@ -237,10 +238,11 @@ static void onenand_system_reset(DeviceState *dev)
 static inline int onenand_load_main(OneNANDState *s, int sec, int secn,
                 void *dest)
 {
-    if (s->bdrv_cur)
-        return bdrv_read(s->bdrv_cur, sec, dest, secn) < 0;
-    else if (sec + secn > s->secs_cur)
+    if (s->blk_cur) {
+        return blk_read(s->blk_cur, sec, dest, secn) < 0;
+    } else if (sec + secn > s->secs_cur) {
         return 1;
+    }
 
     memcpy(dest, s->current + (sec << 9), secn << 9);
 
@@ -256,9 +258,9 @@ static inline int onenand_prog_main(OneNANDState *s, int sec, int secn,
         uint32_t size = (uint32_t)secn * 512;
         const uint8_t *sp = (const uint8_t *)src;
         uint8_t *dp = 0;
-        if (s->bdrv_cur) {
+        if (s->blk_cur) {
             dp = g_malloc(size);
-            if (!dp || bdrv_read(s->bdrv_cur, sec, dp, secn) < 0) {
+            if (!dp || blk_read(s->blk_cur, sec, dp, secn) < 0) {
                 result = 1;
             }
         } else {
@@ -273,11 +275,11 @@ static inline int onenand_prog_main(OneNANDState *s, int sec, int secn,
             for (i = 0; i < size; i++) {
                 dp[i] &= sp[i];
             }
-            if (s->bdrv_cur) {
-                result = bdrv_write(s->bdrv_cur, sec, dp, secn) < 0;
+            if (s->blk_cur) {
+                result = blk_write(s->blk_cur, sec, dp, secn) < 0;
             }
         }
-        if (dp && s->bdrv_cur) {
+        if (dp && s->blk_cur) {
             g_free(dp);
         }
     }
@@ -290,14 +292,16 @@ static inline int onenand_load_spare(OneNANDState *s, int sec, int secn,
 {
     uint8_t buf[512];
 
-    if (s->bdrv_cur) {
-        if (bdrv_read(s->bdrv_cur, s->secs_cur + (sec >> 5), buf, 1) < 0)
+    if (s->blk_cur) {
+        if (blk_read(s->blk_cur, s->secs_cur + (sec >> 5), buf, 1) < 0) {
             return 1;
+        }
         memcpy(dest, buf + ((sec & 31) << 4), secn << 4);
-    } else if (sec + secn > s->secs_cur)
+    } else if (sec + secn > s->secs_cur) {
         return 1;
-    else
+    } else {
         memcpy(dest, s->current + (s->secs_cur << 9) + (sec << 4), secn << 4);
+    }
  
     return 0;
 }
@@ -309,11 +313,10 @@ static inline int onenand_prog_spare(OneNANDState *s, int sec, int secn,
     if (secn > 0) {
         const uint8_t *sp = (const uint8_t *)src;
         uint8_t *dp = 0, *dpp = 0;
-        if (s->bdrv_cur) {
+        if (s->blk_cur) {
             dp = g_malloc(512);
-            if (!dp || bdrv_read(s->bdrv_cur,
-                                 s->secs_cur + (sec >> 5),
-                                 dp, 1) < 0) {
+            if (!dp
+                || blk_read(s->blk_cur, s->secs_cur + (sec >> 5), dp, 1) < 0) {
                 result = 1;
             } else {
                 dpp = dp + ((sec & 31) << 4);
@@ -330,9 +333,9 @@ static inline int onenand_prog_spare(OneNANDState *s, int sec, int secn,
             for (i = 0; i < (secn << 4); i++) {
                 dpp[i] &= sp[i];
             }
-            if (s->bdrv_cur) {
-                result = bdrv_write(s->bdrv_cur, s->secs_cur + (sec >> 5),
-                                    dp, 1) < 0;
+            if (s->blk_cur) {
+                result = blk_write(s->blk_cur, s->secs_cur + (sec >> 5),
+                                   dp, 1) < 0;
             }
         }
         g_free(dp);
@@ -354,16 +357,16 @@ static inline int onenand_erase(OneNANDState *s, int sec, int num)
     }
     memset(blankbuf, 0xff, 512);
     for (; num > 0; num--, sec++) {
-        if (s->bdrv_cur) {
+        if (s->blk_cur) {
             int erasesec = s->secs_cur + (sec >> 5);
-            if (bdrv_write(s->bdrv_cur, sec, blankbuf, 1) < 0) {
+            if (blk_write(s->blk_cur, sec, blankbuf, 1) < 0) {
                 goto fail;
             }
-            if (bdrv_read(s->bdrv_cur, erasesec, tmpbuf, 1) < 0) {
+            if (blk_read(s->blk_cur, erasesec, tmpbuf, 1) < 0) {
                 goto fail;
             }
             memcpy(tmpbuf + ((sec & 31) << 4), blankbuf, 1 << 4);
-            if (bdrv_write(s->bdrv_cur, erasesec, tmpbuf, 1) < 0) {
+            if (blk_write(s->blk_cur, erasesec, tmpbuf, 1) < 0) {
                 goto fail;
             }
         } else {
@@ -576,7 +579,7 @@ static void onenand_command(OneNANDState *s)
 
     case 0x65:	/* OTP Access */
         s->intstatus |= ONEN_INT;
-        s->bdrv_cur = NULL;
+        s->blk_cur = NULL;
         s->current = s->otp;
         s->secs_cur = 1 << (BLOCK_SHIFT - 9);
         s->addr[ONEN_BUF_BLOCK] = 0;
@@ -776,20 +779,20 @@ static int onenand_initfn(SysBusDevice *sbd)
         ? (1 << (6 + ((s->id.dev >> 4) & 7))) : 0;
     memory_region_init_io(&s->iomem, OBJECT(s), &onenand_ops, s, "onenand",
                           0x10000 << s->shift);
-    if (!s->bdrv) {
+    if (!s->blk) {
         s->image = memset(g_malloc(size + (size >> 5)),
                           0xff, size + (size >> 5));
     } else {
-        if (bdrv_is_read_only(s->bdrv)) {
+        if (blk_is_read_only(s->blk)) {
             error_report("Can't use a read-only drive");
             return -1;
         }
-        s->bdrv_cur = s->bdrv;
+        s->blk_cur = s->blk;
     }
     s->otp = memset(g_malloc((64 + 2) << PAGE_SHIFT),
                     0xff, (64 + 2) << PAGE_SHIFT);
     memory_region_init_ram(&s->ram, OBJECT(s), "onenand.ram",
-                           0xc000 << s->shift);
+                           0xc000 << s->shift, &error_abort);
     vmstate_register_ram_global(&s->ram);
     ram = memory_region_get_ram_ptr(&s->ram);
     s->boot[0] = ram + (0x0000 << s->shift);
@@ -815,7 +818,7 @@ static Property onenand_properties[] = {
     DEFINE_PROP_UINT16("device_id", OneNANDState, id.dev, 0),
     DEFINE_PROP_UINT16("version_id", OneNANDState, id.ver, 0),
     DEFINE_PROP_INT32("shift", OneNANDState, shift, 0),
-    DEFINE_PROP_DRIVE("drive", OneNANDState, bdrv),
+    DEFINE_PROP_DRIVE("drive", OneNANDState, blk),
     DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/hw/block/pflash_cfi01.c b/hw/block/pflash_cfi01.c
index f9507b4e5..89d380e59 100644
--- a/hw/block/pflash_cfi01.c
+++ b/hw/block/pflash_cfi01.c
@@ -38,7 +38,7 @@
 
 #include "hw/hw.h"
 #include "hw/block/flash.h"
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "qemu/timer.h"
 #include "qemu/bitops.h"
 #include "exec/address-spaces.h"
@@ -69,7 +69,7 @@ struct pflash_t {
     SysBusDevice parent_obj;
     /*< public >*/
 
-    BlockDriverState *bs;
+    BlockBackend *blk;
     uint32_t nb_blocs;
     uint64_t sector_len;
     uint8_t bank_width;
@@ -94,10 +94,13 @@ struct pflash_t {
     void *storage;
 };
 
+static int pflash_post_load(void *opaque, int version_id);
+
 static const VMStateDescription vmstate_pflash = {
     .name = "pflash_cfi01",
     .version_id = 1,
     .minimum_version_id = 1,
+    .post_load = pflash_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_UINT8(wcycle, pflash_t),
         VMSTATE_UINT8(cmd, pflash_t),
@@ -209,11 +212,11 @@ static uint32_t pflash_devid_query(pflash_t *pfl, hwaddr offset)
     switch (boff & 0xFF) {
     case 0:
         resp = pfl->ident0;
-        DPRINTF("%s: Manufacturer Code %04x\n", __func__, ret);
+        DPRINTF("%s: Manufacturer Code %04x\n", __func__, resp);
         break;
     case 1:
         resp = pfl->ident1;
-        DPRINTF("%s: Device ID Code %04x\n", __func__, ret);
+        DPRINTF("%s: Device ID Code %04x\n", __func__, resp);
         break;
     default:
         DPRINTF("%s: Read Device Information offset=%x\n", __func__,
@@ -392,13 +395,13 @@ static void pflash_update(pflash_t *pfl, int offset,
                           int size)
 {
     int offset_end;
-    if (pfl->bs) {
+    if (pfl->blk) {
         offset_end = offset + size;
         /* round to sectors */
         offset = offset >> 9;
         offset_end = (offset_end + 511) >> 9;
-        bdrv_write(pfl->bs, offset, pfl->storage + (offset << 9),
-                   offset_end - offset);
+        blk_write(pfl->blk, offset, pfl->storage + (offset << 9),
+                  offset_end - offset);
     }
 }
 
@@ -750,6 +753,7 @@ static void pflash_cfi01_realize(DeviceState *dev, Error **errp)
     int ret;
     uint64_t blocks_per_device, device_len;
     int num_devices;
+    Error *local_err = NULL;
 
     total_len = pfl->sector_len * pfl->nb_blocs;
 
@@ -770,25 +774,29 @@ static void pflash_cfi01_realize(DeviceState *dev, Error **errp)
     memory_region_init_rom_device(
         &pfl->mem, OBJECT(dev),
         pfl->be ? &pflash_cfi01_ops_be : &pflash_cfi01_ops_le, pfl,
-        pfl->name, total_len);
+        pfl->name, total_len, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
     vmstate_register_ram(&pfl->mem, DEVICE(pfl));
     pfl->storage = memory_region_get_ram_ptr(&pfl->mem);
     sysbus_init_mmio(SYS_BUS_DEVICE(dev), &pfl->mem);
 
-    if (pfl->bs) {
+    if (pfl->blk) {
         /* read the initial flash content */
-        ret = bdrv_read(pfl->bs, 0, pfl->storage, total_len >> 9);
+        ret = blk_read(pfl->blk, 0, pfl->storage, total_len >> 9);
 
         if (ret < 0) {
             vmstate_unregister_ram(&pfl->mem, DEVICE(pfl));
-            memory_region_destroy(&pfl->mem);
             error_setg(errp, "failed to read the initial flash content");
             return;
         }
     }
 
-    if (pfl->bs) {
-        pfl->ro = bdrv_is_read_only(pfl->bs);
+    if (pfl->blk) {
+        pfl->ro = blk_is_read_only(pfl->blk);
     } else {
         pfl->ro = 0;
     }
@@ -890,7 +898,7 @@ static void pflash_cfi01_realize(DeviceState *dev, Error **errp)
 }
 
 static Property pflash_cfi01_properties[] = {
-    DEFINE_PROP_DRIVE("drive", struct pflash_t, bs),
+    DEFINE_PROP_DRIVE("drive", struct pflash_t, blk),
     /* num-blocks is the number of blocks actually visible to the guest,
      * ie the total size of the device divided by the sector length.
      * If we're emulating flash devices wired in parallel the actual
@@ -954,14 +962,14 @@ type_init(pflash_cfi01_register_types)
 pflash_t *pflash_cfi01_register(hwaddr base,
                                 DeviceState *qdev, const char *name,
                                 hwaddr size,
-                                BlockDriverState *bs,
+                                BlockBackend *blk,
                                 uint32_t sector_len, int nb_blocs,
                                 int bank_width, uint16_t id0, uint16_t id1,
                                 uint16_t id2, uint16_t id3, int be)
 {
     DeviceState *dev = qdev_create(NULL, TYPE_CFI_PFLASH01);
 
-    if (bs && qdev_prop_set_drive(dev, "drive", bs)) {
+    if (blk && qdev_prop_set_drive(dev, "drive", blk)) {
         abort();
     }
     qdev_prop_set_uint32(dev, "num-blocks", nb_blocs);
@@ -983,3 +991,14 @@ MemoryRegion *pflash_cfi01_get_memory(pflash_t *fl)
 {
     return &fl->mem;
 }
+
+static int pflash_post_load(void *opaque, int version_id)
+{
+    pflash_t *pfl = opaque;
+
+    if (!pfl->ro) {
+        DPRINTF("%s: updating bdrv for %s\n", __func__, pfl->name);
+        pflash_update(pfl, 0, pfl->sector_len * pfl->nb_blocs);
+    }
+    return 0;
+}
diff --git a/hw/block/pflash_cfi02.c b/hw/block/pflash_cfi02.c
index 8d4b828ed..8513a17e9 100644
--- a/hw/block/pflash_cfi02.c
+++ b/hw/block/pflash_cfi02.c
@@ -38,7 +38,7 @@
 #include "hw/hw.h"
 #include "hw/block/flash.h"
 #include "qemu/timer.h"
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 #include "qemu/host-utils.h"
 #include "hw/sysbus.h"
@@ -63,7 +63,7 @@ struct pflash_t {
     SysBusDevice parent_obj;
     /*< public >*/
 
-    BlockDriverState *bs;
+    BlockBackend *blk;
     uint32_t sector_len;
     uint32_t nb_blocs;
     uint32_t chip_len;
@@ -249,13 +249,13 @@ static void pflash_update(pflash_t *pfl, int offset,
                           int size)
 {
     int offset_end;
-    if (pfl->bs) {
+    if (pfl->blk) {
         offset_end = offset + size;
         /* round to sectors */
         offset = offset >> 9;
         offset_end = (offset_end + 511) >> 9;
-        bdrv_write(pfl->bs, offset, pfl->storage + (offset << 9),
-                   offset_end - offset);
+        blk_write(pfl->blk, offset, pfl->storage + (offset << 9),
+                  offset_end - offset);
     }
 }
 
@@ -597,6 +597,7 @@ static void pflash_cfi02_realize(DeviceState *dev, Error **errp)
     pflash_t *pfl = CFI_PFLASH02(dev);
     uint32_t chip_len;
     int ret;
+    Error *local_err = NULL;
 
     chip_len = pfl->sector_len * pfl->nb_blocs;
     /* XXX: to be fixed */
@@ -608,16 +609,20 @@ static void pflash_cfi02_realize(DeviceState *dev, Error **errp)
 
     memory_region_init_rom_device(&pfl->orig_mem, OBJECT(pfl), pfl->be ?
                                   &pflash_cfi02_ops_be : &pflash_cfi02_ops_le,
-                                  pfl, pfl->name, chip_len);
+                                  pfl, pfl->name, chip_len, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
     vmstate_register_ram(&pfl->orig_mem, DEVICE(pfl));
     pfl->storage = memory_region_get_ram_ptr(&pfl->orig_mem);
     pfl->chip_len = chip_len;
-    if (pfl->bs) {
+    if (pfl->blk) {
         /* read the initial flash content */
-        ret = bdrv_read(pfl->bs, 0, pfl->storage, chip_len >> 9);
+        ret = blk_read(pfl->blk, 0, pfl->storage, chip_len >> 9);
         if (ret < 0) {
             vmstate_unregister_ram(&pfl->orig_mem, DEVICE(pfl));
-            memory_region_destroy(&pfl->orig_mem);
             error_setg(errp, "failed to read the initial flash content");
             return;
         }
@@ -627,8 +632,8 @@ static void pflash_cfi02_realize(DeviceState *dev, Error **errp)
     pfl->rom_mode = 1;
     sysbus_init_mmio(SYS_BUS_DEVICE(dev), &pfl->mem);
 
-    if (pfl->bs) {
-        pfl->ro = bdrv_is_read_only(pfl->bs);
+    if (pfl->blk) {
+        pfl->ro = blk_is_read_only(pfl->blk);
     } else {
         pfl->ro = 0;
     }
@@ -717,7 +722,7 @@ static void pflash_cfi02_realize(DeviceState *dev, Error **errp)
 }
 
 static Property pflash_cfi02_properties[] = {
-    DEFINE_PROP_DRIVE("drive", struct pflash_t, bs),
+    DEFINE_PROP_DRIVE("drive", struct pflash_t, blk),
     DEFINE_PROP_UINT32("num-blocks", struct pflash_t, nb_blocs, 0),
     DEFINE_PROP_UINT32("sector-length", struct pflash_t, sector_len, 0),
     DEFINE_PROP_UINT8("width", struct pflash_t, width, 0),
@@ -758,7 +763,7 @@ type_init(pflash_cfi02_register_types)
 pflash_t *pflash_cfi02_register(hwaddr base,
                                 DeviceState *qdev, const char *name,
                                 hwaddr size,
-                                BlockDriverState *bs, uint32_t sector_len,
+                                BlockBackend *blk, uint32_t sector_len,
                                 int nb_blocs, int nb_mappings, int width,
                                 uint16_t id0, uint16_t id1,
                                 uint16_t id2, uint16_t id3,
@@ -767,7 +772,7 @@ pflash_t *pflash_cfi02_register(hwaddr base,
 {
     DeviceState *dev = qdev_create(NULL, TYPE_CFI_PFLASH02);
 
-    if (bs && qdev_prop_set_drive(dev, "drive", bs)) {
+    if (blk && qdev_prop_set_drive(dev, "drive", blk)) {
         abort();
     }
     qdev_prop_set_uint32(dev, "num-blocks", nb_blocs);
diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c
index c241c5002..b19b102b4 100644
--- a/hw/block/virtio-blk.c
+++ b/hw/block/virtio-blk.c
@@ -16,12 +16,11 @@
 #include "qemu/error-report.h"
 #include "trace.h"
 #include "hw/block/block.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "hw/virtio/virtio-blk.h"
-#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
-# include "dataplane/virtio-blk.h"
-# include "migration/migration.h"
-#endif
+#include "dataplane/virtio-blk.h"
+#include "migration/migration.h"
 #include "block/scsi.h"
 #ifdef __linux__
 # include <scsi/sg.h>
@@ -66,7 +65,8 @@ static void virtio_blk_req_complete(VirtIOBlockReq *req, unsigned char status)
 static int virtio_blk_handle_rw_error(VirtIOBlockReq *req, int error,
     bool is_read)
 {
-    BlockErrorAction action = bdrv_get_error_action(req->dev->bs, is_read, error);
+    BlockErrorAction action = blk_get_error_action(req->dev->blk,
+                                                   is_read, error);
     VirtIOBlock *s = req->dev;
 
     if (action == BLOCK_ERROR_ACTION_STOP) {
@@ -74,11 +74,11 @@ static int virtio_blk_handle_rw_error(VirtIOBlockReq *req, int error,
         s->rq = req;
     } else if (action == BLOCK_ERROR_ACTION_REPORT) {
         virtio_blk_req_complete(req, VIRTIO_BLK_S_IOERR);
-        bdrv_acct_done(s->bs, &req->acct);
+        block_acct_done(blk_get_stats(s->blk), &req->acct);
         virtio_blk_free_request(req);
     }
 
-    bdrv_error_action(s->bs, action, is_read, error);
+    blk_error_action(s->blk, action, is_read, error);
     return action != BLOCK_ERROR_ACTION_IGNORE;
 }
 
@@ -96,7 +96,7 @@ static void virtio_blk_rw_complete(void *opaque, int ret)
     }
 
     virtio_blk_req_complete(req, VIRTIO_BLK_S_OK);
-    bdrv_acct_done(req->dev->bs, &req->acct);
+    block_acct_done(blk_get_stats(req->dev->blk), &req->acct);
     virtio_blk_free_request(req);
 }
 
@@ -111,7 +111,7 @@ static void virtio_blk_flush_complete(void *opaque, int ret)
     }
 
     virtio_blk_req_complete(req, VIRTIO_BLK_S_OK);
-    bdrv_acct_done(req->dev->bs, &req->acct);
+    block_acct_done(blk_get_stats(req->dev->blk), &req->acct);
     virtio_blk_free_request(req);
 }
 
@@ -157,7 +157,7 @@ int virtio_blk_handle_scsi_req(VirtIOBlock *blk,
      */
     scsi = (void *)elem->in_sg[elem->in_num - 2].iov_base;
 
-    if (!blk->blk.scsi) {
+    if (!blk->conf.scsi) {
         status = VIRTIO_BLK_S_UNSUPP;
         goto fail;
     }
@@ -211,7 +211,7 @@ int virtio_blk_handle_scsi_req(VirtIOBlock *blk,
     hdr.sbp = elem->in_sg[elem->in_num - 3].iov_base;
     hdr.mx_sb_len = elem->in_sg[elem->in_num - 3].iov_len;
 
-    status = bdrv_ioctl(blk->bs, SG_IO, &hdr);
+    status = blk_ioctl(blk->blk, SG_IO, &hdr);
     if (status) {
         status = VIRTIO_BLK_S_UNSUPP;
         goto fail;
@@ -257,7 +257,7 @@ static void virtio_blk_handle_scsi(VirtIOBlockReq *req)
     virtio_blk_free_request(req);
 }
 
-void virtio_submit_multiwrite(BlockDriverState *bs, MultiReqBuffer *mrb)
+void virtio_submit_multiwrite(BlockBackend *blk, MultiReqBuffer *mrb)
 {
     int i, ret;
 
@@ -265,7 +265,7 @@ void virtio_submit_multiwrite(BlockDriverState *bs, MultiReqBuffer *mrb)
         return;
     }
 
-    ret = bdrv_aio_multiwrite(bs, mrb->blkreq, mrb->num_writes);
+    ret = blk_aio_multiwrite(blk, mrb->blkreq, mrb->num_writes);
     if (ret != 0) {
         for (i = 0; i < mrb->num_writes; i++) {
             if (mrb->blkreq[i].error) {
@@ -279,13 +279,14 @@ void virtio_submit_multiwrite(BlockDriverState *bs, MultiReqBuffer *mrb)
 
 static void virtio_blk_handle_flush(VirtIOBlockReq *req, MultiReqBuffer *mrb)
 {
-    bdrv_acct_start(req->dev->bs, &req->acct, 0, BDRV_ACCT_FLUSH);
+    block_acct_start(blk_get_stats(req->dev->blk), &req->acct, 0,
+                     BLOCK_ACCT_FLUSH);
 
     /*
      * Make sure all outstanding writes are posted to the backing device.
      */
-    virtio_submit_multiwrite(req->dev->bs, mrb);
-    bdrv_aio_flush(req->dev->bs, virtio_blk_flush_complete, req);
+    virtio_submit_multiwrite(req->dev->blk, mrb);
+    blk_aio_flush(req->dev->blk, virtio_blk_flush_complete, req);
 }
 
 static bool virtio_blk_sect_range_ok(VirtIOBlock *dev,
@@ -297,10 +298,10 @@ static bool virtio_blk_sect_range_ok(VirtIOBlock *dev,
     if (sector & dev->sector_mask) {
         return false;
     }
-    if (size % dev->conf->logical_block_size) {
+    if (size % dev->conf.conf.logical_block_size) {
         return false;
     }
-    bdrv_get_geometry(dev->bs, &total_sectors);
+    blk_get_geometry(dev->blk, &total_sectors);
     if (sector > total_sectors || nb_sectors > total_sectors - sector) {
         return false;
     }
@@ -322,10 +323,11 @@ static void virtio_blk_handle_write(VirtIOBlockReq *req, MultiReqBuffer *mrb)
         return;
     }
 
-    bdrv_acct_start(req->dev->bs, &req->acct, req->qiov.size, BDRV_ACCT_WRITE);
+    block_acct_start(blk_get_stats(req->dev->blk), &req->acct, req->qiov.size,
+                     BLOCK_ACCT_WRITE);
 
     if (mrb->num_writes == 32) {
-        virtio_submit_multiwrite(req->dev->bs, mrb);
+        virtio_submit_multiwrite(req->dev->blk, mrb);
     }
 
     blkreq = &mrb->blkreq[mrb->num_writes];
@@ -353,10 +355,11 @@ static void virtio_blk_handle_read(VirtIOBlockReq *req)
         return;
     }
 
-    bdrv_acct_start(req->dev->bs, &req->acct, req->qiov.size, BDRV_ACCT_READ);
-    bdrv_aio_readv(req->dev->bs, sector, &req->qiov,
-                   req->qiov.size / BDRV_SECTOR_SIZE,
-                   virtio_blk_rw_complete, req);
+    block_acct_start(blk_get_stats(req->dev->blk), &req->acct, req->qiov.size,
+                     BLOCK_ACCT_READ);
+    blk_aio_readv(req->dev->blk, sector, &req->qiov,
+                  req->qiov.size / BDRV_SECTOR_SIZE,
+                  virtio_blk_rw_complete, req);
 }
 
 void virtio_blk_handle_request(VirtIOBlockReq *req, MultiReqBuffer *mrb)
@@ -404,19 +407,19 @@ void virtio_blk_handle_request(VirtIOBlockReq *req, MultiReqBuffer *mrb)
          * NB: per existing s/n string convention the string is
          * terminated by '\0' only when shorter than buffer.
          */
-        strncpy(req->elem.in_sg[0].iov_base,
-                s->blk.serial ? s->blk.serial : "",
-                MIN(req->elem.in_sg[0].iov_len, VIRTIO_BLK_ID_BYTES));
+        const char *serial = s->conf.serial ? s->conf.serial : "";
+        size_t size = MIN(strlen(serial) + 1,
+                          MIN(iov_size(in_iov, in_num),
+                              VIRTIO_BLK_ID_BYTES));
+        iov_from_buf(in_iov, in_num, 0, serial, size);
         virtio_blk_req_complete(req, VIRTIO_BLK_S_OK);
         virtio_blk_free_request(req);
     } else if (type & VIRTIO_BLK_T_OUT) {
-        qemu_iovec_init_external(&req->qiov, &req->elem.out_sg[1],
-                                 req->elem.out_num - 1);
+        qemu_iovec_init_external(&req->qiov, iov, out_num);
         virtio_blk_handle_write(req, mrb);
     } else if (type == VIRTIO_BLK_T_IN || type == VIRTIO_BLK_T_BARRIER) {
         /* VIRTIO_BLK_T_IN is 0, so we can't just & it. */
-        qemu_iovec_init_external(&req->qiov, &req->elem.in_sg[0],
-                                 req->elem.in_num - 1);
+        qemu_iovec_init_external(&req->qiov, in_iov, in_num);
         virtio_blk_handle_read(req);
     } else {
         virtio_blk_req_complete(req, VIRTIO_BLK_S_UNSUPP);
@@ -432,7 +435,6 @@ static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq)
         .num_writes = 0,
     };
 
-#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
     /* Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
      * dataplane here instead of waiting for .set_status().
      */
@@ -440,13 +442,12 @@ static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq)
         virtio_blk_data_plane_start(s->dataplane);
         return;
     }
-#endif
 
     while ((req = virtio_blk_get_request(s))) {
         virtio_blk_handle_request(req, &mrb);
     }
 
-    virtio_submit_multiwrite(s->bs, &mrb);
+    virtio_submit_multiwrite(s->blk, &mrb);
 
     /*
      * FIXME: Want to check for completions before returning to guest mode,
@@ -469,11 +470,12 @@ static void virtio_blk_dma_restart_bh(void *opaque)
     s->rq = NULL;
 
     while (req) {
+        VirtIOBlockReq *next = req->next;
         virtio_blk_handle_request(req, &mrb);
-        req = req->next;
+        req = next;
     }
 
-    virtio_submit_multiwrite(s->bs, &mrb);
+    virtio_submit_multiwrite(s->blk, &mrb);
 }
 
 static void virtio_blk_dma_restart_cb(void *opaque, int running,
@@ -486,7 +488,7 @@ static void virtio_blk_dma_restart_cb(void *opaque, int running,
     }
 
     if (!s->bh) {
-        s->bh = aio_bh_new(bdrv_get_aio_context(s->blk.conf.bs),
+        s->bh = aio_bh_new(blk_get_aio_context(s->conf.conf.blk),
                            virtio_blk_dma_restart_bh, s);
         qemu_bh_schedule(s->bh);
     }
@@ -496,18 +498,16 @@ static void virtio_blk_reset(VirtIODevice *vdev)
 {
     VirtIOBlock *s = VIRTIO_BLK(vdev);
 
-#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
     if (s->dataplane) {
         virtio_blk_data_plane_stop(s->dataplane);
     }
-#endif
 
     /*
      * This should cancel pending requests, but can't do nicely until there
      * are per-device request lists.
      */
-    bdrv_drain_all();
-    bdrv_set_enable_write_cache(s->bs, s->original_wce);
+    blk_drain_all();
+    blk_set_enable_write_cache(s->blk, s->original_wce);
 }
 
 /* coalesce internal state, copy to pci i/o region 0
@@ -515,19 +515,20 @@ static void virtio_blk_reset(VirtIODevice *vdev)
 static void virtio_blk_update_config(VirtIODevice *vdev, uint8_t *config)
 {
     VirtIOBlock *s = VIRTIO_BLK(vdev);
+    BlockConf *conf = &s->conf.conf;
     struct virtio_blk_config blkcfg;
     uint64_t capacity;
-    int blk_size = s->conf->logical_block_size;
+    int blk_size = conf->logical_block_size;
 
-    bdrv_get_geometry(s->bs, &capacity);
+    blk_get_geometry(s->blk, &capacity);
     memset(&blkcfg, 0, sizeof(blkcfg));
     virtio_stq_p(vdev, &blkcfg.capacity, capacity);
     virtio_stl_p(vdev, &blkcfg.seg_max, 128 - 2);
-    virtio_stw_p(vdev, &blkcfg.cylinders, s->conf->cyls);
+    virtio_stw_p(vdev, &blkcfg.cylinders, conf->cyls);
     virtio_stl_p(vdev, &blkcfg.blk_size, blk_size);
-    virtio_stw_p(vdev, &blkcfg.min_io_size, s->conf->min_io_size / blk_size);
-    virtio_stw_p(vdev, &blkcfg.opt_io_size, s->conf->opt_io_size / blk_size);
-    blkcfg.heads = s->conf->heads;
+    virtio_stw_p(vdev, &blkcfg.min_io_size, conf->min_io_size / blk_size);
+    virtio_stw_p(vdev, &blkcfg.opt_io_size, conf->opt_io_size / blk_size);
+    blkcfg.heads = conf->heads;
     /*
      * We must ensure that the block device capacity is a multiple of
      * the logical block size. If that is not the case, let's use
@@ -539,15 +540,15 @@ static void virtio_blk_update_config(VirtIODevice *vdev, uint8_t *config)
      * divided by 512 - instead it is the amount of blk_size blocks
      * per track (cylinder).
      */
-    if (bdrv_getlength(s->bs) /  s->conf->heads / s->conf->secs % blk_size) {
-        blkcfg.sectors = s->conf->secs & ~s->sector_mask;
+    if (blk_getlength(s->blk) /  conf->heads / conf->secs % blk_size) {
+        blkcfg.sectors = conf->secs & ~s->sector_mask;
     } else {
-        blkcfg.sectors = s->conf->secs;
+        blkcfg.sectors = conf->secs;
     }
     blkcfg.size_max = 0;
-    blkcfg.physical_block_exp = get_physical_block_exp(s->conf);
+    blkcfg.physical_block_exp = get_physical_block_exp(conf);
     blkcfg.alignment_offset = 0;
-    blkcfg.wce = bdrv_enable_write_cache(s->bs);
+    blkcfg.wce = blk_enable_write_cache(s->blk);
     memcpy(config, &blkcfg, sizeof(struct virtio_blk_config));
 }
 
@@ -558,9 +559,9 @@ static void virtio_blk_set_config(VirtIODevice *vdev, const uint8_t *config)
 
     memcpy(&blkcfg, config, sizeof(blkcfg));
 
-    aio_context_acquire(bdrv_get_aio_context(s->bs));
-    bdrv_set_enable_write_cache(s->bs, blkcfg.wce != 0);
-    aio_context_release(bdrv_get_aio_context(s->bs));
+    aio_context_acquire(blk_get_aio_context(s->blk));
+    blk_set_enable_write_cache(s->blk, blkcfg.wce != 0);
+    aio_context_release(blk_get_aio_context(s->blk));
 }
 
 static uint32_t virtio_blk_get_features(VirtIODevice *vdev, uint32_t features)
@@ -573,14 +574,15 @@ static uint32_t virtio_blk_get_features(VirtIODevice *vdev, uint32_t features)
     features |= (1 << VIRTIO_BLK_F_BLK_SIZE);
     features |= (1 << VIRTIO_BLK_F_SCSI);
 
-    if (s->blk.config_wce) {
+    if (s->conf.config_wce) {
         features |= (1 << VIRTIO_BLK_F_CONFIG_WCE);
     }
-    if (bdrv_enable_write_cache(s->bs))
+    if (blk_enable_write_cache(s->blk)) {
         features |= (1 << VIRTIO_BLK_F_WCE);
-
-    if (bdrv_is_read_only(s->bs))
+    }
+    if (blk_is_read_only(s->blk)) {
         features |= 1 << VIRTIO_BLK_F_RO;
+    }
 
     return features;
 }
@@ -590,12 +592,10 @@ static void virtio_blk_set_status(VirtIODevice *vdev, uint8_t status)
     VirtIOBlock *s = VIRTIO_BLK(vdev);
     uint32_t features;
 
-#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
     if (s->dataplane && !(status & (VIRTIO_CONFIG_S_DRIVER |
                                     VIRTIO_CONFIG_S_DRIVER_OK))) {
         virtio_blk_data_plane_stop(s->dataplane);
     }
-#endif
 
     if (!(status & VIRTIO_CONFIG_S_DRIVER_OK)) {
         return;
@@ -616,13 +616,13 @@ static void virtio_blk_set_status(VirtIODevice *vdev, uint8_t status)
      *     Guest writes 1 to the WCE configuration field (writeback mode)
      *     Guest sets DRIVER_OK bit in status field
      *
-     * s->bs would erroneously be placed in writethrough mode.
+     * s->blk would erroneously be placed in writethrough mode.
      */
     if (!(features & (1 << VIRTIO_BLK_F_CONFIG_WCE))) {
-        aio_context_acquire(bdrv_get_aio_context(s->bs));
-        bdrv_set_enable_write_cache(s->bs,
-                                    !!(features & (1 << VIRTIO_BLK_F_WCE)));
-        aio_context_release(bdrv_get_aio_context(s->bs));
+        aio_context_acquire(blk_get_aio_context(s->blk));
+        blk_set_enable_write_cache(s->blk,
+                                   !!(features & (1 << VIRTIO_BLK_F_WCE)));
+        aio_context_release(blk_get_aio_context(s->blk));
     }
 }
 
@@ -690,7 +690,6 @@ static const BlockDevOps virtio_block_ops = {
     .resize_cb = virtio_blk_resize,
 };
 
-#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
 /* Disable dataplane thread during live migration since it does not
  * update the dirty memory bitmap yet.
  */
@@ -712,8 +711,8 @@ static void virtio_blk_migration_state_changed(Notifier *notifier, void *data)
         if (s->dataplane) {
             return;
         }
-        bdrv_drain_all(); /* complete in-flight non-dataplane requests */
-        virtio_blk_data_plane_create(VIRTIO_DEVICE(s), &s->blk,
+        blk_drain_all(); /* complete in-flight non-dataplane requests */
+        virtio_blk_data_plane_create(VIRTIO_DEVICE(s), &s->conf,
                                      &s->dataplane, &err);
         if (err != NULL) {
             error_report("%s", error_get_pretty(err));
@@ -721,46 +720,42 @@ static void virtio_blk_migration_state_changed(Notifier *notifier, void *data)
         }
     }
 }
-#endif /* CONFIG_VIRTIO_BLK_DATA_PLANE */
 
 static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VirtIOBlock *s = VIRTIO_BLK(dev);
-    VirtIOBlkConf *blk = &(s->blk);
-#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
+    VirtIOBlkConf *conf = &s->conf;
     Error *err = NULL;
-#endif
     static int virtio_blk_id;
 
-    if (!blk->conf.bs) {
+    if (!conf->conf.blk) {
         error_setg(errp, "drive property not set");
         return;
     }
-    if (!bdrv_is_inserted(blk->conf.bs)) {
+    if (!blk_is_inserted(conf->conf.blk)) {
         error_setg(errp, "Device needs media, but drive is empty");
         return;
     }
 
-    blkconf_serial(&blk->conf, &blk->serial);
-    s->original_wce = bdrv_enable_write_cache(blk->conf.bs);
-    if (blkconf_geometry(&blk->conf, NULL, 65535, 255, 255) < 0) {
-        error_setg(errp, "Error setting geometry");
+    blkconf_serial(&conf->conf, &conf->serial);
+    s->original_wce = blk_enable_write_cache(conf->conf.blk);
+    blkconf_geometry(&conf->conf, NULL, 65535, 255, 255, &err);
+    if (err) {
+        error_propagate(errp, err);
         return;
     }
 
     virtio_init(vdev, "virtio-blk", VIRTIO_ID_BLOCK,
                 sizeof(struct virtio_blk_config));
 
-    s->bs = blk->conf.bs;
-    s->conf = &blk->conf;
+    s->blk = conf->conf.blk;
     s->rq = NULL;
-    s->sector_mask = (s->conf->logical_block_size / BDRV_SECTOR_SIZE) - 1;
+    s->sector_mask = (s->conf.conf.logical_block_size / BDRV_SECTOR_SIZE) - 1;
 
     s->vq = virtio_add_queue(vdev, 128, virtio_blk_handle_output);
     s->complete_request = virtio_blk_complete_request;
-#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
-    virtio_blk_data_plane_create(vdev, blk, &s->dataplane, &err);
+    virtio_blk_data_plane_create(vdev, conf, &s->dataplane, &err);
     if (err != NULL) {
         error_propagate(errp, err);
         virtio_cleanup(vdev);
@@ -768,17 +763,14 @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
     }
     s->migration_state_notifier.notify = virtio_blk_migration_state_changed;
     add_migration_state_change_notifier(&s->migration_state_notifier);
-#endif
 
     s->change = qemu_add_vm_change_state_handler(virtio_blk_dma_restart_cb, s);
     register_savevm(dev, "virtio-blk", virtio_blk_id++, 2,
                     virtio_blk_save, virtio_blk_load, s);
-    bdrv_set_dev_ops(s->bs, &virtio_block_ops, s);
-    bdrv_set_guest_block_size(s->bs, s->conf->logical_block_size);
+    blk_set_dev_ops(s->blk, &virtio_block_ops, s);
+    blk_set_guest_block_size(s->blk, s->conf.conf.logical_block_size);
 
-    bdrv_iostatus_enable(s->bs);
-
-    add_boot_device_path(s->conf->bootindex, dev, "/disk@0,0");
+    blk_iostatus_enable(s->blk);
 }
 
 static void virtio_blk_device_unrealize(DeviceState *dev, Error **errp)
@@ -786,14 +778,12 @@ static void virtio_blk_device_unrealize(DeviceState *dev, Error **errp)
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VirtIOBlock *s = VIRTIO_BLK(dev);
 
-#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
     remove_migration_state_change_notifier(&s->migration_state_notifier);
     virtio_blk_data_plane_destroy(s->dataplane);
     s->dataplane = NULL;
-#endif
     qemu_del_vm_change_state_handler(s->change);
     unregister_savevm(dev, "virtio-blk", s);
-    blockdev_mark_auto_del(s->bs);
+    blockdev_mark_auto_del(s->blk);
     virtio_cleanup(vdev);
 }
 
@@ -802,22 +792,23 @@ static void virtio_blk_instance_init(Object *obj)
     VirtIOBlock *s = VIRTIO_BLK(obj);
 
     object_property_add_link(obj, "iothread", TYPE_IOTHREAD,
-                             (Object **)&s->blk.iothread,
+                             (Object **)&s->conf.iothread,
                              qdev_prop_allow_set_link_before_realize,
                              OBJ_PROP_LINK_UNREF_ON_RELEASE, NULL);
+    device_add_bootindex_property(obj, &s->conf.conf.bootindex,
+                                  "bootindex", "/disk@0,0",
+                                  DEVICE(obj), NULL);
 }
 
 static Property virtio_blk_properties[] = {
-    DEFINE_BLOCK_PROPERTIES(VirtIOBlock, blk.conf),
-    DEFINE_BLOCK_CHS_PROPERTIES(VirtIOBlock, blk.conf),
-    DEFINE_PROP_STRING("serial", VirtIOBlock, blk.serial),
-    DEFINE_PROP_BIT("config-wce", VirtIOBlock, blk.config_wce, 0, true),
+    DEFINE_BLOCK_PROPERTIES(VirtIOBlock, conf.conf),
+    DEFINE_BLOCK_CHS_PROPERTIES(VirtIOBlock, conf.conf),
+    DEFINE_PROP_STRING("serial", VirtIOBlock, conf.serial),
+    DEFINE_PROP_BIT("config-wce", VirtIOBlock, conf.config_wce, 0, true),
 #ifdef __linux__
-    DEFINE_PROP_BIT("scsi", VirtIOBlock, blk.scsi, 0, true),
-#endif
-#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
-    DEFINE_PROP_BIT("x-data-plane", VirtIOBlock, blk.data_plane, 0, false),
+    DEFINE_PROP_BIT("scsi", VirtIOBlock, conf.scsi, 0, true),
 #endif
+    DEFINE_PROP_BIT("x-data-plane", VirtIOBlock, conf.data_plane, 0, false),
     DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/hw/block/xen_disk.c b/hw/block/xen_disk.c
index aed5b5b3e..21842a01e 100644
--- a/hw/block/xen_disk.c
+++ b/hw/block/xen_disk.c
@@ -39,6 +39,7 @@
 #include "hw/xen/xen_backend.h"
 #include "xen_blkif.h"
 #include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 
 /* ------------------------------------------------------------- */
 
@@ -58,6 +59,13 @@ struct PersistentGrant {
 
 typedef struct PersistentGrant PersistentGrant;
 
+struct PersistentRegion {
+    void *addr;
+    int num;
+};
+
+typedef struct PersistentRegion PersistentRegion;
+
 struct ioreq {
     blkif_request_t     req;
     int16_t             status;
@@ -117,12 +125,13 @@ struct XenBlkDev {
     gboolean            feature_discard;
     gboolean            feature_persistent;
     GTree               *persistent_gnts;
+    GSList              *persistent_regions;
     unsigned int        persistent_gnt_count;
     unsigned int        max_grants;
 
     /* qemu block driver */
     DriveInfo           *dinfo;
-    BlockDriverState    *bs;
+    BlockBackend        *blk;
     QEMUBH              *bh;
 };
 
@@ -176,6 +185,23 @@ static void destroy_grant(gpointer pgnt)
     g_free(grant);
 }
 
+static void remove_persistent_region(gpointer data, gpointer dev)
+{
+    PersistentRegion *region = data;
+    struct XenBlkDev *blkdev = dev;
+    XenGnttab gnt = blkdev->xendev.gnttabdev;
+
+    if (xc_gnttab_munmap(gnt, region->addr, region->num) != 0) {
+        xen_be_printf(&blkdev->xendev, 0,
+                      "xc_gnttab_munmap region %p failed: %s\n",
+                      region->addr, strerror(errno));
+    }
+    xen_be_printf(&blkdev->xendev, 3,
+                  "unmapped grant region %p with %d pages\n",
+                  region->addr, region->num);
+    g_free(region);
+}
+
 static struct ioreq *ioreq_start(struct XenBlkDev *blkdev)
 {
     struct ioreq *ioreq = NULL;
@@ -342,6 +368,7 @@ static int ioreq_map(struct ioreq *ioreq)
     void *page[BLKIF_MAX_SEGMENTS_PER_REQUEST];
     int i, j, new_maps = 0;
     PersistentGrant *grant;
+    PersistentRegion *region;
     /* domids and refs variables will contain the information necessary
      * to map the grants that are needed to fulfill this request.
      *
@@ -420,7 +447,22 @@ static int ioreq_map(struct ioreq *ioreq)
             }
         }
     }
-    if (ioreq->blkdev->feature_persistent) {
+    if (ioreq->blkdev->feature_persistent && new_maps != 0 &&
+        (!batch_maps || (ioreq->blkdev->persistent_gnt_count + new_maps <=
+        ioreq->blkdev->max_grants))) {
+        /*
+         * If we are using persistent grants and batch mappings only
+         * add the new maps to the list of persistent grants if the whole
+         * area can be persistently mapped.
+         */
+        if (batch_maps) {
+            region = g_malloc0(sizeof(*region));
+            region->addr = ioreq->pages;
+            region->num = new_maps;
+            ioreq->blkdev->persistent_regions = g_slist_append(
+                                            ioreq->blkdev->persistent_regions,
+                                            region);
+        }
         while ((ioreq->blkdev->persistent_gnt_count < ioreq->blkdev->max_grants)
               && new_maps) {
             /* Go through the list of newly mapped grants and add as many
@@ -446,6 +488,7 @@ static int ioreq_map(struct ioreq *ioreq)
                           grant);
             ioreq->blkdev->persistent_gnt_count++;
         }
+        assert(!batch_maps || new_maps == 0);
     }
     for (i = 0; i < ioreq->v.niov; i++) {
         ioreq->v.iov[i].iov_base += (uintptr_t)page[i];
@@ -479,7 +522,7 @@ static void qemu_aio_complete(void *opaque, int ret)
     if (ioreq->postsync) {
         ioreq->postsync = 0;
         ioreq->aio_inflight++;
-        bdrv_aio_flush(ioreq->blkdev->bs, qemu_aio_complete, ioreq);
+        blk_aio_flush(ioreq->blkdev->blk, qemu_aio_complete, ioreq);
         return;
     }
 
@@ -493,7 +536,7 @@ static void qemu_aio_complete(void *opaque, int ret)
             break;
         }
     case BLKIF_OP_READ:
-        bdrv_acct_done(ioreq->blkdev->bs, &ioreq->acct);
+        block_acct_done(blk_get_stats(ioreq->blkdev->blk), &ioreq->acct);
         break;
     case BLKIF_OP_DISCARD:
     default:
@@ -512,17 +555,18 @@ static int ioreq_runio_qemu_aio(struct ioreq *ioreq)
 
     ioreq->aio_inflight++;
     if (ioreq->presync) {
-        bdrv_aio_flush(ioreq->blkdev->bs, qemu_aio_complete, ioreq);
+        blk_aio_flush(ioreq->blkdev->blk, qemu_aio_complete, ioreq);
         return 0;
     }
 
     switch (ioreq->req.operation) {
     case BLKIF_OP_READ:
-        bdrv_acct_start(blkdev->bs, &ioreq->acct, ioreq->v.size, BDRV_ACCT_READ);
+        block_acct_start(blk_get_stats(blkdev->blk), &ioreq->acct,
+                         ioreq->v.size, BLOCK_ACCT_READ);
         ioreq->aio_inflight++;
-        bdrv_aio_readv(blkdev->bs, ioreq->start / BLOCK_SIZE,
-                       &ioreq->v, ioreq->v.size / BLOCK_SIZE,
-                       qemu_aio_complete, ioreq);
+        blk_aio_readv(blkdev->blk, ioreq->start / BLOCK_SIZE,
+                      &ioreq->v, ioreq->v.size / BLOCK_SIZE,
+                      qemu_aio_complete, ioreq);
         break;
     case BLKIF_OP_WRITE:
     case BLKIF_OP_FLUSH_DISKCACHE:
@@ -530,17 +574,18 @@ static int ioreq_runio_qemu_aio(struct ioreq *ioreq)
             break;
         }
 
-        bdrv_acct_start(blkdev->bs, &ioreq->acct, ioreq->v.size, BDRV_ACCT_WRITE);
+        block_acct_start(blk_get_stats(blkdev->blk), &ioreq->acct,
+                         ioreq->v.size, BLOCK_ACCT_WRITE);
         ioreq->aio_inflight++;
-        bdrv_aio_writev(blkdev->bs, ioreq->start / BLOCK_SIZE,
-                        &ioreq->v, ioreq->v.size / BLOCK_SIZE,
-                        qemu_aio_complete, ioreq);
+        blk_aio_writev(blkdev->blk, ioreq->start / BLOCK_SIZE,
+                       &ioreq->v, ioreq->v.size / BLOCK_SIZE,
+                       qemu_aio_complete, ioreq);
         break;
     case BLKIF_OP_DISCARD:
     {
         struct blkif_request_discard *discard_req = (void *)&ioreq->req;
         ioreq->aio_inflight++;
-        bdrv_aio_discard(blkdev->bs,
+        blk_aio_discard(blkdev->blk,
                         discard_req->sector_number, discard_req->nr_sectors,
                         qemu_aio_complete, ioreq);
         break;
@@ -589,6 +634,7 @@ static int blk_send_response_one(struct ioreq *ioreq)
         break;
     default:
         dst = NULL;
+        return 0;
     }
     memcpy(dst, &resp, sizeof(resp));
     blkdev->rings.common.rsp_prod_pvt++;
@@ -851,47 +897,49 @@ static int blk_connect(struct XenDevice *xendev)
     blkdev->dinfo = drive_get(IF_XEN, 0, index);
     if (!blkdev->dinfo) {
         Error *local_err = NULL;
+        BlockBackend *blk;
+        BlockDriver *drv;
+        BlockDriverState *bs;
+
         /* setup via xenbus -> create new block driver instance */
         xen_be_printf(&blkdev->xendev, 2, "create new bdrv (xenbus setup)\n");
-        blkdev->bs = bdrv_new(blkdev->dev, &local_err);
-        if (local_err) {
-            blkdev->bs = NULL;
-        }
-        if (blkdev->bs) {
-            BlockDriver *drv = bdrv_find_whitelisted_format(blkdev->fileproto,
-                                                           readonly);
-            if (bdrv_open(&blkdev->bs, blkdev->filename, NULL, NULL, qflags,
-                          drv, &local_err) != 0)
-            {
-                xen_be_printf(&blkdev->xendev, 0, "error: %s\n",
-                              error_get_pretty(local_err));
-                error_free(local_err);
-                bdrv_unref(blkdev->bs);
-                blkdev->bs = NULL;
-            }
+        blk = blk_new_with_bs(blkdev->dev, NULL);
+        if (!blk) {
+            return -1;
         }
-        if (!blkdev->bs) {
+        blkdev->blk = blk;
+
+        bs = blk_bs(blk);
+        drv = bdrv_find_whitelisted_format(blkdev->fileproto, readonly);
+        if (bdrv_open(&bs, blkdev->filename, NULL, NULL, qflags,
+                      drv, &local_err) != 0) {
+            xen_be_printf(&blkdev->xendev, 0, "error: %s\n",
+                          error_get_pretty(local_err));
+            error_free(local_err);
+            blk_unref(blk);
+            blkdev->blk = NULL;
             return -1;
         }
+        assert(bs == blk_bs(blk));
     } else {
         /* setup via qemu cmdline -> already setup for us */
         xen_be_printf(&blkdev->xendev, 2, "get configured bdrv (cmdline setup)\n");
-        blkdev->bs = blkdev->dinfo->bdrv;
-        if (bdrv_is_read_only(blkdev->bs) && !readonly) {
+        blkdev->blk = blk_by_legacy_dinfo(blkdev->dinfo);
+        if (blk_is_read_only(blkdev->blk) && !readonly) {
             xen_be_printf(&blkdev->xendev, 0, "Unexpected read-only drive");
-            blkdev->bs = NULL;
+            blkdev->blk = NULL;
             return -1;
         }
-        /* blkdev->bs is not create by us, we get a reference
-         * so we can bdrv_unref() unconditionally */
-        bdrv_ref(blkdev->bs);
+        /* blkdev->blk is not create by us, we get a reference
+         * so we can blk_unref() unconditionally */
+        blk_ref(blkdev->blk);
     }
-    bdrv_attach_dev_nofail(blkdev->bs, blkdev);
-    blkdev->file_size = bdrv_getlength(blkdev->bs);
+    blk_attach_dev_nofail(blkdev->blk, blkdev);
+    blkdev->file_size = blk_getlength(blkdev->blk);
     if (blkdev->file_size < 0) {
-        xen_be_printf(&blkdev->xendev, 1, "bdrv_getlength: %d (%s) | drv %s\n",
+        xen_be_printf(&blkdev->xendev, 1, "blk_getlength: %d (%s) | drv %s\n",
                       (int)blkdev->file_size, strerror(-blkdev->file_size),
-                      bdrv_get_format_name(blkdev->bs) ?: "-");
+                      bdrv_get_format_name(blk_bs(blkdev->blk)) ?: "-");
         blkdev->file_size = 0;
     }
 
@@ -965,7 +1013,10 @@ static int blk_connect(struct XenDevice *xendev)
         blkdev->max_grants = max_requests * BLKIF_MAX_SEGMENTS_PER_REQUEST;
         blkdev->persistent_gnts = g_tree_new_full((GCompareDataFunc)int_cmp,
                                              NULL, NULL,
+                                             batch_maps ?
+                                             (GDestroyNotify)g_free :
                                              (GDestroyNotify)destroy_grant);
+        blkdev->persistent_regions = NULL;
         blkdev->persistent_gnt_count = 0;
     }
 
@@ -982,10 +1033,10 @@ static void blk_disconnect(struct XenDevice *xendev)
 {
     struct XenBlkDev *blkdev = container_of(xendev, struct XenBlkDev, xendev);
 
-    if (blkdev->bs) {
-        bdrv_detach_dev(blkdev->bs, blkdev);
-        bdrv_unref(blkdev->bs);
-        blkdev->bs = NULL;
+    if (blkdev->blk) {
+        blk_detach_dev(blkdev->blk, blkdev);
+        blk_unref(blkdev->blk);
+        blkdev->blk = NULL;
     }
     xen_be_unbind_evtchn(&blkdev->xendev);
 
@@ -994,6 +1045,26 @@ static void blk_disconnect(struct XenDevice *xendev)
         blkdev->cnt_map--;
         blkdev->sring = NULL;
     }
+
+    /*
+     * Unmap persistent grants before switching to the closed state
+     * so the frontend can free them.
+     *
+     * In the !batch_maps case g_tree_destroy will take care of unmapping
+     * the grant, but in the batch_maps case we need to iterate over every
+     * region in persistent_regions and unmap it.
+     */
+    if (blkdev->feature_persistent) {
+        g_tree_destroy(blkdev->persistent_gnts);
+        assert(batch_maps || blkdev->persistent_gnt_count == 0);
+        if (batch_maps) {
+            blkdev->persistent_gnt_count = 0;
+            g_slist_foreach(blkdev->persistent_regions,
+                            (GFunc)remove_persistent_region, blkdev);
+            g_slist_free(blkdev->persistent_regions);
+        }
+        blkdev->feature_persistent = false;
+    }
 }
 
 static int blk_free(struct XenDevice *xendev)
@@ -1001,15 +1072,10 @@ static int blk_free(struct XenDevice *xendev)
     struct XenBlkDev *blkdev = container_of(xendev, struct XenBlkDev, xendev);
     struct ioreq *ioreq;
 
-    if (blkdev->bs || blkdev->sring) {
+    if (blkdev->blk || blkdev->sring) {
         blk_disconnect(xendev);
     }
 
-    /* Free persistent grants */
-    if (blkdev->feature_persistent) {
-        g_tree_destroy(blkdev->persistent_gnts);
-    }
-
     while (!QLIST_EMPTY(&blkdev->freelist)) {
         ioreq = QLIST_FIRST(&blkdev->freelist);
         QLIST_REMOVE(ioreq, list);
diff --git a/hw/bt/l2cap.c b/hw/bt/l2cap.c
index 2301d6f87..591e04778 100644
--- a/hw/bt/l2cap.c
+++ b/hw/bt/l2cap.c
@@ -429,7 +429,7 @@ static struct l2cap_chan_s *l2cap_channel_open(struct l2cap_instance_s *l2cap,
                 status = L2CAP_CS_NO_INFO;
             } else {
                 g_free(ch);
-
+                ch = NULL;
                 result = L2CAP_CR_NO_MEM;
                 status = L2CAP_CS_NO_INFO;
             }
diff --git a/hw/char/parallel.c b/hw/char/parallel.c
index 7ac90a512..c2b553f0d 100644
--- a/hw/char/parallel.c
+++ b/hw/char/parallel.c
@@ -477,6 +477,23 @@ static const MemoryRegionPortio isa_parallel_portio_sw_list[] = {
     PORTIO_END_OF_LIST(),
 };
 
+
+static const VMStateDescription vmstate_parallel_isa = {
+    .name = "parallel_isa",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_UINT8(state.dataw, ISAParallelState),
+        VMSTATE_UINT8(state.datar, ISAParallelState),
+        VMSTATE_UINT8(state.status, ISAParallelState),
+        VMSTATE_UINT8(state.control, ISAParallelState),
+        VMSTATE_INT32(state.irq_pending, ISAParallelState),
+        VMSTATE_INT32(state.epp_timeout, ISAParallelState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+
 static void parallel_isa_realizefn(DeviceState *dev, Error **errp)
 {
     static int index;
@@ -606,6 +623,7 @@ static void parallel_isa_class_initfn(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
 
     dc->realize = parallel_isa_realizefn;
+    dc->vmsd = &vmstate_parallel_isa;
     dc->props = parallel_isa_properties;
     set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
 }
diff --git a/hw/char/sclpconsole-lm.c b/hw/char/sclpconsole-lm.c
index 80dd0a9e1..a9f5e62f2 100644
--- a/hw/char/sclpconsole-lm.c
+++ b/hw/char/sclpconsole-lm.c
@@ -52,7 +52,8 @@ typedef struct SCLPConsoleLM {
  * event_pending is set when a newline character is encountered
  *
  * The maximum command line length is limited by the maximum
- * space available in an SCCB
+ * space available in an SCCB. Line mode console input is sent
+ * truncated to the guest in case it doesn't fit into the SCCB.
  */
 
 static int chr_can_read(void *opaque)
@@ -61,10 +62,8 @@ static int chr_can_read(void *opaque)
 
     if (scon->event.event_pending) {
         return 0;
-    } else if (SIZE_CONSOLE_BUFFER - scon->length) {
-        return 1;
     }
-    return 0;
+    return 1;
 }
 
 static void chr_read(void *opaque, const uint8_t *buf, int size)
@@ -78,6 +77,10 @@ static void chr_read(void *opaque, const uint8_t *buf, int size)
         sclp_service_interrupt(0);
         return;
     }
+    if (scon->length == SIZE_CONSOLE_BUFFER) {
+        /* Eat the character, but still process CR and LF.  */
+        return;
+    }
     scon->buf[scon->length] = *buf;
     scon->length += 1;
     if (scon->echo) {
@@ -125,6 +128,7 @@ static int get_console_data(SCLPEvent *event, uint8_t *buf, size_t *size,
     cons->length = 0;
     /* data provided and no more data pending */
     event->event_pending = false;
+    qemu_notify_event();
     return 0;
 }
 
diff --git a/hw/char/sclpconsole.c b/hw/char/sclpconsole.c
index fca105db4..79891dfc5 100644
--- a/hw/char/sclpconsole.c
+++ b/hw/char/sclpconsole.c
@@ -36,6 +36,7 @@ typedef struct SCLPConsole {
     uint32_t iov_bs;        /* offset in buf for char layer read operation */
     uint32_t iov_data_len;  /* length of byte stream in buffer             */
     uint32_t iov_sclp_rest; /* length of byte stream not read via SCLP     */
+    bool notify;            /* qemu_notify_event() req'd if true           */
 } SCLPConsole;
 
 /* character layer call-back functions */
@@ -44,8 +45,12 @@ typedef struct SCLPConsole {
 static int chr_can_read(void *opaque)
 {
     SCLPConsole *scon = opaque;
+    int avail = SIZE_BUFFER_VT220 - scon->iov_data_len;
 
-    return SIZE_BUFFER_VT220 - scon->iov_data_len;
+    if (avail == 0) {
+        scon->notify = true;
+    }
+    return avail;
 }
 
 /* Send data from a char device over to the guest */
@@ -113,6 +118,10 @@ static void get_console_data(SCLPEvent *event, uint8_t *buf, size_t *size,
         cons->iov_sclp += avail;
         /* more data pending */
     }
+    if (cons->notify) {
+        cons->notify = false;
+        qemu_notify_event();
+    }
 }
 
 static int read_event_data(SCLPEvent *event, EventBufferHeader *evt_buf_hdr,
@@ -229,6 +238,7 @@ static void console_reset(DeviceState *dev)
    scon->iov_bs = 0;
    scon->iov_data_len = 0;
    scon->iov_sclp_rest = 0;
+   scon->notify = false;
 }
 
 static int console_exit(SCLPEvent *event)
diff --git a/hw/char/serial-pci.c b/hw/char/serial-pci.c
index c133c33e7..f05c9b4d8 100644
--- a/hw/char/serial-pci.c
+++ b/hw/char/serial-pci.c
@@ -135,7 +135,6 @@ static void serial_pci_exit(PCIDevice *dev)
     SerialState *s = &pci->state;
 
     serial_exit_core(s);
-    memory_region_destroy(&s->io);
     qemu_free_irq(s->irq);
 }
 
@@ -149,10 +148,8 @@ static void multi_serial_pci_exit(PCIDevice *dev)
         s = pci->state + i;
         serial_exit_core(s);
         memory_region_del_subregion(&pci->iobar, &s->io);
-        memory_region_destroy(&s->io);
         g_free(pci->name[i]);
     }
-    memory_region_destroy(&pci->iobar);
     qemu_free_irqs(pci->irqs, pci->ports);
 }
 
diff --git a/hw/char/serial.c b/hw/char/serial.c
index 764e1846c..ebcacdc87 100644
--- a/hw/char/serial.c
+++ b/hw/char/serial.c
@@ -272,6 +272,36 @@ static gboolean serial_xmit(GIOChannel *chan, GIOCondition cond, void *opaque)
 }
 
 
+/* Setter for FCR.
+   is_load flag means, that value is set while loading VM state
+   and interrupt should not be invoked */
+static void serial_write_fcr(SerialState *s, uint8_t val)
+{
+    /* Set fcr - val only has the bits that are supposed to "stick" */
+    s->fcr = val;
+
+    if (val & UART_FCR_FE) {
+        s->iir |= UART_IIR_FE;
+        /* Set recv_fifo trigger Level */
+        switch (val & 0xC0) {
+        case UART_FCR_ITL_1:
+            s->recv_fifo_itl = 1;
+            break;
+        case UART_FCR_ITL_2:
+            s->recv_fifo_itl = 4;
+            break;
+        case UART_FCR_ITL_3:
+            s->recv_fifo_itl = 8;
+            break;
+        case UART_FCR_ITL_4:
+            s->recv_fifo_itl = 14;
+            break;
+        }
+    } else {
+        s->iir &= ~UART_IIR_FE;
+    }
+}
+
 static void serial_ioport_write(void *opaque, hwaddr addr, uint64_t val,
                                 unsigned size)
 {
@@ -327,20 +357,16 @@ static void serial_ioport_write(void *opaque, hwaddr addr, uint64_t val,
         }
         break;
     case 2:
-        val = val & 0xFF;
-
-        if (s->fcr == val)
-            break;
-
         /* Did the enable/disable flag change? If so, make sure FIFOs get flushed */
-        if ((val ^ s->fcr) & UART_FCR_FE)
+        if ((val ^ s->fcr) & UART_FCR_FE) {
             val |= UART_FCR_XFR | UART_FCR_RFR;
+        }
 
         /* FIFO clear */
 
         if (val & UART_FCR_RFR) {
             timer_del(s->fifo_timeout_timer);
-            s->timeout_ipending=0;
+            s->timeout_ipending = 0;
             fifo8_reset(&s->recv_fifo);
         }
 
@@ -348,28 +374,7 @@ static void serial_ioport_write(void *opaque, hwaddr addr, uint64_t val,
             fifo8_reset(&s->xmit_fifo);
         }
 
-        if (val & UART_FCR_FE) {
-            s->iir |= UART_IIR_FE;
-            /* Set recv_fifo trigger Level */
-            switch (val & 0xC0) {
-            case UART_FCR_ITL_1:
-                s->recv_fifo_itl = 1;
-                break;
-            case UART_FCR_ITL_2:
-                s->recv_fifo_itl = 4;
-                break;
-            case UART_FCR_ITL_3:
-                s->recv_fifo_itl = 8;
-                break;
-            case UART_FCR_ITL_4:
-                s->recv_fifo_itl = 14;
-                break;
-            }
-        } else
-            s->iir &= ~UART_IIR_FE;
-
-        /* Set fcr - or at least the bits in it that are supposed to "stick" */
-        s->fcr = val & 0xC9;
+        serial_write_fcr(s, val & 0xC9);
         serial_update_irq(s);
         break;
     case 3:
@@ -590,6 +595,14 @@ static void serial_pre_save(void *opaque)
     s->fcr_vmstate = s->fcr;
 }
 
+static int serial_pre_load(void *opaque)
+{
+    SerialState *s = opaque;
+    s->thr_ipending = -1;
+    s->poll_msl = -1;
+    return 0;
+}
+
 static int serial_post_load(void *opaque, int version_id)
 {
     SerialState *s = opaque;
@@ -597,17 +610,139 @@ static int serial_post_load(void *opaque, int version_id)
     if (version_id < 3) {
         s->fcr_vmstate = 0;
     }
+    if (s->thr_ipending == -1) {
+        s->thr_ipending = ((s->iir & UART_IIR_ID) == UART_IIR_THRI);
+    }
+    s->last_break_enable = (s->lcr >> 6) & 1;
     /* Initialize fcr via setter to perform essential side-effects */
-    serial_ioport_write(s, 0x02, s->fcr_vmstate, 1);
+    serial_write_fcr(s, s->fcr_vmstate);
     serial_update_parameters(s);
     return 0;
 }
 
+static bool serial_thr_ipending_needed(void *opaque)
+{
+    SerialState *s = opaque;
+    bool expected_value = ((s->iir & UART_IIR_ID) == UART_IIR_THRI);
+    return s->thr_ipending != expected_value;
+}
+
+const VMStateDescription vmstate_serial_thr_ipending = {
+    .name = "serial/thr_ipending",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT32(thr_ipending, SerialState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool serial_tsr_needed(void *opaque)
+{
+    SerialState *s = (SerialState *)opaque;
+    return s->tsr_retry != 0;
+}
+
+const VMStateDescription vmstate_serial_tsr = {
+    .name = "serial/tsr",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT32(tsr_retry, SerialState),
+        VMSTATE_UINT8(thr, SerialState),
+        VMSTATE_UINT8(tsr, SerialState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool serial_recv_fifo_needed(void *opaque)
+{
+    SerialState *s = (SerialState *)opaque;
+    return !fifo8_is_empty(&s->recv_fifo);
+
+}
+
+const VMStateDescription vmstate_serial_recv_fifo = {
+    .name = "serial/recv_fifo",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_STRUCT(recv_fifo, SerialState, 1, vmstate_fifo8, Fifo8),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool serial_xmit_fifo_needed(void *opaque)
+{
+    SerialState *s = (SerialState *)opaque;
+    return !fifo8_is_empty(&s->xmit_fifo);
+}
+
+const VMStateDescription vmstate_serial_xmit_fifo = {
+    .name = "serial/xmit_fifo",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_STRUCT(xmit_fifo, SerialState, 1, vmstate_fifo8, Fifo8),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool serial_fifo_timeout_timer_needed(void *opaque)
+{
+    SerialState *s = (SerialState *)opaque;
+    return timer_pending(s->fifo_timeout_timer);
+}
+
+const VMStateDescription vmstate_serial_fifo_timeout_timer = {
+    .name = "serial/fifo_timeout_timer",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_TIMER(fifo_timeout_timer, SerialState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool serial_timeout_ipending_needed(void *opaque)
+{
+    SerialState *s = (SerialState *)opaque;
+    return s->timeout_ipending != 0;
+}
+
+const VMStateDescription vmstate_serial_timeout_ipending = {
+    .name = "serial/timeout_ipending",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT32(timeout_ipending, SerialState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool serial_poll_needed(void *opaque)
+{
+    SerialState *s = (SerialState *)opaque;
+    return s->poll_msl >= 0;
+}
+
+const VMStateDescription vmstate_serial_poll = {
+    .name = "serial/poll",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT32(poll_msl, SerialState),
+        VMSTATE_TIMER(modem_status_poll, SerialState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 const VMStateDescription vmstate_serial = {
     .name = "serial",
     .version_id = 3,
     .minimum_version_id = 2,
     .pre_save = serial_pre_save,
+    .pre_load = serial_pre_load,
     .post_load = serial_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_UINT16_V(divider, SerialState, 2),
@@ -621,6 +756,32 @@ const VMStateDescription vmstate_serial = {
         VMSTATE_UINT8(scr, SerialState),
         VMSTATE_UINT8_V(fcr_vmstate, SerialState, 3),
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (VMStateSubsection[]) {
+        {
+            .vmsd = &vmstate_serial_thr_ipending,
+            .needed = &serial_thr_ipending_needed,
+        } , {
+            .vmsd = &vmstate_serial_tsr,
+            .needed = &serial_tsr_needed,
+        } , {
+            .vmsd = &vmstate_serial_recv_fifo,
+            .needed = &serial_recv_fifo_needed,
+        } , {
+            .vmsd = &vmstate_serial_xmit_fifo,
+            .needed = &serial_xmit_fifo_needed,
+        } , {
+            .vmsd = &vmstate_serial_fifo_timeout_timer,
+            .needed = &serial_fifo_timeout_timer_needed,
+        } , {
+            .vmsd = &vmstate_serial_timeout_ipending,
+            .needed = &serial_timeout_ipending_needed,
+        } , {
+            .vmsd = &vmstate_serial_poll,
+            .needed = &serial_poll_needed,
+        } , {
+            /* empty */
+        }
     }
 };
 
@@ -642,6 +803,10 @@ static void serial_reset(void *opaque)
     s->char_transmit_time = (get_ticks_per_sec() / 9600) * 10;
     s->poll_msl = 0;
 
+    s->timeout_ipending = 0;
+    timer_del(s->fifo_timeout_timer);
+    timer_del(s->modem_status_poll);
+
     fifo8_reset(&s->recv_fifo);
     fifo8_reset(&s->xmit_fifo);
 
@@ -650,6 +815,9 @@ static void serial_reset(void *opaque)
     s->thr_ipending = 0;
     s->last_break_enable = 0;
     qemu_irq_lower(s->irq);
+
+    serial_update_msl(s);
+    s->msr &= ~UART_MSR_ANY_DELTA;
 }
 
 void serial_realize_core(SerialState *s, Error **errp)
@@ -668,6 +836,7 @@ void serial_realize_core(SerialState *s, Error **errp)
                           serial_event, s);
     fifo8_create(&s->recv_fifo, UART_FIFO_LENGTH);
     fifo8_create(&s->xmit_fifo, UART_FIFO_LENGTH);
+    serial_reset(s);
 }
 
 void serial_exit_core(SerialState *s)
@@ -779,7 +948,5 @@ SerialState *serial_mm_init(MemoryRegion *address_space,
     memory_region_init_io(&s->io, NULL, &serial_mm_ops[end], s,
                           "serial", 8 << it_shift);
     memory_region_add_subregion(address_space, base, &s->io);
-
-    serial_update_msl(s);
     return s;
 }
diff --git a/hw/char/virtio-serial-bus.c b/hw/char/virtio-serial-bus.c
index 23123b730..a7b1b6898 100644
--- a/hw/char/virtio-serial-bus.c
+++ b/hw/char/virtio-serial-bus.c
@@ -26,6 +26,10 @@
 #include "hw/virtio/virtio-serial.h"
 #include "hw/virtio/virtio-access.h"
 
+struct VirtIOSerialDevices {
+    QLIST_HEAD(, VirtIOSerial) devices;
+} vserdevices;
+
 static VirtIOSerialPort *find_port_by_id(VirtIOSerial *vser, uint32_t id)
 {
     VirtIOSerialPort *port;
@@ -52,6 +56,22 @@ static VirtIOSerialPort *find_port_by_vq(VirtIOSerial *vser, VirtQueue *vq)
     return NULL;
 }
 
+static VirtIOSerialPort *find_port_by_name(char *name)
+{
+    VirtIOSerial *vser;
+
+    QLIST_FOREACH(vser, &vserdevices.devices, next) {
+        VirtIOSerialPort *port;
+
+        QTAILQ_FOREACH(port, &vser->ports, next) {
+            if (!strcmp(port->name, name)) {
+                return port;
+            }
+        }
+    }
+    return NULL;
+}
+
 static bool use_multiport(VirtIOSerial *vser)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(vser);
@@ -851,6 +871,12 @@ static void virtser_port_device_realize(DeviceState *dev, Error **errp)
         return;
     }
 
+    if (port->name != NULL && find_port_by_name(port->name)) {
+        error_setg(errp, "virtio-serial-bus: A port already exists by name %s",
+                   port->name);
+        return;
+    }
+
     if (port->id == VIRTIO_CONSOLE_BAD_ID) {
         if (plugging_port0) {
             port->id = 0;
@@ -878,6 +904,12 @@ static void virtser_port_device_realize(DeviceState *dev, Error **errp)
     }
 
     port->elem.out_num = 0;
+}
+
+static void virtser_port_device_plug(HotplugHandler *hotplug_dev,
+                                     DeviceState *dev, Error **errp)
+{
+    VirtIOSerialPort *port = VIRTIO_SERIAL_PORT(dev);
 
     QTAILQ_INSERT_TAIL(&port->vser->ports, port, next);
     port->ivq = port->vser->ivqs[port->id];
@@ -886,7 +918,7 @@ static void virtser_port_device_realize(DeviceState *dev, Error **errp)
     add_port(port->vser, port->id);
 
     /* Send an update to the guest about this new port added */
-    virtio_notify_config(vdev);
+    virtio_notify_config(VIRTIO_DEVICE(hotplug_dev));
 }
 
 static void virtser_port_device_unrealize(DeviceState *dev, Error **errp)
@@ -909,7 +941,6 @@ static void virtio_serial_device_realize(DeviceState *dev, Error **errp)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VirtIOSerial *vser = VIRTIO_SERIAL(dev);
-    BusState *bus;
     uint32_t i, max_supported_ports;
 
     if (!vser->serial.max_virtserial_ports) {
@@ -931,8 +962,7 @@ static void virtio_serial_device_realize(DeviceState *dev, Error **errp)
     /* Spawn a new virtio-serial bus on which the ports will ride as devices */
     qbus_create_inplace(&vser->bus, sizeof(vser->bus), TYPE_VIRTIO_SERIAL_BUS,
                         dev, vdev->bus_name);
-    bus = BUS(&vser->bus);
-    bus->allow_hotplug = 1;
+    qbus_set_hotplug_handler(BUS(&vser->bus), DEVICE(vser), errp);
     vser->bus.vser = vser;
     QTAILQ_INIT(&vser->ports);
 
@@ -983,6 +1013,8 @@ static void virtio_serial_device_realize(DeviceState *dev, Error **errp)
      */
     register_savevm(dev, "virtio-console", -1, 3, virtio_serial_save,
                     virtio_serial_load, vser);
+
+    QLIST_INSERT_HEAD(&vserdevices.devices, vser, next);
 }
 
 static void virtio_serial_port_class_init(ObjectClass *klass, void *data)
@@ -993,7 +1025,6 @@ static void virtio_serial_port_class_init(ObjectClass *klass, void *data)
     k->bus_type = TYPE_VIRTIO_SERIAL_BUS;
     k->realize = virtser_port_device_realize;
     k->unrealize = virtser_port_device_unrealize;
-    k->unplug = qdev_simple_unplug_cb;
     k->props = virtser_props;
 }
 
@@ -1011,6 +1042,8 @@ static void virtio_serial_device_unrealize(DeviceState *dev, Error **errp)
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VirtIOSerial *vser = VIRTIO_SERIAL(dev);
 
+    QLIST_REMOVE(vser, next);
+
     unregister_savevm(dev, "virtio-console", vser);
 
     g_free(vser->ivqs);
@@ -1034,6 +1067,9 @@ static void virtio_serial_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
+    HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass);
+
+    QLIST_INIT(&vserdevices.devices);
 
     dc->props = virtio_serial_properties;
     set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
@@ -1045,6 +1081,8 @@ static void virtio_serial_class_init(ObjectClass *klass, void *data)
     vdc->reset = vser_reset;
     vdc->save = virtio_serial_save_device;
     vdc->load = virtio_serial_load_device;
+    hc->plug = virtser_port_device_plug;
+    hc->unplug = qdev_simple_device_unplug_cb;
 }
 
 static const TypeInfo virtio_device_info = {
@@ -1052,6 +1090,10 @@ static const TypeInfo virtio_device_info = {
     .parent = TYPE_VIRTIO_DEVICE,
     .instance_size = sizeof(VirtIOSerial),
     .class_init = virtio_serial_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_HOTPLUG_HANDLER },
+        { }
+    }
 };
 
 static void virtio_serial_register_types(void)
diff --git a/hw/core/Makefile.objs b/hw/core/Makefile.objs
index 5377d052e..9dce1bc53 100644
--- a/hw/core/Makefile.objs
+++ b/hw/core/Makefile.objs
@@ -4,6 +4,7 @@ common-obj-y += fw-path-provider.o
 # irq.o needed for qdev GPIO handling:
 common-obj-y += irq.o
 common-obj-y += hotplug.o
+common-obj-y += nmi.o
 
 common-obj-$(CONFIG_EMPTY_SLOT) += empty_slot.o
 common-obj-$(CONFIG_XILINX_AXI) += stream.o
@@ -13,3 +14,4 @@ common-obj-$(CONFIG_SOFTMMU) += machine.o
 common-obj-$(CONFIG_SOFTMMU) += null-machine.o
 common-obj-$(CONFIG_SOFTMMU) += loader.o
 common-obj-$(CONFIG_SOFTMMU) += qdev-properties-system.o
+common-obj-$(CONFIG_SOFTMMU) += platform-bus.o
diff --git a/hw/core/hotplug.c b/hw/core/hotplug.c
index 5573d9d2d..4e0107455 100644
--- a/hw/core/hotplug.c
+++ b/hw/core/hotplug.c
@@ -23,6 +23,17 @@ void hotplug_handler_plug(HotplugHandler *plug_handler,
     }
 }
 
+void hotplug_handler_unplug_request(HotplugHandler *plug_handler,
+                                    DeviceState *plugged_dev,
+                                    Error **errp)
+{
+    HotplugHandlerClass *hdc = HOTPLUG_HANDLER_GET_CLASS(plug_handler);
+
+    if (hdc->unplug_request) {
+        hdc->unplug_request(plug_handler, plugged_dev, errp);
+    }
+}
+
 void hotplug_handler_unplug(HotplugHandler *plug_handler,
                             DeviceState *plugged_dev,
                             Error **errp)
diff --git a/hw/core/irq.c b/hw/core/irq.c
index cffced040..8a62a36d5 100644
--- a/hw/core/irq.c
+++ b/hw/core/irq.c
@@ -140,16 +140,10 @@ void qemu_irq_intercept_in(qemu_irq *gpio_in, qemu_irq_handler handler, int n)
     for (i = 0; i < n; i++) {
         *old_irqs[i] = *gpio_in[i];
         gpio_in[i]->handler = handler;
-        gpio_in[i]->opaque = old_irqs;
+        gpio_in[i]->opaque = &old_irqs[i];
     }
 }
 
-void qemu_irq_intercept_out(qemu_irq **gpio_out, qemu_irq_handler handler, int n)
-{
-    qemu_irq *old_irqs = *gpio_out;
-    *gpio_out = qemu_allocate_irqs(handler, old_irqs, n);
-}
-
 static const TypeInfo irq_type_info = {
    .name = TYPE_IRQ,
    .parent = TYPE_OBJECT,
diff --git a/hw/core/loader.c b/hw/core/loader.c
index 2bf6b8ff8..7527fd303 100644
--- a/hw/core/loader.c
+++ b/hw/core/loader.c
@@ -80,6 +80,13 @@ int load_image(const char *filename, uint8_t *addr)
     if (fd < 0)
         return -1;
     size = lseek(fd, 0, SEEK_END);
+    if (size == -1) {
+        fprintf(stderr, "file %-20s: get size error: %s\n",
+                filename, strerror(errno));
+        close(fd);
+        return -1;
+    }
+
     lseek(fd, 0, SEEK_SET);
     if (read(fd, addr, size) != size) {
         close(fd);
@@ -89,6 +96,27 @@ int load_image(const char *filename, uint8_t *addr)
     return size;
 }
 
+/* return the size or -1 if error */
+ssize_t load_image_size(const char *filename, void *addr, size_t size)
+{
+    int fd;
+    ssize_t actsize;
+
+    fd = open(filename, O_RDONLY | O_BINARY);
+    if (fd < 0) {
+        return -1;
+    }
+
+    actsize = read(fd, addr, size);
+    if (actsize < 0) {
+        close(fd);
+        return -1;
+    }
+    close(fd);
+
+    return actsize;
+}
+
 /* read()-like version */
 ssize_t read_targphys(const char *name,
                       int fd, hwaddr dst_addr, size_t nbytes)
@@ -456,7 +484,9 @@ static ssize_t gunzip(void *dst, size_t dstlen, uint8_t *src,
 
 /* Load a U-Boot image.  */
 static int load_uboot_image(const char *filename, hwaddr *ep, hwaddr *loadaddr,
-                            int *is_linux, uint8_t image_type)
+                            int *is_linux, uint8_t image_type,
+                            uint64_t (*translate_fn)(void *, uint64_t),
+                            void *translate_opaque)
 {
     int fd;
     int size;
@@ -490,6 +520,9 @@ static int load_uboot_image(const char *filename, hwaddr *ep, hwaddr *loadaddr,
     switch (hdr->ih_type) {
     case IH_TYPE_KERNEL:
         address = hdr->ih_load;
+        if (translate_fn) {
+            address = translate_fn(translate_opaque, address);
+        }
         if (loadaddr) {
             *loadaddr = hdr->ih_load;
         }
@@ -566,15 +599,67 @@ out:
 }
 
 int load_uimage(const char *filename, hwaddr *ep, hwaddr *loadaddr,
-                int *is_linux)
+                int *is_linux,
+                uint64_t (*translate_fn)(void *, uint64_t),
+                void *translate_opaque)
 {
-    return load_uboot_image(filename, ep, loadaddr, is_linux, IH_TYPE_KERNEL);
+    return load_uboot_image(filename, ep, loadaddr, is_linux, IH_TYPE_KERNEL,
+                            translate_fn, translate_opaque);
 }
 
 /* Load a ramdisk.  */
 int load_ramdisk(const char *filename, hwaddr addr, uint64_t max_sz)
 {
-    return load_uboot_image(filename, NULL, &addr, NULL, IH_TYPE_RAMDISK);
+    return load_uboot_image(filename, NULL, &addr, NULL, IH_TYPE_RAMDISK,
+                            NULL, NULL);
+}
+
+/* This simply prevents g_malloc in the function below from allocating
+ * a huge amount of memory, by placing a limit on the maximum
+ * uncompressed image size that load_image_gzipped will read.
+ */
+#define LOAD_IMAGE_MAX_GUNZIP_BYTES (256 << 20)
+
+/* Load a gzip-compressed kernel. */
+int load_image_gzipped(const char *filename, hwaddr addr, uint64_t max_sz)
+{
+    uint8_t *compressed_data = NULL;
+    uint8_t *data = NULL;
+    gsize len;
+    ssize_t bytes;
+    int ret = -1;
+
+    if (!g_file_get_contents(filename, (char **) &compressed_data, &len,
+                             NULL)) {
+        goto out;
+    }
+
+    /* Is it a gzip-compressed file? */
+    if (len < 2 ||
+        compressed_data[0] != 0x1f ||
+        compressed_data[1] != 0x8b) {
+        goto out;
+    }
+
+    if (max_sz > LOAD_IMAGE_MAX_GUNZIP_BYTES) {
+        max_sz = LOAD_IMAGE_MAX_GUNZIP_BYTES;
+    }
+
+    data = g_malloc(max_sz);
+    bytes = gunzip(data, max_sz, compressed_data, len);
+    if (bytes < 0) {
+        fprintf(stderr, "%s: unable to decompress gzipped kernel file\n",
+                filename);
+        goto out;
+    }
+
+    rom_add_blob_fixed(filename, data, bytes, addr);
+    ret = bytes;
+
+ out:
+    g_free(compressed_data);
+    g_free(data);
+    return ret;
 }
 
 /*
@@ -632,7 +717,7 @@ static void *rom_set_mr(Rom *rom, Object *owner, const char *name)
     void *data;
 
     rom->mr = g_malloc(sizeof(*rom->mr));
-    memory_region_init_ram(rom->mr, owner, name, rom->datasize);
+    memory_region_init_ram(rom->mr, owner, name, rom->datasize, &error_abort);
     memory_region_set_readonly(rom->mr, true);
     vmstate_register_ram_global(rom->mr);
 
@@ -670,6 +755,12 @@ int rom_add_file(const char *file, const char *fw_dir,
     }
     rom->addr     = addr;
     rom->romsize  = lseek(fd, 0, SEEK_END);
+    if (rom->romsize == -1) {
+        fprintf(stderr, "rom: file %-20s: get size error: %s\n",
+                rom->name, strerror(errno));
+        goto err;
+    }
+
     rom->datasize = rom->romsize;
     rom->data     = g_malloc0(rom->datasize);
     lseek(fd, 0, SEEK_SET);
@@ -720,12 +811,12 @@ err:
     return -1;
 }
 
-void *rom_add_blob(const char *name, const void *blob, size_t len,
+ram_addr_t rom_add_blob(const char *name, const void *blob, size_t len,
                    hwaddr addr, const char *fw_file_name,
                    FWCfgReadCallback fw_callback, void *callback_opaque)
 {
     Rom *rom;
-    void *data = NULL;
+    ram_addr_t ret = RAM_ADDR_MAX;
 
     rom           = g_malloc0(sizeof(*rom));
     rom->name     = g_strdup(name);
@@ -737,11 +828,13 @@ void *rom_add_blob(const char *name, const void *blob, size_t len,
     rom_insert(rom);
     if (fw_file_name && fw_cfg) {
         char devpath[100];
+        void *data;
 
         snprintf(devpath, sizeof(devpath), "/rom@%s", fw_file_name);
 
         if (rom_file_has_mr) {
             data = rom_set_mr(rom, OBJECT(fw_cfg), devpath);
+            ret = memory_region_get_ram_addr(rom->mr);
         } else {
             data = rom->data;
         }
@@ -750,7 +843,7 @@ void *rom_add_blob(const char *name, const void *blob, size_t len,
                                  fw_callback, callback_opaque,
                                  data, rom->romsize);
     }
-    return data;
+    return ret;
 }
 
 /* This function is specific for elf program because we don't need to allocate
@@ -955,7 +1048,7 @@ void do_info_roms(Monitor *mon, const QDict *qdict)
         if (rom->mr) {
             monitor_printf(mon, "%s"
                            " size=0x%06zx name=\"%s\"\n",
-                           rom->mr->name,
+                           memory_region_name(rom->mr),
                            rom->romsize,
                            rom->name);
         } else if (!rom->fw_file) {
diff --git a/hw/core/machine.c b/hw/core/machine.c
index 7a66c57ab..19d3e3a70 100644
--- a/hw/core/machine.c
+++ b/hw/core/machine.c
@@ -12,6 +12,9 @@
 
 #include "hw/boards.h"
 #include "qapi/visitor.h"
+#include "hw/sysbus.h"
+#include "sysemu/sysemu.h"
+#include "qemu/error-report.h"
 
 static char *machine_get_accel(Object *obj, Error **errp)
 {
@@ -24,6 +27,7 @@ static void machine_set_accel(Object *obj, const char *value, Error **errp)
 {
     MachineState *ms = MACHINE(obj);
 
+    g_free(ms->accel);
     ms->accel = g_strdup(value);
 }
 
@@ -79,6 +83,7 @@ static void machine_set_kernel(Object *obj, const char *value, Error **errp)
 {
     MachineState *ms = MACHINE(obj);
 
+    g_free(ms->kernel_filename);
     ms->kernel_filename = g_strdup(value);
 }
 
@@ -93,6 +98,7 @@ static void machine_set_initrd(Object *obj, const char *value, Error **errp)
 {
     MachineState *ms = MACHINE(obj);
 
+    g_free(ms->initrd_filename);
     ms->initrd_filename = g_strdup(value);
 }
 
@@ -107,6 +113,7 @@ static void machine_set_append(Object *obj, const char *value, Error **errp)
 {
     MachineState *ms = MACHINE(obj);
 
+    g_free(ms->kernel_cmdline);
     ms->kernel_cmdline = g_strdup(value);
 }
 
@@ -121,6 +128,7 @@ static void machine_set_dtb(Object *obj, const char *value, Error **errp)
 {
     MachineState *ms = MACHINE(obj);
 
+    g_free(ms->dtb);
     ms->dtb = g_strdup(value);
 }
 
@@ -135,6 +143,7 @@ static void machine_set_dumpdtb(Object *obj, const char *value, Error **errp)
 {
     MachineState *ms = MACHINE(obj);
 
+    g_free(ms->dumpdtb);
     ms->dumpdtb = g_strdup(value);
 }
 
@@ -176,6 +185,7 @@ static void machine_set_dt_compatible(Object *obj, const char *value, Error **er
 {
     MachineState *ms = MACHINE(obj);
 
+    g_free(ms->dt_compatible);
     ms->dt_compatible = g_strdup(value);
 }
 
@@ -232,11 +242,53 @@ static void machine_set_firmware(Object *obj, const char *value, Error **errp)
 {
     MachineState *ms = MACHINE(obj);
 
+    g_free(ms->firmware);
     ms->firmware = g_strdup(value);
 }
 
+static bool machine_get_iommu(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return ms->iommu;
+}
+
+static void machine_set_iommu(Object *obj, bool value, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->iommu = value;
+}
+
+static int error_on_sysbus_device(SysBusDevice *sbdev, void *opaque)
+{
+    error_report("Option '-device %s' cannot be handled by this machine",
+                 object_class_get_name(object_get_class(OBJECT(sbdev))));
+    exit(1);
+}
+
+static void machine_init_notify(Notifier *notifier, void *data)
+{
+    Object *machine = qdev_get_machine();
+    ObjectClass *oc = object_get_class(machine);
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    if (mc->has_dynamic_sysbus) {
+        /* Our machine can handle dynamic sysbus devices, we're all good */
+        return;
+    }
+
+    /*
+     * Loop through all dynamically created devices and check whether there
+     * are sysbus devices among them. If there are, error out.
+     */
+    foreach_dynamic_sysbus_device(error_on_sysbus_device, NULL);
+}
+
 static void machine_initfn(Object *obj)
 {
+    MachineState *ms = MACHINE(obj);
+
     object_property_add_str(obj, "accel",
                             machine_get_accel, machine_set_accel, NULL);
     object_property_add_bool(obj, "kernel-irqchip",
@@ -270,10 +322,21 @@ static void machine_initfn(Object *obj)
                              machine_set_dump_guest_core,
                              NULL);
     object_property_add_bool(obj, "mem-merge",
-                             machine_get_mem_merge, machine_set_mem_merge, NULL);
-    object_property_add_bool(obj, "usb", machine_get_usb, machine_set_usb, NULL);
+                             machine_get_mem_merge,
+                             machine_set_mem_merge, NULL);
+    object_property_add_bool(obj, "usb",
+                             machine_get_usb,
+                             machine_set_usb, NULL);
     object_property_add_str(obj, "firmware",
-                            machine_get_firmware, machine_set_firmware, NULL);
+                            machine_get_firmware,
+                            machine_set_firmware, NULL);
+    object_property_add_bool(obj, "iommu",
+                             machine_get_iommu,
+                             machine_set_iommu, NULL);
+
+    /* Register notifier when init is done for sysbus sanity checks */
+    ms->sysbus_notifier.notify = machine_init_notify;
+    qemu_add_machine_init_done_notifier(&ms->sysbus_notifier);
 }
 
 static void machine_finalize(Object *obj)
diff --git a/hw/core/nmi.c b/hw/core/nmi.c
new file mode 100644
index 000000000..3dff02065
--- /dev/null
+++ b/hw/core/nmi.c
@@ -0,0 +1,84 @@
+/*
+ *  NMI monitor handler class and helpers.
+ *
+ *  Copyright IBM Corp., 2014
+ *
+ *  Author: Alexey Kardashevskiy <aik@ozlabs.ru>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License,
+ *  or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/nmi.h"
+#include "qapi/qmp/qerror.h"
+
+struct do_nmi_s {
+    int cpu_index;
+    Error *errp;
+    bool handled;
+};
+
+static void nmi_children(Object *o, struct do_nmi_s *ns);
+
+static int do_nmi(Object *o, void *opaque)
+{
+    struct do_nmi_s *ns = opaque;
+    NMIState *n = (NMIState *) object_dynamic_cast(o, TYPE_NMI);
+
+    if (n) {
+        NMIClass *nc = NMI_GET_CLASS(n);
+
+        ns->handled = true;
+        nc->nmi_monitor_handler(n, ns->cpu_index, &ns->errp);
+        if (ns->errp) {
+            return -1;
+        }
+    }
+    nmi_children(o, ns);
+
+    return 0;
+}
+
+static void nmi_children(Object *o, struct do_nmi_s *ns)
+{
+    object_child_foreach(o, do_nmi, ns);
+}
+
+void nmi_monitor_handle(int cpu_index, Error **errp)
+{
+    struct do_nmi_s ns = {
+        .cpu_index = cpu_index,
+        .errp = NULL,
+        .handled = false
+    };
+
+    nmi_children(object_get_root(), &ns);
+    if (ns.handled) {
+        error_propagate(errp, ns.errp);
+    } else {
+        error_set(errp, QERR_UNSUPPORTED);
+    }
+}
+
+static const TypeInfo nmi_info = {
+    .name          = TYPE_NMI,
+    .parent        = TYPE_INTERFACE,
+    .class_size    = sizeof(NMIClass),
+};
+
+static void nmi_register_types(void)
+{
+    type_register_static(&nmi_info);
+}
+
+type_init(nmi_register_types)
diff --git a/hw/core/platform-bus.c b/hw/core/platform-bus.c
new file mode 100644
index 000000000..0f052b333
--- /dev/null
+++ b/hw/core/platform-bus.c
@@ -0,0 +1,253 @@
+/*
+ *  Platform Bus device to support dynamic Sysbus devices
+ *
+ * Copyright (C) 2014 Freescale Semiconductor, Inc. All rights reserved.
+ *
+ * Author: Alexander Graf, <agraf@suse.de>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/platform-bus.h"
+#include "monitor/monitor.h"
+#include "exec/address-spaces.h"
+#include "sysemu/sysemu.h"
+
+
+/*
+ * Returns the PlatformBus IRQ number for a SysBusDevice irq number or -1 if
+ * the IRQ is not mapped on this Platform bus.
+ */
+int platform_bus_get_irqn(PlatformBusDevice *pbus, SysBusDevice *sbdev,
+                          int n)
+{
+    qemu_irq sbirq = sysbus_get_connected_irq(sbdev, n);
+    int i;
+
+    for (i = 0; i < pbus->num_irqs; i++) {
+        if (pbus->irqs[i] == sbirq) {
+            return i;
+        }
+    }
+
+    /* IRQ not mapped on platform bus */
+    return -1;
+}
+
+/*
+ * Returns the PlatformBus MMIO region offset for Region n of a SysBusDevice or
+ * -1 if the region is not mapped on this Platform bus.
+ */
+hwaddr platform_bus_get_mmio_addr(PlatformBusDevice *pbus, SysBusDevice *sbdev,
+                                  int n)
+{
+    MemoryRegion *pbus_mr = &pbus->mmio;
+    MemoryRegion *sbdev_mr = sysbus_mmio_get_region(sbdev, n);
+    Object *pbus_mr_obj = OBJECT(pbus_mr);
+    Object *parent_mr;
+
+    if (!memory_region_is_mapped(sbdev_mr)) {
+        /* Region is not mapped? */
+        return -1;
+    }
+
+    parent_mr = object_property_get_link(OBJECT(sbdev_mr), "container", NULL);
+
+    assert(parent_mr);
+    if (parent_mr != pbus_mr_obj) {
+        /* MMIO region is not mapped on platform bus */
+        return -1;
+    }
+
+    return object_property_get_int(OBJECT(sbdev_mr), "addr", NULL);
+}
+
+static int platform_bus_count_irqs(SysBusDevice *sbdev, void *opaque)
+{
+    PlatformBusDevice *pbus = opaque;
+    qemu_irq sbirq;
+    int n, i;
+
+    for (n = 0; ; n++) {
+        if (!sysbus_has_irq(sbdev, n)) {
+            break;
+        }
+
+        sbirq = sysbus_get_connected_irq(sbdev, n);
+        for (i = 0; i < pbus->num_irqs; i++) {
+            if (pbus->irqs[i] == sbirq) {
+                bitmap_set(pbus->used_irqs, i, 1);
+                break;
+            }
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Loop through all sysbus devices and look for unassigned IRQ lines as well as
+ * unassociated MMIO regions. Connect them to the platform bus if available.
+ */
+static void plaform_bus_refresh_irqs(PlatformBusDevice *pbus)
+{
+    bitmap_zero(pbus->used_irqs, pbus->num_irqs);
+    foreach_dynamic_sysbus_device(platform_bus_count_irqs, pbus);
+    pbus->done_gathering = true;
+}
+
+static int platform_bus_map_irq(PlatformBusDevice *pbus, SysBusDevice *sbdev,
+                                int n)
+{
+    int max_irqs = pbus->num_irqs;
+    int irqn;
+
+    if (sysbus_is_irq_connected(sbdev, n)) {
+        /* IRQ is already mapped, nothing to do */
+        return 0;
+    }
+
+    irqn = find_first_zero_bit(pbus->used_irqs, max_irqs);
+    if (irqn >= max_irqs) {
+        hw_error("Platform Bus: Can not fit IRQ line");
+        return -1;
+    }
+
+    set_bit(irqn, pbus->used_irqs);
+    sysbus_connect_irq(sbdev, n, pbus->irqs[irqn]);
+
+    return 0;
+}
+
+static int platform_bus_map_mmio(PlatformBusDevice *pbus, SysBusDevice *sbdev,
+                                 int n)
+{
+    MemoryRegion *sbdev_mr = sysbus_mmio_get_region(sbdev, n);
+    uint64_t size = memory_region_size(sbdev_mr);
+    uint64_t alignment = (1ULL << (63 - clz64(size + size - 1)));
+    uint64_t off;
+    bool found_region = false;
+
+    if (memory_region_is_mapped(sbdev_mr)) {
+        /* Region is already mapped, nothing to do */
+        return 0;
+    }
+
+    /*
+     * Look for empty space in the MMIO space that is naturally aligned with
+     * the target device's memory region
+     */
+    for (off = 0; off < pbus->mmio_size; off += alignment) {
+        if (!memory_region_find(&pbus->mmio, off, size).mr) {
+            found_region = true;
+            break;
+        }
+    }
+
+    if (!found_region) {
+        hw_error("Platform Bus: Can not fit MMIO region of size %"PRIx64, size);
+    }
+
+    /* Map the device's region into our Platform Bus MMIO space */
+    memory_region_add_subregion(&pbus->mmio, off, sbdev_mr);
+
+    return 0;
+}
+
+/*
+ * For each sysbus device, look for unassigned IRQ lines as well as
+ * unassociated MMIO regions. Connect them to the platform bus if available.
+ */
+static int link_sysbus_device(SysBusDevice *sbdev, void *opaque)
+{
+    PlatformBusDevice *pbus = opaque;
+    int i;
+
+    for (i = 0; sysbus_has_irq(sbdev, i); i++) {
+        platform_bus_map_irq(pbus, sbdev, i);
+    }
+
+    for (i = 0; sysbus_has_mmio(sbdev, i); i++) {
+        platform_bus_map_mmio(pbus, sbdev, i);
+    }
+
+    return 0;
+}
+
+static void platform_bus_init_notify(Notifier *notifier, void *data)
+{
+    PlatformBusDevice *pb = container_of(notifier, PlatformBusDevice, notifier);
+
+    /*
+     * Generate a bitmap of used IRQ lines, as the user might have specified
+     * them on the command line.
+     */
+    plaform_bus_refresh_irqs(pb);
+
+    foreach_dynamic_sysbus_device(link_sysbus_device, pb);
+}
+
+static void platform_bus_realize(DeviceState *dev, Error **errp)
+{
+    PlatformBusDevice *pbus;
+    SysBusDevice *d;
+    int i;
+
+    d = SYS_BUS_DEVICE(dev);
+    pbus = PLATFORM_BUS_DEVICE(dev);
+
+    memory_region_init(&pbus->mmio, NULL, "platform bus", pbus->mmio_size);
+    sysbus_init_mmio(d, &pbus->mmio);
+
+    pbus->used_irqs = bitmap_new(pbus->num_irqs);
+    pbus->irqs = g_new0(qemu_irq, pbus->num_irqs);
+    for (i = 0; i < pbus->num_irqs; i++) {
+        sysbus_init_irq(d, &pbus->irqs[i]);
+    }
+
+    /*
+     * Register notifier that allows us to gather dangling devices once the
+     * machine is completely assembled
+     */
+    pbus->notifier.notify = platform_bus_init_notify;
+    qemu_add_machine_init_done_notifier(&pbus->notifier);
+}
+
+static Property platform_bus_properties[] = {
+    DEFINE_PROP_UINT32("num_irqs", PlatformBusDevice, num_irqs, 0),
+    DEFINE_PROP_UINT32("mmio_size", PlatformBusDevice, mmio_size, 0),
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void platform_bus_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = platform_bus_realize;
+    dc->props = platform_bus_properties;
+}
+
+static const TypeInfo platform_bus_info = {
+    .name          = TYPE_PLATFORM_BUS_DEVICE,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(PlatformBusDevice),
+    .class_init    = platform_bus_class_init,
+};
+
+static void platform_bus_register_types(void)
+{
+    type_register_static(&platform_bus_info);
+}
+
+type_init(platform_bus_register_types)
diff --git a/hw/core/qdev-properties-system.c b/hw/core/qdev-properties-system.c
index ae0900f65..65901ef10 100644
--- a/hw/core/qdev-properties-system.c
+++ b/hw/core/qdev-properties-system.c
@@ -13,6 +13,7 @@
 #include "net/net.h"
 #include "hw/qdev.h"
 #include "qapi/qmp/qerror.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "hw/block/block.h"
 #include "net/hub.h"
@@ -68,16 +69,16 @@ static void set_pointer(Object *obj, Visitor *v, Property *prop,
 
 static int parse_drive(DeviceState *dev, const char *str, void **ptr)
 {
-    BlockDriverState *bs;
+    BlockBackend *blk;
 
-    bs = bdrv_find(str);
-    if (bs == NULL) {
+    blk = blk_by_name(str);
+    if (!blk) {
         return -ENOENT;
     }
-    if (bdrv_attach_dev(bs, dev) < 0) {
+    if (blk_attach_dev(blk, dev) < 0) {
         return -EEXIST;
     }
-    *ptr = bs;
+    *ptr = blk;
     return 0;
 }
 
@@ -85,17 +86,17 @@ static void release_drive(Object *obj, const char *name, void *opaque)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
-    BlockDriverState **ptr = qdev_get_prop_ptr(dev, prop);
+    BlockBackend **ptr = qdev_get_prop_ptr(dev, prop);
 
     if (*ptr) {
-        bdrv_detach_dev(*ptr, dev);
+        blk_detach_dev(*ptr, dev);
         blockdev_auto_del(*ptr);
     }
 }
 
 static char *print_drive(void *ptr)
 {
-    return g_strdup(bdrv_get_device_name(ptr));
+    return g_strdup(blk_name(ptr));
 }
 
 static void get_drive(Object *obj, Visitor *v, void *opaque,
@@ -112,7 +113,7 @@ static void set_drive(Object *obj, Visitor *v, void *opaque,
 
 PropertyInfo qdev_prop_drive = {
     .name  = "str",
-    .legacy_name  = "drive",
+    .description = "ID of a drive to use as a backend",
     .get   = get_drive,
     .set   = set_drive,
     .release = release_drive,
@@ -169,7 +170,7 @@ static void set_chr(Object *obj, Visitor *v, void *opaque,
 
 PropertyInfo qdev_prop_chr = {
     .name  = "str",
-    .legacy_name  = "chr",
+    .description = "ID of a chardev to use as a backend",
     .get   = get_chr,
     .set   = set_chr,
     .release = release_chr,
@@ -248,7 +249,7 @@ static void set_netdev(Object *obj, Visitor *v, void *opaque,
 
 PropertyInfo qdev_prop_netdev = {
     .name  = "str",
-    .legacy_name  = "netdev",
+    .description = "ID of a netdev to use as a backend",
     .get   = get_netdev,
     .set   = set_netdev,
 };
@@ -328,19 +329,18 @@ static void set_vlan(Object *obj, Visitor *v, void *opaque,
 
 PropertyInfo qdev_prop_vlan = {
     .name  = "int32",
-    .legacy_name  = "vlan",
+    .description = "Integer VLAN id to connect to",
     .print = print_vlan,
     .get   = get_vlan,
     .set   = set_vlan,
 };
 
 int qdev_prop_set_drive(DeviceState *dev, const char *name,
-                        BlockDriverState *value)
+                        BlockBackend *value)
 {
     Error *err = NULL;
-    const char *bdrv_name = value ? bdrv_get_device_name(value) : "";
-    object_property_set_str(OBJECT(dev), bdrv_name,
-                            name, &err);
+    object_property_set_str(OBJECT(dev),
+                            value ? blk_name(value) : "", name, &err);
     if (err) {
         qerror_report_err(err);
         error_free(err);
@@ -350,7 +350,7 @@ int qdev_prop_set_drive(DeviceState *dev, const char *name,
 }
 
 void qdev_prop_set_drive_nofail(DeviceState *dev, const char *name,
-                                BlockDriverState *value)
+                                BlockBackend *value)
 {
     if (qdev_prop_set_drive(dev, name, value) < 0) {
         exit(1);
@@ -388,28 +388,12 @@ void qdev_set_nic_properties(DeviceState *dev, NICInfo *nd)
 static int qdev_add_one_global(QemuOpts *opts, void *opaque)
 {
     GlobalProperty *g;
-    ObjectClass *oc;
 
     g = g_malloc0(sizeof(*g));
     g->driver   = qemu_opt_get(opts, "driver");
     g->property = qemu_opt_get(opts, "property");
     g->value    = qemu_opt_get(opts, "value");
-    oc = object_class_dynamic_cast(object_class_by_name(g->driver),
-                                   TYPE_DEVICE);
-    if (oc) {
-        DeviceClass *dc = DEVICE_CLASS(oc);
-
-        if (dc->hotpluggable) {
-            /* If hotpluggable then skip not_used checking. */
-            g->not_used = false;
-        } else {
-            /* Maybe a typo. */
-            g->not_used = true;
-        }
-    } else {
-        /* Maybe a typo. */
-        g->not_used = true;
-    }
+    g->user_provided = true;
     qdev_prop_register_global(g);
     return 0;
 }
diff --git a/hw/core/qdev-properties.c b/hw/core/qdev-properties.c
index 3d12560f4..2e47f70eb 100644
--- a/hw/core/qdev-properties.c
+++ b/hw/core/qdev-properties.c
@@ -1,7 +1,7 @@
 #include "net/net.h"
 #include "hw/qdev.h"
 #include "qapi/qmp/qerror.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "hw/block/block.h"
 #include "net/hub.h"
 #include "qapi/visitor.h"
@@ -120,7 +120,7 @@ static void prop_set_bit(Object *obj, Visitor *v, void *opaque,
 
 PropertyInfo qdev_prop_bit = {
     .name  = "bool",
-    .legacy_name  = "on/off",
+    .description = "on/off",
     .get   = prop_get_bit,
     .set   = prop_set_bit,
 };
@@ -455,7 +455,7 @@ inval:
 
 PropertyInfo qdev_prop_macaddr = {
     .name  = "str",
-    .legacy_name  = "macaddr",
+    .description = "Ethernet 6-byte MAC Address, example: 52:54:00:12:34:56",
     .get   = get_mac,
     .set   = set_mac,
 };
@@ -477,7 +477,8 @@ QEMU_BUILD_BUG_ON(sizeof(BiosAtaTranslation) != sizeof(int));
 
 PropertyInfo qdev_prop_bios_chs_trans = {
     .name = "BiosAtaTranslation",
-    .legacy_name = "bios-chs-trans",
+    .description = "Logical CHS translation algorithm, "
+                   "auto/none/lba/large/rechs",
     .enum_table = BiosAtaTranslation_lookup,
     .get = get_enum,
     .set = set_enum,
@@ -551,7 +552,7 @@ static int print_pci_devfn(DeviceState *dev, Property *prop, char *dest,
 
 PropertyInfo qdev_prop_pci_devfn = {
     .name  = "int32",
-    .legacy_name  = "pci-devfn",
+    .description = "Slot and optional function number, example: 06.0 or 06",
     .print = print_pci_devfn,
     .get   = get_int32,
     .set   = set_pci_devfn,
@@ -598,7 +599,7 @@ static void set_blocksize(Object *obj, Visitor *v, void *opaque,
 
 PropertyInfo qdev_prop_blocksize = {
     .name  = "uint16",
-    .legacy_name  = "blocksize",
+    .description = "A power of two between 512 and 32768",
     .get   = get_uint16,
     .set   = set_blocksize,
 };
@@ -706,7 +707,8 @@ inval:
 
 PropertyInfo qdev_prop_pci_host_devaddr = {
     .name = "str",
-    .legacy_name = "pci-host-devaddr",
+    .description = "Address (bus/device/function) of "
+                   "the host device, example: 04:10.0",
     .get = get_pci_host_devaddr,
     .set = set_pci_host_devaddr,
 };
@@ -955,19 +957,35 @@ void qdev_prop_register_global_list(GlobalProperty *props)
     }
 }
 
-int qdev_prop_check_global(void)
+int qdev_prop_check_globals(void)
 {
     GlobalProperty *prop;
     int ret = 0;
 
     QTAILQ_FOREACH(prop, &global_props, next) {
-        if (!prop->not_used) {
+        ObjectClass *oc;
+        DeviceClass *dc;
+        if (prop->used) {
+            continue;
+        }
+        if (!prop->user_provided) {
+            continue;
+        }
+        oc = object_class_by_name(prop->driver);
+        oc = object_class_dynamic_cast(oc, TYPE_DEVICE);
+        if (!oc) {
+            error_report("Warning: global %s.%s has invalid class name",
+                       prop->driver, prop->property);
+            ret = 1;
+            continue;
+        }
+        dc = DEVICE_CLASS(oc);
+        if (!dc->hotpluggable && !prop->used) {
+            error_report("Warning: global %s.%s=%s not used",
+                       prop->driver, prop->property, prop->value);
+            ret = 1;
             continue;
         }
-        ret = 1;
-        error_report("Warning: \"-global %s.%s=%s\" not used",
-                     prop->driver, prop->property, prop->value);
-
     }
     return ret;
 }
@@ -983,7 +1001,7 @@ void qdev_prop_set_globals_for_type(DeviceState *dev, const char *typename,
         if (strcmp(typename, prop->driver) != 0) {
             continue;
         }
-        prop->not_used = false;
+        prop->used = true;
         object_property_parse(OBJECT(dev), prop->value, prop->property, &err);
         if (err != NULL) {
             error_propagate(errp, err);
diff --git a/hw/core/qdev.c b/hw/core/qdev.c
index da1ba48c9..35fd00d26 100644
--- a/hw/core/qdev.c
+++ b/hw/core/qdev.c
@@ -85,10 +85,6 @@ static void bus_add_child(BusState *bus, DeviceState *child)
     char name[32];
     BusChild *kid = g_malloc0(sizeof(*kid));
 
-    if (qdev_hotplug) {
-        assert(bus->allow_hotplug);
-    }
-
     kid->index = bus->max_index++;
     kid->child = child;
     object_ref(OBJECT(kid->child));
@@ -112,6 +108,24 @@ void qdev_set_parent_bus(DeviceState *dev, BusState *bus)
     bus_add_child(bus, dev);
 }
 
+static void qbus_set_hotplug_handler_internal(BusState *bus, Object *handler,
+                                              Error **errp)
+{
+
+    object_property_set_link(OBJECT(bus), OBJECT(handler),
+                             QDEV_HOTPLUG_HANDLER_PROPERTY, errp);
+}
+
+void qbus_set_hotplug_handler(BusState *bus, DeviceState *handler, Error **errp)
+{
+    qbus_set_hotplug_handler_internal(bus, OBJECT(handler), errp);
+}
+
+void qbus_set_bus_hotplug_handler(BusState *bus, Error **errp)
+{
+    qbus_set_hotplug_handler_internal(bus, OBJECT(bus), errp);
+}
+
 /* Create a new device.  This only initializes the device state structure
    and allows properties to be set.  qdev_init should be called to
    initialize the actual device emulation.  */
@@ -209,11 +223,30 @@ void qdev_set_legacy_instance_id(DeviceState *dev, int alias_id,
     dev->alias_required_for_version = required_for_version;
 }
 
+static HotplugHandler *qdev_get_hotplug_handler(DeviceState *dev)
+{
+    HotplugHandler *hotplug_ctrl = NULL;
+
+    if (dev->parent_bus && dev->parent_bus->hotplug_handler) {
+        hotplug_ctrl = dev->parent_bus->hotplug_handler;
+    } else if (object_dynamic_cast(qdev_get_machine(), TYPE_MACHINE)) {
+        MachineState *machine = MACHINE(qdev_get_machine());
+        MachineClass *mc = MACHINE_GET_CLASS(machine);
+
+        if (mc->get_hotplug_handler) {
+            hotplug_ctrl = mc->get_hotplug_handler(machine, dev);
+        }
+    }
+    return hotplug_ctrl;
+}
+
 void qdev_unplug(DeviceState *dev, Error **errp)
 {
     DeviceClass *dc = DEVICE_GET_CLASS(dev);
+    HotplugHandler *hotplug_ctrl;
+    HotplugHandlerClass *hdc;
 
-    if (dev->parent_bus && !dev->parent_bus->allow_hotplug) {
+    if (dev->parent_bus && !qbus_is_hotpluggable(dev->parent_bus)) {
         error_set(errp, QERR_BUS_NO_HOTPLUG, dev->parent_bus->name);
         return;
     }
@@ -226,13 +259,18 @@ void qdev_unplug(DeviceState *dev, Error **errp)
 
     qdev_hot_removed = true;
 
-    if (dev->parent_bus && dev->parent_bus->hotplug_handler) {
-        hotplug_handler_unplug(dev->parent_bus->hotplug_handler, dev, errp);
+    hotplug_ctrl = qdev_get_hotplug_handler(dev);
+    /* hotpluggable device MUST have HotplugHandler, if it doesn't
+     * then something is very wrong with it */
+    g_assert(hotplug_ctrl);
+
+    /* If device supports async unplug just request it to be done,
+     * otherwise just remove it synchronously */
+    hdc = HOTPLUG_HANDLER_GET_CLASS(hotplug_ctrl);
+    if (hdc->unplug_request) {
+        hotplug_handler_unplug_request(hotplug_ctrl, dev, errp);
     } else {
-        assert(dc->unplug != NULL);
-        if (dc->unplug(dev) < 0) { /* legacy handler */
-            error_set(errp, QERR_UNDEFINED_ERROR);
-        }
+        hotplug_handler_unplug(hotplug_ctrl, dev, errp);
     }
 }
 
@@ -269,14 +307,13 @@ void qbus_reset_all_fn(void *opaque)
 }
 
 /* can be used as ->unplug() callback for the simple cases */
-int qdev_simple_unplug_cb(DeviceState *dev)
+void qdev_simple_device_unplug_cb(HotplugHandler *hotplug_dev,
+                                  DeviceState *dev, Error **errp)
 {
     /* just zap it */
     object_unparent(OBJECT(dev));
-    return 0;
 }
 
-
 /* Like qdev_init(), but terminate program via error_report() instead of
    returning an error value.  This is okay during machine creation.
    Don't use for hotplug, because there callers need to recover from
@@ -337,10 +374,20 @@ static NamedGPIOList *qdev_get_named_gpio_list(DeviceState *dev,
 void qdev_init_gpio_in_named(DeviceState *dev, qemu_irq_handler handler,
                              const char *name, int n)
 {
+    int i;
     NamedGPIOList *gpio_list = qdev_get_named_gpio_list(dev, name);
+    char *propname = g_strdup_printf("%s[*]", name ? name : "unnamed-gpio-in");
 
+    assert(gpio_list->num_out == 0 || !name);
     gpio_list->in = qemu_extend_irqs(gpio_list->in, gpio_list->num_in, handler,
                                      dev, n);
+
+    for (i = gpio_list->num_in; i < gpio_list->num_in + n; i++) {
+        object_property_add_child(OBJECT(dev), propname,
+                                  OBJECT(gpio_list->in[i]), &error_abort);
+    }
+    g_free(propname);
+
     gpio_list->num_in += n;
 }
 
@@ -352,11 +399,22 @@ void qdev_init_gpio_in(DeviceState *dev, qemu_irq_handler handler, int n)
 void qdev_init_gpio_out_named(DeviceState *dev, qemu_irq *pins,
                               const char *name, int n)
 {
+    int i;
     NamedGPIOList *gpio_list = qdev_get_named_gpio_list(dev, name);
-
-    assert(gpio_list->num_out == 0);
-    gpio_list->num_out = n;
-    gpio_list->out = pins;
+    char *propname = g_strdup_printf("%s[*]", name ? name : "unnamed-gpio-out");
+
+    assert(gpio_list->num_in == 0 || !name);
+    gpio_list->num_out += n;
+
+    for (i = 0; i < n; ++i) {
+        memset(&pins[i], 0, sizeof(*pins));
+        object_property_add_link(OBJECT(dev), propname, TYPE_IRQ,
+                                 (Object **)&pins[i],
+                                 object_property_allow_set_link,
+                                 OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                                 &error_abort);
+    }
+    g_free(propname);
 }
 
 void qdev_init_gpio_out(DeviceState *dev, qemu_irq *pins, int n)
@@ -380,10 +438,55 @@ qemu_irq qdev_get_gpio_in(DeviceState *dev, int n)
 void qdev_connect_gpio_out_named(DeviceState *dev, const char *name, int n,
                                  qemu_irq pin)
 {
-    NamedGPIOList *gpio_list = qdev_get_named_gpio_list(dev, name);
+    char *propname = g_strdup_printf("%s[%d]",
+                                     name ? name : "unnamed-gpio-out", n);
+    if (pin) {
+        /* We need a name for object_property_set_link to work.  If the
+         * object has a parent, object_property_add_child will come back
+         * with an error without doing anything.  If it has none, it will
+         * never fail.  So we can just call it with a NULL Error pointer.
+         */
+        object_property_add_child(qdev_get_machine(), "non-qdev-gpio[*]",
+                                  OBJECT(pin), NULL);
+    }
+    object_property_set_link(OBJECT(dev), OBJECT(pin), propname, &error_abort);
+    g_free(propname);
+}
+
+qemu_irq qdev_get_gpio_out_connector(DeviceState *dev, const char *name, int n)
+{
+    char *propname = g_strdup_printf("%s[%d]",
+                                     name ? name : "unnamed-gpio-out", n);
+
+    qemu_irq ret = (qemu_irq)object_property_get_link(OBJECT(dev), propname,
+                                                      NULL);
+
+    return ret;
+}
+
+/* disconnect a GPIO ouput, returning the disconnected input (if any) */
+
+static qemu_irq qdev_disconnect_gpio_out_named(DeviceState *dev,
+                                               const char *name, int n)
+{
+    char *propname = g_strdup_printf("%s[%d]",
+                                     name ? name : "unnamed-gpio-out", n);
+
+    qemu_irq ret = (qemu_irq)object_property_get_link(OBJECT(dev), propname,
+                                                      NULL);
+    if (ret) {
+        object_property_set_link(OBJECT(dev), NULL, propname, NULL);
+    }
+    g_free(propname);
+    return ret;
+}
 
-    assert(n >= 0 && n < gpio_list->num_out);
-    gpio_list->out[n] = pin;
+qemu_irq qdev_intercept_gpio_out(DeviceState *dev, qemu_irq icpt,
+                                 const char *name, int n)
+{
+    qemu_irq disconnected = qdev_disconnect_gpio_out_named(dev, name, n);
+    qdev_connect_gpio_out_named(dev, name, n, icpt);
+    return disconnected;
 }
 
 void qdev_connect_gpio_out(DeviceState * dev, int n, qemu_irq pin)
@@ -391,6 +494,32 @@ void qdev_connect_gpio_out(DeviceState * dev, int n, qemu_irq pin)
     qdev_connect_gpio_out_named(dev, NULL, n, pin);
 }
 
+void qdev_pass_gpios(DeviceState *dev, DeviceState *container,
+                     const char *name)
+{
+    int i;
+    NamedGPIOList *ngl = qdev_get_named_gpio_list(dev, name);
+
+    for (i = 0; i < ngl->num_in; i++) {
+        const char *nm = ngl->name ? ngl->name : "unnamed-gpio-in";
+        char *propname = g_strdup_printf("%s[%d]", nm, i);
+
+        object_property_add_alias(OBJECT(container), propname,
+                                  OBJECT(dev), propname,
+                                  &error_abort);
+    }
+    for (i = 0; i < ngl->num_out; i++) {
+        const char *nm = ngl->name ? ngl->name : "unnamed-gpio-out";
+        char *propname = g_strdup_printf("%s[%d]", nm, i);
+
+        object_property_add_alias(OBJECT(container), propname,
+                                  OBJECT(dev), propname,
+                                  &error_abort);
+    }
+    QLIST_REMOVE(ngl, node);
+    QLIST_INSERT_HEAD(&container->gpios, ngl, node);
+}
+
 BusState *qdev_get_child_bus(DeviceState *dev, const char *name)
 {
     BusState *bus;
@@ -766,6 +895,11 @@ void qdev_property_add_static(DeviceState *dev, Property *prop,
         error_propagate(errp, local_err);
         return;
     }
+
+    object_property_set_description(obj, prop->name,
+                                    prop->info->description,
+                                    &error_abort);
+
     if (prop->qtype == QTYPE_NONE) {
         return;
     }
@@ -801,6 +935,27 @@ void qdev_alias_all_properties(DeviceState *target, Object *source)
     } while (class != object_class_by_name(TYPE_DEVICE));
 }
 
+static int qdev_add_hotpluggable_device(Object *obj, void *opaque)
+{
+    GSList **list = opaque;
+    DeviceState *dev = DEVICE(obj);
+
+    if (dev->realized && object_property_get_bool(obj, "hotpluggable", NULL)) {
+        *list = g_slist_append(*list, dev);
+    }
+
+    return 0;
+}
+
+GSList *qdev_build_hotpluggable_device_list(Object *peripheral)
+{
+    GSList *list = NULL;
+
+    object_child_foreach(peripheral, qdev_add_hotpluggable_device, &list);
+
+    return list;
+}
+
 static bool device_get_realized(Object *obj, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
@@ -811,6 +966,7 @@ static void device_set_realized(Object *obj, bool value, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     DeviceClass *dc = DEVICE_GET_CLASS(dev);
+    HotplugHandler *hotplug_ctrl;
     BusState *bus;
     Error *local_err = NULL;
 
@@ -820,13 +976,13 @@ static void device_set_realized(Object *obj, bool value, Error **errp)
     }
 
     if (value && !dev->realized) {
-        if (!obj->parent && local_err == NULL) {
+        if (!obj->parent) {
             static int unattached_count;
             gchar *name = g_strdup_printf("device[%d]", unattached_count++);
 
             object_property_add_child(container_get(qdev_get_machine(),
                                                     "/unattached"),
-                                      name, obj, &local_err);
+                                      name, obj, &error_abort);
             g_free(name);
         }
 
@@ -834,65 +990,78 @@ static void device_set_realized(Object *obj, bool value, Error **errp)
             dc->realize(dev, &local_err);
         }
 
-        if (dev->parent_bus && dev->parent_bus->hotplug_handler &&
-            local_err == NULL) {
-            hotplug_handler_plug(dev->parent_bus->hotplug_handler,
-                                 dev, &local_err);
-        } else if (local_err == NULL &&
-                   object_dynamic_cast(qdev_get_machine(), TYPE_MACHINE)) {
-            HotplugHandler *hotplug_ctrl;
-            MachineState *machine = MACHINE(qdev_get_machine());
-            MachineClass *mc = MACHINE_GET_CLASS(machine);
-
-            if (mc->get_hotplug_handler) {
-                hotplug_ctrl = mc->get_hotplug_handler(machine, dev);
-                if (hotplug_ctrl) {
-                    hotplug_handler_plug(hotplug_ctrl, dev, &local_err);
-                }
-            }
+        if (local_err != NULL) {
+            goto fail;
+        }
+
+        hotplug_ctrl = qdev_get_hotplug_handler(dev);
+        if (hotplug_ctrl) {
+            hotplug_handler_plug(hotplug_ctrl, dev, &local_err);
+        }
+
+        if (local_err != NULL) {
+            goto post_realize_fail;
         }
 
-        if (qdev_get_vmsd(dev) && local_err == NULL) {
+        if (qdev_get_vmsd(dev)) {
             vmstate_register_with_alias_id(dev, -1, qdev_get_vmsd(dev), dev,
                                            dev->instance_id_alias,
                                            dev->alias_required_for_version);
         }
-        if (local_err == NULL) {
-            QLIST_FOREACH(bus, &dev->child_bus, sibling) {
-                object_property_set_bool(OBJECT(bus), true, "realized",
+
+        QLIST_FOREACH(bus, &dev->child_bus, sibling) {
+            object_property_set_bool(OBJECT(bus), true, "realized",
                                          &local_err);
-                if (local_err != NULL) {
-                    break;
-                }
+            if (local_err != NULL) {
+                goto child_realize_fail;
             }
         }
-        if (dev->hotplugged && local_err == NULL) {
+        if (dev->hotplugged) {
             device_reset(dev);
         }
         dev->pending_deleted_event = false;
     } else if (!value && dev->realized) {
+        Error **local_errp = NULL;
         QLIST_FOREACH(bus, &dev->child_bus, sibling) {
+            local_errp = local_err ? NULL : &local_err;
             object_property_set_bool(OBJECT(bus), false, "realized",
-                                     &local_err);
-            if (local_err != NULL) {
-                break;
-            }
+                                     local_errp);
         }
-        if (qdev_get_vmsd(dev) && local_err == NULL) {
+        if (qdev_get_vmsd(dev)) {
             vmstate_unregister(dev, qdev_get_vmsd(dev), dev);
         }
-        if (dc->unrealize && local_err == NULL) {
-            dc->unrealize(dev, &local_err);
+        if (dc->unrealize) {
+            local_errp = local_err ? NULL : &local_err;
+            dc->unrealize(dev, local_errp);
         }
         dev->pending_deleted_event = true;
     }
 
     if (local_err != NULL) {
-        error_propagate(errp, local_err);
-        return;
+        goto fail;
     }
 
     dev->realized = value;
+    return;
+
+child_realize_fail:
+    QLIST_FOREACH(bus, &dev->child_bus, sibling) {
+        object_property_set_bool(OBJECT(bus), false, "realized",
+                                 NULL);
+    }
+
+    if (qdev_get_vmsd(dev)) {
+        vmstate_unregister(dev, qdev_get_vmsd(dev), dev);
+    }
+
+post_realize_fail:
+    if (dc->unrealize) {
+        dc->unrealize(dev, NULL);
+    }
+
+fail:
+    error_propagate(errp, local_err);
+    return;
 }
 
 static bool device_get_hotpluggable(Object *obj, Error **errp)
@@ -901,7 +1070,7 @@ static bool device_get_hotpluggable(Object *obj, Error **errp)
     DeviceState *dev = DEVICE(obj);
 
     return dc->hotpluggable && (dev->parent_bus == NULL ||
-                                dev->parent_bus->allow_hotplug);
+                                qbus_is_hotpluggable(dev->parent_bus));
 }
 
 static bool device_get_hotplugged(Object *obj, Error **err)
diff --git a/hw/core/sysbus.c b/hw/core/sysbus.c
index f4e760d6e..84af59379 100644
--- a/hw/core/sysbus.c
+++ b/hw/core/sysbus.c
@@ -24,6 +24,51 @@
 static void sysbus_dev_print(Monitor *mon, DeviceState *dev, int indent);
 static char *sysbus_get_fw_dev_path(DeviceState *dev);
 
+typedef struct SysBusFind {
+    void *opaque;
+    FindSysbusDeviceFunc *func;
+} SysBusFind;
+
+/* Run func() for every sysbus device, traverse the tree for everything else */
+static int find_sysbus_device(Object *obj, void *opaque)
+{
+    SysBusFind *find = opaque;
+    Object *dev;
+    SysBusDevice *sbdev;
+
+    dev = object_dynamic_cast(obj, TYPE_SYS_BUS_DEVICE);
+    sbdev = (SysBusDevice *)dev;
+
+    if (!sbdev) {
+        /* Container, traverse it for children */
+        return object_child_foreach(obj, find_sysbus_device, opaque);
+    }
+
+    find->func(sbdev, find->opaque);
+
+    return 0;
+}
+
+/*
+ * Loop through all dynamically created sysbus devices and call
+ * func() for each instance.
+ */
+void foreach_dynamic_sysbus_device(FindSysbusDeviceFunc *func, void *opaque)
+{
+    Object *container;
+    SysBusFind find = {
+        .func = func,
+        .opaque = opaque,
+    };
+
+    /* Loop through all sysbus devices that were spawened outside the machine */
+    container = container_get(qdev_get_machine(), "/peripheral");
+    find_sysbus_device(container, &find);
+    container = container_get(qdev_get_machine(), "/peripheral-anon");
+    find_sysbus_device(container, &find);
+}
+
+
 static void system_bus_class_init(ObjectClass *klass, void *data)
 {
     BusClass *k = BUS_CLASS(klass);
@@ -39,13 +84,36 @@ static const TypeInfo system_bus_info = {
     .class_init = system_bus_class_init,
 };
 
+/* Check whether an IRQ source exists */
+bool sysbus_has_irq(SysBusDevice *dev, int n)
+{
+    char *prop = g_strdup_printf("%s[%d]", SYSBUS_DEVICE_GPIO_IRQ, n);
+    ObjectProperty *r;
+
+    r = object_property_find(OBJECT(dev), prop, NULL);
+    return (r != NULL);
+}
+
+bool sysbus_is_irq_connected(SysBusDevice *dev, int n)
+{
+    return !!sysbus_get_connected_irq(dev, n);
+}
+
+qemu_irq sysbus_get_connected_irq(SysBusDevice *dev, int n)
+{
+    DeviceState *d = DEVICE(dev);
+    return qdev_get_gpio_out_connector(d, SYSBUS_DEVICE_GPIO_IRQ, n);
+}
+
 void sysbus_connect_irq(SysBusDevice *dev, int n, qemu_irq irq)
 {
-    assert(n >= 0 && n < dev->num_irq);
-    dev->irqs[n] = NULL;
-    if (dev->irqp[n]) {
-        *dev->irqp[n] = irq;
-    }
+    qdev_connect_gpio_out_named(DEVICE(dev), SYSBUS_DEVICE_GPIO_IRQ, n, irq);
+}
+
+/* Check whether an MMIO region exists */
+bool sysbus_has_mmio(SysBusDevice *dev, unsigned int n)
+{
+    return (n < dev->num_mmio);
 }
 
 static void sysbus_mmio_map_common(SysBusDevice *dev, int n, hwaddr addr,
@@ -89,22 +157,13 @@ void sysbus_mmio_map_overlap(SysBusDevice *dev, int n, hwaddr addr,
 /* Request an IRQ source.  The actual IRQ object may be populated later.  */
 void sysbus_init_irq(SysBusDevice *dev, qemu_irq *p)
 {
-    int n;
-
-    assert(dev->num_irq < QDEV_MAX_IRQ);
-    n = dev->num_irq++;
-    dev->irqp[n] = p;
+    qdev_init_gpio_out_named(DEVICE(dev), p, SYSBUS_DEVICE_GPIO_IRQ, 1);
 }
 
 /* Pass IRQs from a target device.  */
 void sysbus_pass_irq(SysBusDevice *dev, SysBusDevice *target)
 {
-    int i;
-    assert(dev->num_irq == 0);
-    dev->num_irq = target->num_irq;
-    for (i = 0; i < dev->num_irq; i++) {
-        dev->irqp[i] = target->irqp[i];
-    }
+    qdev_pass_gpios(DEVICE(target), DEVICE(dev), SYSBUS_DEVICE_GPIO_IRQ);
 }
 
 void sysbus_init_mmio(SysBusDevice *dev, MemoryRegion *memory)
@@ -210,7 +269,6 @@ static void sysbus_dev_print(Monitor *mon, DeviceState *dev, int indent)
     hwaddr size;
     int i;
 
-    monitor_printf(mon, "%*sirq %d\n", indent, "", s->num_irq);
     for (i = 0; i < s->num_mmio; i++) {
         size = memory_region_size(s->mmio[i].memory);
         monitor_printf(mon, "%*smmio " TARGET_FMT_plx "/" TARGET_FMT_plx "\n",
@@ -242,11 +300,6 @@ void sysbus_add_io(SysBusDevice *dev, hwaddr addr,
     memory_region_add_subregion(get_system_io(), addr, mem);
 }
 
-void sysbus_del_io(SysBusDevice *dev, MemoryRegion *mem)
-{
-    memory_region_del_subregion(get_system_io(), mem);
-}
-
 MemoryRegion *sysbus_address_space(SysBusDevice *dev)
 {
     return get_system_memory();
@@ -257,13 +310,6 @@ static void sysbus_device_class_init(ObjectClass *klass, void *data)
     DeviceClass *k = DEVICE_CLASS(klass);
     k->init = sysbus_device_init;
     k->bus_type = TYPE_SYSTEM_BUS;
-    /*
-     * device_add plugs devices into suitable bus.  For "real" buses,
-     * that actually connects the device.  For sysbus, the connections
-     * need to be made separately, and device_add can't do that.  The
-     * device would be left unconnected, and could not possibly work.
-     */
-    k->cannot_instantiate_with_device_add_yet = true;
 }
 
 static const TypeInfo sysbus_device_type_info = {
diff --git a/hw/cpu/icc_bus.c b/hw/cpu/icc_bus.c
index 7f44c59b2..6646ea2b3 100644
--- a/hw/cpu/icc_bus.c
+++ b/hw/cpu/icc_bus.c
@@ -24,18 +24,10 @@
 
 /* icc-bridge implementation */
 
-static void icc_bus_init(Object *obj)
-{
-    BusState *b = BUS(obj);
-
-    b->allow_hotplug = true;
-}
-
 static const TypeInfo icc_bus_info = {
     .name = TYPE_ICC_BUS,
     .parent = TYPE_BUS,
     .instance_size = sizeof(ICCBus),
-    .instance_init = icc_bus_init,
 };
 
 
@@ -81,11 +73,11 @@ typedef struct ICCBridgeState {
     MemoryRegion apic_container;
 } ICCBridgeState;
 
-#define ICC_BRIGDE(obj) OBJECT_CHECK(ICCBridgeState, (obj), TYPE_ICC_BRIDGE)
+#define ICC_BRIDGE(obj) OBJECT_CHECK(ICCBridgeState, (obj), TYPE_ICC_BRIDGE)
 
 static void icc_bridge_init(Object *obj)
 {
-    ICCBridgeState *s = ICC_BRIGDE(obj);
+    ICCBridgeState *s = ICC_BRIDGE(obj);
     SysBusDevice *sb = SYS_BUS_DEVICE(obj);
 
     qbus_create_inplace(&s->icc_bus, sizeof(s->icc_bus), TYPE_ICC_BUS,
diff --git a/hw/cris/axis_dev88.c b/hw/cris/axis_dev88.c
index 184933801..047919686 100644
--- a/hw/cris/axis_dev88.c
+++ b/hw/cris/axis_dev88.c
@@ -30,7 +30,7 @@
 #include "hw/loader.h"
 #include "elf.h"
 #include "boot.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 #include "sysemu/qtest.h"
 
@@ -270,19 +270,21 @@ void axisdev88_init(MachineState *machine)
     env = &cpu->env;
 
     /* allocate RAM */
-    memory_region_init_ram(phys_ram, NULL, "axisdev88.ram", ram_size);
+    memory_region_init_ram(phys_ram, NULL, "axisdev88.ram", ram_size,
+                           &error_abort);
     vmstate_register_ram_global(phys_ram);
     memory_region_add_subregion(address_space_mem, 0x40000000, phys_ram);
 
     /* The ETRAX-FS has 128Kb on chip ram, the docs refer to it as the 
        internal memory.  */
-    memory_region_init_ram(phys_intmem, NULL, "axisdev88.chipram", INTMEM_SIZE);
+    memory_region_init_ram(phys_intmem, NULL, "axisdev88.chipram", INTMEM_SIZE,
+                           &error_abort);
     vmstate_register_ram_global(phys_intmem);
     memory_region_add_subregion(address_space_mem, 0x38000000, phys_intmem);
 
       /* Attach a NAND flash to CS1.  */
     nand = drive_get(IF_MTD, 0, 0);
-    nand_state.nand = nand_init(nand ? nand->bdrv : NULL,
+    nand_state.nand = nand_init(nand ? blk_by_legacy_dinfo(nand) : NULL,
                                 NAND_MFR_STMICRO, 0x39);
     memory_region_init_io(&nand_state.iomem, NULL, &nand_ops, &nand_state,
                           "nand", 0x05000000);
diff --git a/hw/display/blizzard.c b/hw/display/blizzard.c
index 55c0ddf00..92b1fac56 100644
--- a/hw/display/blizzard.c
+++ b/hw/display/blizzard.c
@@ -134,14 +134,6 @@ static const int blizzard_iformat_bpp[0x10] = {
     0, 0, 0, 0, 0, 0,
 };
 
-static inline void blizzard_rgb2yuv(int r, int g, int b,
-                int *y, int *u, int *v)
-{
-    *y = 0x10 + ((0x838 * r + 0x1022 * g + 0x322 * b) >> 13);
-    *u = 0x80 + ((0xe0e * b - 0x04c1 * r - 0x94e * g) >> 13);
-    *v = 0x80 + ((0xe0e * r - 0x0bc7 * g - 0x247 * b) >> 13);
-}
-
 static void blizzard_window(BlizzardState *s)
 {
     DisplaySurface *surface = qemu_console_surface(s->con);
diff --git a/hw/display/cg3.c b/hw/display/cg3.c
index 65ef7a7fe..1e6ff2b54 100644
--- a/hw/display/cg3.c
+++ b/hw/display/cg3.c
@@ -279,7 +279,8 @@ static void cg3_initfn(Object *obj)
     SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     CG3State *s = CG3(obj);
 
-    memory_region_init_ram(&s->rom, NULL, "cg3.prom", FCODE_MAX_ROM_SIZE);
+    memory_region_init_ram(&s->rom, NULL, "cg3.prom", FCODE_MAX_ROM_SIZE,
+                           &error_abort);
     memory_region_set_readonly(&s->rom, true);
     sysbus_init_mmio(sbd, &s->rom);
 
@@ -306,7 +307,8 @@ static void cg3_realizefn(DeviceState *dev, Error **errp)
         }
     }
 
-    memory_region_init_ram(&s->vram_mem, NULL, "cg3.vram", s->vram_size);
+    memory_region_init_ram(&s->vram_mem, NULL, "cg3.vram", s->vram_size,
+                           &error_abort);
     vmstate_register_ram_global(&s->vram_mem);
     sysbus_init_mmio(sbd, &s->vram_mem);
 
diff --git a/hw/display/cirrus_vga.c b/hw/display/cirrus_vga.c
index db330e954..27252646b 100644
--- a/hw/display/cirrus_vga.c
+++ b/hw/display/cirrus_vga.c
@@ -29,6 +29,7 @@
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "ui/console.h"
+#include "ui/pixel_ops.h"
 #include "vga_int.h"
 #include "hw/loader.h"
 
@@ -172,20 +173,6 @@
 
 #define CIRRUS_PNPMMIO_SIZE         0x1000
 
-#define BLTUNSAFE(s) \
-    ( \
-        ( /* check dst is within bounds */ \
-            (s)->cirrus_blt_height * ABS((s)->cirrus_blt_dstpitch) \
-                + ((s)->cirrus_blt_dstaddr & (s)->cirrus_addr_mask) > \
-                    (s)->vga.vram_size \
-        ) || \
-        ( /* check src is within bounds */ \
-            (s)->cirrus_blt_height * ABS((s)->cirrus_blt_srcpitch) \
-                + ((s)->cirrus_blt_srcaddr & (s)->cirrus_addr_mask) > \
-                    (s)->vga.vram_size \
-        ) \
-    )
-
 struct CirrusVGAState;
 typedef void (*cirrus_bitblt_rop_t) (struct CirrusVGAState *s,
                                      uint8_t * dst, const uint8_t * src,
@@ -278,6 +265,50 @@ static void cirrus_update_memory_access(CirrusVGAState *s);
  *
  ***************************************/
 
+static bool blit_region_is_unsafe(struct CirrusVGAState *s,
+                                  int32_t pitch, int32_t addr)
+{
+    if (pitch < 0) {
+        int64_t min = addr
+            + ((int64_t)s->cirrus_blt_height-1) * pitch;
+        int32_t max = addr
+            + s->cirrus_blt_width;
+        if (min < 0 || max >= s->vga.vram_size) {
+            return true;
+        }
+    } else {
+        int64_t max = addr
+            + ((int64_t)s->cirrus_blt_height-1) * pitch
+            + s->cirrus_blt_width;
+        if (max >= s->vga.vram_size) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static bool blit_is_unsafe(struct CirrusVGAState *s)
+{
+    /* should be the case, see cirrus_bitblt_start */
+    assert(s->cirrus_blt_width > 0);
+    assert(s->cirrus_blt_height > 0);
+
+    if (s->cirrus_blt_width > CIRRUS_BLTBUFSIZE) {
+        return true;
+    }
+
+    if (blit_region_is_unsafe(s, s->cirrus_blt_dstpitch,
+                              s->cirrus_blt_dstaddr & s->cirrus_addr_mask)) {
+        return true;
+    }
+    if (blit_region_is_unsafe(s, s->cirrus_blt_srcpitch,
+                              s->cirrus_blt_srcaddr & s->cirrus_addr_mask)) {
+        return true;
+    }
+
+    return false;
+}
+
 static void cirrus_bitblt_rop_nop(CirrusVGAState *s,
                                   uint8_t *dst,const uint8_t *src,
                                   int dstpitch,int srcpitch,
@@ -635,7 +666,7 @@ static int cirrus_bitblt_common_patterncopy(CirrusVGAState * s,
 
     dst = s->vga.vram_ptr + (s->cirrus_blt_dstaddr & s->cirrus_addr_mask);
 
-    if (BLTUNSAFE(s))
+    if (blit_is_unsafe(s))
         return 0;
 
     (*s->cirrus_rop) (s, dst, src,
@@ -653,8 +684,9 @@ static int cirrus_bitblt_solidfill(CirrusVGAState *s, int blt_rop)
 {
     cirrus_fill_t rop_func;
 
-    if (BLTUNSAFE(s))
+    if (blit_is_unsafe(s)) {
         return 0;
+    }
     rop_func = cirrus_fill[rop_to_index[blt_rop]][s->cirrus_blt_pixelwidth - 1];
     rop_func(s, s->vga.vram_ptr + (s->cirrus_blt_dstaddr & s->cirrus_addr_mask),
              s->cirrus_blt_dstpitch,
@@ -751,7 +783,7 @@ static void cirrus_do_copy(CirrusVGAState *s, int dst, int src, int w, int h)
 
 static int cirrus_bitblt_videotovideo_copy(CirrusVGAState * s)
 {
-    if (BLTUNSAFE(s))
+    if (blit_is_unsafe(s))
         return 0;
 
     cirrus_do_copy(s, s->cirrus_blt_dstaddr - s->vga.start_addr,
@@ -2170,20 +2202,44 @@ static void cirrus_cursor_invalidate(VGACommonState *s1)
     }
 }
 
-#define DEPTH 8
-#include "cirrus_vga_template.h"
-
-#define DEPTH 16
-#include "cirrus_vga_template.h"
-
-#define DEPTH 32
-#include "cirrus_vga_template.h"
+static void vga_draw_cursor_line(uint8_t *d1,
+                                 const uint8_t *src1,
+                                 int poffset, int w,
+                                 unsigned int color0,
+                                 unsigned int color1,
+                                 unsigned int color_xor)
+{
+    const uint8_t *plane0, *plane1;
+    int x, b0, b1;
+    uint8_t *d;
+
+    d = d1;
+    plane0 = src1;
+    plane1 = src1 + poffset;
+    for (x = 0; x < w; x++) {
+        b0 = (plane0[x >> 3] >> (7 - (x & 7))) & 1;
+        b1 = (plane1[x >> 3] >> (7 - (x & 7))) & 1;
+        switch (b0 | (b1 << 1)) {
+        case 0:
+            break;
+        case 1:
+            ((uint32_t *)d)[0] ^= color_xor;
+            break;
+        case 2:
+            ((uint32_t *)d)[0] = color0;
+            break;
+        case 3:
+            ((uint32_t *)d)[0] = color1;
+            break;
+        }
+        d += 4;
+    }
+}
 
 static void cirrus_cursor_draw_line(VGACommonState *s1, uint8_t *d1, int scr_y)
 {
     CirrusVGAState *s = container_of(s1, CirrusVGAState, vga);
-    DisplaySurface *surface = qemu_console_surface(s->vga.con);
-    int w, h, bpp, x1, x2, poffset;
+    int w, h, x1, x2, poffset;
     unsigned int color0, color1;
     const uint8_t *palette, *src;
     uint32_t content;
@@ -2212,6 +2268,8 @@ static void cirrus_cursor_draw_line(VGACommonState *s1, uint8_t *d1, int scr_y)
     } else {
         src += (s->vga.sr[0x13] & 0x3f) * 256;
         src += (scr_y - s->hw_cursor_y) * 4;
+
+
         poffset = 128;
         content = ((uint32_t *)src)[0] |
             ((uint32_t *)(src + 128))[0];
@@ -2229,30 +2287,14 @@ static void cirrus_cursor_draw_line(VGACommonState *s1, uint8_t *d1, int scr_y)
         x2 = s->vga.last_scr_width;
     w = x2 - x1;
     palette = s->cirrus_hidden_palette;
-    color0 = s->vga.rgb_to_pixel(c6_to_8(palette[0x0 * 3]),
-                                 c6_to_8(palette[0x0 * 3 + 1]),
-                                 c6_to_8(palette[0x0 * 3 + 2]));
-    color1 = s->vga.rgb_to_pixel(c6_to_8(palette[0xf * 3]),
-                                 c6_to_8(palette[0xf * 3 + 1]),
-                                 c6_to_8(palette[0xf * 3 + 2]));
-    bpp = surface_bytes_per_pixel(surface);
-    d1 += x1 * bpp;
-    switch (surface_bits_per_pixel(surface)) {
-    default:
-        break;
-    case 8:
-        vga_draw_cursor_line_8(d1, src, poffset, w, color0, color1, 0xff);
-        break;
-    case 15:
-        vga_draw_cursor_line_16(d1, src, poffset, w, color0, color1, 0x7fff);
-        break;
-    case 16:
-        vga_draw_cursor_line_16(d1, src, poffset, w, color0, color1, 0xffff);
-        break;
-    case 32:
-        vga_draw_cursor_line_32(d1, src, poffset, w, color0, color1, 0xffffff);
-        break;
-    }
+    color0 = rgb_to_pixel32(c6_to_8(palette[0x0 * 3]),
+                            c6_to_8(palette[0x0 * 3 + 1]),
+                            c6_to_8(palette[0x0 * 3 + 2]));
+    color1 = rgb_to_pixel32(c6_to_8(palette[0xf * 3]),
+                            c6_to_8(palette[0xf * 3 + 1]),
+                            c6_to_8(palette[0xf * 3 + 2]));
+    d1 += x1 * 4;
+    vga_draw_cursor_line(d1, src, poffset, w, color0, color1, 0xffffff);
 }
 
 /***************************************
diff --git a/hw/display/cirrus_vga_template.h b/hw/display/cirrus_vga_template.h
deleted file mode 100644
index 3b2828058..000000000
--- a/hw/display/cirrus_vga_template.h
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * QEMU Cirrus VGA Emulator templates
- *
- * Copyright (c) 2003 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#if DEPTH == 8
-#define BPP 1
-#elif DEPTH == 15 || DEPTH == 16
-#define BPP 2
-#elif DEPTH == 32
-#define BPP 4
-#else
-#error unsupported depth
-#endif
-
-static void glue(vga_draw_cursor_line_, DEPTH)(uint8_t *d1,
-                                               const uint8_t *src1,
-                                               int poffset, int w,
-                                               unsigned int color0,
-                                               unsigned int color1,
-                                               unsigned int color_xor)
-{
-    const uint8_t *plane0, *plane1;
-    int x, b0, b1;
-    uint8_t *d;
-
-    d = d1;
-    plane0 = src1;
-    plane1 = src1 + poffset;
-    for (x = 0; x < w; x++) {
-        b0 = (plane0[x >> 3] >> (7 - (x & 7))) & 1;
-        b1 = (plane1[x >> 3] >> (7 - (x & 7))) & 1;
-#if DEPTH == 8
-        switch (b0 | (b1 << 1)) {
-        case 0:
-            break;
-        case 1:
-            d[0] ^= color_xor;
-            break;
-        case 2:
-            d[0] = color0;
-            break;
-        case 3:
-            d[0] = color1;
-            break;
-        }
-#elif DEPTH == 16
-        switch (b0 | (b1 << 1)) {
-        case 0:
-            break;
-        case 1:
-            ((uint16_t *)d)[0] ^= color_xor;
-            break;
-        case 2:
-            ((uint16_t *)d)[0] = color0;
-            break;
-        case 3:
-            ((uint16_t *)d)[0] = color1;
-            break;
-        }
-#elif DEPTH == 32
-        switch (b0 | (b1 << 1)) {
-        case 0:
-            break;
-        case 1:
-            ((uint32_t *)d)[0] ^= color_xor;
-            break;
-        case 2:
-            ((uint32_t *)d)[0] = color0;
-            break;
-        case 3:
-            ((uint32_t *)d)[0] = color1;
-            break;
-        }
-#else
-#error unsupported depth
-#endif
-        d += BPP;
-    }
-}
-
-#undef DEPTH
-#undef BPP
diff --git a/hw/display/pxa2xx_lcd.c b/hw/display/pxa2xx_lcd.c
index 611fb174c..ac3c01882 100644
--- a/hw/display/pxa2xx_lcd.c
+++ b/hw/display/pxa2xx_lcd.c
@@ -279,14 +279,6 @@ static inline void pxa2xx_dma_ber_set(PXA2xxLCDState *s, int ch)
         s->liidr = s->dma_ch[ch].id;
 }
 
-/* Set Read Status interrupt high and poke associated registers */
-static inline void pxa2xx_dma_rdst_set(PXA2xxLCDState *s)
-{
-    s->status[0] |= LCSR0_RDST;
-    if (s->irqlevel && !(s->control[0] & LCCR0_RDSTM))
-        s->status[0] |= LCSR0_SINT;
-}
-
 /* Load new Frame Descriptors from DMA */
 static void pxa2xx_descriptor_load(PXA2xxLCDState *s)
 {
diff --git a/hw/display/qxl-render.c b/hw/display/qxl-render.c
index cc2c2b1db..e812ddd6e 100644
--- a/hw/display/qxl-render.c
+++ b/hw/display/qxl-render.c
@@ -116,13 +116,14 @@ static void qxl_render_update_area_unlocked(PCIQXLDevice *qxl)
                qxl->guest_primary.bytes_pp,
                qxl->guest_primary.bits_pp);
         if (qxl->guest_primary.qxl_stride > 0) {
+            pixman_format_code_t format =
+                qemu_default_pixman_format(qxl->guest_primary.bits_pp, true);
             surface = qemu_create_displaysurface_from
                 (qxl->guest_primary.surface.width,
                  qxl->guest_primary.surface.height,
-                 qxl->guest_primary.bits_pp,
+                 format,
                  qxl->guest_primary.abs_stride,
-                 qxl->guest_primary.data,
-                 false);
+                 qxl->guest_primary.data);
         } else {
             surface = qemu_create_displaysurface
                 (qxl->guest_primary.surface.width,
@@ -138,7 +139,9 @@ static void qxl_render_update_area_unlocked(PCIQXLDevice *qxl)
         if (qemu_spice_rect_is_empty(qxl->dirty+i)) {
             break;
         }
-        if (qxl->dirty[i].left > qxl->dirty[i].right ||
+        if (qxl->dirty[i].left < 0 ||
+            qxl->dirty[i].top < 0 ||
+            qxl->dirty[i].left > qxl->dirty[i].right ||
             qxl->dirty[i].top > qxl->dirty[i].bottom ||
             qxl->dirty[i].right > qxl->guest_primary.surface.width ||
             qxl->dirty[i].bottom > qxl->guest_primary.surface.height) {
diff --git a/hw/display/qxl.c b/hw/display/qxl.c
index d43aa49eb..b540dd656 100644
--- a/hw/display/qxl.c
+++ b/hw/display/qxl.c
@@ -132,6 +132,8 @@ static void qxl_reset_memslots(PCIQXLDevice *d);
 static void qxl_reset_surfaces(PCIQXLDevice *d);
 static void qxl_ring_set_dirty(PCIQXLDevice *qxl);
 
+static void qxl_hw_update(void *opaque);
+
 void qxl_set_guest_bug(PCIQXLDevice *qxl, const char *msg, ...)
 {
     trace_qxl_set_guest_bug(qxl->id);
@@ -1076,6 +1078,10 @@ static const QXLInterface qxl_interface = {
     .client_monitors_config = interface_client_monitors_config,
 };
 
+static const GraphicHwOps qxl_ops = {
+    .gfx_update  = qxl_hw_update,
+};
+
 static void qxl_enter_vga_mode(PCIQXLDevice *d)
 {
     if (d->mode == QXL_MODE_VGA) {
@@ -1085,6 +1091,7 @@ static void qxl_enter_vga_mode(PCIQXLDevice *d)
 #if SPICE_SERVER_VERSION >= 0x000c03 /* release 0.12.3 */
     spice_qxl_driver_unload(&d->ssd.qxl);
 #endif
+    graphic_console_set_hwops(d->ssd.dcl.con, d->vga.hw_ops, &d->vga);
     qemu_spice_create_host_primary(&d->ssd);
     d->mode = QXL_MODE_VGA;
     vga_dirty_log_start(&d->vga);
@@ -1097,6 +1104,7 @@ static void qxl_exit_vga_mode(PCIQXLDevice *d)
         return;
     }
     trace_qxl_exit_vga_mode(d->id);
+    graphic_console_set_hwops(d->ssd.dcl.con, &qxl_ops, d);
     vga_dirty_log_stop(&d->vga);
     qxl_destroy_primary(d, QXL_SYNC);
 }
@@ -1583,6 +1591,11 @@ async_common:
             qxl_set_guest_bug(d,
                     "QXL_IO_UPDATE_AREA: invalid area (%ux%u)x(%ux%u)\n",
                     update.left, update.top, update.right, update.bottom);
+            if (update.left == update.right || update.top == update.bottom) {
+                /* old drivers may provide empty area, keep going */
+                qxl_clear_guest_bug(d);
+                goto cancel_async;
+            }
             break;
         }
         if (async == QXL_ASYNC) {
@@ -1756,41 +1769,8 @@ static void qxl_send_events(PCIQXLDevice *d, uint32_t events)
 static void qxl_hw_update(void *opaque)
 {
     PCIQXLDevice *qxl = opaque;
-    VGACommonState *vga = &qxl->vga;
-
-    switch (qxl->mode) {
-    case QXL_MODE_VGA:
-        vga->hw_ops->gfx_update(vga);
-        break;
-    case QXL_MODE_COMPAT:
-    case QXL_MODE_NATIVE:
-        qxl_render_update(qxl);
-        break;
-    default:
-        break;
-    }
-}
-
-static void qxl_hw_invalidate(void *opaque)
-{
-    PCIQXLDevice *qxl = opaque;
-    VGACommonState *vga = &qxl->vga;
 
-    if (qxl->mode == QXL_MODE_VGA) {
-        vga->hw_ops->invalidate(vga);
-        return;
-    }
-}
-
-static void qxl_hw_text_update(void *opaque, console_ch_t *chardata)
-{
-    PCIQXLDevice *qxl = opaque;
-    VGACommonState *vga = &qxl->vga;
-
-    if (qxl->mode == QXL_MODE_VGA) {
-        vga->hw_ops->text_update(vga, chardata);
-        return;
-    }
+    qxl_render_update(qxl);
 }
 
 static void qxl_dirty_surfaces(PCIQXLDevice *qxl)
@@ -1979,14 +1959,14 @@ static int qxl_init_common(PCIQXLDevice *qxl)
 
     qxl->rom_size = qxl_rom_size();
     memory_region_init_ram(&qxl->rom_bar, OBJECT(qxl), "qxl.vrom",
-                           qxl->rom_size);
+                           qxl->rom_size, &error_abort);
     vmstate_register_ram(&qxl->rom_bar, &qxl->pci.qdev);
     init_qxl_rom(qxl);
     init_qxl_ram(qxl);
 
     qxl->guest_surfaces.cmds = g_new0(QXLPHYSICAL, qxl->ssd.num_surfaces);
     memory_region_init_ram(&qxl->vram_bar, OBJECT(qxl), "qxl.vram",
-                           qxl->vram_size);
+                           qxl->vram_size, &error_abort);
     vmstate_register_ram(&qxl->vram_bar, &qxl->pci.qdev);
     memory_region_init_alias(&qxl->vram32_bar, OBJECT(qxl), "qxl.vram32",
                              &qxl->vram_bar, 0, qxl->vram32_size);
@@ -2049,12 +2029,6 @@ static int qxl_init_common(PCIQXLDevice *qxl)
     return 0;
 }
 
-static const GraphicHwOps qxl_ops = {
-    .invalidate  = qxl_hw_invalidate,
-    .gfx_update  = qxl_hw_update,
-    .text_update = qxl_hw_text_update,
-};
-
 static int qxl_init_primary(PCIDevice *dev)
 {
     PCIQXLDevice *qxl = DO_UPCAST(PCIQXLDevice, pci, dev);
@@ -2063,6 +2037,7 @@ static int qxl_init_primary(PCIDevice *dev)
 
     qxl->id = 0;
     qxl_init_ramsize(qxl);
+    vga->vbe_size = qxl->vgamem_size;
     vga->vram_size_mb = qxl->vga.vram_size >> 20;
     vga_common_init(vga, OBJECT(dev), true);
     vga_init(vga, OBJECT(dev),
@@ -2094,7 +2069,7 @@ static int qxl_init_secondary(PCIDevice *dev)
     qxl->id = device_id++;
     qxl_init_ramsize(qxl);
     memory_region_init_ram(&qxl->vga.vram, OBJECT(dev), "qxl.vgavram",
-                           qxl->vga.vram_size);
+                           qxl->vga.vram_size, &error_abort);
     vmstate_register_ram(&qxl->vga.vram, &qxl->pci.qdev);
     qxl->vga.vram_ptr = memory_region_get_ram_ptr(&qxl->vga.vram);
     qxl->vga.con = graphic_console_init(DEVICE(dev), 0, &qxl_ops, qxl);
diff --git a/hw/display/sm501.c b/hw/display/sm501.c
index eedf2d48e..c72154b6f 100644
--- a/hw/display/sm501.c
+++ b/hw/display/sm501.c
@@ -1410,7 +1410,7 @@ void sm501_init(MemoryRegion *address_space_mem, uint32_t base,
 
     /* allocate local memory */
     memory_region_init_ram(&s->local_mem_region, NULL, "sm501.local",
-                           local_mem_bytes);
+                           local_mem_bytes, &error_abort);
     vmstate_register_ram_global(&s->local_mem_region);
     s->local_mem = memory_region_get_ram_ptr(&s->local_mem_region);
     memory_region_add_subregion(address_space_mem, base, &s->local_mem_region);
diff --git a/hw/display/tc6393xb.c b/hw/display/tc6393xb.c
index f4011d2db..4306adc95 100644
--- a/hw/display/tc6393xb.c
+++ b/hw/display/tc6393xb.c
@@ -15,6 +15,7 @@
 #include "hw/block/flash.h"
 #include "ui/console.h"
 #include "ui/pixel_ops.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 
 #define IRQ_TC6393_NAND		0
@@ -576,12 +577,14 @@ TC6393xbState *tc6393xb_init(MemoryRegion *sysmem, uint32_t base, qemu_irq irq)
     s->sub_irqs = qemu_allocate_irqs(tc6393xb_sub_irq, s, TC6393XB_NR_IRQS);
 
     nand = drive_get(IF_MTD, 0, 0);
-    s->flash = nand_init(nand ? nand->bdrv : NULL, NAND_MFR_TOSHIBA, 0x76);
+    s->flash = nand_init(nand ? blk_by_legacy_dinfo(nand) : NULL,
+                         NAND_MFR_TOSHIBA, 0x76);
 
     memory_region_init_io(&s->iomem, NULL, &tc6393xb_ops, s, "tc6393xb", 0x10000);
     memory_region_add_subregion(sysmem, base, &s->iomem);
 
-    memory_region_init_ram(&s->vram, NULL, "tc6393xb.vram", 0x100000);
+    memory_region_init_ram(&s->vram, NULL, "tc6393xb.vram", 0x100000,
+                           &error_abort);
     vmstate_register_ram_global(&s->vram);
     s->vram_ptr = memory_region_get_ram_ptr(&s->vram);
     memory_region_add_subregion(sysmem, base + 0x100000, &s->vram);
diff --git a/hw/display/tcx.c b/hw/display/tcx.c
index 28c742cc2..a9f9f66d1 100644
--- a/hw/display/tcx.c
+++ b/hw/display/tcx.c
@@ -33,10 +33,20 @@
 
 #define MAXX 1024
 #define MAXY 768
-#define TCX_DAC_NREGS 16
-#define TCX_THC_NREGS_8  0x081c
-#define TCX_THC_NREGS_24 0x1000
+#define TCX_DAC_NREGS    16
+#define TCX_THC_NREGS    0x1000
+#define TCX_DHC_NREGS    0x4000
 #define TCX_TEC_NREGS    0x1000
+#define TCX_ALT_NREGS    0x8000
+#define TCX_STIP_NREGS   0x800000
+#define TCX_BLIT_NREGS   0x800000
+#define TCX_RSTIP_NREGS  0x800000
+#define TCX_RBLIT_NREGS  0x800000
+
+#define TCX_THC_MISC     0x818
+#define TCX_THC_CURSXY   0x8fc
+#define TCX_THC_CURSMASK 0x900
+#define TCX_THC_CURSBITS 0x980
 
 #define TYPE_TCX "SUNW,tcx"
 #define TCX(obj) OBJECT_CHECK(TCXState, (obj), TYPE_TCX)
@@ -45,6 +55,7 @@ typedef struct TCXState {
     SysBusDevice parent_obj;
 
     QemuConsole *con;
+    qemu_irq irq;
     uint8_t *vram;
     uint32_t *vram24, *cplane;
     hwaddr prom_addr;
@@ -52,17 +63,30 @@ typedef struct TCXState {
     MemoryRegion vram_mem;
     MemoryRegion vram_8bit;
     MemoryRegion vram_24bit;
+    MemoryRegion stip;
+    MemoryRegion blit;
     MemoryRegion vram_cplane;
-    MemoryRegion dac;
+    MemoryRegion rstip;
+    MemoryRegion rblit;
     MemoryRegion tec;
+    MemoryRegion dac;
+    MemoryRegion thc;
+    MemoryRegion dhc;
+    MemoryRegion alt;
     MemoryRegion thc24;
-    MemoryRegion thc8;
+
     ram_addr_t vram24_offset, cplane_offset;
+    uint32_t tmpblit;
     uint32_t vram_size;
-    uint32_t palette[256];
-    uint8_t r[256], g[256], b[256];
+    uint32_t palette[260];
+    uint8_t r[260], g[260], b[260];
     uint16_t width, height, depth;
     uint8_t dac_index, dac_state;
+    uint32_t thcmisc;
+    uint32_t cursmask[32];
+    uint32_t cursbits[32];
+    uint16_t cursx;
+    uint16_t cursy;
 } TCXState;
 
 static void tcx_set_dirty(TCXState *s)
@@ -70,10 +94,36 @@ static void tcx_set_dirty(TCXState *s)
     memory_region_set_dirty(&s->vram_mem, 0, MAXX * MAXY);
 }
 
-static void tcx24_set_dirty(TCXState *s)
+static inline int tcx24_check_dirty(TCXState *s, ram_addr_t page,
+                                    ram_addr_t page24, ram_addr_t cpage)
 {
-    memory_region_set_dirty(&s->vram_mem, s->vram24_offset, MAXX * MAXY * 4);
-    memory_region_set_dirty(&s->vram_mem, s->cplane_offset, MAXX * MAXY * 4);
+    int ret;
+
+    ret = memory_region_get_dirty(&s->vram_mem, page, TARGET_PAGE_SIZE,
+                                  DIRTY_MEMORY_VGA);
+    ret |= memory_region_get_dirty(&s->vram_mem, page24, TARGET_PAGE_SIZE * 4,
+                                   DIRTY_MEMORY_VGA);
+    ret |= memory_region_get_dirty(&s->vram_mem, cpage, TARGET_PAGE_SIZE * 4,
+                                   DIRTY_MEMORY_VGA);
+    return ret;
+}
+
+static inline void tcx24_reset_dirty(TCXState *ts, ram_addr_t page_min,
+                               ram_addr_t page_max, ram_addr_t page24,
+                              ram_addr_t cpage)
+{
+    memory_region_reset_dirty(&ts->vram_mem,
+                              page_min,
+                              (page_max - page_min) + TARGET_PAGE_SIZE,
+                              DIRTY_MEMORY_VGA);
+    memory_region_reset_dirty(&ts->vram_mem,
+                              page24 + page_min * 4,
+                              (page_max - page_min) * 4 + TARGET_PAGE_SIZE,
+                              DIRTY_MEMORY_VGA);
+    memory_region_reset_dirty(&ts->vram_mem,
+                              cpage + page_min * 4,
+                              (page_max - page_min) * 4 + TARGET_PAGE_SIZE,
+                              DIRTY_MEMORY_VGA);
 }
 
 static void update_palette_entries(TCXState *s, int start, int end)
@@ -102,11 +152,7 @@ static void update_palette_entries(TCXState *s, int start, int end)
             break;
         }
     }
-    if (s->depth == 24) {
-        tcx24_set_dirty(s);
-    } else {
-        tcx_set_dirty(s);
-    }
+    tcx_set_dirty(s);
 }
 
 static void tcx_draw_line32(TCXState *s1, uint8_t *d,
@@ -116,7 +162,7 @@ static void tcx_draw_line32(TCXState *s1, uint8_t *d,
     uint8_t val;
     uint32_t *p = (uint32_t *)d;
 
-    for(x = 0; x < width; x++) {
+    for (x = 0; x < width; x++) {
         val = *s++;
         *p++ = s1->palette[val];
     }
@@ -129,7 +175,7 @@ static void tcx_draw_line16(TCXState *s1, uint8_t *d,
     uint8_t val;
     uint16_t *p = (uint16_t *)d;
 
-    for(x = 0; x < width; x++) {
+    for (x = 0; x < width; x++) {
         val = *s++;
         *p++ = s1->palette[val];
     }
@@ -147,6 +193,83 @@ static void tcx_draw_line8(TCXState *s1, uint8_t *d,
     }
 }
 
+static void tcx_draw_cursor32(TCXState *s1, uint8_t *d,
+                              int y, int width)
+{
+    int x, len;
+    uint32_t mask, bits;
+    uint32_t *p = (uint32_t *)d;
+
+    y = y - s1->cursy;
+    mask = s1->cursmask[y];
+    bits = s1->cursbits[y];
+    len = MIN(width - s1->cursx, 32);
+    p = &p[s1->cursx];
+    for (x = 0; x < len; x++) {
+        if (mask & 0x80000000) {
+            if (bits & 0x80000000) {
+                *p = s1->palette[259];
+            } else {
+                *p = s1->palette[258];
+            }
+        }
+        p++;
+        mask <<= 1;
+        bits <<= 1;
+    }
+}
+
+static void tcx_draw_cursor16(TCXState *s1, uint8_t *d,
+                              int y, int width)
+{
+    int x, len;
+    uint32_t mask, bits;
+    uint16_t *p = (uint16_t *)d;
+
+    y = y - s1->cursy;
+    mask = s1->cursmask[y];
+    bits = s1->cursbits[y];
+    len = MIN(width - s1->cursx, 32);
+    p = &p[s1->cursx];
+    for (x = 0; x < len; x++) {
+        if (mask & 0x80000000) {
+            if (bits & 0x80000000) {
+                *p = s1->palette[259];
+            } else {
+                *p = s1->palette[258];
+            }
+        }
+        p++;
+        mask <<= 1;
+        bits <<= 1;
+    }
+}
+
+static void tcx_draw_cursor8(TCXState *s1, uint8_t *d,
+                              int y, int width)
+{
+    int x, len;
+    uint32_t mask, bits;
+
+    y = y - s1->cursy;
+    mask = s1->cursmask[y];
+    bits = s1->cursbits[y];
+    len = MIN(width - s1->cursx, 32);
+    d = &d[s1->cursx];
+    for (x = 0; x < len; x++) {
+        if (mask & 0x80000000) {
+            if (bits & 0x80000000) {
+                *d = s1->palette[259];
+            } else {
+                *d = s1->palette[258];
+            }
+        }
+        d++;
+        mask <<= 1;
+        bits <<= 1;
+    }
+}
+
 /*
   XXX Could be much more optimal:
   * detect if line/page/whole screen is in 24 bit mode
@@ -162,11 +285,10 @@ static inline void tcx24_draw_line32(TCXState *s1, uint8_t *d,
     uint8_t val, *p8;
     uint32_t *p = (uint32_t *)d;
     uint32_t dval;
-
     bgr = is_surface_bgr(surface);
     for(x = 0; x < width; x++, s++, s24++) {
-        if ((be32_to_cpu(*cplane++) & 0xff000000) == 0x03000000) {
-            // 24-bit direct, BGR order
+        if (be32_to_cpu(*cplane) & 0x03000000) {
+            /* 24-bit direct, BGR order */
             p8 = (uint8_t *)s24;
             p8++;
             b = *p8++;
@@ -177,47 +299,18 @@ static inline void tcx24_draw_line32(TCXState *s1, uint8_t *d,
             else
                 dval = rgb_to_pixel32(r, g, b);
         } else {
+            /* 8-bit pseudocolor */
             val = *s;
             dval = s1->palette[val];
         }
         *p++ = dval;
+        cplane++;
     }
 }
 
-static inline int check_dirty(TCXState *s, ram_addr_t page, ram_addr_t page24,
-                              ram_addr_t cpage)
-{
-    int ret;
-
-    ret = memory_region_get_dirty(&s->vram_mem, page, TARGET_PAGE_SIZE,
-                                  DIRTY_MEMORY_VGA);
-    ret |= memory_region_get_dirty(&s->vram_mem, page24, TARGET_PAGE_SIZE * 4,
-                                   DIRTY_MEMORY_VGA);
-    ret |= memory_region_get_dirty(&s->vram_mem, cpage, TARGET_PAGE_SIZE * 4,
-                                   DIRTY_MEMORY_VGA);
-    return ret;
-}
-
-static inline void reset_dirty(TCXState *ts, ram_addr_t page_min,
-                               ram_addr_t page_max, ram_addr_t page24,
-                              ram_addr_t cpage)
-{
-    memory_region_reset_dirty(&ts->vram_mem,
-                              page_min,
-                              (page_max - page_min) + TARGET_PAGE_SIZE,
-                              DIRTY_MEMORY_VGA);
-    memory_region_reset_dirty(&ts->vram_mem,
-                              page24 + page_min * 4,
-                              (page_max - page_min) * 4 + TARGET_PAGE_SIZE,
-                              DIRTY_MEMORY_VGA);
-    memory_region_reset_dirty(&ts->vram_mem,
-                              cpage + page_min * 4,
-                              (page_max - page_min) * 4 + TARGET_PAGE_SIZE,
-                              DIRTY_MEMORY_VGA);
-}
-
 /* Fixed line length 1024 allows us to do nice tricks not possible on
    VGA... */
+
 static void tcx_update_display(void *opaque)
 {
     TCXState *ts = opaque;
@@ -226,6 +319,7 @@ static void tcx_update_display(void *opaque)
     int y, y_start, dd, ds;
     uint8_t *d, *s;
     void (*f)(TCXState *s1, uint8_t *dst, const uint8_t *src, int width);
+    void (*fc)(TCXState *s1, uint8_t *dst, int y, int width);
 
     if (surface_bits_per_pixel(surface) == 0) {
         return;
@@ -243,20 +337,23 @@ static void tcx_update_display(void *opaque)
     switch (surface_bits_per_pixel(surface)) {
     case 32:
         f = tcx_draw_line32;
+        fc = tcx_draw_cursor32;
         break;
     case 15:
     case 16:
         f = tcx_draw_line16;
+        fc = tcx_draw_cursor16;
         break;
     default:
     case 8:
         f = tcx_draw_line8;
+        fc = tcx_draw_cursor8;
         break;
     case 0:
         return;
     }
 
-    for(y = 0; y < ts->height; y += 4, page += TARGET_PAGE_SIZE) {
+    for (y = 0; y < ts->height; page += TARGET_PAGE_SIZE) {
         if (memory_region_get_dirty(&ts->vram_mem, page, TARGET_PAGE_SIZE,
                                     DIRTY_MEMORY_VGA)) {
             if (y_start < 0)
@@ -265,18 +362,38 @@ static void tcx_update_display(void *opaque)
                 page_min = page;
             if (page > page_max)
                 page_max = page;
+
             f(ts, d, s, ts->width);
+            if (y >= ts->cursy && y < ts->cursy + 32 && ts->cursx < ts->width) {
+                fc(ts, d, y, ts->width);
+            }
             d += dd;
             s += ds;
+            y++;
+
             f(ts, d, s, ts->width);
+            if (y >= ts->cursy && y < ts->cursy + 32 && ts->cursx < ts->width) {
+                fc(ts, d, y, ts->width);
+            }
             d += dd;
             s += ds;
+            y++;
+
             f(ts, d, s, ts->width);
+            if (y >= ts->cursy && y < ts->cursy + 32 && ts->cursx < ts->width) {
+                fc(ts, d, y, ts->width);
+            }
             d += dd;
             s += ds;
+            y++;
+
             f(ts, d, s, ts->width);
+            if (y >= ts->cursy && y < ts->cursy + 32 && ts->cursx < ts->width) {
+                fc(ts, d, y, ts->width);
+            }
             d += dd;
             s += ds;
+            y++;
         } else {
             if (y_start >= 0) {
                 /* flush to display */
@@ -286,6 +403,7 @@ static void tcx_update_display(void *opaque)
             }
             d += dd * 4;
             s += ds * 4;
+            y += 4;
         }
     }
     if (y_start >= 0) {
@@ -328,9 +446,9 @@ static void tcx24_update_display(void *opaque)
     dd = surface_stride(surface);
     ds = 1024;
 
-    for(y = 0; y < ts->height; y += 4, page += TARGET_PAGE_SIZE,
+    for (y = 0; y < ts->height; page += TARGET_PAGE_SIZE,
             page24 += TARGET_PAGE_SIZE, cpage += TARGET_PAGE_SIZE) {
-        if (check_dirty(ts, page, page24, cpage)) {
+        if (tcx24_check_dirty(ts, page, page24, cpage)) {
             if (y_start < 0)
                 y_start = y;
             if (page < page_min)
@@ -338,25 +456,41 @@ static void tcx24_update_display(void *opaque)
             if (page > page_max)
                 page_max = page;
             tcx24_draw_line32(ts, d, s, ts->width, cptr, s24);
+            if (y >= ts->cursy && y < ts->cursy+32 && ts->cursx < ts->width) {
+                tcx_draw_cursor32(ts, d, y, ts->width);
+            }
             d += dd;
             s += ds;
             cptr += ds;
             s24 += ds;
+            y++;
             tcx24_draw_line32(ts, d, s, ts->width, cptr, s24);
+            if (y >= ts->cursy && y < ts->cursy+32 && ts->cursx < ts->width) {
+                tcx_draw_cursor32(ts, d, y, ts->width);
+            }
             d += dd;
             s += ds;
             cptr += ds;
             s24 += ds;
+            y++;
             tcx24_draw_line32(ts, d, s, ts->width, cptr, s24);
+            if (y >= ts->cursy && y < ts->cursy+32 && ts->cursx < ts->width) {
+                tcx_draw_cursor32(ts, d, y, ts->width);
+            }
             d += dd;
             s += ds;
             cptr += ds;
             s24 += ds;
+            y++;
             tcx24_draw_line32(ts, d, s, ts->width, cptr, s24);
+            if (y >= ts->cursy && y < ts->cursy+32 && ts->cursx < ts->width) {
+                tcx_draw_cursor32(ts, d, y, ts->width);
+            }
             d += dd;
             s += ds;
             cptr += ds;
             s24 += ds;
+            y++;
         } else {
             if (y_start >= 0) {
                 /* flush to display */
@@ -368,6 +502,7 @@ static void tcx24_update_display(void *opaque)
             s += ds * 4;
             cptr += ds * 4;
             s24 += ds * 4;
+            y += 4;
         }
     }
     if (y_start >= 0) {
@@ -377,7 +512,7 @@ static void tcx24_update_display(void *opaque)
     }
     /* reset modified pages */
     if (page_max >= page_min) {
-        reset_dirty(ts, page_min, page_max, page24, cpage);
+        tcx24_reset_dirty(ts, page_min, page_max, page24, cpage);
     }
 }
 
@@ -394,7 +529,6 @@ static void tcx24_invalidate_display(void *opaque)
     TCXState *s = opaque;
 
     tcx_set_dirty(s);
-    tcx24_set_dirty(s);
     qemu_console_resize(s->con, s->width, s->height);
 }
 
@@ -403,12 +537,7 @@ static int vmstate_tcx_post_load(void *opaque, int version_id)
     TCXState *s = opaque;
 
     update_palette_entries(s, 0, 256);
-    if (s->depth == 24) {
-        tcx24_set_dirty(s);
-    } else {
-        tcx_set_dirty(s);
-    }
-
+    tcx_set_dirty(s);
     return 0;
 }
 
@@ -435,56 +564,87 @@ static void tcx_reset(DeviceState *d)
     TCXState *s = TCX(d);
 
     /* Initialize palette */
-    memset(s->r, 0, 256);
-    memset(s->g, 0, 256);
-    memset(s->b, 0, 256);
+    memset(s->r, 0, 260);
+    memset(s->g, 0, 260);
+    memset(s->b, 0, 260);
     s->r[255] = s->g[255] = s->b[255] = 255;
-    update_palette_entries(s, 0, 256);
+    s->r[256] = s->g[256] = s->b[256] = 255;
+    s->r[258] = s->g[258] = s->b[258] = 255;
+    update_palette_entries(s, 0, 260);
     memset(s->vram, 0, MAXX*MAXY);
     memory_region_reset_dirty(&s->vram_mem, 0, MAXX * MAXY * (1 + 4 + 4),
                               DIRTY_MEMORY_VGA);
     s->dac_index = 0;
     s->dac_state = 0;
+    s->cursx = 0xf000; /* Put cursor off screen */
+    s->cursy = 0xf000;
 }
 
 static uint64_t tcx_dac_readl(void *opaque, hwaddr addr,
                               unsigned size)
 {
-    return 0;
+    TCXState *s = opaque;
+    uint32_t val = 0;
+
+    switch (s->dac_state) {
+    case 0:
+        val = s->r[s->dac_index] << 24;
+        s->dac_state++;
+        break;
+    case 1:
+        val = s->g[s->dac_index] << 24;
+        s->dac_state++;
+        break;
+    case 2:
+        val = s->b[s->dac_index] << 24;
+        s->dac_index = (s->dac_index + 1) & 0xff; /* Index autoincrement */
+    default:
+        s->dac_state = 0;
+        break;
+    }
+
+    return val;
 }
 
 static void tcx_dac_writel(void *opaque, hwaddr addr, uint64_t val,
                            unsigned size)
 {
     TCXState *s = opaque;
+    unsigned index;
 
     switch (addr) {
-    case 0:
+    case 0: /* Address */
         s->dac_index = val >> 24;
         s->dac_state = 0;
         break;
-    case 4:
+    case 4:  /* Pixel colours */
+    case 12: /* Overlay (cursor) colours */
+        if (addr & 8) {
+            index = (s->dac_index & 3) + 256;
+        } else {
+            index = s->dac_index;
+        }
         switch (s->dac_state) {
         case 0:
-            s->r[s->dac_index] = val >> 24;
-            update_palette_entries(s, s->dac_index, s->dac_index + 1);
+            s->r[index] = val >> 24;
+            update_palette_entries(s, index, index + 1);
             s->dac_state++;
             break;
         case 1:
-            s->g[s->dac_index] = val >> 24;
-            update_palette_entries(s, s->dac_index, s->dac_index + 1);
+            s->g[index] = val >> 24;
+            update_palette_entries(s, index, index + 1);
             s->dac_state++;
             break;
         case 2:
-            s->b[s->dac_index] = val >> 24;
-            update_palette_entries(s, s->dac_index, s->dac_index + 1);
-            s->dac_index = (s->dac_index + 1) & 255; // Index autoincrement
+            s->b[index] = val >> 24;
+            update_palette_entries(s, index, index + 1);
+            s->dac_index = (s->dac_index + 1) & 0xff; /* Index autoincrement */
         default:
             s->dac_state = 0;
             break;
         }
         break;
-    default:
+    default: /* Control registers */
         break;
     }
 }
@@ -499,20 +659,266 @@ static const MemoryRegionOps tcx_dac_ops = {
     },
 };
 
-static uint64_t dummy_readl(void *opaque, hwaddr addr,
+static uint64_t tcx_stip_readl(void *opaque, hwaddr addr,
+                               unsigned size)
+{
+    return 0;
+}
+
+static void tcx_stip_writel(void *opaque, hwaddr addr,
+                            uint64_t val, unsigned size)
+{
+    TCXState *s = opaque;
+    int i;
+    uint32_t col;
+
+    if (!(addr & 4)) {
+        s->tmpblit = val;
+    } else {
+        addr = (addr >> 3) & 0xfffff;
+        col = cpu_to_be32(s->tmpblit);
+        if (s->depth == 24) {
+            for (i = 0; i < 32; i++)  {
+                if (val & 0x80000000) {
+                    s->vram[addr + i] = s->tmpblit;
+                    s->vram24[addr + i] = col;
+                }
+                val <<= 1;
+            }
+        } else {
+            for (i = 0; i < 32; i++)  {
+                if (val & 0x80000000) {
+                    s->vram[addr + i] = s->tmpblit;
+                }
+                val <<= 1;
+            }
+        }
+        memory_region_set_dirty(&s->vram_mem, addr, 32);
+    }
+}
+
+static void tcx_rstip_writel(void *opaque, hwaddr addr,
+                             uint64_t val, unsigned size)
+{
+    TCXState *s = opaque;
+    int i;
+    uint32_t col;
+
+    if (!(addr & 4)) {
+        s->tmpblit = val;
+    } else {
+        addr = (addr >> 3) & 0xfffff;
+        col = cpu_to_be32(s->tmpblit);
+        if (s->depth == 24) {
+            for (i = 0; i < 32; i++) {
+                if (val & 0x80000000) {
+                    s->vram[addr + i] = s->tmpblit;
+                    s->vram24[addr + i] = col;
+                    s->cplane[addr + i] = col;
+                }
+                val <<= 1;
+            }
+        } else {
+            for (i = 0; i < 32; i++)  {
+                if (val & 0x80000000) {
+                    s->vram[addr + i] = s->tmpblit;
+                }
+                val <<= 1;
+            }
+        }
+        memory_region_set_dirty(&s->vram_mem, addr, 32);
+    }
+}
+
+static const MemoryRegionOps tcx_stip_ops = {
+    .read = tcx_stip_readl,
+    .write = tcx_stip_writel,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
+
+static const MemoryRegionOps tcx_rstip_ops = {
+    .read = tcx_stip_readl,
+    .write = tcx_rstip_writel,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
+
+static uint64_t tcx_blit_readl(void *opaque, hwaddr addr,
+                               unsigned size)
+{
+    return 0;
+}
+
+static void tcx_blit_writel(void *opaque, hwaddr addr,
+                            uint64_t val, unsigned size)
+{
+    TCXState *s = opaque;
+    uint32_t adsr, len;
+    int i;
+
+    if (!(addr & 4)) {
+        s->tmpblit = val;
+    } else {
+        addr = (addr >> 3) & 0xfffff;
+        adsr = val & 0xffffff;
+        len = ((val >> 24) & 0x1f) + 1;
+        if (adsr == 0xffffff) {
+            memset(&s->vram[addr], s->tmpblit, len);
+            if (s->depth == 24) {
+                val = s->tmpblit & 0xffffff;
+                val = cpu_to_be32(val);
+                for (i = 0; i < len; i++) {
+                    s->vram24[addr + i] = val;
+                }
+            }
+        } else {
+            memcpy(&s->vram[addr], &s->vram[adsr], len);
+            if (s->depth == 24) {
+                memcpy(&s->vram24[addr], &s->vram24[adsr], len * 4);
+            }
+        }
+        memory_region_set_dirty(&s->vram_mem, addr, len);
+    }
+}
+
+static void tcx_rblit_writel(void *opaque, hwaddr addr,
+                         uint64_t val, unsigned size)
+{
+    TCXState *s = opaque;
+    uint32_t adsr, len;
+    int i;
+
+    if (!(addr & 4)) {
+        s->tmpblit = val;
+    } else {
+        addr = (addr >> 3) & 0xfffff;
+        adsr = val & 0xffffff;
+        len = ((val >> 24) & 0x1f) + 1;
+        if (adsr == 0xffffff) {
+            memset(&s->vram[addr], s->tmpblit, len);
+            if (s->depth == 24) {
+                val = s->tmpblit & 0xffffff;
+                val = cpu_to_be32(val);
+                for (i = 0; i < len; i++) {
+                    s->vram24[addr + i] = val;
+                    s->cplane[addr + i] = val;
+                }
+            }
+        } else {
+            memcpy(&s->vram[addr], &s->vram[adsr], len);
+            if (s->depth == 24) {
+                memcpy(&s->vram24[addr], &s->vram24[adsr], len * 4);
+                memcpy(&s->cplane[addr], &s->cplane[adsr], len * 4);
+            }
+        }
+        memory_region_set_dirty(&s->vram_mem, addr, len);
+    }
+}
+
+static const MemoryRegionOps tcx_blit_ops = {
+    .read = tcx_blit_readl,
+    .write = tcx_blit_writel,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
+
+static const MemoryRegionOps tcx_rblit_ops = {
+    .read = tcx_blit_readl,
+    .write = tcx_rblit_writel,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
+
+static void tcx_invalidate_cursor_position(TCXState *s)
+{
+    int ymin, ymax, start, end;
+
+    /* invalidate only near the cursor */
+    ymin = s->cursy;
+    if (ymin >= s->height) {
+        return;
+    }
+    ymax = MIN(s->height, ymin + 32);
+    start = ymin * 1024;
+    end   = ymax * 1024;
+
+    memory_region_set_dirty(&s->vram_mem, start, end-start);
+}
+
+static uint64_t tcx_thc_readl(void *opaque, hwaddr addr,
+                            unsigned size)
+{
+    TCXState *s = opaque;
+    uint64_t val;
+
+    if (addr == TCX_THC_MISC) {
+        val = s->thcmisc | 0x02000000;
+    } else {
+        val = 0;
+    }
+    return val;
+}
+
+static void tcx_thc_writel(void *opaque, hwaddr addr,
+                         uint64_t val, unsigned size)
+{
+    TCXState *s = opaque;
+
+    if (addr == TCX_THC_CURSXY) {
+        tcx_invalidate_cursor_position(s);
+        s->cursx = val >> 16;
+        s->cursy = val;
+        tcx_invalidate_cursor_position(s);
+    } else if (addr >= TCX_THC_CURSMASK && addr < TCX_THC_CURSMASK + 128) {
+        s->cursmask[(addr - TCX_THC_CURSMASK) >> 2] = val;
+        tcx_invalidate_cursor_position(s);
+    } else if (addr >= TCX_THC_CURSBITS && addr < TCX_THC_CURSBITS + 128) {
+        s->cursbits[(addr - TCX_THC_CURSBITS) >> 2] = val;
+        tcx_invalidate_cursor_position(s);
+    } else if (addr == TCX_THC_MISC) {
+        s->thcmisc = val;
+    }
+
+}
+
+static const MemoryRegionOps tcx_thc_ops = {
+    .read = tcx_thc_readl,
+    .write = tcx_thc_writel,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
+
+static uint64_t tcx_dummy_readl(void *opaque, hwaddr addr,
                             unsigned size)
 {
     return 0;
 }
 
-static void dummy_writel(void *opaque, hwaddr addr,
+static void tcx_dummy_writel(void *opaque, hwaddr addr,
                          uint64_t val, unsigned size)
 {
+    return;
 }
 
-static const MemoryRegionOps dummy_ops = {
-    .read = dummy_readl,
-    .write = dummy_writel,
+static const MemoryRegionOps tcx_dummy_ops = {
+    .read = tcx_dummy_readl,
+    .write = tcx_dummy_writel,
     .endianness = DEVICE_NATIVE_ENDIAN,
     .valid = {
         .min_access_size = 4,
@@ -535,24 +941,55 @@ static void tcx_initfn(Object *obj)
     SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     TCXState *s = TCX(obj);
 
-    memory_region_init_ram(&s->rom, NULL, "tcx.prom", FCODE_MAX_ROM_SIZE);
+    memory_region_init_ram(&s->rom, NULL, "tcx.prom", FCODE_MAX_ROM_SIZE,
+                           &error_abort);
     memory_region_set_readonly(&s->rom, true);
     sysbus_init_mmio(sbd, &s->rom);
 
-    /* DAC */
+    /* 2/STIP : Stippler */
+    memory_region_init_io(&s->stip, OBJECT(s), &tcx_stip_ops, s, "tcx.stip",
+                          TCX_STIP_NREGS);
+    sysbus_init_mmio(sbd, &s->stip);
+
+    /* 3/BLIT : Blitter */
+    memory_region_init_io(&s->blit, OBJECT(s), &tcx_blit_ops, s, "tcx.blit",
+                          TCX_BLIT_NREGS);
+    sysbus_init_mmio(sbd, &s->blit);
+
+    /* 5/RSTIP : Raw Stippler */
+    memory_region_init_io(&s->rstip, OBJECT(s), &tcx_rstip_ops, s, "tcx.rstip",
+                          TCX_RSTIP_NREGS);
+    sysbus_init_mmio(sbd, &s->rstip);
+
+    /* 6/RBLIT : Raw Blitter */
+    memory_region_init_io(&s->rblit, OBJECT(s), &tcx_rblit_ops, s, "tcx.rblit",
+                          TCX_RBLIT_NREGS);
+    sysbus_init_mmio(sbd, &s->rblit);
+
+    /* 7/TEC : ??? */
+    memory_region_init_io(&s->tec, OBJECT(s), &tcx_dummy_ops, s,
+                          "tcx.tec", TCX_TEC_NREGS);
+    sysbus_init_mmio(sbd, &s->tec);
+
+    /* 8/CMAP : DAC */
     memory_region_init_io(&s->dac, OBJECT(s), &tcx_dac_ops, s,
                           "tcx.dac", TCX_DAC_NREGS);
     sysbus_init_mmio(sbd, &s->dac);
 
-    /* TEC (dummy) */
-    memory_region_init_io(&s->tec, OBJECT(s), &dummy_ops, s,
-                          "tcx.tec", TCX_TEC_NREGS);
-    sysbus_init_mmio(sbd, &s->tec);
+    /* 9/THC : Cursor */
+    memory_region_init_io(&s->thc, OBJECT(s), &tcx_thc_ops, s, "tcx.thc",
+                          TCX_THC_NREGS);
+    sysbus_init_mmio(sbd, &s->thc);
 
-    /* THC: NetBSD writes here even with 8-bit display: dummy */
-    memory_region_init_io(&s->thc24, OBJECT(s), &dummy_ops, s, "tcx.thc24",
-                          TCX_THC_NREGS_24);
-    sysbus_init_mmio(sbd, &s->thc24);
+    /* 11/DHC : ??? */
+    memory_region_init_io(&s->dhc, OBJECT(s), &tcx_dummy_ops, s, "tcx.dhc",
+                          TCX_DHC_NREGS);
+    sysbus_init_mmio(sbd, &s->dhc);
+
+    /* 12/ALT : ??? */
+    memory_region_init_io(&s->alt, OBJECT(s), &tcx_dummy_ops, s, "tcx.alt",
+                          TCX_ALT_NREGS);
+    sysbus_init_mmio(sbd, &s->alt);
 
     return;
 }
@@ -567,11 +1004,11 @@ static void tcx_realizefn(DeviceState *dev, Error **errp)
     char *fcode_filename;
 
     memory_region_init_ram(&s->vram_mem, OBJECT(s), "tcx.vram",
-                           s->vram_size * (1 + 4 + 4));
+                           s->vram_size * (1 + 4 + 4), &error_abort);
     vmstate_register_ram_global(&s->vram_mem);
     vram_base = memory_region_get_ram_ptr(&s->vram_mem);
 
-    /* FCode ROM */
+    /* 10/ROM : FCode ROM */
     vmstate_register_ram_global(&s->rom);
     fcode_filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, TCX_ROM_FILE);
     if (fcode_filename) {
@@ -582,7 +1019,7 @@ static void tcx_realizefn(DeviceState *dev, Error **errp)
         }
     }
 
-    /* 8-bit plane */
+    /* 0/DFB8 : 8-bit plane */
     s->vram = vram_base;
     size = s->vram_size;
     memory_region_init_alias(&s->vram_8bit, OBJECT(s), "tcx.vram.8bit",
@@ -591,34 +1028,39 @@ static void tcx_realizefn(DeviceState *dev, Error **errp)
     vram_offset += size;
     vram_base += size;
 
-    if (s->depth == 24) {
-        /* 24-bit plane */
-        size = s->vram_size * 4;
-        s->vram24 = (uint32_t *)vram_base;
-        s->vram24_offset = vram_offset;
-        memory_region_init_alias(&s->vram_24bit, OBJECT(s), "tcx.vram.24bit",
-                                 &s->vram_mem, vram_offset, size);
-        sysbus_init_mmio(sbd, &s->vram_24bit);
-        vram_offset += size;
-        vram_base += size;
-
-        /* Control plane */
-        size = s->vram_size * 4;
-        s->cplane = (uint32_t *)vram_base;
-        s->cplane_offset = vram_offset;
-        memory_region_init_alias(&s->vram_cplane, OBJECT(s), "tcx.vram.cplane",
-                                 &s->vram_mem, vram_offset, size);
-        sysbus_init_mmio(sbd, &s->vram_cplane);
+    /* 1/DFB24 : 24bit plane */
+    size = s->vram_size * 4;
+    s->vram24 = (uint32_t *)vram_base;
+    s->vram24_offset = vram_offset;
+    memory_region_init_alias(&s->vram_24bit, OBJECT(s), "tcx.vram.24bit",
+                             &s->vram_mem, vram_offset, size);
+    sysbus_init_mmio(sbd, &s->vram_24bit);
+    vram_offset += size;
+    vram_base += size;
+
+    /* 4/RDFB32 : Raw Framebuffer */
+    size = s->vram_size * 4;
+    s->cplane = (uint32_t *)vram_base;
+    s->cplane_offset = vram_offset;
+    memory_region_init_alias(&s->vram_cplane, OBJECT(s), "tcx.vram.cplane",
+                             &s->vram_mem, vram_offset, size);
+    sysbus_init_mmio(sbd, &s->vram_cplane);
 
-        s->con = graphic_console_init(DEVICE(dev), 0, &tcx24_ops, s);
-    } else {
-        /* THC 8 bit (dummy) */
-        memory_region_init_io(&s->thc8, OBJECT(s), &dummy_ops, s, "tcx.thc8",
-                              TCX_THC_NREGS_8);
-        sysbus_init_mmio(sbd, &s->thc8);
+    /* 9/THC24bits : NetBSD writes here even with 8-bit display: dummy */
+    if (s->depth == 8) {
+        memory_region_init_io(&s->thc24, OBJECT(s), &tcx_dummy_ops, s,
+                              "tcx.thc24", TCX_THC_NREGS);
+        sysbus_init_mmio(sbd, &s->thc24);
+    }
+
+    sysbus_init_irq(sbd, &s->irq);
 
+    if (s->depth == 8) {
         s->con = graphic_console_init(DEVICE(dev), 0, &tcx_ops, s);
+    } else {
+        s->con = graphic_console_init(DEVICE(dev), 0, &tcx24_ops, s);
     }
+    s->thcmisc = 0;
 
     qemu_console_resize(s->con, s->width, s->height);
 }
diff --git a/hw/display/vga_template.h b/hw/display/vga-helpers.h
index 90ec9c208..94f6de204 100644
--- a/hw/display/vga_template.h
+++ b/hw/display/vga-helpers.h
@@ -22,41 +22,9 @@
  * THE SOFTWARE.
  */
 
-#if DEPTH == 8
-#define BPP 1
-#define PIXEL_TYPE uint8_t
-#elif DEPTH == 15 || DEPTH == 16
-#define BPP 2
-#define PIXEL_TYPE uint16_t
-#elif DEPTH == 32
-#define BPP 4
-#define PIXEL_TYPE uint32_t
-#else
-#error unsupport depth
-#endif
-
-#ifdef BGR_FORMAT
-#define PIXEL_NAME glue(DEPTH, bgr)
-#else
-#define PIXEL_NAME DEPTH
-#endif /* BGR_FORMAT */
-
-#if DEPTH != 15 && !defined(BGR_FORMAT)
-
-static inline void glue(vga_draw_glyph_line_, DEPTH)(uint8_t *d,
-                                                     uint32_t font_data,
-                                                     uint32_t xorcol,
-                                                     uint32_t bgcol)
+static inline void vga_draw_glyph_line(uint8_t *d, uint32_t font_data,
+                                       uint32_t xorcol, uint32_t bgcol)
 {
-#if BPP == 1
-        ((uint32_t *)d)[0] = (dmask16[(font_data >> 4)] & xorcol) ^ bgcol;
-        ((uint32_t *)d)[1] = (dmask16[(font_data >> 0) & 0xf] & xorcol) ^ bgcol;
-#elif BPP == 2
-        ((uint32_t *)d)[0] = (dmask4[(font_data >> 6)] & xorcol) ^ bgcol;
-        ((uint32_t *)d)[1] = (dmask4[(font_data >> 4) & 3] & xorcol) ^ bgcol;
-        ((uint32_t *)d)[2] = (dmask4[(font_data >> 2) & 3] & xorcol) ^ bgcol;
-        ((uint32_t *)d)[3] = (dmask4[(font_data >> 0) & 3] & xorcol) ^ bgcol;
-#else
         ((uint32_t *)d)[0] = (-((font_data >> 7)) & xorcol) ^ bgcol;
         ((uint32_t *)d)[1] = (-((font_data >> 6) & 1) & xorcol) ^ bgcol;
         ((uint32_t *)d)[2] = (-((font_data >> 5) & 1) & xorcol) ^ bgcol;
@@ -65,25 +33,24 @@ static inline void glue(vga_draw_glyph_line_, DEPTH)(uint8_t *d,
         ((uint32_t *)d)[5] = (-((font_data >> 2) & 1) & xorcol) ^ bgcol;
         ((uint32_t *)d)[6] = (-((font_data >> 1) & 1) & xorcol) ^ bgcol;
         ((uint32_t *)d)[7] = (-((font_data >> 0) & 1) & xorcol) ^ bgcol;
-#endif
 }
 
-static void glue(vga_draw_glyph8_, DEPTH)(uint8_t *d, int linesize,
-                                          const uint8_t *font_ptr, int h,
-                                          uint32_t fgcol, uint32_t bgcol)
+static void vga_draw_glyph8(uint8_t *d, int linesize,
+                            const uint8_t *font_ptr, int h,
+                            uint32_t fgcol, uint32_t bgcol)
 {
     uint32_t font_data, xorcol;
 
     xorcol = bgcol ^ fgcol;
     do {
         font_data = font_ptr[0];
-        glue(vga_draw_glyph_line_, DEPTH)(d, font_data, xorcol, bgcol);
+        vga_draw_glyph_line(d, font_data, xorcol, bgcol);
         font_ptr += 4;
         d += linesize;
     } while (--h);
 }
 
-static void glue(vga_draw_glyph16_, DEPTH)(uint8_t *d, int linesize,
+static void vga_draw_glyph16(uint8_t *d, int linesize,
                                           const uint8_t *font_ptr, int h,
                                           uint32_t fgcol, uint32_t bgcol)
 {
@@ -92,48 +59,24 @@ static void glue(vga_draw_glyph16_, DEPTH)(uint8_t *d, int linesize,
     xorcol = bgcol ^ fgcol;
     do {
         font_data = font_ptr[0];
-        glue(vga_draw_glyph_line_, DEPTH)(d,
-                                          expand4to8[font_data >> 4],
-                                          xorcol, bgcol);
-        glue(vga_draw_glyph_line_, DEPTH)(d + 8 * BPP,
-                                          expand4to8[font_data & 0x0f],
-                                          xorcol, bgcol);
+        vga_draw_glyph_line(d, expand4to8[font_data >> 4],
+                            xorcol, bgcol);
+        vga_draw_glyph_line(d + 32, expand4to8[font_data & 0x0f],
+                            xorcol, bgcol);
         font_ptr += 4;
         d += linesize;
     } while (--h);
 }
 
-static void glue(vga_draw_glyph9_, DEPTH)(uint8_t *d, int linesize,
-                                          const uint8_t *font_ptr, int h,
-                                          uint32_t fgcol, uint32_t bgcol, int dup9)
+static void vga_draw_glyph9(uint8_t *d, int linesize,
+                            const uint8_t *font_ptr, int h,
+                            uint32_t fgcol, uint32_t bgcol, int dup9)
 {
     uint32_t font_data, xorcol, v;
 
     xorcol = bgcol ^ fgcol;
     do {
         font_data = font_ptr[0];
-#if BPP == 1
-        stl_p((uint32_t *)d, (dmask16[(font_data >> 4)] & xorcol) ^ bgcol);
-        v = (dmask16[(font_data >> 0) & 0xf] & xorcol) ^ bgcol;
-        stl_p(((uint32_t *)d)+1, v);
-        if (dup9)
-            ((uint8_t *)d)[8] = v >> (24 * (1 - BIG));
-        else
-            ((uint8_t *)d)[8] = bgcol;
-
-#elif BPP == 2
-        stl_p(((uint32_t *)d)+0, (dmask4[(font_data >> 6)] & xorcol) ^ bgcol);
-        stl_p(((uint32_t *)d)+1,
-              (dmask4[(font_data >> 4) & 3] & xorcol) ^ bgcol);
-        stl_p(((uint32_t *)d)+2,
-              (dmask4[(font_data >> 2) & 3] & xorcol) ^ bgcol);
-        v = (dmask4[(font_data >> 0) & 3] & xorcol) ^ bgcol;
-        stl_p(((uint32_t *)d)+3, v);
-        if (dup9)
-            ((uint16_t *)d)[8] = v >> (16 * (1 - BIG));
-        else
-            ((uint16_t *)d)[8] = bgcol;
-#else
         ((uint32_t *)d)[0] = (-((font_data >> 7)) & xorcol) ^ bgcol;
         ((uint32_t *)d)[1] = (-((font_data >> 6) & 1) & xorcol) ^ bgcol;
         ((uint32_t *)d)[2] = (-((font_data >> 5) & 1) & xorcol) ^ bgcol;
@@ -147,7 +90,6 @@ static void glue(vga_draw_glyph9_, DEPTH)(uint8_t *d, int linesize,
             ((uint32_t *)d)[8] = v;
         else
             ((uint32_t *)d)[8] = bgcol;
-#endif
         font_ptr += 4;
         d += linesize;
     } while (--h);
@@ -156,8 +98,8 @@ static void glue(vga_draw_glyph9_, DEPTH)(uint8_t *d, int linesize,
 /*
  * 4 color mode
  */
-static void glue(vga_draw_line2_, DEPTH)(VGACommonState *s1, uint8_t *d,
-                                         const uint8_t *s, int width)
+static void vga_draw_line2(VGACommonState *s1, uint8_t *d,
+                           const uint8_t *s, int width)
 {
     uint32_t plane_mask, *palette, data, v;
     int x;
@@ -170,36 +112,30 @@ static void glue(vga_draw_line2_, DEPTH)(VGACommonState *s1, uint8_t *d,
         data &= plane_mask;
         v = expand2[GET_PLANE(data, 0)];
         v |= expand2[GET_PLANE(data, 2)] << 2;
-        ((PIXEL_TYPE *)d)[0] = palette[v >> 12];
-        ((PIXEL_TYPE *)d)[1] = palette[(v >> 8) & 0xf];
-        ((PIXEL_TYPE *)d)[2] = palette[(v >> 4) & 0xf];
-        ((PIXEL_TYPE *)d)[3] = palette[(v >> 0) & 0xf];
+        ((uint32_t *)d)[0] = palette[v >> 12];
+        ((uint32_t *)d)[1] = palette[(v >> 8) & 0xf];
+        ((uint32_t *)d)[2] = palette[(v >> 4) & 0xf];
+        ((uint32_t *)d)[3] = palette[(v >> 0) & 0xf];
 
         v = expand2[GET_PLANE(data, 1)];
         v |= expand2[GET_PLANE(data, 3)] << 2;
-        ((PIXEL_TYPE *)d)[4] = palette[v >> 12];
-        ((PIXEL_TYPE *)d)[5] = palette[(v >> 8) & 0xf];
-        ((PIXEL_TYPE *)d)[6] = palette[(v >> 4) & 0xf];
-        ((PIXEL_TYPE *)d)[7] = palette[(v >> 0) & 0xf];
-        d += BPP * 8;
+        ((uint32_t *)d)[4] = palette[v >> 12];
+        ((uint32_t *)d)[5] = palette[(v >> 8) & 0xf];
+        ((uint32_t *)d)[6] = palette[(v >> 4) & 0xf];
+        ((uint32_t *)d)[7] = palette[(v >> 0) & 0xf];
+        d += 32;
         s += 4;
     }
 }
 
-#if BPP == 1
-#define PUT_PIXEL2(d, n, v) ((uint16_t *)d)[(n)] = (v)
-#elif BPP == 2
-#define PUT_PIXEL2(d, n, v) ((uint32_t *)d)[(n)] = (v)
-#else
 #define PUT_PIXEL2(d, n, v) \
 ((uint32_t *)d)[2*(n)] = ((uint32_t *)d)[2*(n)+1] = (v)
-#endif
 
 /*
  * 4 color mode, dup2 horizontal
  */
-static void glue(vga_draw_line2d2_, DEPTH)(VGACommonState *s1, uint8_t *d,
-                                           const uint8_t *s, int width)
+static void vga_draw_line2d2(VGACommonState *s1, uint8_t *d,
+                             const uint8_t *s, int width)
 {
     uint32_t plane_mask, *palette, data, v;
     int x;
@@ -223,7 +159,7 @@ static void glue(vga_draw_line2d2_, DEPTH)(VGACommonState *s1, uint8_t *d,
         PUT_PIXEL2(d, 5, palette[(v >> 8) & 0xf]);
         PUT_PIXEL2(d, 6, palette[(v >> 4) & 0xf]);
         PUT_PIXEL2(d, 7, palette[(v >> 0) & 0xf]);
-        d += BPP * 16;
+        d += 64;
         s += 4;
     }
 }
@@ -231,8 +167,8 @@ static void glue(vga_draw_line2d2_, DEPTH)(VGACommonState *s1, uint8_t *d,
 /*
  * 16 color mode
  */
-static void glue(vga_draw_line4_, DEPTH)(VGACommonState *s1, uint8_t *d,
-                                         const uint8_t *s, int width)
+static void vga_draw_line4(VGACommonState *s1, uint8_t *d,
+                           const uint8_t *s, int width)
 {
     uint32_t plane_mask, data, v, *palette;
     int x;
@@ -247,15 +183,15 @@ static void glue(vga_draw_line4_, DEPTH)(VGACommonState *s1, uint8_t *d,
         v |= expand4[GET_PLANE(data, 1)] << 1;
         v |= expand4[GET_PLANE(data, 2)] << 2;
         v |= expand4[GET_PLANE(data, 3)] << 3;
-        ((PIXEL_TYPE *)d)[0] = palette[v >> 28];
-        ((PIXEL_TYPE *)d)[1] = palette[(v >> 24) & 0xf];
-        ((PIXEL_TYPE *)d)[2] = palette[(v >> 20) & 0xf];
-        ((PIXEL_TYPE *)d)[3] = palette[(v >> 16) & 0xf];
-        ((PIXEL_TYPE *)d)[4] = palette[(v >> 12) & 0xf];
-        ((PIXEL_TYPE *)d)[5] = palette[(v >> 8) & 0xf];
-        ((PIXEL_TYPE *)d)[6] = palette[(v >> 4) & 0xf];
-        ((PIXEL_TYPE *)d)[7] = palette[(v >> 0) & 0xf];
-        d += BPP * 8;
+        ((uint32_t *)d)[0] = palette[v >> 28];
+        ((uint32_t *)d)[1] = palette[(v >> 24) & 0xf];
+        ((uint32_t *)d)[2] = palette[(v >> 20) & 0xf];
+        ((uint32_t *)d)[3] = palette[(v >> 16) & 0xf];
+        ((uint32_t *)d)[4] = palette[(v >> 12) & 0xf];
+        ((uint32_t *)d)[5] = palette[(v >> 8) & 0xf];
+        ((uint32_t *)d)[6] = palette[(v >> 4) & 0xf];
+        ((uint32_t *)d)[7] = palette[(v >> 0) & 0xf];
+        d += 32;
         s += 4;
     }
 }
@@ -263,8 +199,8 @@ static void glue(vga_draw_line4_, DEPTH)(VGACommonState *s1, uint8_t *d,
 /*
  * 16 color mode, dup2 horizontal
  */
-static void glue(vga_draw_line4d2_, DEPTH)(VGACommonState *s1, uint8_t *d,
-                                           const uint8_t *s, int width)
+static void vga_draw_line4d2(VGACommonState *s1, uint8_t *d,
+                             const uint8_t *s, int width)
 {
     uint32_t plane_mask, data, v, *palette;
     int x;
@@ -287,7 +223,7 @@ static void glue(vga_draw_line4d2_, DEPTH)(VGACommonState *s1, uint8_t *d,
         PUT_PIXEL2(d, 5, palette[(v >> 8) & 0xf]);
         PUT_PIXEL2(d, 6, palette[(v >> 4) & 0xf]);
         PUT_PIXEL2(d, 7, palette[(v >> 0) & 0xf]);
-        d += BPP * 16;
+        d += 64;
         s += 4;
     }
 }
@@ -297,8 +233,8 @@ static void glue(vga_draw_line4d2_, DEPTH)(VGACommonState *s1, uint8_t *d,
  *
  * XXX: add plane_mask support (never used in standard VGA modes)
  */
-static void glue(vga_draw_line8d2_, DEPTH)(VGACommonState *s1, uint8_t *d,
-                                           const uint8_t *s, int width)
+static void vga_draw_line8d2(VGACommonState *s1, uint8_t *d,
+                             const uint8_t *s, int width)
 {
     uint32_t *palette;
     int x;
@@ -310,7 +246,7 @@ static void glue(vga_draw_line8d2_, DEPTH)(VGACommonState *s1, uint8_t *d,
         PUT_PIXEL2(d, 1, palette[s[1]]);
         PUT_PIXEL2(d, 2, palette[s[2]]);
         PUT_PIXEL2(d, 3, palette[s[3]]);
-        d += BPP * 8;
+        d += 32;
         s += 4;
     }
 }
@@ -320,8 +256,8 @@ static void glue(vga_draw_line8d2_, DEPTH)(VGACommonState *s1, uint8_t *d,
  *
  * XXX: add plane_mask support (never used in standard VGA modes)
  */
-static void glue(vga_draw_line8_, DEPTH)(VGACommonState *s1, uint8_t *d,
-                                         const uint8_t *s, int width)
+static void vga_draw_line8(VGACommonState *s1, uint8_t *d,
+                           const uint8_t *s, int width)
 {
     uint32_t *palette;
     int x;
@@ -329,107 +265,141 @@ static void glue(vga_draw_line8_, DEPTH)(VGACommonState *s1, uint8_t *d,
     palette = s1->last_palette;
     width >>= 3;
     for(x = 0; x < width; x++) {
-        ((PIXEL_TYPE *)d)[0] = palette[s[0]];
-        ((PIXEL_TYPE *)d)[1] = palette[s[1]];
-        ((PIXEL_TYPE *)d)[2] = palette[s[2]];
-        ((PIXEL_TYPE *)d)[3] = palette[s[3]];
-        ((PIXEL_TYPE *)d)[4] = palette[s[4]];
-        ((PIXEL_TYPE *)d)[5] = palette[s[5]];
-        ((PIXEL_TYPE *)d)[6] = palette[s[6]];
-        ((PIXEL_TYPE *)d)[7] = palette[s[7]];
-        d += BPP * 8;
+        ((uint32_t *)d)[0] = palette[s[0]];
+        ((uint32_t *)d)[1] = palette[s[1]];
+        ((uint32_t *)d)[2] = palette[s[2]];
+        ((uint32_t *)d)[3] = palette[s[3]];
+        ((uint32_t *)d)[4] = palette[s[4]];
+        ((uint32_t *)d)[5] = palette[s[5]];
+        ((uint32_t *)d)[6] = palette[s[6]];
+        ((uint32_t *)d)[7] = palette[s[7]];
+        d += 32;
         s += 8;
     }
 }
 
-#endif /* DEPTH != 15 */
-
-
-/* XXX: optimize */
-
 /*
  * 15 bit color
  */
-static void glue(vga_draw_line15_, PIXEL_NAME)(VGACommonState *s1, uint8_t *d,
-                                          const uint8_t *s, int width)
+static void vga_draw_line15_le(VGACommonState *s1, uint8_t *d,
+                               const uint8_t *s, int width)
 {
-#if DEPTH == 15 && defined(HOST_WORDS_BIGENDIAN) == defined(TARGET_WORDS_BIGENDIAN)
-    memcpy(d, s, width * 2);
-#else
     int w;
     uint32_t v, r, g, b;
 
     w = width;
     do {
-        v = lduw_p((void *)s);
+        v = lduw_le_p((void *)s);
         r = (v >> 7) & 0xf8;
         g = (v >> 2) & 0xf8;
         b = (v << 3) & 0xf8;
-        ((PIXEL_TYPE *)d)[0] = glue(rgb_to_pixel, PIXEL_NAME)(r, g, b);
+        ((uint32_t *)d)[0] = rgb_to_pixel32(r, g, b);
         s += 2;
-        d += BPP;
+        d += 4;
+    } while (--w != 0);
+}
+
+static void vga_draw_line15_be(VGACommonState *s1, uint8_t *d,
+                               const uint8_t *s, int width)
+{
+    int w;
+    uint32_t v, r, g, b;
+
+    w = width;
+    do {
+        v = lduw_be_p((void *)s);
+        r = (v >> 7) & 0xf8;
+        g = (v >> 2) & 0xf8;
+        b = (v << 3) & 0xf8;
+        ((uint32_t *)d)[0] = rgb_to_pixel32(r, g, b);
+        s += 2;
+        d += 4;
     } while (--w != 0);
-#endif
 }
 
 /*
  * 16 bit color
  */
-static void glue(vga_draw_line16_, PIXEL_NAME)(VGACommonState *s1, uint8_t *d,
-                                          const uint8_t *s, int width)
+static void vga_draw_line16_le(VGACommonState *s1, uint8_t *d,
+                               const uint8_t *s, int width)
 {
-#if DEPTH == 16 && defined(HOST_WORDS_BIGENDIAN) == defined(TARGET_WORDS_BIGENDIAN)
-    memcpy(d, s, width * 2);
-#else
     int w;
     uint32_t v, r, g, b;
 
     w = width;
     do {
-        v = lduw_p((void *)s);
+        v = lduw_le_p((void *)s);
         r = (v >> 8) & 0xf8;
         g = (v >> 3) & 0xfc;
         b = (v << 3) & 0xf8;
-        ((PIXEL_TYPE *)d)[0] = glue(rgb_to_pixel, PIXEL_NAME)(r, g, b);
+        ((uint32_t *)d)[0] = rgb_to_pixel32(r, g, b);
         s += 2;
-        d += BPP;
+        d += 4;
+    } while (--w != 0);
+}
+
+static void vga_draw_line16_be(VGACommonState *s1, uint8_t *d,
+                               const uint8_t *s, int width)
+{
+    int w;
+    uint32_t v, r, g, b;
+
+    w = width;
+    do {
+        v = lduw_be_p((void *)s);
+        r = (v >> 8) & 0xf8;
+        g = (v >> 3) & 0xfc;
+        b = (v << 3) & 0xf8;
+        ((uint32_t *)d)[0] = rgb_to_pixel32(r, g, b);
+        s += 2;
+        d += 4;
     } while (--w != 0);
-#endif
 }
 
 /*
  * 24 bit color
  */
-static void glue(vga_draw_line24_, PIXEL_NAME)(VGACommonState *s1, uint8_t *d,
-                                          const uint8_t *s, int width)
+static void vga_draw_line24_le(VGACommonState *s1, uint8_t *d,
+                               const uint8_t *s, int width)
 {
     int w;
     uint32_t r, g, b;
 
     w = width;
     do {
-#if defined(TARGET_WORDS_BIGENDIAN)
-        r = s[0];
-        g = s[1];
-        b = s[2];
-#else
         b = s[0];
         g = s[1];
         r = s[2];
-#endif
-        ((PIXEL_TYPE *)d)[0] = glue(rgb_to_pixel, PIXEL_NAME)(r, g, b);
+        ((uint32_t *)d)[0] = rgb_to_pixel32(r, g, b);
         s += 3;
-        d += BPP;
+        d += 4;
+    } while (--w != 0);
+}
+
+static void vga_draw_line24_be(VGACommonState *s1, uint8_t *d,
+                               const uint8_t *s, int width)
+{
+    int w;
+    uint32_t r, g, b;
+
+    w = width;
+    do {
+        r = s[0];
+        g = s[1];
+        b = s[2];
+        ((uint32_t *)d)[0] = rgb_to_pixel32(r, g, b);
+        s += 3;
+        d += 4;
     } while (--w != 0);
 }
 
 /*
  * 32 bit color
  */
-static void glue(vga_draw_line32_, PIXEL_NAME)(VGACommonState *s1, uint8_t *d,
-                                          const uint8_t *s, int width)
+static void vga_draw_line32_le(VGACommonState *s1, uint8_t *d,
+                               const uint8_t *s, int width)
 {
-#if DEPTH == 32 && defined(HOST_WORDS_BIGENDIAN) == defined(TARGET_WORDS_BIGENDIAN) && !defined(BGR_FORMAT)
+#ifndef HOST_WORDS_BIGENDIAN
     memcpy(d, s, width * 4);
 #else
     int w;
@@ -437,25 +407,33 @@ static void glue(vga_draw_line32_, PIXEL_NAME)(VGACommonState *s1, uint8_t *d,
 
     w = width;
     do {
-#if defined(TARGET_WORDS_BIGENDIAN)
-        r = s[1];
-        g = s[2];
-        b = s[3];
-#else
         b = s[0];
         g = s[1];
         r = s[2];
-#endif
-        ((PIXEL_TYPE *)d)[0] = glue(rgb_to_pixel, PIXEL_NAME)(r, g, b);
+        ((uint32_t *)d)[0] = rgb_to_pixel32(r, g, b);
         s += 4;
-        d += BPP;
+        d += 4;
     } while (--w != 0);
 #endif
 }
 
-#undef PUT_PIXEL2
-#undef DEPTH
-#undef BPP
-#undef PIXEL_TYPE
-#undef PIXEL_NAME
-#undef BGR_FORMAT
+static void vga_draw_line32_be(VGACommonState *s1, uint8_t *d,
+                               const uint8_t *s, int width)
+{
+#ifdef HOST_WORDS_BIGENDIAN
+    memcpy(d, s, width * 4);
+#else
+    int w;
+    uint32_t r, g, b;
+
+    w = width;
+    do {
+        r = s[1];
+        g = s[2];
+        b = s[3];
+        ((uint32_t *)d)[0] = rgb_to_pixel32(r, g, b);
+        s += 4;
+        d += 4;
+    } while (--w != 0);
+#endif
+}
diff --git a/hw/display/vga-pci.c b/hw/display/vga-pci.c
index 0351d9470..db922f184 100644
--- a/hw/display/vga-pci.c
+++ b/hw/display/vga-pci.c
@@ -35,10 +35,18 @@
 #define PCI_VGA_IOPORT_SIZE   (0x3e0 - 0x3c0)
 #define PCI_VGA_BOCHS_OFFSET  0x500
 #define PCI_VGA_BOCHS_SIZE    (0x0b * 2)
+#define PCI_VGA_QEXT_OFFSET   0x600
+#define PCI_VGA_QEXT_SIZE     (2 * 4)
 #define PCI_VGA_MMIO_SIZE     0x1000
 
+#define PCI_VGA_QEXT_REG_SIZE         (0 * 4)
+#define PCI_VGA_QEXT_REG_BYTEORDER    (1 * 4)
+#define  PCI_VGA_QEXT_LITTLE_ENDIAN   0x1e1e1e1e
+#define  PCI_VGA_QEXT_BIG_ENDIAN      0xbebebebe
+
 enum vga_pci_flags {
     PCI_VGA_FLAG_ENABLE_MMIO = 1,
+    PCI_VGA_FLAG_ENABLE_QEXT = 2,
 };
 
 typedef struct PCIVGAState {
@@ -48,6 +56,7 @@ typedef struct PCIVGAState {
     MemoryRegion mmio;
     MemoryRegion ioport;
     MemoryRegion bochs;
+    MemoryRegion qext;
 } PCIVGAState;
 
 static const VMStateDescription vmstate_vga_pci = {
@@ -140,6 +149,46 @@ static const MemoryRegionOps pci_vga_bochs_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
+static uint64_t pci_vga_qext_read(void *ptr, hwaddr addr, unsigned size)
+{
+    PCIVGAState *d = ptr;
+
+    switch (addr) {
+    case PCI_VGA_QEXT_REG_SIZE:
+        return PCI_VGA_QEXT_SIZE;
+    case PCI_VGA_QEXT_REG_BYTEORDER:
+        return d->vga.big_endian_fb ?
+            PCI_VGA_QEXT_BIG_ENDIAN : PCI_VGA_QEXT_LITTLE_ENDIAN;
+    default:
+        return 0;
+    }
+}
+
+static void pci_vga_qext_write(void *ptr, hwaddr addr,
+                               uint64_t val, unsigned size)
+{
+    PCIVGAState *d = ptr;
+
+    switch (addr) {
+    case PCI_VGA_QEXT_REG_BYTEORDER:
+        if (val == PCI_VGA_QEXT_BIG_ENDIAN) {
+            d->vga.big_endian_fb = true;
+        }
+        if (val == PCI_VGA_QEXT_LITTLE_ENDIAN) {
+            d->vga.big_endian_fb = false;
+        }
+        break;
+    }
+}
+
+static const MemoryRegionOps pci_vga_qext_ops = {
+    .read = pci_vga_qext_read,
+    .write = pci_vga_qext_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
 static int pci_std_vga_initfn(PCIDevice *dev)
 {
     PCIVGAState *d = DO_UPCAST(PCIVGAState, dev, dev);
@@ -167,6 +216,15 @@ static int pci_std_vga_initfn(PCIDevice *dev)
                                     &d->ioport);
         memory_region_add_subregion(&d->mmio, PCI_VGA_BOCHS_OFFSET,
                                     &d->bochs);
+
+        if (d->flags & (1 << PCI_VGA_FLAG_ENABLE_QEXT)) {
+            memory_region_init_io(&d->qext, NULL, &pci_vga_qext_ops, d,
+                                  "qemu extended regs", PCI_VGA_QEXT_SIZE);
+            memory_region_add_subregion(&d->mmio, PCI_VGA_QEXT_OFFSET,
+                                        &d->qext);
+            pci_set_byte(&d->dev.config[PCI_REVISION_ID], 2);
+        }
+
         pci_register_bar(&d->dev, 2, PCI_BASE_ADDRESS_SPACE_MEMORY, &d->mmio);
     }
 
@@ -199,6 +257,14 @@ static int pci_secondary_vga_initfn(PCIDevice *dev)
     memory_region_add_subregion(&d->mmio, PCI_VGA_BOCHS_OFFSET,
                                 &d->bochs);
 
+    if (d->flags & (1 << PCI_VGA_FLAG_ENABLE_QEXT)) {
+        memory_region_init_io(&d->qext, NULL, &pci_vga_qext_ops, d,
+                              "qemu extended regs", PCI_VGA_QEXT_SIZE);
+        memory_region_add_subregion(&d->mmio, PCI_VGA_QEXT_OFFSET,
+                                    &d->qext);
+        pci_set_byte(&d->dev.config[PCI_REVISION_ID], 2);
+    }
+
     pci_register_bar(&d->dev, 0, PCI_BASE_ADDRESS_MEM_PREFETCH, &s->vram);
     pci_register_bar(&d->dev, 2, PCI_BASE_ADDRESS_SPACE_MEMORY, &d->mmio);
 
@@ -215,11 +281,15 @@ static void pci_secondary_vga_reset(DeviceState *dev)
 static Property vga_pci_properties[] = {
     DEFINE_PROP_UINT32("vgamem_mb", PCIVGAState, vga.vram_size_mb, 16),
     DEFINE_PROP_BIT("mmio", PCIVGAState, flags, PCI_VGA_FLAG_ENABLE_MMIO, true),
+    DEFINE_PROP_BIT("qemu-extended-regs",
+                    PCIVGAState, flags, PCI_VGA_FLAG_ENABLE_QEXT, true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
 static Property secondary_pci_properties[] = {
     DEFINE_PROP_UINT32("vgamem_mb", PCIVGAState, vga.vram_size_mb, 16),
+    DEFINE_PROP_BIT("qemu-extended-regs",
+                    PCIVGAState, flags, PCI_VGA_FLAG_ENABLE_QEXT, true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/hw/display/vga.c b/hw/display/vga.c
index 4b089a36a..52eaf0565 100644
--- a/hw/display/vga.c
+++ b/hw/display/vga.c
@@ -136,47 +136,24 @@ static const uint32_t mask16[16] = {
 #define PAT(x) cbswap_32(x)
 #endif
 
-static const uint32_t dmask16[16] = {
-    PAT(0x00000000),
-    PAT(0x000000ff),
-    PAT(0x0000ff00),
-    PAT(0x0000ffff),
-    PAT(0x00ff0000),
-    PAT(0x00ff00ff),
-    PAT(0x00ffff00),
-    PAT(0x00ffffff),
-    PAT(0xff000000),
-    PAT(0xff0000ff),
-    PAT(0xff00ff00),
-    PAT(0xff00ffff),
-    PAT(0xffff0000),
-    PAT(0xffff00ff),
-    PAT(0xffffff00),
-    PAT(0xffffffff),
-};
-
-static const uint32_t dmask4[4] = {
-    PAT(0x00000000),
-    PAT(0x0000ffff),
-    PAT(0xffff0000),
-    PAT(0xffffffff),
-};
-
 static uint32_t expand4[256];
 static uint16_t expand2[256];
 static uint8_t expand4to8[16];
 
 static void vga_update_memory_access(VGACommonState *s)
 {
-    MemoryRegion *region, *old_region = s->chain4_alias;
     hwaddr base, offset, size;
 
     if (s->legacy_address_space == NULL) {
         return;
     }
 
-    s->chain4_alias = NULL;
-
+    if (s->has_chain4_alias) {
+        memory_region_del_subregion(s->legacy_address_space, &s->chain4_alias);
+        object_unparent(OBJECT(&s->chain4_alias));
+        s->has_chain4_alias = false;
+        s->plane_updated = 0xf;
+    }
     if ((s->sr[VGA_SEQ_PLANE_WRITE] & VGA_SR02_ALL_PLANES) ==
         VGA_SR02_ALL_PLANES && s->sr[VGA_SEQ_MEMORY_MODE] & VGA_SR04_CHN_4M) {
         offset = 0;
@@ -201,18 +178,11 @@ static void vga_update_memory_access(VGACommonState *s)
             break;
         }
         base += isa_mem_base;
-        region = g_malloc(sizeof(*region));
-        memory_region_init_alias(region, memory_region_owner(&s->vram),
+        memory_region_init_alias(&s->chain4_alias, memory_region_owner(&s->vram),
                                  "vga.chain4", &s->vram, offset, size);
         memory_region_add_subregion_overlap(s->legacy_address_space, base,
-                                            region, 2);
-        s->chain4_alias = region;
-    }
-    if (old_region) {
-        memory_region_del_subregion(s->legacy_address_space, old_region);
-        memory_region_destroy(old_region);
-        g_free(old_region);
-        s->plane_updated = 0xf;
+                                            &s->chain4_alias, 2);
+        s->has_chain4_alias = true;
     }
 }
 
@@ -580,6 +550,93 @@ void vga_ioport_write(void *opaque, uint32_t addr, uint32_t val)
     }
 }
 
+/*
+ * Sanity check vbe register writes.
+ *
+ * As we don't have a way to signal errors to the guest in the bochs
+ * dispi interface we'll go adjust the registers to the closest valid
+ * value.
+ */
+static void vbe_fixup_regs(VGACommonState *s)
+{
+    uint16_t *r = s->vbe_regs;
+    uint32_t bits, linelength, maxy, offset;
+
+    if (!(r[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED)) {
+        /* vbe is turned off -- nothing to do */
+        return;
+    }
+
+    /* check depth */
+    switch (r[VBE_DISPI_INDEX_BPP]) {
+    case 4:
+    case 8:
+    case 16:
+    case 24:
+    case 32:
+        bits = r[VBE_DISPI_INDEX_BPP];
+        break;
+    case 15:
+        bits = 16;
+        break;
+    default:
+        bits = r[VBE_DISPI_INDEX_BPP] = 8;
+        break;
+    }
+
+    /* check width */
+    r[VBE_DISPI_INDEX_XRES] &= ~7u;
+    if (r[VBE_DISPI_INDEX_XRES] == 0) {
+        r[VBE_DISPI_INDEX_XRES] = 8;
+    }
+    if (r[VBE_DISPI_INDEX_XRES] > VBE_DISPI_MAX_XRES) {
+        r[VBE_DISPI_INDEX_XRES] = VBE_DISPI_MAX_XRES;
+    }
+    r[VBE_DISPI_INDEX_VIRT_WIDTH] &= ~7u;
+    if (r[VBE_DISPI_INDEX_VIRT_WIDTH] > VBE_DISPI_MAX_XRES) {
+        r[VBE_DISPI_INDEX_VIRT_WIDTH] = VBE_DISPI_MAX_XRES;
+    }
+    if (r[VBE_DISPI_INDEX_VIRT_WIDTH] < r[VBE_DISPI_INDEX_XRES]) {
+        r[VBE_DISPI_INDEX_VIRT_WIDTH] = r[VBE_DISPI_INDEX_XRES];
+    }
+
+    /* check height */
+    linelength = r[VBE_DISPI_INDEX_VIRT_WIDTH] * bits / 8;
+    maxy = s->vbe_size / linelength;
+    if (r[VBE_DISPI_INDEX_YRES] == 0) {
+        r[VBE_DISPI_INDEX_YRES] = 1;
+    }
+    if (r[VBE_DISPI_INDEX_YRES] > VBE_DISPI_MAX_YRES) {
+        r[VBE_DISPI_INDEX_YRES] = VBE_DISPI_MAX_YRES;
+    }
+    if (r[VBE_DISPI_INDEX_YRES] > maxy) {
+        r[VBE_DISPI_INDEX_YRES] = maxy;
+    }
+
+    /* check offset */
+    if (r[VBE_DISPI_INDEX_X_OFFSET] > VBE_DISPI_MAX_XRES) {
+        r[VBE_DISPI_INDEX_X_OFFSET] = VBE_DISPI_MAX_XRES;
+    }
+    if (r[VBE_DISPI_INDEX_Y_OFFSET] > VBE_DISPI_MAX_YRES) {
+        r[VBE_DISPI_INDEX_Y_OFFSET] = VBE_DISPI_MAX_YRES;
+    }
+    offset = r[VBE_DISPI_INDEX_X_OFFSET] * bits / 8;
+    offset += r[VBE_DISPI_INDEX_Y_OFFSET] * linelength;
+    if (offset + r[VBE_DISPI_INDEX_YRES] * linelength > s->vbe_size) {
+        r[VBE_DISPI_INDEX_Y_OFFSET] = 0;
+        offset = r[VBE_DISPI_INDEX_X_OFFSET] * bits / 8;
+        if (offset + r[VBE_DISPI_INDEX_YRES] * linelength > s->vbe_size) {
+            r[VBE_DISPI_INDEX_X_OFFSET] = 0;
+            offset = 0;
+        }
+    }
+
+    /* update vga state */
+    r[VBE_DISPI_INDEX_VIRT_HEIGHT] = maxy;
+    s->vbe_line_offset = linelength;
+    s->vbe_start_addr  = offset / 4;
+}
+
 static uint32_t vbe_ioport_read_index(void *opaque, uint32_t addr)
 {
     VGACommonState *s = opaque;
@@ -614,7 +671,7 @@ uint32_t vbe_ioport_read_data(void *opaque, uint32_t addr)
             val = s->vbe_regs[s->vbe_index];
         }
     } else if (s->vbe_index == VBE_DISPI_INDEX_VIDEO_MEMORY_64K) {
-        val = s->vram_size / (64 * 1024);
+        val = s->vbe_size / (64 * 1024);
     } else {
         val = 0;
     }
@@ -649,22 +706,13 @@ void vbe_ioport_write_data(void *opaque, uint32_t addr, uint32_t val)
             }
             break;
         case VBE_DISPI_INDEX_XRES:
-            if ((val <= VBE_DISPI_MAX_XRES) && ((val & 7) == 0)) {
-                s->vbe_regs[s->vbe_index] = val;
-            }
-            break;
         case VBE_DISPI_INDEX_YRES:
-            if (val <= VBE_DISPI_MAX_YRES) {
-                s->vbe_regs[s->vbe_index] = val;
-            }
-            break;
         case VBE_DISPI_INDEX_BPP:
-            if (val == 0)
-                val = 8;
-            if (val == 4 || val == 8 || val == 15 ||
-                val == 16 || val == 24 || val == 32) {
-                s->vbe_regs[s->vbe_index] = val;
-            }
+        case VBE_DISPI_INDEX_VIRT_WIDTH:
+        case VBE_DISPI_INDEX_X_OFFSET:
+        case VBE_DISPI_INDEX_Y_OFFSET:
+            s->vbe_regs[s->vbe_index] = val;
+            vbe_fixup_regs(s);
             break;
         case VBE_DISPI_INDEX_BANK:
             if (s->vbe_regs[VBE_DISPI_INDEX_BPP] == 4) {
@@ -681,28 +729,19 @@ void vbe_ioport_write_data(void *opaque, uint32_t addr, uint32_t val)
                 !(s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED)) {
                 int h, shift_control;
 
-                s->vbe_regs[VBE_DISPI_INDEX_VIRT_WIDTH] =
-                    s->vbe_regs[VBE_DISPI_INDEX_XRES];
-                s->vbe_regs[VBE_DISPI_INDEX_VIRT_HEIGHT] =
-                    s->vbe_regs[VBE_DISPI_INDEX_YRES];
+                s->vbe_regs[VBE_DISPI_INDEX_VIRT_WIDTH] = 0;
                 s->vbe_regs[VBE_DISPI_INDEX_X_OFFSET] = 0;
                 s->vbe_regs[VBE_DISPI_INDEX_Y_OFFSET] = 0;
+                s->vbe_regs[VBE_DISPI_INDEX_ENABLE] |= VBE_DISPI_ENABLED;
+                vbe_fixup_regs(s);
 
-                if (s->vbe_regs[VBE_DISPI_INDEX_BPP] == 4)
-                    s->vbe_line_offset = s->vbe_regs[VBE_DISPI_INDEX_XRES] >> 1;
-                else
-                    s->vbe_line_offset = s->vbe_regs[VBE_DISPI_INDEX_XRES] *
-                        ((s->vbe_regs[VBE_DISPI_INDEX_BPP] + 7) >> 3);
-                s->vbe_start_addr = 0;
-
-                /* clear the screen (should be done in BIOS) */
+                /* clear the screen */
                 if (!(val & VBE_DISPI_NOCLEARMEM)) {
                     memset(s->vram_ptr, 0,
                            s->vbe_regs[VBE_DISPI_INDEX_YRES] * s->vbe_line_offset);
                 }
 
-                /* we initialize the VGA graphic mode (should be done
-                   in BIOS) */
+                /* we initialize the VGA graphic mode */
                 /* graphic mode + memory map 1 */
                 s->gr[VGA_GFX_MISC] = (s->gr[VGA_GFX_MISC] & ~0x0c) | 0x04 |
                     VGA_GR06_GRAPHICS_MODE;
@@ -735,47 +774,12 @@ void vbe_ioport_write_data(void *opaque, uint32_t addr, uint32_t val)
                     (shift_control << 5);
                 s->cr[VGA_CRTC_MAX_SCAN] &= ~0x9f; /* no double scan */
             } else {
-                /* XXX: the bios should do that */
                 s->bank_offset = 0;
             }
             s->dac_8bit = (val & VBE_DISPI_8BIT_DAC) > 0;
             s->vbe_regs[s->vbe_index] = val;
             vga_update_memory_access(s);
             break;
-        case VBE_DISPI_INDEX_VIRT_WIDTH:
-            {
-                int w, h, line_offset;
-
-                if (val < s->vbe_regs[VBE_DISPI_INDEX_XRES])
-                    return;
-                w = val;
-                if (s->vbe_regs[VBE_DISPI_INDEX_BPP] == 4)
-                    line_offset = w >> 1;
-                else
-                    line_offset = w * ((s->vbe_regs[VBE_DISPI_INDEX_BPP] + 7) >> 3);
-                h = s->vram_size / line_offset;
-                /* XXX: support weird bochs semantics ? */
-                if (h < s->vbe_regs[VBE_DISPI_INDEX_YRES])
-                    return;
-                s->vbe_regs[VBE_DISPI_INDEX_VIRT_WIDTH] = w;
-                s->vbe_regs[VBE_DISPI_INDEX_VIRT_HEIGHT] = h;
-                s->vbe_line_offset = line_offset;
-            }
-            break;
-        case VBE_DISPI_INDEX_X_OFFSET:
-        case VBE_DISPI_INDEX_Y_OFFSET:
-            {
-                int x;
-                s->vbe_regs[s->vbe_index] = val;
-                s->vbe_start_addr = s->vbe_line_offset * s->vbe_regs[VBE_DISPI_INDEX_Y_OFFSET];
-                x = s->vbe_regs[VBE_DISPI_INDEX_X_OFFSET];
-                if (s->vbe_regs[VBE_DISPI_INDEX_BPP] == 4)
-                    s->vbe_start_addr += x >> 1;
-                else
-                    s->vbe_start_addr += x * ((s->vbe_regs[VBE_DISPI_INDEX_BPP] + 7) >> 3);
-                s->vbe_start_addr >>= 2;
-            }
-            break;
         default:
             break;
         }
@@ -974,95 +978,10 @@ void vga_mem_writeb(VGACommonState *s, hwaddr addr, uint32_t val)
     }
 }
 
-typedef void vga_draw_glyph8_func(uint8_t *d, int linesize,
-                             const uint8_t *font_ptr, int h,
-                             uint32_t fgcol, uint32_t bgcol);
-typedef void vga_draw_glyph9_func(uint8_t *d, int linesize,
-                                  const uint8_t *font_ptr, int h,
-                                  uint32_t fgcol, uint32_t bgcol, int dup9);
 typedef void vga_draw_line_func(VGACommonState *s1, uint8_t *d,
                                 const uint8_t *s, int width);
 
-#define DEPTH 8
-#include "vga_template.h"
-
-#define DEPTH 15
-#include "vga_template.h"
-
-#define BGR_FORMAT
-#define DEPTH 15
-#include "vga_template.h"
-
-#define DEPTH 16
-#include "vga_template.h"
-
-#define BGR_FORMAT
-#define DEPTH 16
-#include "vga_template.h"
-
-#define DEPTH 32
-#include "vga_template.h"
-
-#define BGR_FORMAT
-#define DEPTH 32
-#include "vga_template.h"
-
-static unsigned int rgb_to_pixel8_dup(unsigned int r, unsigned int g, unsigned b)
-{
-    unsigned int col;
-    col = rgb_to_pixel8(r, g, b);
-    col |= col << 8;
-    col |= col << 16;
-    return col;
-}
-
-static unsigned int rgb_to_pixel15_dup(unsigned int r, unsigned int g, unsigned b)
-{
-    unsigned int col;
-    col = rgb_to_pixel15(r, g, b);
-    col |= col << 16;
-    return col;
-}
-
-static unsigned int rgb_to_pixel15bgr_dup(unsigned int r, unsigned int g,
-                                          unsigned int b)
-{
-    unsigned int col;
-    col = rgb_to_pixel15bgr(r, g, b);
-    col |= col << 16;
-    return col;
-}
-
-static unsigned int rgb_to_pixel16_dup(unsigned int r, unsigned int g, unsigned b)
-{
-    unsigned int col;
-    col = rgb_to_pixel16(r, g, b);
-    col |= col << 16;
-    return col;
-}
-
-static unsigned int rgb_to_pixel16bgr_dup(unsigned int r, unsigned int g,
-                                          unsigned int b)
-{
-    unsigned int col;
-    col = rgb_to_pixel16bgr(r, g, b);
-    col |= col << 16;
-    return col;
-}
-
-static unsigned int rgb_to_pixel32_dup(unsigned int r, unsigned int g, unsigned b)
-{
-    unsigned int col;
-    col = rgb_to_pixel32(r, g, b);
-    return col;
-}
-
-static unsigned int rgb_to_pixel32bgr_dup(unsigned int r, unsigned int g, unsigned b)
-{
-    unsigned int col;
-    col = rgb_to_pixel32bgr(r, g, b);
-    return col;
-}
+#include "vga-helpers.h"
 
 /* return true if the palette was modified */
 static int update_palette16(VGACommonState *s)
@@ -1080,9 +999,9 @@ static int update_palette16(VGACommonState *s)
             v = ((s->ar[VGA_ATC_COLOR_PAGE] & 0xc) << 4) | (v & 0x3f);
         }
         v = v * 3;
-        col = s->rgb_to_pixel(c6_to_8(s->palette[v]),
-                              c6_to_8(s->palette[v + 1]),
-                              c6_to_8(s->palette[v + 2]));
+        col = rgb_to_pixel32(c6_to_8(s->palette[v]),
+                             c6_to_8(s->palette[v + 1]),
+                             c6_to_8(s->palette[v + 2]));
         if (col != palette[i]) {
             full_update = 1;
             palette[i] = col;
@@ -1102,13 +1021,13 @@ static int update_palette256(VGACommonState *s)
     v = 0;
     for(i = 0; i < 256; i++) {
         if (s->dac_8bit) {
-          col = s->rgb_to_pixel(s->palette[v],
-                                s->palette[v + 1],
-                                s->palette[v + 2]);
+            col = rgb_to_pixel32(s->palette[v],
+                                 s->palette[v + 1],
+                                 s->palette[v + 2]);
         } else {
-          col = s->rgb_to_pixel(c6_to_8(s->palette[v]),
-                                c6_to_8(s->palette[v + 1]),
-                                c6_to_8(s->palette[v + 2]));
+            col = rgb_to_pixel32(c6_to_8(s->palette[v]),
+                                 c6_to_8(s->palette[v + 1]),
+                                 c6_to_8(s->palette[v + 2]));
         }
         if (col != palette[i]) {
             full_update = 1;
@@ -1170,56 +1089,6 @@ static int update_basic_params(VGACommonState *s)
     return full_update;
 }
 
-#define NB_DEPTHS 7
-
-static inline int get_depth_index(DisplaySurface *s)
-{
-    switch (surface_bits_per_pixel(s)) {
-    default:
-    case 8:
-        return 0;
-    case 15:
-        return 1;
-    case 16:
-        return 2;
-    case 32:
-        if (is_surface_bgr(s)) {
-            return 4;
-        } else {
-            return 3;
-        }
-    }
-}
-
-static vga_draw_glyph8_func * const vga_draw_glyph8_table[NB_DEPTHS] = {
-    vga_draw_glyph8_8,
-    vga_draw_glyph8_16,
-    vga_draw_glyph8_16,
-    vga_draw_glyph8_32,
-    vga_draw_glyph8_32,
-    vga_draw_glyph8_16,
-    vga_draw_glyph8_16,
-};
-
-static vga_draw_glyph8_func * const vga_draw_glyph16_table[NB_DEPTHS] = {
-    vga_draw_glyph16_8,
-    vga_draw_glyph16_16,
-    vga_draw_glyph16_16,
-    vga_draw_glyph16_32,
-    vga_draw_glyph16_32,
-    vga_draw_glyph16_16,
-    vga_draw_glyph16_16,
-};
-
-static vga_draw_glyph9_func * const vga_draw_glyph9_table[NB_DEPTHS] = {
-    vga_draw_glyph9_8,
-    vga_draw_glyph9_16,
-    vga_draw_glyph9_16,
-    vga_draw_glyph9_32,
-    vga_draw_glyph9_32,
-    vga_draw_glyph9_16,
-    vga_draw_glyph9_16,
-};
 
 static const uint8_t cursor_glyph[32 * 4] = {
     0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -1271,18 +1140,6 @@ static void vga_get_text_resolution(VGACommonState *s, int *pwidth, int *pheight
     *pcheight = cheight;
 }
 
-typedef unsigned int rgb_to_pixel_dup_func(unsigned int r, unsigned int g, unsigned b);
-
-static rgb_to_pixel_dup_func * const rgb_to_pixel_dup_table[NB_DEPTHS] = {
-    rgb_to_pixel8_dup,
-    rgb_to_pixel15_dup,
-    rgb_to_pixel16_dup,
-    rgb_to_pixel32_dup,
-    rgb_to_pixel32bgr_dup,
-    rgb_to_pixel15bgr_dup,
-    rgb_to_pixel16bgr_dup,
-};
-
 /*
  * Text mode update
  * Missing:
@@ -1299,11 +1156,9 @@ static void vga_draw_text(VGACommonState *s, int full_update)
     uint32_t offset, fgcol, bgcol, v, cursor_offset;
     uint8_t *d1, *d, *src, *dest, *cursor_ptr;
     const uint8_t *font_ptr, *font_base[2];
-    int dup9, line_offset, depth_index;
+    int dup9, line_offset;
     uint32_t *palette;
     uint32_t *ch_attr_ptr;
-    vga_draw_glyph8_func *vga_draw_glyph8;
-    vga_draw_glyph9_func *vga_draw_glyph9;
     int64_t now = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
 
     /* compute font data address (in plane 2) */
@@ -1321,7 +1176,7 @@ static void vga_draw_text(VGACommonState *s, int full_update)
         s->font_offsets[1] = offset;
         full_update = 1;
     }
-    if (s->plane_updated & (1 << 2) || s->chain4_alias) {
+    if (s->plane_updated & (1 << 2) || s->has_chain4_alias) {
         /* if the plane 2 was modified since the last display, it
            indicates the font may have been modified */
         s->plane_updated = 0;
@@ -1355,8 +1210,6 @@ static void vga_draw_text(VGACommonState *s, int full_update)
         s->last_cw = cw;
         full_update = 1;
     }
-    s->rgb_to_pixel =
-        rgb_to_pixel_dup_table[get_depth_index(surface)];
     full_update |= update_palette16(s);
     palette = s->last_palette;
     x_incr = cw * surface_bytes_per_pixel(surface);
@@ -1390,13 +1243,6 @@ static void vga_draw_text(VGACommonState *s, int full_update)
         s->cursor_visible_phase = !s->cursor_visible_phase;
     }
 
-    depth_index = get_depth_index(surface);
-    if (cw == 16)
-        vga_draw_glyph8 = vga_draw_glyph16_table[depth_index];
-    else
-        vga_draw_glyph8 = vga_draw_glyph8_table[depth_index];
-    vga_draw_glyph9 = vga_draw_glyph9_table[depth_index];
-
     dest = surface_data(surface);
     linesize = surface_stride(surface);
     ch_attr_ptr = s->last_ch_attr;
@@ -1426,7 +1272,10 @@ static void vga_draw_text(VGACommonState *s, int full_update)
                 font_ptr += 32 * 4 * ch;
                 bgcol = palette[cattr >> 4];
                 fgcol = palette[cattr & 0x0f];
-                if (cw != 9) {
+                if (cw == 16) {
+                    vga_draw_glyph16(d1, linesize,
+                                     font_ptr, cheight, fgcol, bgcol);
+                } else if (cw != 9) {
                     vga_draw_glyph8(d1, linesize,
                                     font_ptr, cheight, fgcol, bgcol);
                 } else {
@@ -1451,7 +1300,10 @@ static void vga_draw_text(VGACommonState *s, int full_update)
                     if (line_last >= line_start && line_start < cheight) {
                         h = line_last - line_start + 1;
                         d = d1 + linesize * line_start;
-                        if (cw != 9) {
+                        if (cw == 16) {
+                            vga_draw_glyph16(d, linesize,
+                                             cursor_glyph, h, fgcol, bgcol);
+                        } else if (cw != 9) {
                             vga_draw_glyph8(d, linesize,
                                             cursor_glyph, h, fgcol, bgcol);
                         } else {
@@ -1486,93 +1338,32 @@ enum {
     VGA_DRAW_LINE4D2,
     VGA_DRAW_LINE8D2,
     VGA_DRAW_LINE8,
-    VGA_DRAW_LINE15,
-    VGA_DRAW_LINE16,
-    VGA_DRAW_LINE24,
-    VGA_DRAW_LINE32,
+    VGA_DRAW_LINE15_LE,
+    VGA_DRAW_LINE16_LE,
+    VGA_DRAW_LINE24_LE,
+    VGA_DRAW_LINE32_LE,
+    VGA_DRAW_LINE15_BE,
+    VGA_DRAW_LINE16_BE,
+    VGA_DRAW_LINE24_BE,
+    VGA_DRAW_LINE32_BE,
     VGA_DRAW_LINE_NB,
 };
 
-static vga_draw_line_func * const vga_draw_line_table[NB_DEPTHS * VGA_DRAW_LINE_NB] = {
-    vga_draw_line2_8,
-    vga_draw_line2_16,
-    vga_draw_line2_16,
-    vga_draw_line2_32,
-    vga_draw_line2_32,
-    vga_draw_line2_16,
-    vga_draw_line2_16,
-
-    vga_draw_line2d2_8,
-    vga_draw_line2d2_16,
-    vga_draw_line2d2_16,
-    vga_draw_line2d2_32,
-    vga_draw_line2d2_32,
-    vga_draw_line2d2_16,
-    vga_draw_line2d2_16,
-
-    vga_draw_line4_8,
-    vga_draw_line4_16,
-    vga_draw_line4_16,
-    vga_draw_line4_32,
-    vga_draw_line4_32,
-    vga_draw_line4_16,
-    vga_draw_line4_16,
-
-    vga_draw_line4d2_8,
-    vga_draw_line4d2_16,
-    vga_draw_line4d2_16,
-    vga_draw_line4d2_32,
-    vga_draw_line4d2_32,
-    vga_draw_line4d2_16,
-    vga_draw_line4d2_16,
-
-    vga_draw_line8d2_8,
-    vga_draw_line8d2_16,
-    vga_draw_line8d2_16,
-    vga_draw_line8d2_32,
-    vga_draw_line8d2_32,
-    vga_draw_line8d2_16,
-    vga_draw_line8d2_16,
-
-    vga_draw_line8_8,
-    vga_draw_line8_16,
-    vga_draw_line8_16,
-    vga_draw_line8_32,
-    vga_draw_line8_32,
-    vga_draw_line8_16,
-    vga_draw_line8_16,
-
-    vga_draw_line15_8,
-    vga_draw_line15_15,
-    vga_draw_line15_16,
-    vga_draw_line15_32,
-    vga_draw_line15_32bgr,
-    vga_draw_line15_15bgr,
-    vga_draw_line15_16bgr,
-
-    vga_draw_line16_8,
-    vga_draw_line16_15,
-    vga_draw_line16_16,
-    vga_draw_line16_32,
-    vga_draw_line16_32bgr,
-    vga_draw_line16_15bgr,
-    vga_draw_line16_16bgr,
-
-    vga_draw_line24_8,
-    vga_draw_line24_15,
-    vga_draw_line24_16,
-    vga_draw_line24_32,
-    vga_draw_line24_32bgr,
-    vga_draw_line24_15bgr,
-    vga_draw_line24_16bgr,
-
-    vga_draw_line32_8,
-    vga_draw_line32_15,
-    vga_draw_line32_16,
-    vga_draw_line32_32,
-    vga_draw_line32_32bgr,
-    vga_draw_line32_15bgr,
-    vga_draw_line32_16bgr,
+static vga_draw_line_func * const vga_draw_line_table[VGA_DRAW_LINE_NB] = {
+    vga_draw_line2,
+    vga_draw_line2d2,
+    vga_draw_line4,
+    vga_draw_line4d2,
+    vga_draw_line8d2,
+    vga_draw_line8,
+    vga_draw_line15_le,
+    vga_draw_line16_le,
+    vga_draw_line24_le,
+    vga_draw_line32_le,
+    vga_draw_line15_be,
+    vga_draw_line16_be,
+    vga_draw_line24_be,
+    vga_draw_line32_be,
 };
 
 static int vga_get_bpp(VGACommonState *s)
@@ -1644,11 +1435,11 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
     int disp_width, multi_scan, multi_run;
     uint8_t *d;
     uint32_t v, addr1, addr;
-    vga_draw_line_func *vga_draw_line;
-#if defined(HOST_WORDS_BIGENDIAN) == defined(TARGET_WORDS_BIGENDIAN)
-    static const bool byteswap = false;
+    vga_draw_line_func *vga_draw_line = NULL;
+#ifdef HOST_WORDS_BIGENDIAN
+    bool byteswap = !s->big_endian_fb;
 #else
-    static const bool byteswap = true;
+    bool byteswap = s->big_endian_fb;
 #endif
 
     full_update |= update_basic_params(s);
@@ -1691,11 +1482,14 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
     if (s->line_offset != s->last_line_offset ||
         disp_width != s->last_width ||
         height != s->last_height ||
-        s->last_depth != depth) {
+        s->last_depth != depth ||
+        s->last_byteswap != byteswap) {
         if (depth == 32 || (depth == 16 && !byteswap)) {
+            pixman_format_code_t format =
+                qemu_default_pixman_format(depth, !byteswap);
             surface = qemu_create_displaysurface_from(disp_width,
-                    height, depth, s->line_offset,
-                    s->vram_ptr + (s->start_addr * 4), byteswap);
+                    height, format, s->line_offset,
+                    s->vram_ptr + (s->start_addr * 4));
             dpy_gfx_replace_surface(s->con, surface);
         } else {
             qemu_console_resize(s->con, disp_width, height);
@@ -1707,19 +1501,19 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
         s->last_height = height;
         s->last_line_offset = s->line_offset;
         s->last_depth = depth;
+        s->last_byteswap = byteswap;
         full_update = 1;
     } else if (is_buffer_shared(surface) &&
                (full_update || surface_data(surface) != s->vram_ptr
                 + (s->start_addr * 4))) {
+        pixman_format_code_t format =
+            qemu_default_pixman_format(depth, !byteswap);
         surface = qemu_create_displaysurface_from(disp_width,
-                height, depth, s->line_offset,
-                s->vram_ptr + (s->start_addr * 4), byteswap);
+                height, format, s->line_offset,
+                s->vram_ptr + (s->start_addr * 4));
         dpy_gfx_replace_surface(s->con, surface);
     }
 
-    s->rgb_to_pixel =
-        rgb_to_pixel_dup_table[get_depth_index(surface)];
-
     if (shift_control == 0) {
         full_update |= update_palette16(s);
         if (s->sr[VGA_SEQ_CLOCK_MODE] & 8) {
@@ -1750,25 +1544,24 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
             bits = 8;
             break;
         case 15:
-            v = VGA_DRAW_LINE15;
+            v = s->big_endian_fb ? VGA_DRAW_LINE15_BE : VGA_DRAW_LINE15_LE;
             bits = 16;
             break;
         case 16:
-            v = VGA_DRAW_LINE16;
+            v = s->big_endian_fb ? VGA_DRAW_LINE16_BE : VGA_DRAW_LINE16_LE;
             bits = 16;
             break;
         case 24:
-            v = VGA_DRAW_LINE24;
+            v = s->big_endian_fb ? VGA_DRAW_LINE24_BE : VGA_DRAW_LINE24_LE;
             bits = 24;
             break;
         case 32:
-            v = VGA_DRAW_LINE32;
+            v = s->big_endian_fb ? VGA_DRAW_LINE32_BE : VGA_DRAW_LINE32_LE;
             bits = 32;
             break;
         }
     }
-    vga_draw_line = vga_draw_line_table[v * NB_DEPTHS +
-                                        get_depth_index(surface)];
+    vga_draw_line = vga_draw_line_table[v];
 
     if (!is_buffer_shared(surface) && s->cursor_invalidate) {
         s->cursor_invalidate(s);
@@ -1858,7 +1651,7 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
 static void vga_draw_blank(VGACommonState *s, int full_update)
 {
     DisplaySurface *surface = qemu_console_surface(s->con);
-    int i, w, val;
+    int i, w;
     uint8_t *d;
 
     if (!full_update)
@@ -1866,17 +1659,10 @@ static void vga_draw_blank(VGACommonState *s, int full_update)
     if (s->last_scr_width <= 0 || s->last_scr_height <= 0)
         return;
 
-    s->rgb_to_pixel =
-        rgb_to_pixel_dup_table[get_depth_index(surface)];
-    if (surface_bits_per_pixel(surface) == 8) {
-        val = s->rgb_to_pixel(0, 0, 0);
-    } else {
-        val = 0;
-    }
     w = s->last_scr_width * surface_bytes_per_pixel(surface);
     d = surface_data(surface);
     for(i = 0; i < s->last_scr_height; i++) {
-        memset(d, val, w);
+        memset(d, 0, w);
         d += surface_stride(surface);
     }
     dpy_gfx_update(s->con, 0, 0,
@@ -1979,6 +1765,7 @@ void vga_common_reset(VGACommonState *s)
     s->cursor_start = 0;
     s->cursor_end = 0;
     s->cursor_offset = 0;
+    s->big_endian_fb = s->default_endian_fb;
     memset(s->invalidated_y_table, '\0', sizeof(s->invalidated_y_table));
     memset(s->last_palette, '\0', sizeof(s->last_palette));
     memset(s->last_ch_attr, '\0', sizeof(s->last_ch_attr));
@@ -2210,6 +1997,28 @@ static int vga_common_post_load(void *opaque, int version_id)
     return 0;
 }
 
+static bool vga_endian_state_needed(void *opaque)
+{
+    VGACommonState *s = opaque;
+
+    /*
+     * Only send the endian state if it's different from the
+     * default one, thus ensuring backward compatibility for
+     * migration of the common case
+     */
+    return s->default_endian_fb != s->big_endian_fb;
+}
+
+const VMStateDescription vmstate_vga_endian = {
+    .name = "vga.endian",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_BOOL(big_endian_fb, VGACommonState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 const VMStateDescription vmstate_vga_common = {
     .name = "vga",
     .version_id = 2,
@@ -2246,6 +2055,14 @@ const VMStateDescription vmstate_vga_common = {
         VMSTATE_UINT32(vbe_line_offset, VGACommonState),
         VMSTATE_UINT32(vbe_bank_mask, VGACommonState),
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (VMStateSubsection []) {
+        {
+            .vmsd = &vmstate_vga_endian,
+            .needed = vga_endian_state_needed,
+        }, {
+            /* empty */
+        }
     }
 };
 
@@ -2289,9 +2106,13 @@ void vga_common_init(VGACommonState *s, Object *obj, bool global_vmstate)
         s->vram_size <<= 1;
     }
     s->vram_size_mb = s->vram_size >> 20;
+    if (!s->vbe_size) {
+        s->vbe_size = s->vram_size;
+    }
 
     s->is_vbe_vmstate = 1;
-    memory_region_init_ram(&s->vram, obj, "vga.vram", s->vram_size);
+    memory_region_init_ram(&s->vram, obj, "vga.vram", s->vram_size,
+                           &error_abort);
     vmstate_register_ram(&s->vram, global_vmstate ? NULL : DEVICE(obj));
     xen_register_framebuffer(&s->vram);
     s->vram_ptr = memory_region_get_ram_ptr(&s->vram);
@@ -2310,6 +2131,17 @@ void vga_common_init(VGACommonState *s, Object *obj, bool global_vmstate)
         s->update_retrace_info = vga_precise_update_retrace_info;
         break;
     }
+
+    /*
+     * Set default fb endian based on target, could probably be turned
+     * into a device attribute set by the machine/platform to remove
+     * all target endian dependencies from this file.
+     */
+#ifdef TARGET_WORDS_BIGENDIAN
+    s->default_endian_fb = true;
+#else
+    s->default_endian_fb = false;
+#endif
     vga_dirty_log_start(s);
 }
 
diff --git a/hw/display/vga_int.h b/hw/display/vga_int.h
index 5320abdc0..ed69e064a 100644
--- a/hw/display/vga_int.h
+++ b/hw/display/vga_int.h
@@ -93,8 +93,10 @@ typedef struct VGACommonState {
     MemoryRegion vram_vbe;
     uint32_t vram_size;
     uint32_t vram_size_mb; /* property */
+    uint32_t vbe_size;
     uint32_t latch;
-    MemoryRegion *chain4_alias;
+    bool has_chain4_alias;
+    MemoryRegion chain4_alias;
     uint8_t sr_index;
     uint8_t sr[256];
     uint8_t gr_index;
@@ -148,15 +150,16 @@ typedef struct VGACommonState {
     uint32_t last_width, last_height; /* in chars or pixels */
     uint32_t last_scr_width, last_scr_height; /* in pixels */
     uint32_t last_depth; /* in bits */
+    bool last_byteswap;
     uint8_t cursor_start, cursor_end;
     bool cursor_visible_phase;
     int64_t cursor_blink_time;
     uint32_t cursor_offset;
-    unsigned int (*rgb_to_pixel)(unsigned int r,
-                                 unsigned int g, unsigned b);
     const GraphicHwOps *hw_ops;
     bool full_update_text;
     bool full_update_gfx;
+    bool big_endian_fb;
+    bool default_endian_fb;
     /* hardware mouse cursor support */
     uint32_t invalidated_y_table[VGA_MAX_HEIGHT / 32];
     void (*cursor_invalidate)(struct VGACommonState *s);
diff --git a/hw/display/vmware_vga.c b/hw/display/vmware_vga.c
index 591b64543..1751f1979 100644
--- a/hw/display/vmware_vga.c
+++ b/hw/display/vmware_vga.c
@@ -292,47 +292,74 @@ enum {
     SVGA_CURSOR_ON_RESTORE_TO_FB = 3,
 };
 
-static inline void vmsvga_update_rect(struct vmsvga_state_s *s,
-                int x, int y, int w, int h)
+static inline bool vmsvga_verify_rect(DisplaySurface *surface,
+                                      const char *name,
+                                      int x, int y, int w, int h)
 {
-    DisplaySurface *surface = qemu_console_surface(s->vga.con);
-    int line;
-    int bypl;
-    int width;
-    int start;
-    uint8_t *src;
-    uint8_t *dst;
-
     if (x < 0) {
-        fprintf(stderr, "%s: update x was < 0 (%d)\n", __func__, x);
-        w += x;
-        x = 0;
+        fprintf(stderr, "%s: x was < 0 (%d)\n", name, x);
+        return false;
+    }
+    if (x > SVGA_MAX_WIDTH) {
+        fprintf(stderr, "%s: x was > %d (%d)\n", name, SVGA_MAX_WIDTH, x);
+        return false;
     }
     if (w < 0) {
-        fprintf(stderr, "%s: update w was < 0 (%d)\n", __func__, w);
-        w = 0;
+        fprintf(stderr, "%s: w was < 0 (%d)\n", name, w);
+        return false;
+    }
+    if (w > SVGA_MAX_WIDTH) {
+        fprintf(stderr, "%s: w was > %d (%d)\n", name, SVGA_MAX_WIDTH, w);
+        return false;
     }
     if (x + w > surface_width(surface)) {
-        fprintf(stderr, "%s: update width too large x: %d, w: %d\n",
-                __func__, x, w);
-        x = MIN(x, surface_width(surface));
-        w = surface_width(surface) - x;
+        fprintf(stderr, "%s: width was > %d (x: %d, w: %d)\n",
+                name, surface_width(surface), x, w);
+        return false;
     }
 
     if (y < 0) {
-        fprintf(stderr, "%s: update y was < 0 (%d)\n",  __func__, y);
-        h += y;
-        y = 0;
+        fprintf(stderr, "%s: y was < 0 (%d)\n", name, y);
+        return false;
+    }
+    if (y > SVGA_MAX_HEIGHT) {
+        fprintf(stderr, "%s: y was > %d (%d)\n", name, SVGA_MAX_HEIGHT, y);
+        return false;
     }
     if (h < 0) {
-        fprintf(stderr, "%s: update h was < 0 (%d)\n",  __func__, h);
-        h = 0;
+        fprintf(stderr, "%s: h was < 0 (%d)\n", name, h);
+        return false;
+    }
+    if (h > SVGA_MAX_HEIGHT) {
+        fprintf(stderr, "%s: h was > %d (%d)\n", name, SVGA_MAX_HEIGHT, h);
+        return false;
     }
     if (y + h > surface_height(surface)) {
-        fprintf(stderr, "%s: update height too large y: %d, h: %d\n",
-                __func__, y, h);
-        y = MIN(y, surface_height(surface));
-        h = surface_height(surface) - y;
+        fprintf(stderr, "%s: update height > %d (y: %d, h: %d)\n",
+                name, surface_height(surface), y, h);
+        return false;
+    }
+
+    return true;
+}
+
+static inline void vmsvga_update_rect(struct vmsvga_state_s *s,
+                                      int x, int y, int w, int h)
+{
+    DisplaySurface *surface = qemu_console_surface(s->vga.con);
+    int line;
+    int bypl;
+    int width;
+    int start;
+    uint8_t *src;
+    uint8_t *dst;
+
+    if (!vmsvga_verify_rect(surface, __func__, x, y, w, h)) {
+        /* go for a fullscreen update as fallback */
+        x = 0;
+        y = 0;
+        w = surface_width(surface);
+        h = surface_height(surface);
     }
 
     bypl = surface_stride(surface);
@@ -377,7 +404,7 @@ static inline void vmsvga_update_rect_flush(struct vmsvga_state_s *s)
 }
 
 #ifdef HW_RECT_ACCEL
-static inline void vmsvga_copy_rect(struct vmsvga_state_s *s,
+static inline int vmsvga_copy_rect(struct vmsvga_state_s *s,
                 int x0, int y0, int x1, int y1, int w, int h)
 {
     DisplaySurface *surface = qemu_console_surface(s->vga.con);
@@ -388,6 +415,13 @@ static inline void vmsvga_copy_rect(struct vmsvga_state_s *s,
     int line = h;
     uint8_t *ptr[2];
 
+    if (!vmsvga_verify_rect(surface, "vmsvga_copy_rect/src", x0, y0, w, h)) {
+        return -1;
+    }
+    if (!vmsvga_verify_rect(surface, "vmsvga_copy_rect/dst", x1, y1, w, h)) {
+        return -1;
+    }
+
     if (y1 > y0) {
         ptr[0] = vram + bypp * x0 + bypl * (y0 + h - 1);
         ptr[1] = vram + bypp * x1 + bypl * (y1 + h - 1);
@@ -403,11 +437,12 @@ static inline void vmsvga_copy_rect(struct vmsvga_state_s *s,
     }
 
     vmsvga_update_rect_delayed(s, x1, y1, w, h);
+    return 0;
 }
 #endif
 
 #ifdef HW_FILL_ACCEL
-static inline void vmsvga_fill_rect(struct vmsvga_state_s *s,
+static inline int vmsvga_fill_rect(struct vmsvga_state_s *s,
                 uint32_t c, int x, int y, int w, int h)
 {
     DisplaySurface *surface = qemu_console_surface(s->vga.con);
@@ -420,6 +455,10 @@ static inline void vmsvga_fill_rect(struct vmsvga_state_s *s,
     uint8_t *src;
     uint8_t col[4];
 
+    if (!vmsvga_verify_rect(surface, __func__, x, y, w, h)) {
+        return -1;
+    }
+
     col[0] = c;
     col[1] = c >> 8;
     col[2] = c >> 16;
@@ -444,6 +483,7 @@ static inline void vmsvga_fill_rect(struct vmsvga_state_s *s,
     }
 
     vmsvga_update_rect_delayed(s, x, y, w, h);
+    return 0;
 }
 #endif
 
@@ -576,12 +616,12 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
             width = vmsvga_fifo_read(s);
             height = vmsvga_fifo_read(s);
 #ifdef HW_FILL_ACCEL
-            vmsvga_fill_rect(s, colour, x, y, width, height);
-            break;
-#else
+            if (vmsvga_fill_rect(s, colour, x, y, width, height) == 0) {
+                break;
+            }
+#endif
             args = 0;
             goto badcmd;
-#endif
 
         case SVGA_CMD_RECT_COPY:
             len -= 7;
@@ -596,12 +636,12 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
             width = vmsvga_fifo_read(s);
             height = vmsvga_fifo_read(s);
 #ifdef HW_RECT_ACCEL
-            vmsvga_copy_rect(s, x, y, dx, dy, width, height);
-            break;
-#else
+            if (vmsvga_copy_rect(s, x, y, dx, dy, width, height) == 0) {
+                break;
+            }
+#endif
             args = 0;
             goto badcmd;
-#endif
 
         case SVGA_CMD_DEFINE_CURSOR:
             len -= 8;
@@ -1052,10 +1092,12 @@ static inline void vmsvga_check_size(struct vmsvga_state_s *s)
         s->new_height != surface_height(surface) ||
         s->new_depth != surface_bits_per_pixel(surface)) {
         int stride = (s->new_depth * s->new_width) / 8;
+        pixman_format_code_t format =
+            qemu_default_pixman_format(s->new_depth, true);
         trace_vmware_setmode(s->new_width, s->new_height, s->new_depth);
         surface = qemu_create_displaysurface_from(s->new_width, s->new_height,
-                                                  s->new_depth, stride,
-                                                  s->vga.vram_ptr, false);
+                                                  format, stride,
+                                                  s->vga.vram_ptr);
         dpy_gfx_replace_surface(s->vga.con, surface);
         s->invalidated = 1;
     }
@@ -1201,7 +1243,8 @@ static void vmsvga_init(DeviceState *dev, struct vmsvga_state_s *s,
     s->vga.con = graphic_console_init(dev, 0, &vmsvga_ops, s);
 
     s->fifo_size = SVGA_FIFO_SIZE;
-    memory_region_init_ram(&s->fifo_ram, NULL, "vmsvga.fifo", s->fifo_size);
+    memory_region_init_ram(&s->fifo_ram, NULL, "vmsvga.fifo", s->fifo_size,
+                           &error_abort);
     vmstate_register_ram_global(&s->fifo_ram);
     s->fifo_ptr = memory_region_get_ram_ptr(&s->fifo_ram);
 
diff --git a/hw/display/xenfb.c b/hw/display/xenfb.c
index 07ddc9deb..8a61e959a 100644
--- a/hw/display/xenfb.c
+++ b/hw/display/xenfb.c
@@ -713,15 +713,17 @@ static void xenfb_update(void *opaque)
 
     /* resize if needed */
     if (xenfb->do_resize) {
+        pixman_format_code_t format;
+
         xenfb->do_resize = 0;
         switch (xenfb->depth) {
         case 16:
         case 32:
             /* console.c supported depth -> buffer can be used directly */
+            format = qemu_default_pixman_format(xenfb->depth, true);
             surface = qemu_create_displaysurface_from
-                (xenfb->width, xenfb->height, xenfb->depth,
-                 xenfb->row_stride, xenfb->pixels + xenfb->offset,
-                 false);
+                (xenfb->width, xenfb->height, format,
+                 xenfb->row_stride, xenfb->pixels + xenfb->offset);
             break;
         default:
             /* we must convert stuff */
diff --git a/hw/dma/i8257.c b/hw/dma/i8257.c
index dd370ed7e..a414029be 100644
--- a/hw/dma/i8257.c
+++ b/hw/dma/i8257.c
@@ -24,6 +24,7 @@
 #include "hw/hw.h"
 #include "hw/isa/isa.h"
 #include "qemu/main-loop.h"
+#include "trace.h"
 
 /* #define DEBUG_DMA */
 
@@ -473,8 +474,7 @@ static void dma_reset(void *opaque)
 
 static int dma_phony_handler (void *opaque, int nchan, int dma_pos, int dma_len)
 {
-    dolog ("unregistered DMA channel used nchan=%d dma_pos=%d dma_len=%d\n",
-           nchan, dma_pos, dma_len);
+    trace_i8257_unregistered_dma(nchan, dma_pos, dma_len);
     return dma_pos;
 }
 
diff --git a/hw/dma/xilinx_axidma.c b/hw/dma/xilinx_axidma.c
index ee60d3ff3..d06002dde 100644
--- a/hw/dma/xilinx_axidma.c
+++ b/hw/dma/xilinx_axidma.c
@@ -553,10 +553,12 @@ static void xilinx_axidma_realize(DeviceState *dev, Error **errp)
     int i;
 
     for (i = 0; i < 2; i++) {
-        s->streams[i].nr = i;
-        s->streams[i].bh = qemu_bh_new(timer_hit, &s->streams[i]);
-        s->streams[i].ptimer = ptimer_init(s->streams[i].bh);
-        ptimer_set_freq(s->streams[i].ptimer, s->freqhz);
+        struct Stream *st = &s->streams[i];
+
+        st->nr = i;
+        st->bh = qemu_bh_new(timer_hit, st);
+        st->ptimer = ptimer_init(st->bh);
+        ptimer_set_freq(st->ptimer, s->freqhz);
     }
     return;
 
diff --git a/hw/gpio/Makefile.objs b/hw/gpio/Makefile.objs
index 2c8b51f09..1abcf1798 100644
--- a/hw/gpio/Makefile.objs
+++ b/hw/gpio/Makefile.objs
@@ -2,5 +2,6 @@ common-obj-$(CONFIG_MAX7310) += max7310.o
 common-obj-$(CONFIG_PL061) += pl061.o
 common-obj-$(CONFIG_PUV3) += puv3_gpio.o
 common-obj-$(CONFIG_ZAURUS) += zaurus.o
+common-obj-$(CONFIG_E500) += mpc8xxx.o
 
 obj-$(CONFIG_OMAP) += omap_gpio.o
diff --git a/hw/gpio/mpc8xxx.c b/hw/gpio/mpc8xxx.c
new file mode 100644
index 000000000..1aeaaaaf0
--- /dev/null
+++ b/hw/gpio/mpc8xxx.c
@@ -0,0 +1,217 @@
+/*
+ *  GPIO Controller for a lot of Freescale SoCs
+ *
+ * Copyright (C) 2014 Freescale Semiconductor, Inc. All rights reserved.
+ *
+ * Author: Alexander Graf, <agraf@suse.de>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/sysbus.h"
+
+#define TYPE_MPC8XXX_GPIO "mpc8xxx_gpio"
+#define MPC8XXX_GPIO(obj) OBJECT_CHECK(MPC8XXXGPIOState, (obj), TYPE_MPC8XXX_GPIO)
+
+typedef struct MPC8XXXGPIOState {
+    SysBusDevice parent_obj;
+
+    MemoryRegion iomem;
+    qemu_irq irq;
+    qemu_irq out[32];
+
+    uint32_t dir;
+    uint32_t odr;
+    uint32_t dat;
+    uint32_t ier;
+    uint32_t imr;
+    uint32_t icr;
+} MPC8XXXGPIOState;
+
+static const VMStateDescription vmstate_mpc8xxx_gpio = {
+    .name = "mpc8xxx_gpio",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(dir, MPC8XXXGPIOState),
+        VMSTATE_UINT32(odr, MPC8XXXGPIOState),
+        VMSTATE_UINT32(dat, MPC8XXXGPIOState),
+        VMSTATE_UINT32(ier, MPC8XXXGPIOState),
+        VMSTATE_UINT32(imr, MPC8XXXGPIOState),
+        VMSTATE_UINT32(icr, MPC8XXXGPIOState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void mpc8xxx_gpio_update(MPC8XXXGPIOState *s)
+{
+    qemu_set_irq(s->irq, !!(s->ier & s->imr));
+}
+
+static uint64_t mpc8xxx_gpio_read(void *opaque, hwaddr offset,
+                                  unsigned size)
+{
+    MPC8XXXGPIOState *s = (MPC8XXXGPIOState *)opaque;
+
+    if (size != 4) {
+        /* All registers are 32bit */
+        return 0;
+    }
+
+    switch (offset) {
+    case 0x0: /* Direction */
+        return s->dir;
+    case 0x4: /* Open Drain */
+        return s->odr;
+    case 0x8: /* Data */
+        return s->dat;
+    case 0xC: /* Interrupt Event */
+        return s->ier;
+    case 0x10: /* Interrupt Mask */
+        return s->imr;
+    case 0x14: /* Interrupt Control */
+        return s->icr;
+    default:
+        return 0;
+    }
+}
+
+static void mpc8xxx_write_data(MPC8XXXGPIOState *s, uint32_t new_data)
+{
+    uint32_t old_data = s->dat;
+    uint32_t diff = old_data ^ new_data;
+    int i;
+
+    for (i = 0; i < 32; i++) {
+        uint32_t mask = 0x80000000 >> i;
+        if (!(diff & mask)) {
+            continue;
+        }
+
+        if (s->dir & mask) {
+            /* Output */
+            qemu_set_irq(s->out[i], (new_data & mask) != 0);
+        }
+    }
+
+    s->dat = new_data;
+}
+
+static void mpc8xxx_gpio_write(void *opaque, hwaddr offset,
+                        uint64_t value, unsigned size)
+{
+    MPC8XXXGPIOState *s = (MPC8XXXGPIOState *)opaque;
+
+    if (size != 4) {
+        /* All registers are 32bit */
+        return;
+    }
+
+    switch (offset) {
+    case 0x0: /* Direction */
+        s->dir = value;
+        break;
+    case 0x4: /* Open Drain */
+        s->odr = value;
+        break;
+    case 0x8: /* Data */
+        mpc8xxx_write_data(s, value);
+        break;
+    case 0xC: /* Interrupt Event */
+        s->ier &= ~value;
+        break;
+    case 0x10: /* Interrupt Mask */
+        s->imr = value;
+        break;
+    case 0x14: /* Interrupt Control */
+        s->icr = value;
+        break;
+    }
+
+    mpc8xxx_gpio_update(s);
+}
+
+static void mpc8xxx_gpio_reset(MPC8XXXGPIOState *s)
+{
+    s->dir = 0;
+    s->odr = 0;
+    s->dat = 0;
+    s->ier = 0;
+    s->imr = 0;
+    s->icr = 0;
+}
+
+static void mpc8xxx_gpio_set_irq(void * opaque, int irq, int level)
+{
+    MPC8XXXGPIOState *s = (MPC8XXXGPIOState *)opaque;
+    uint32_t mask;
+
+    mask = 0x80000000 >> irq;
+    if ((s->dir & mask) == 0) {
+        uint32_t old_value = s->dat & mask;
+
+        s->dat &= ~mask;
+        if (level)
+            s->dat |= mask;
+
+        if (!(s->icr & irq) || (old_value && !level)) {
+            s->ier |= mask;
+        }
+
+        mpc8xxx_gpio_update(s);
+    }
+}
+
+static const MemoryRegionOps mpc8xxx_gpio_ops = {
+    .read = mpc8xxx_gpio_read,
+    .write = mpc8xxx_gpio_write,
+    .endianness = DEVICE_BIG_ENDIAN,
+};
+
+static int mpc8xxx_gpio_initfn(SysBusDevice *sbd)
+{
+    DeviceState *dev = DEVICE(sbd);
+    MPC8XXXGPIOState *s = MPC8XXX_GPIO(dev);
+
+    memory_region_init_io(&s->iomem, OBJECT(s), &mpc8xxx_gpio_ops, s, "mpc8xxx_gpio", 0x1000);
+    sysbus_init_mmio(sbd, &s->iomem);
+    sysbus_init_irq(sbd, &s->irq);
+    qdev_init_gpio_in(dev, mpc8xxx_gpio_set_irq, 32);
+    qdev_init_gpio_out(dev, s->out, 32);
+    mpc8xxx_gpio_reset(s);
+    return 0;
+}
+
+static void mpc8xxx_gpio_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+
+    k->init = mpc8xxx_gpio_initfn;
+    dc->vmsd = &vmstate_mpc8xxx_gpio;
+}
+
+static const TypeInfo mpc8xxx_gpio_info = {
+    .name          = TYPE_MPC8XXX_GPIO,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(MPC8XXXGPIOState),
+    .class_init    = mpc8xxx_gpio_class_init,
+};
+
+static void mpc8xxx_gpio_register_types(void)
+{
+    type_register_static(&mpc8xxx_gpio_info);
+}
+
+type_init(mpc8xxx_gpio_register_types)
diff --git a/hw/gpio/pl061.c b/hw/gpio/pl061.c
index dd4ea293e..bd03e9997 100644
--- a/hw/gpio/pl061.c
+++ b/hw/gpio/pl061.c
@@ -37,7 +37,8 @@ typedef struct PL061State {
     MemoryRegion iomem;
     uint32_t locked;
     uint32_t data;
-    uint32_t old_data;
+    uint32_t old_out_data;
+    uint32_t old_in_data;
     uint32_t dir;
     uint32_t isense;
     uint32_t ibe;
@@ -63,12 +64,13 @@ typedef struct PL061State {
 
 static const VMStateDescription vmstate_pl061 = {
     .name = "pl061",
-    .version_id = 2,
-    .minimum_version_id = 1,
+    .version_id = 3,
+    .minimum_version_id = 3,
     .fields = (VMStateField[]) {
         VMSTATE_UINT32(locked, PL061State),
         VMSTATE_UINT32(data, PL061State),
-        VMSTATE_UINT32(old_data, PL061State),
+        VMSTATE_UINT32(old_out_data, PL061State),
+        VMSTATE_UINT32(old_in_data, PL061State),
         VMSTATE_UINT32(dir, PL061State),
         VMSTATE_UINT32(isense, PL061State),
         VMSTATE_UINT32(ibe, PL061State),
@@ -98,23 +100,52 @@ static void pl061_update(PL061State *s)
     uint8_t out;
     int i;
 
+    DPRINTF("dir = %d, data = %d\n", s->dir, s->data);
+
     /* Outputs float high.  */
     /* FIXME: This is board dependent.  */
     out = (s->data & s->dir) | ~s->dir;
-    changed = s->old_data ^ out;
-    if (!changed)
-        return;
+    changed = s->old_out_data ^ out;
+    if (changed) {
+        s->old_out_data = out;
+        for (i = 0; i < 8; i++) {
+            mask = 1 << i;
+            if (changed & mask) {
+                DPRINTF("Set output %d = %d\n", i, (out & mask) != 0);
+                qemu_set_irq(s->out[i], (out & mask) != 0);
+            }
+        }
+    }
 
-    s->old_data = out;
-    for (i = 0; i < 8; i++) {
-        mask = 1 << i;
-        if (changed & mask) {
-            DPRINTF("Set output %d = %d\n", i, (out & mask) != 0);
-            qemu_set_irq(s->out[i], (out & mask) != 0);
+    /* Inputs */
+    changed = (s->old_in_data ^ s->data) & ~s->dir;
+    if (changed) {
+        s->old_in_data = s->data;
+        for (i = 0; i < 8; i++) {
+            mask = 1 << i;
+            if (changed & mask) {
+                DPRINTF("Changed input %d = %d\n", i, (s->data & mask) != 0);
+
+                if (!(s->isense & mask)) {
+                    /* Edge interrupt */
+                    if (s->ibe & mask) {
+                        /* Any edge triggers the interrupt */
+                        s->istate |= mask;
+                    } else {
+                        /* Edge is selected by IEV */
+                        s->istate |= ~(s->data ^ s->iev) & mask;
+                    }
+                }
+            }
         }
     }
 
-    /* FIXME: Implement input interrupts.  */
+    /* Level interrupt */
+    s->istate |= ~(s->data ^ s->iev) & s->isense;
+
+    DPRINTF("istate = %02X\n", s->istate);
+
+    qemu_set_irq(s->irq, (s->istate & s->im) != 0);
 }
 
 static uint64_t pl061_read(void *opaque, hwaddr offset,
diff --git a/hw/i2c/pm_smbus.c b/hw/i2c/pm_smbus.c
index fedb5fb4d..ce1713d26 100644
--- a/hw/i2c/pm_smbus.c
+++ b/hw/i2c/pm_smbus.c
@@ -139,7 +139,8 @@ static void smb_ioport_writeb(void *opaque, hwaddr addr, uint64_t val,
 {
     PMSMBus *s = opaque;
 
-    SMBUS_DPRINTF("SMB writeb port=0x%04x val=0x%02x\n", addr, val);
+    SMBUS_DPRINTF("SMB writeb port=0x%04" HWADDR_PRIx
+                  " val=0x%02" PRIx64 "\n", addr, val);
     switch(addr) {
     case SMBHSTSTS:
         s->smb_stat = (~(val & 0xff)) & s->smb_stat;
@@ -206,7 +207,7 @@ static uint64_t smb_ioport_readb(void *opaque, hwaddr addr, unsigned width)
         val = 0;
         break;
     }
-    SMBUS_DPRINTF("SMB readb port=0x%04x val=0x%02x\n", addr, val);
+    SMBUS_DPRINTF("SMB readb port=0x%04" HWADDR_PRIx " val=0x%02x\n", addr, val);
     return val;
 }
 
diff --git a/hw/i386/Makefile.objs b/hw/i386/Makefile.objs
index 48014abf0..9d419addb 100644
--- a/hw/i386/Makefile.objs
+++ b/hw/i386/Makefile.objs
@@ -2,6 +2,7 @@ obj-$(CONFIG_KVM) += kvm/
 obj-y += multiboot.o smbios.o
 obj-y += pc.o pc_piix.o pc_q35.o
 obj-y += pc_sysfw.o
+obj-y += intel_iommu.o
 obj-$(CONFIG_XEN) += ../xenpv/ xen/
 
 obj-y += kvmvapic.o
@@ -10,7 +11,8 @@ obj-y += bios-linker-loader.o
 hw/i386/acpi-build.o: hw/i386/acpi-build.c hw/i386/acpi-dsdt.hex \
 	hw/i386/ssdt-proc.hex hw/i386/ssdt-pcihp.hex hw/i386/ssdt-misc.hex \
 	hw/i386/acpi-dsdt.hex hw/i386/q35-acpi-dsdt.hex \
-	hw/i386/q35-acpi-dsdt.hex hw/i386/ssdt-mem.hex
+	hw/i386/q35-acpi-dsdt.hex hw/i386/ssdt-mem.hex \
+	hw/i386/ssdt-tpm.hex
 
 iasl-option=$(shell if test -z "`$(1) $(2) 2>&1 > /dev/null`" \
     ; then echo "$(2)"; else echo "$(3)"; fi ;)
diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c
index 816c6d9b4..b37a39782 100644
--- a/hw/i386/acpi-build.c
+++ b/hw/i386/acpi-build.c
@@ -40,6 +40,8 @@
 #include "hw/loader.h"
 #include "hw/isa/isa.h"
 #include "hw/acpi/memory_hotplug.h"
+#include "sysemu/tpm.h"
+#include "hw/acpi/tpm.h"
 
 /* Supported chipsets: */
 #include "hw/acpi/piix4.h"
@@ -47,12 +49,14 @@
 #include "hw/i386/ich9.h"
 #include "hw/pci/pci_bus.h"
 #include "hw/pci-host/q35.h"
+#include "hw/i386/intel_iommu.h"
 
 #include "hw/i386/q35-acpi-dsdt.hex"
 #include "hw/i386/acpi-dsdt.hex"
 
 #include "qapi/qmp/qint.h"
 #include "qom/qom-qobject.h"
+#include "exec/ram_addr.h"
 
 /* These are used to size the ACPI tables for -M pc-i440fx-1.7 and
  * -M pc-i440fx-2.0.  Even if the actual amount of AML generated grows
@@ -88,6 +92,7 @@ typedef struct AcpiPmInfo {
 
 typedef struct AcpiMiscInfo {
     bool has_hpet;
+    bool has_tpm;
     DECLARE_BITMAP(slot_hotplug_enable, PCI_SLOT_MAX);
     const unsigned char *dsdt_code;
     unsigned dsdt_size;
@@ -210,6 +215,7 @@ static void acpi_get_pm_info(AcpiPmInfo *pm)
 static void acpi_get_misc_info(AcpiMiscInfo *info)
 {
     info->has_hpet = hpet_find();
+    info->has_tpm = tpm_find();
     info->pvpanic_port = pvpanic_port();
 }
 
@@ -244,6 +250,7 @@ static void acpi_get_pci_info(PcPciInfo *info)
 
 #define ACPI_BUILD_TABLE_FILE "etc/acpi/tables"
 #define ACPI_BUILD_RSDP_FILE "etc/acpi/rsdp"
+#define ACPI_BUILD_TPMLOG_FILE "etc/tpm/log"
 
 static void
 build_header(GArray *linker, GArray *table_data,
@@ -546,6 +553,12 @@ static void fadt_setup(AcpiFadtDescriptorRev1 *fadt, AcpiPmInfo *pm)
                               (1 << ACPI_FADT_F_SLP_BUTTON) |
                               (1 << ACPI_FADT_F_RTC_S4));
     fadt->flags |= cpu_to_le32(1 << ACPI_FADT_F_USE_PLATFORM_CLOCK);
+    /* APIC destination mode ("Flat Logical") has an upper limit of 8 CPUs
+     * For more than 8 CPUs, "Clustered Logical" mode has to be used
+     */
+    if (max_cpus > 8) {
+        fadt->flags |= cpu_to_le32(1 << ACPI_FADT_F_FORCE_APIC_CLUSTER_MODEL);
+    }
 }
 
 
@@ -698,6 +711,7 @@ static inline char acpi_get_hex(uint32_t val)
 
 #include "hw/i386/ssdt-misc.hex"
 #include "hw/i386/ssdt-pcihp.hex"
+#include "hw/i386/ssdt-tpm.hex"
 
 static void
 build_append_notify_method(GArray *device, const char *name,
@@ -762,7 +776,7 @@ static void *acpi_set_bsel(PCIBus *bus, void *opaque)
     unsigned *bsel_alloc = opaque;
     unsigned *bus_bsel;
 
-    if (bus->qbus.allow_hotplug) {
+    if (qbus_is_hotpluggable(BUS(bus))) {
         bus_bsel = g_malloc(sizeof *bus_bsel);
 
         *bus_bsel = (*bsel_alloc)++;
@@ -1201,6 +1215,40 @@ build_hpet(GArray *table_data, GArray *linker)
                  (void *)hpet, "HPET", sizeof(*hpet), 1);
 }
 
+static void
+build_tpm_tcpa(GArray *table_data, GArray *linker, GArray *tcpalog)
+{
+    Acpi20Tcpa *tcpa = acpi_data_push(table_data, sizeof *tcpa);
+    uint64_t log_area_start_address = acpi_data_len(tcpalog);
+
+    tcpa->platform_class = cpu_to_le16(TPM_TCPA_ACPI_CLASS_CLIENT);
+    tcpa->log_area_minimum_length = cpu_to_le32(TPM_LOG_AREA_MINIMUM_SIZE);
+    tcpa->log_area_start_address = cpu_to_le64(log_area_start_address);
+
+    bios_linker_loader_alloc(linker, ACPI_BUILD_TPMLOG_FILE, 1,
+                             false /* high memory */);
+
+    /* log area start address to be filled by Guest linker */
+    bios_linker_loader_add_pointer(linker, ACPI_BUILD_TABLE_FILE,
+                                   ACPI_BUILD_TPMLOG_FILE,
+                                   table_data, &tcpa->log_area_start_address,
+                                   sizeof(tcpa->log_area_start_address));
+
+    build_header(linker, table_data,
+                 (void *)tcpa, "TCPA", sizeof(*tcpa), 2);
+
+    acpi_data_push(tcpalog, TPM_LOG_AREA_MINIMUM_SIZE);
+}
+
+static void
+build_tpm_ssdt(GArray *table_data, GArray *linker)
+{
+    void *tpm_ptr;
+
+    tpm_ptr = acpi_data_push(table_data, sizeof(ssdt_tpm_aml));
+    memcpy(tpm_ptr, ssdt_tpm_aml, sizeof(ssdt_tpm_aml));
+}
+
 typedef enum {
     MEM_AFFINITY_NOFLAGS      = 0,
     MEM_AFFINITY_ENABLED      = (1 << 0),
@@ -1222,8 +1270,7 @@ acpi_build_srat_memory(AcpiSratMemoryAffinity *numamem, uint64_t base,
 }
 
 static void
-build_srat(GArray *table_data, GArray *linker,
-           AcpiCpuInfo *cpu, PcGuestInfo *guest_info)
+build_srat(GArray *table_data, GArray *linker, PcGuestInfo *guest_info)
 {
     AcpiSystemResourceAffinityTable *srat;
     AcpiSratProcessorAffinity *core;
@@ -1253,11 +1300,7 @@ build_srat(GArray *table_data, GArray *linker,
         core->proximity_lo = curnode;
         memset(core->proximity_hi, 0, 3);
         core->local_sapic_eid = 0;
-        if (test_bit(i, cpu->found_cpus)) {
-            core->flags = cpu_to_le32(1);
-        } else {
-            core->flags = cpu_to_le32(0);
-        }
+        core->flags = cpu_to_le32(1);
     }
 
 
@@ -1350,6 +1393,30 @@ build_mcfg_q35(GArray *table_data, GArray *linker, AcpiMcfgInfo *info)
 }
 
 static void
+build_dmar_q35(GArray *table_data, GArray *linker)
+{
+    int dmar_start = table_data->len;
+
+    AcpiTableDmar *dmar;
+    AcpiDmarHardwareUnit *drhd;
+
+    dmar = acpi_data_push(table_data, sizeof(*dmar));
+    dmar->host_address_width = VTD_HOST_ADDRESS_WIDTH - 1;
+    dmar->flags = 0;    /* No intr_remap for now */
+
+    /* DMAR Remapping Hardware Unit Definition structure */
+    drhd = acpi_data_push(table_data, sizeof(*drhd));
+    drhd->type = cpu_to_le16(ACPI_DMAR_TYPE_HARDWARE_UNIT);
+    drhd->length = cpu_to_le16(sizeof(*drhd));   /* No device scope now */
+    drhd->flags = ACPI_DMAR_INCLUDE_PCI_ALL;
+    drhd->pci_segment = cpu_to_le16(0);
+    drhd->address = cpu_to_le64(Q35_HOST_BRIDGE_IOMMU_ADDR);
+
+    build_header(linker, table_data, (void *)(table_data->data + dmar_start),
+                 "DMAR", table_data->len - dmar_start, 1);
+}
+
+static void
 build_dsdt(GArray *table_data, GArray *linker, AcpiMiscInfo *misc)
 {
     AcpiTableHeader *dsdt;
@@ -1393,7 +1460,7 @@ build_rsdp(GArray *rsdp_table, GArray *linker, unsigned rsdt)
 {
     AcpiRsdpDescriptor *rsdp = acpi_data_push(rsdp_table, sizeof *rsdp);
 
-    bios_linker_loader_alloc(linker, ACPI_BUILD_RSDP_FILE, 1,
+    bios_linker_loader_alloc(linker, ACPI_BUILD_RSDP_FILE, 16,
                              true /* fseg memory */);
 
     memcpy(&rsdp->signature, "RSD PTR ", 8);
@@ -1416,6 +1483,7 @@ typedef
 struct AcpiBuildTables {
     GArray *table_data;
     GArray *rsdp;
+    GArray *tcpalog;
     GArray *linker;
 } AcpiBuildTables;
 
@@ -1423,23 +1491,23 @@ static inline void acpi_build_tables_init(AcpiBuildTables *tables)
 {
     tables->rsdp = g_array_new(false, true /* clear */, 1);
     tables->table_data = g_array_new(false, true /* clear */, 1);
+    tables->tcpalog = g_array_new(false, true /* clear */, 1);
     tables->linker = bios_linker_loader_init();
 }
 
 static inline void acpi_build_tables_cleanup(AcpiBuildTables *tables, bool mfre)
 {
     void *linker_data = bios_linker_loader_cleanup(tables->linker);
-    if (mfre) {
-        g_free(linker_data);
-    }
+    g_free(linker_data);
     g_array_free(tables->rsdp, mfre);
-    g_array_free(tables->table_data, mfre);
+    g_array_free(tables->table_data, true);
+    g_array_free(tables->tcpalog, mfre);
 }
 
 typedef
 struct AcpiBuildState {
     /* Copy of table in RAM (for patching). */
-    uint8_t *table_ram;
+    ram_addr_t table_ram;
     uint32_t table_size;
     /* Is table patched? */
     uint8_t patched;
@@ -1470,6 +1538,16 @@ static bool acpi_get_mcfg(AcpiMcfgInfo *mcfg)
     return true;
 }
 
+static bool acpi_has_iommu(void)
+{
+    bool ambiguous;
+    Object *intel_iommu;
+
+    intel_iommu = object_resolve_path_type("", TYPE_INTEL_IOMMU_DEVICE,
+                                           &ambiguous);
+    return intel_iommu && !ambiguous;
+}
+
 static
 void acpi_build(PcGuestInfo *guest_info, AcpiBuildTables *tables)
 {
@@ -1531,14 +1609,25 @@ void acpi_build(PcGuestInfo *guest_info, AcpiBuildTables *tables)
         acpi_add_table(table_offsets, tables->table_data);
         build_hpet(tables->table_data, tables->linker);
     }
+    if (misc.has_tpm) {
+        acpi_add_table(table_offsets, tables->table_data);
+        build_tpm_tcpa(tables->table_data, tables->linker, tables->tcpalog);
+
+        acpi_add_table(table_offsets, tables->table_data);
+        build_tpm_ssdt(tables->table_data, tables->linker);
+    }
     if (guest_info->numa_nodes) {
         acpi_add_table(table_offsets, tables->table_data);
-        build_srat(tables->table_data, tables->linker, &cpu, guest_info);
+        build_srat(tables->table_data, tables->linker, guest_info);
     }
     if (acpi_get_mcfg(&mcfg)) {
         acpi_add_table(table_offsets, tables->table_data);
         build_mcfg_q35(tables->table_data, tables->linker, &mcfg);
     }
+    if (acpi_has_iommu()) {
+        acpi_add_table(table_offsets, tables->table_data);
+        build_dmar_q35(tables->table_data, tables->linker);
+    }
 
     /* Add tables supplied by user (if any) */
     for (u = acpi_table_first(); u; u = acpi_table_next(u)) {
@@ -1623,9 +1712,12 @@ static void acpi_build_update(void *build_opaque, uint32_t offset)
     acpi_build(build_state->guest_info, &tables);
 
     assert(acpi_data_len(tables.table_data) == build_state->table_size);
-    memcpy(build_state->table_ram, tables.table_data->data,
+    memcpy(qemu_get_ram_ptr(build_state->table_ram), tables.table_data->data,
            build_state->table_size);
 
+    cpu_physical_memory_set_dirty_range_nocode(build_state->table_ram,
+                                               build_state->table_size);
+
     acpi_build_tables_cleanup(&tables, true);
 }
 
@@ -1635,7 +1727,7 @@ static void acpi_build_reset(void *build_opaque)
     build_state->patched = 0;
 }
 
-static void *acpi_add_rom_blob(AcpiBuildState *build_state, GArray *blob,
+static ram_addr_t acpi_add_rom_blob(AcpiBuildState *build_state, GArray *blob,
                                const char *name)
 {
     return rom_add_blob(name, blob->data, acpi_data_len(blob), -1, name,
@@ -1684,10 +1776,14 @@ void acpi_setup(PcGuestInfo *guest_info)
     /* Now expose it all to Guest */
     build_state->table_ram = acpi_add_rom_blob(build_state, tables.table_data,
                                                ACPI_BUILD_TABLE_FILE);
+    assert(build_state->table_ram != RAM_ADDR_MAX);
     build_state->table_size = acpi_data_len(tables.table_data);
 
     acpi_add_rom_blob(NULL, tables.linker, "etc/table-loader");
 
+    fw_cfg_add_file(guest_info->fw_cfg, ACPI_BUILD_TPMLOG_FILE,
+                    tables.tcpalog->data, acpi_data_len(tables.tcpalog));
+
     /*
      * RSDP is small so it's easy to keep it immutable, no need to
      * bother with ROM blobs.
diff --git a/hw/i386/acpi-defs.h b/hw/i386/acpi-defs.h
index e93babb02..c4468f84e 100644
--- a/hw/i386/acpi-defs.h
+++ b/hw/i386/acpi-defs.h
@@ -314,4 +314,55 @@ struct AcpiTableMcfg {
 } QEMU_PACKED;
 typedef struct AcpiTableMcfg AcpiTableMcfg;
 
+/*
+ * TCPA Description Table
+ */
+struct Acpi20Tcpa {
+    ACPI_TABLE_HEADER_DEF                    /* ACPI common table header */
+    uint16_t platform_class;
+    uint32_t log_area_minimum_length;
+    uint64_t log_area_start_address;
+} QEMU_PACKED;
+typedef struct Acpi20Tcpa Acpi20Tcpa;
+
+/* DMAR - DMA Remapping table r2.2 */
+struct AcpiTableDmar {
+    ACPI_TABLE_HEADER_DEF
+    uint8_t host_address_width; /* Maximum DMA physical addressability */
+    uint8_t flags;
+    uint8_t reserved[10];
+} QEMU_PACKED;
+typedef struct AcpiTableDmar AcpiTableDmar;
+
+/* Masks for Flags field above */
+#define ACPI_DMAR_INTR_REMAP        1
+#define ACPI_DMAR_X2APIC_OPT_OUT    (1 << 1)
+
+/* Values for sub-structure type for DMAR */
+enum {
+    ACPI_DMAR_TYPE_HARDWARE_UNIT = 0,       /* DRHD */
+    ACPI_DMAR_TYPE_RESERVED_MEMORY = 1,     /* RMRR */
+    ACPI_DMAR_TYPE_ATSR = 2,                /* ATSR */
+    ACPI_DMAR_TYPE_HARDWARE_AFFINITY = 3,   /* RHSR */
+    ACPI_DMAR_TYPE_ANDD = 4,                /* ANDD */
+    ACPI_DMAR_TYPE_RESERVED = 5             /* Reserved for furture use */
+};
+
+/*
+ * Sub-structures for DMAR
+ */
+/* Type 0: Hardware Unit Definition */
+struct AcpiDmarHardwareUnit {
+    uint16_t type;
+    uint16_t length;
+    uint8_t flags;
+    uint8_t reserved;
+    uint16_t pci_segment;   /* The PCI Segment associated with this unit */
+    uint64_t address;   /* Base address of remapping hardware register-set */
+} QEMU_PACKED;
+typedef struct AcpiDmarHardwareUnit AcpiDmarHardwareUnit;
+
+/* Masks for Flags field above */
+#define ACPI_DMAR_INCLUDE_PCI_ALL   1
+
 #endif
diff --git a/hw/i386/acpi-dsdt-mem-hotplug.dsl b/hw/i386/acpi-dsdt-mem-hotplug.dsl
new file mode 100644
index 000000000..2a36c4799
--- /dev/null
+++ b/hw/i386/acpi-dsdt-mem-hotplug.dsl
@@ -0,0 +1,176 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+    External(MEMORY_SLOT_NOTIFY_METHOD, MethodObj)
+
+    Scope(\_SB.PCI0) {
+        Device(MEMORY_HOTPLUG_DEVICE) {
+            Name(_HID, "PNP0A06")
+            Name(_UID, "Memory hotplug resources")
+            External(MEMORY_SLOTS_NUMBER, IntObj)
+
+            /* Memory hotplug IO registers */
+            OperationRegion(MEMORY_HOTPLUG_IO_REGION, SystemIO,
+                            ACPI_MEMORY_HOTPLUG_BASE,
+                            ACPI_MEMORY_HOTPLUG_IO_LEN)
+
+            Name(_CRS, ResourceTemplate() {
+                IO(Decode16, ACPI_MEMORY_HOTPLUG_BASE, ACPI_MEMORY_HOTPLUG_BASE,
+                   0, ACPI_MEMORY_HOTPLUG_IO_LEN, IO)
+            })
+
+            Method(_STA, 0) {
+                If (LEqual(MEMORY_SLOTS_NUMBER, Zero)) {
+                    Return(0x0)
+                }
+                /* present, functioning, decoding, not shown in UI */
+                Return(0xB)
+            }
+
+            Field(MEMORY_HOTPLUG_IO_REGION, DWordAcc, NoLock, Preserve) {
+                MEMORY_SLOT_ADDR_LOW, 32,  // read only
+                MEMORY_SLOT_ADDR_HIGH, 32, // read only
+                MEMORY_SLOT_SIZE_LOW, 32,  // read only
+                MEMORY_SLOT_SIZE_HIGH, 32, // read only
+                MEMORY_SLOT_PROXIMITY, 32, // read only
+            }
+            Field(MEMORY_HOTPLUG_IO_REGION, ByteAcc, NoLock, Preserve) {
+                Offset(20),
+                MEMORY_SLOT_ENABLED,  1, // 1 if enabled, read only
+                MEMORY_SLOT_INSERT_EVENT, 1, // (read) 1 if has a insert event. (write) 1 to clear event
+            }
+
+            Mutex (MEMORY_SLOT_LOCK, 0)
+            Field (MEMORY_HOTPLUG_IO_REGION, DWordAcc, NoLock, Preserve) {
+                MEMORY_SLOT_SLECTOR, 32,  // DIMM selector, write only
+                MEMORY_SLOT_OST_EVENT, 32,  // _OST event code, write only
+                MEMORY_SLOT_OST_STATUS, 32,  // _OST status code, write only
+            }
+
+            Method(MEMORY_SLOT_SCAN_METHOD, 0) {
+                If (LEqual(MEMORY_SLOTS_NUMBER, Zero)) {
+                     Return(Zero)
+                }
+
+                Store(Zero, Local0) // Mem devs iterrator
+                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
+                while (LLess(Local0, MEMORY_SLOTS_NUMBER)) {
+                    Store(Local0, MEMORY_SLOT_SLECTOR) // select Local0 DIMM
+                    If (LEqual(MEMORY_SLOT_INSERT_EVENT, One)) { // Memory device needs check
+                        MEMORY_SLOT_NOTIFY_METHOD(Local0, 1)
+                        Store(1, MEMORY_SLOT_INSERT_EVENT)
+                    }
+                    // TODO: handle memory eject request
+                    Add(Local0, One, Local0) // goto next DIMM
+                }
+                Release(MEMORY_SLOT_LOCK)
+                Return(One)
+            }
+
+            Method(MEMORY_SLOT_STATUS_METHOD, 1) {
+                Store(Zero, Local0)
+
+                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
+                Store(ToInteger(Arg0), MEMORY_SLOT_SLECTOR) // select DIMM
+
+                If (LEqual(MEMORY_SLOT_ENABLED, One)) {
+                    Store(0xF, Local0)
+                }
+
+                Release(MEMORY_SLOT_LOCK)
+                Return(Local0)
+            }
+
+            Method(MEMORY_SLOT_CRS_METHOD, 1, Serialized) {
+                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
+                Store(ToInteger(Arg0), MEMORY_SLOT_SLECTOR) // select DIMM
+
+                Name(MR64, ResourceTemplate() {
+                    QWordMemory(ResourceProducer, PosDecode, MinFixed, MaxFixed,
+                    Cacheable, ReadWrite,
+                    0x0000000000000000,        // Address Space Granularity
+                    0x0000000000000000,        // Address Range Minimum
+                    0xFFFFFFFFFFFFFFFE,        // Address Range Maximum
+                    0x0000000000000000,        // Address Translation Offset
+                    0xFFFFFFFFFFFFFFFF,        // Address Length
+                    ,, MW64, AddressRangeMemory, TypeStatic)
+                })
+
+                CreateDWordField(MR64, 14, MINL)
+                CreateDWordField(MR64, 18, MINH)
+                CreateDWordField(MR64, 38, LENL)
+                CreateDWordField(MR64, 42, LENH)
+                CreateDWordField(MR64, 22, MAXL)
+                CreateDWordField(MR64, 26, MAXH)
+
+                Store(MEMORY_SLOT_ADDR_HIGH, MINH)
+                Store(MEMORY_SLOT_ADDR_LOW, MINL)
+                Store(MEMORY_SLOT_SIZE_HIGH, LENH)
+                Store(MEMORY_SLOT_SIZE_LOW, LENL)
+
+                // 64-bit math: MAX = MIN + LEN - 1
+                Add(MINL, LENL, MAXL)
+                Add(MINH, LENH, MAXH)
+                If (LLess(MAXL, MINL)) {
+                    Add(MAXH, One, MAXH)
+                }
+                If (LLess(MAXL, One)) {
+                    Subtract(MAXH, One, MAXH)
+                }
+                Subtract(MAXL, One, MAXL)
+
+                If (LEqual(MAXH, Zero)){
+                    Name(MR32, ResourceTemplate() {
+                        DWordMemory(ResourceProducer, PosDecode, MinFixed, MaxFixed,
+                        Cacheable, ReadWrite,
+                        0x00000000,        // Address Space Granularity
+                        0x00000000,        // Address Range Minimum
+                        0xFFFFFFFE,        // Address Range Maximum
+                        0x00000000,        // Address Translation Offset
+                        0xFFFFFFFF,        // Address Length
+                        ,, MW32, AddressRangeMemory, TypeStatic)
+                    })
+                    CreateDWordField(MR32, MW32._MIN, MIN)
+                    CreateDWordField(MR32, MW32._MAX, MAX)
+                    CreateDWordField(MR32, MW32._LEN, LEN)
+                    Store(MINL, MIN)
+                    Store(MAXL, MAX)
+                    Store(LENL, LEN)
+
+                    Release(MEMORY_SLOT_LOCK)
+                    Return(MR32)
+                }
+
+                Release(MEMORY_SLOT_LOCK)
+                Return(MR64)
+            }
+
+            Method(MEMORY_SLOT_PROXIMITY_METHOD, 1) {
+                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
+                Store(ToInteger(Arg0), MEMORY_SLOT_SLECTOR) // select DIMM
+                Store(MEMORY_SLOT_PROXIMITY, Local0)
+                Release(MEMORY_SLOT_LOCK)
+                Return(Local0)
+            }
+
+            Method(MEMORY_SLOT_OST_METHOD, 4) {
+                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
+                Store(ToInteger(Arg0), MEMORY_SLOT_SLECTOR) // select DIMM
+                Store(Arg1, MEMORY_SLOT_OST_EVENT)
+                Store(Arg2, MEMORY_SLOT_OST_STATUS)
+                Release(MEMORY_SLOT_LOCK)
+            }
+        } // Device()
+    } // Scope()
diff --git a/hw/i386/acpi-dsdt.dsl b/hw/i386/acpi-dsdt.dsl
index 6ba017014..a611e07ec 100644
--- a/hw/i386/acpi-dsdt.dsl
+++ b/hw/i386/acpi-dsdt.dsl
@@ -297,13 +297,12 @@ DefinitionBlock (
 #include "hw/acpi/pc-hotplug.h"
 #define CPU_STATUS_BASE PIIX4_CPU_HOTPLUG_IO_BASE
 #include "acpi-dsdt-cpu-hotplug.dsl"
+#include "acpi-dsdt-mem-hotplug.dsl"
 
 
 /****************************************************************
  * General purpose events
  ****************************************************************/
-    External(\_SB.PCI0.MEMORY_HOPTLUG_DEVICE.MEMORY_SLOT_SCAN_METHOD, MethodObj)
-
     Scope(\_GPE) {
         Name(_HID, "ACPI0006")
 
@@ -321,7 +320,7 @@ DefinitionBlock (
         }
         Method(_E03) {
             // Memory hotplug event
-            \_SB.PCI0.MEMORY_HOPTLUG_DEVICE.MEMORY_SLOT_SCAN_METHOD()
+            \_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_SCAN_METHOD()
         }
         Method(_L04) {
         }
diff --git a/hw/i386/acpi-dsdt.hex.generated b/hw/i386/acpi-dsdt.hex.generated
index 6c8a1fcdf..875570e5b 100644
--- a/hw/i386/acpi-dsdt.hex.generated
+++ b/hw/i386/acpi-dsdt.hex.generated
@@ -3,12 +3,12 @@ static unsigned char AcpiDsdtAmlCode[] = {
 0x53,
 0x44,
 0x54,
-0xf7,
-0xa,
+0x8,
+0xe,
 0x0,
 0x0,
 0x1,
-0x2e,
+0xfc,
 0x42,
 0x58,
 0x50,
@@ -31,9 +31,9 @@ static unsigned char AcpiDsdtAmlCode[] = {
 0x4e,
 0x54,
 0x4c,
-0x13,
-0x9,
-0x12,
+0x28,
+0x8,
+0x14,
 0x20,
 0x10,
 0x49,
@@ -2593,6 +2593,791 @@ static unsigned char AcpiDsdtAmlCode[] = {
 0xa,
 0xb,
 0x10,
+0x40,
+0x31,
+0x2e,
+0x5f,
+0x53,
+0x42,
+0x5f,
+0x50,
+0x43,
+0x49,
+0x30,
+0x5b,
+0x82,
+0x43,
+0x30,
+0x4d,
+0x48,
+0x50,
+0x44,
+0x8,
+0x5f,
+0x48,
+0x49,
+0x44,
+0xd,
+0x50,
+0x4e,
+0x50,
+0x30,
+0x41,
+0x30,
+0x36,
+0x0,
+0x8,
+0x5f,
+0x55,
+0x49,
+0x44,
+0xd,
+0x4d,
+0x65,
+0x6d,
+0x6f,
+0x72,
+0x79,
+0x20,
+0x68,
+0x6f,
+0x74,
+0x70,
+0x6c,
+0x75,
+0x67,
+0x20,
+0x72,
+0x65,
+0x73,
+0x6f,
+0x75,
+0x72,
+0x63,
+0x65,
+0x73,
+0x0,
+0x5b,
+0x80,
+0x48,
+0x50,
+0x4d,
+0x52,
+0x1,
+0xb,
+0x0,
+0xa,
+0xa,
+0x18,
+0x8,
+0x5f,
+0x43,
+0x52,
+0x53,
+0x11,
+0xd,
+0xa,
+0xa,
+0x47,
+0x1,
+0x0,
+0xa,
+0x0,
+0xa,
+0x0,
+0x18,
+0x79,
+0x0,
+0x14,
+0x13,
+0x5f,
+0x53,
+0x54,
+0x41,
+0x0,
+0xa0,
+0x9,
+0x93,
+0x4d,
+0x44,
+0x4e,
+0x52,
+0x0,
+0xa4,
+0x0,
+0xa4,
+0xa,
+0xb,
+0x5b,
+0x81,
+0x1f,
+0x48,
+0x50,
+0x4d,
+0x52,
+0x3,
+0x4d,
+0x52,
+0x42,
+0x4c,
+0x20,
+0x4d,
+0x52,
+0x42,
+0x48,
+0x20,
+0x4d,
+0x52,
+0x4c,
+0x4c,
+0x20,
+0x4d,
+0x52,
+0x4c,
+0x48,
+0x20,
+0x4d,
+0x50,
+0x58,
+0x5f,
+0x20,
+0x5b,
+0x81,
+0x13,
+0x48,
+0x50,
+0x4d,
+0x52,
+0x1,
+0x0,
+0x40,
+0xa,
+0x4d,
+0x45,
+0x53,
+0x5f,
+0x1,
+0x4d,
+0x49,
+0x4e,
+0x53,
+0x1,
+0x5b,
+0x1,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0x0,
+0x5b,
+0x81,
+0x15,
+0x48,
+0x50,
+0x4d,
+0x52,
+0x3,
+0x4d,
+0x53,
+0x45,
+0x4c,
+0x20,
+0x4d,
+0x4f,
+0x45,
+0x56,
+0x20,
+0x4d,
+0x4f,
+0x53,
+0x43,
+0x20,
+0x14,
+0x4a,
+0x4,
+0x4d,
+0x53,
+0x43,
+0x4e,
+0x0,
+0xa0,
+0x9,
+0x93,
+0x4d,
+0x44,
+0x4e,
+0x52,
+0x0,
+0xa4,
+0x0,
+0x70,
+0x0,
+0x60,
+0x5b,
+0x23,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xff,
+0xff,
+0xa2,
+0x25,
+0x95,
+0x60,
+0x4d,
+0x44,
+0x4e,
+0x52,
+0x70,
+0x60,
+0x4d,
+0x53,
+0x45,
+0x4c,
+0xa0,
+0x13,
+0x93,
+0x4d,
+0x49,
+0x4e,
+0x53,
+0x1,
+0x4d,
+0x54,
+0x46,
+0x59,
+0x60,
+0x1,
+0x70,
+0x1,
+0x4d,
+0x49,
+0x4e,
+0x53,
+0x72,
+0x60,
+0x1,
+0x60,
+0x5b,
+0x27,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xa4,
+0x1,
+0x14,
+0x2d,
+0x4d,
+0x52,
+0x53,
+0x54,
+0x1,
+0x70,
+0x0,
+0x60,
+0x5b,
+0x23,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xff,
+0xff,
+0x70,
+0x99,
+0x68,
+0x0,
+0x4d,
+0x53,
+0x45,
+0x4c,
+0xa0,
+0xb,
+0x93,
+0x4d,
+0x45,
+0x53,
+0x5f,
+0x1,
+0x70,
+0xa,
+0xf,
+0x60,
+0x5b,
+0x27,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xa4,
+0x60,
+0x14,
+0x41,
+0x18,
+0x4d,
+0x43,
+0x52,
+0x53,
+0x9,
+0x5b,
+0x23,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xff,
+0xff,
+0x70,
+0x99,
+0x68,
+0x0,
+0x4d,
+0x53,
+0x45,
+0x4c,
+0x8,
+0x4d,
+0x52,
+0x36,
+0x34,
+0x11,
+0x33,
+0xa,
+0x30,
+0x8a,
+0x2b,
+0x0,
+0x0,
+0xc,
+0x3,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0xfe,
+0xff,
+0xff,
+0xff,
+0xff,
+0xff,
+0xff,
+0xff,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0xff,
+0xff,
+0xff,
+0xff,
+0xff,
+0xff,
+0xff,
+0xff,
+0x79,
+0x0,
+0x8a,
+0x4d,
+0x52,
+0x36,
+0x34,
+0xa,
+0xe,
+0x4d,
+0x49,
+0x4e,
+0x4c,
+0x8a,
+0x4d,
+0x52,
+0x36,
+0x34,
+0xa,
+0x12,
+0x4d,
+0x49,
+0x4e,
+0x48,
+0x8a,
+0x4d,
+0x52,
+0x36,
+0x34,
+0xa,
+0x26,
+0x4c,
+0x45,
+0x4e,
+0x4c,
+0x8a,
+0x4d,
+0x52,
+0x36,
+0x34,
+0xa,
+0x2a,
+0x4c,
+0x45,
+0x4e,
+0x48,
+0x8a,
+0x4d,
+0x52,
+0x36,
+0x34,
+0xa,
+0x16,
+0x4d,
+0x41,
+0x58,
+0x4c,
+0x8a,
+0x4d,
+0x52,
+0x36,
+0x34,
+0xa,
+0x1a,
+0x4d,
+0x41,
+0x58,
+0x48,
+0x70,
+0x4d,
+0x52,
+0x42,
+0x48,
+0x4d,
+0x49,
+0x4e,
+0x48,
+0x70,
+0x4d,
+0x52,
+0x42,
+0x4c,
+0x4d,
+0x49,
+0x4e,
+0x4c,
+0x70,
+0x4d,
+0x52,
+0x4c,
+0x48,
+0x4c,
+0x45,
+0x4e,
+0x48,
+0x70,
+0x4d,
+0x52,
+0x4c,
+0x4c,
+0x4c,
+0x45,
+0x4e,
+0x4c,
+0x72,
+0x4d,
+0x49,
+0x4e,
+0x4c,
+0x4c,
+0x45,
+0x4e,
+0x4c,
+0x4d,
+0x41,
+0x58,
+0x4c,
+0x72,
+0x4d,
+0x49,
+0x4e,
+0x48,
+0x4c,
+0x45,
+0x4e,
+0x48,
+0x4d,
+0x41,
+0x58,
+0x48,
+0xa0,
+0x14,
+0x95,
+0x4d,
+0x41,
+0x58,
+0x4c,
+0x4d,
+0x49,
+0x4e,
+0x4c,
+0x72,
+0x4d,
+0x41,
+0x58,
+0x48,
+0x1,
+0x4d,
+0x41,
+0x58,
+0x48,
+0xa0,
+0x11,
+0x95,
+0x4d,
+0x41,
+0x58,
+0x4c,
+0x1,
+0x74,
+0x4d,
+0x41,
+0x58,
+0x48,
+0x1,
+0x4d,
+0x41,
+0x58,
+0x48,
+0x74,
+0x4d,
+0x41,
+0x58,
+0x4c,
+0x1,
+0x4d,
+0x41,
+0x58,
+0x4c,
+0xa0,
+0x44,
+0x7,
+0x93,
+0x4d,
+0x41,
+0x58,
+0x48,
+0x0,
+0x8,
+0x4d,
+0x52,
+0x33,
+0x32,
+0x11,
+0x1f,
+0xa,
+0x1c,
+0x87,
+0x17,
+0x0,
+0x0,
+0xc,
+0x3,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0xfe,
+0xff,
+0xff,
+0xff,
+0x0,
+0x0,
+0x0,
+0x0,
+0xff,
+0xff,
+0xff,
+0xff,
+0x79,
+0x0,
+0x8a,
+0x4d,
+0x52,
+0x33,
+0x32,
+0xa,
+0xa,
+0x4d,
+0x49,
+0x4e,
+0x5f,
+0x8a,
+0x4d,
+0x52,
+0x33,
+0x32,
+0xa,
+0xe,
+0x4d,
+0x41,
+0x58,
+0x5f,
+0x8a,
+0x4d,
+0x52,
+0x33,
+0x32,
+0xa,
+0x16,
+0x4c,
+0x45,
+0x4e,
+0x5f,
+0x70,
+0x4d,
+0x49,
+0x4e,
+0x4c,
+0x4d,
+0x49,
+0x4e,
+0x5f,
+0x70,
+0x4d,
+0x41,
+0x58,
+0x4c,
+0x4d,
+0x41,
+0x58,
+0x5f,
+0x70,
+0x4c,
+0x45,
+0x4e,
+0x4c,
+0x4c,
+0x45,
+0x4e,
+0x5f,
+0x5b,
+0x27,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xa4,
+0x4d,
+0x52,
+0x33,
+0x32,
+0x5b,
+0x27,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xa4,
+0x4d,
+0x52,
+0x36,
+0x34,
+0x14,
+0x24,
+0x4d,
+0x50,
+0x58,
+0x4d,
+0x1,
+0x5b,
+0x23,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xff,
+0xff,
+0x70,
+0x99,
+0x68,
+0x0,
+0x4d,
+0x53,
+0x45,
+0x4c,
+0x70,
+0x4d,
+0x50,
+0x58,
+0x5f,
+0x60,
+0x5b,
+0x27,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xa4,
+0x60,
+0x14,
+0x28,
+0x4d,
+0x4f,
+0x53,
+0x54,
+0x4,
+0x5b,
+0x23,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xff,
+0xff,
+0x70,
+0x99,
+0x68,
+0x0,
+0x4d,
+0x53,
+0x45,
+0x4c,
+0x70,
+0x69,
+0x4d,
+0x4f,
+0x45,
+0x56,
+0x70,
+0x6a,
+0x4d,
+0x4f,
+0x53,
+0x43,
+0x5b,
+0x27,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0x10,
 0x45,
 0xd,
 0x5f,
diff --git a/hw/i386/intel_iommu.c b/hw/i386/intel_iommu.c
new file mode 100644
index 000000000..0a4282adf
--- /dev/null
+++ b/hw/i386/intel_iommu.c
@@ -0,0 +1,1963 @@
+/*
+ * QEMU emulation of an Intel IOMMU (VT-d)
+ *   (DMA Remapping device)
+ *
+ * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com>
+ * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/sysbus.h"
+#include "exec/address-spaces.h"
+#include "intel_iommu_internal.h"
+
+/*#define DEBUG_INTEL_IOMMU*/
+#ifdef DEBUG_INTEL_IOMMU
+enum {
+    DEBUG_GENERAL, DEBUG_CSR, DEBUG_INV, DEBUG_MMU, DEBUG_FLOG,
+    DEBUG_CACHE,
+};
+#define VTD_DBGBIT(x)   (1 << DEBUG_##x)
+static int vtd_dbgflags = VTD_DBGBIT(GENERAL) | VTD_DBGBIT(CSR);
+
+#define VTD_DPRINTF(what, fmt, ...) do { \
+    if (vtd_dbgflags & VTD_DBGBIT(what)) { \
+        fprintf(stderr, "(vtd)%s: " fmt "\n", __func__, \
+                ## __VA_ARGS__); } \
+    } while (0)
+#else
+#define VTD_DPRINTF(what, fmt, ...) do {} while (0)
+#endif
+
+static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val,
+                            uint64_t wmask, uint64_t w1cmask)
+{
+    stq_le_p(&s->csr[addr], val);
+    stq_le_p(&s->wmask[addr], wmask);
+    stq_le_p(&s->w1cmask[addr], w1cmask);
+}
+
+static void vtd_define_quad_wo(IntelIOMMUState *s, hwaddr addr, uint64_t mask)
+{
+    stq_le_p(&s->womask[addr], mask);
+}
+
+static void vtd_define_long(IntelIOMMUState *s, hwaddr addr, uint32_t val,
+                            uint32_t wmask, uint32_t w1cmask)
+{
+    stl_le_p(&s->csr[addr], val);
+    stl_le_p(&s->wmask[addr], wmask);
+    stl_le_p(&s->w1cmask[addr], w1cmask);
+}
+
+static void vtd_define_long_wo(IntelIOMMUState *s, hwaddr addr, uint32_t mask)
+{
+    stl_le_p(&s->womask[addr], mask);
+}
+
+/* "External" get/set operations */
+static void vtd_set_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val)
+{
+    uint64_t oldval = ldq_le_p(&s->csr[addr]);
+    uint64_t wmask = ldq_le_p(&s->wmask[addr]);
+    uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]);
+    stq_le_p(&s->csr[addr],
+             ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val));
+}
+
+static void vtd_set_long(IntelIOMMUState *s, hwaddr addr, uint32_t val)
+{
+    uint32_t oldval = ldl_le_p(&s->csr[addr]);
+    uint32_t wmask = ldl_le_p(&s->wmask[addr]);
+    uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]);
+    stl_le_p(&s->csr[addr],
+             ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val));
+}
+
+static uint64_t vtd_get_quad(IntelIOMMUState *s, hwaddr addr)
+{
+    uint64_t val = ldq_le_p(&s->csr[addr]);
+    uint64_t womask = ldq_le_p(&s->womask[addr]);
+    return val & ~womask;
+}
+
+static uint32_t vtd_get_long(IntelIOMMUState *s, hwaddr addr)
+{
+    uint32_t val = ldl_le_p(&s->csr[addr]);
+    uint32_t womask = ldl_le_p(&s->womask[addr]);
+    return val & ~womask;
+}
+
+/* "Internal" get/set operations */
+static uint64_t vtd_get_quad_raw(IntelIOMMUState *s, hwaddr addr)
+{
+    return ldq_le_p(&s->csr[addr]);
+}
+
+static uint32_t vtd_get_long_raw(IntelIOMMUState *s, hwaddr addr)
+{
+    return ldl_le_p(&s->csr[addr]);
+}
+
+static void vtd_set_quad_raw(IntelIOMMUState *s, hwaddr addr, uint64_t val)
+{
+    stq_le_p(&s->csr[addr], val);
+}
+
+static uint32_t vtd_set_clear_mask_long(IntelIOMMUState *s, hwaddr addr,
+                                        uint32_t clear, uint32_t mask)
+{
+    uint32_t new_val = (ldl_le_p(&s->csr[addr]) & ~clear) | mask;
+    stl_le_p(&s->csr[addr], new_val);
+    return new_val;
+}
+
+static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr,
+                                        uint64_t clear, uint64_t mask)
+{
+    uint64_t new_val = (ldq_le_p(&s->csr[addr]) & ~clear) | mask;
+    stq_le_p(&s->csr[addr], new_val);
+    return new_val;
+}
+
+/* GHashTable functions */
+static gboolean vtd_uint64_equal(gconstpointer v1, gconstpointer v2)
+{
+    return *((const uint64_t *)v1) == *((const uint64_t *)v2);
+}
+
+static guint vtd_uint64_hash(gconstpointer v)
+{
+    return (guint)*(const uint64_t *)v;
+}
+
+static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value,
+                                          gpointer user_data)
+{
+    VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value;
+    uint16_t domain_id = *(uint16_t *)user_data;
+    return entry->domain_id == domain_id;
+}
+
+static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value,
+                                        gpointer user_data)
+{
+    VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value;
+    VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data;
+    uint64_t gfn = info->gfn & info->mask;
+    return (entry->domain_id == info->domain_id) &&
+            ((entry->gfn & info->mask) == gfn);
+}
+
+/* Reset all the gen of VTDAddressSpace to zero and set the gen of
+ * IntelIOMMUState to 1.
+ */
+static void vtd_reset_context_cache(IntelIOMMUState *s)
+{
+    VTDAddressSpace **pvtd_as;
+    VTDAddressSpace *vtd_as;
+    uint32_t bus_it;
+    uint32_t devfn_it;
+
+    VTD_DPRINTF(CACHE, "global context_cache_gen=1");
+    for (bus_it = 0; bus_it < VTD_PCI_BUS_MAX; ++bus_it) {
+        pvtd_as = s->address_spaces[bus_it];
+        if (!pvtd_as) {
+            continue;
+        }
+        for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) {
+            vtd_as = pvtd_as[devfn_it];
+            if (!vtd_as) {
+                continue;
+            }
+            vtd_as->context_cache_entry.context_cache_gen = 0;
+        }
+    }
+    s->context_cache_gen = 1;
+}
+
+static void vtd_reset_iotlb(IntelIOMMUState *s)
+{
+    assert(s->iotlb);
+    g_hash_table_remove_all(s->iotlb);
+}
+
+static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id,
+                                       hwaddr addr)
+{
+    uint64_t key;
+
+    key = (addr >> VTD_PAGE_SHIFT_4K) |
+           ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT);
+    return g_hash_table_lookup(s->iotlb, &key);
+
+}
+
+static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id,
+                             uint16_t domain_id, hwaddr addr, uint64_t slpte,
+                             bool read_flags, bool write_flags)
+{
+    VTDIOTLBEntry *entry = g_malloc(sizeof(*entry));
+    uint64_t *key = g_malloc(sizeof(*key));
+    uint64_t gfn = addr >> VTD_PAGE_SHIFT_4K;
+
+    VTD_DPRINTF(CACHE, "update iotlb sid 0x%"PRIx16 " gpa 0x%"PRIx64
+                " slpte 0x%"PRIx64 " did 0x%"PRIx16, source_id, addr, slpte,
+                domain_id);
+    if (g_hash_table_size(s->iotlb) >= VTD_IOTLB_MAX_SIZE) {
+        VTD_DPRINTF(CACHE, "iotlb exceeds size limit, forced to reset");
+        vtd_reset_iotlb(s);
+    }
+
+    entry->gfn = gfn;
+    entry->domain_id = domain_id;
+    entry->slpte = slpte;
+    entry->read_flags = read_flags;
+    entry->write_flags = write_flags;
+    *key = gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT);
+    g_hash_table_replace(s->iotlb, key, entry);
+}
+
+/* Given the reg addr of both the message data and address, generate an
+ * interrupt via MSI.
+ */
+static void vtd_generate_interrupt(IntelIOMMUState *s, hwaddr mesg_addr_reg,
+                                   hwaddr mesg_data_reg)
+{
+    hwaddr addr;
+    uint32_t data;
+
+    assert(mesg_data_reg < DMAR_REG_SIZE);
+    assert(mesg_addr_reg < DMAR_REG_SIZE);
+
+    addr = vtd_get_long_raw(s, mesg_addr_reg);
+    data = vtd_get_long_raw(s, mesg_data_reg);
+
+    VTD_DPRINTF(FLOG, "msi: addr 0x%"PRIx64 " data 0x%"PRIx32, addr, data);
+    stl_le_phys(&address_space_memory, addr, data);
+}
+
+/* Generate a fault event to software via MSI if conditions are met.
+ * Notice that the value of FSTS_REG being passed to it should be the one
+ * before any update.
+ */
+static void vtd_generate_fault_event(IntelIOMMUState *s, uint32_t pre_fsts)
+{
+    if (pre_fsts & VTD_FSTS_PPF || pre_fsts & VTD_FSTS_PFO ||
+        pre_fsts & VTD_FSTS_IQE) {
+        VTD_DPRINTF(FLOG, "there are previous interrupt conditions "
+                    "to be serviced by software, fault event is not generated "
+                    "(FSTS_REG 0x%"PRIx32 ")", pre_fsts);
+        return;
+    }
+    vtd_set_clear_mask_long(s, DMAR_FECTL_REG, 0, VTD_FECTL_IP);
+    if (vtd_get_long_raw(s, DMAR_FECTL_REG) & VTD_FECTL_IM) {
+        VTD_DPRINTF(FLOG, "Interrupt Mask set, fault event is not generated");
+    } else {
+        vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG);
+        vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
+    }
+}
+
+/* Check if the Fault (F) field of the Fault Recording Register referenced by
+ * @index is Set.
+ */
+static bool vtd_is_frcd_set(IntelIOMMUState *s, uint16_t index)
+{
+    /* Each reg is 128-bit */
+    hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
+    addr += 8; /* Access the high 64-bit half */
+
+    assert(index < DMAR_FRCD_REG_NR);
+
+    return vtd_get_quad_raw(s, addr) & VTD_FRCD_F;
+}
+
+/* Update the PPF field of Fault Status Register.
+ * Should be called whenever change the F field of any fault recording
+ * registers.
+ */
+static void vtd_update_fsts_ppf(IntelIOMMUState *s)
+{
+    uint32_t i;
+    uint32_t ppf_mask = 0;
+
+    for (i = 0; i < DMAR_FRCD_REG_NR; i++) {
+        if (vtd_is_frcd_set(s, i)) {
+            ppf_mask = VTD_FSTS_PPF;
+            break;
+        }
+    }
+    vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_PPF, ppf_mask);
+    VTD_DPRINTF(FLOG, "set PPF of FSTS_REG to %d", ppf_mask ? 1 : 0);
+}
+
+static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index)
+{
+    /* Each reg is 128-bit */
+    hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
+    addr += 8; /* Access the high 64-bit half */
+
+    assert(index < DMAR_FRCD_REG_NR);
+
+    vtd_set_clear_mask_quad(s, addr, 0, VTD_FRCD_F);
+    vtd_update_fsts_ppf(s);
+}
+
+/* Must not update F field now, should be done later */
+static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index,
+                            uint16_t source_id, hwaddr addr,
+                            VTDFaultReason fault, bool is_write)
+{
+    uint64_t hi = 0, lo;
+    hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
+
+    assert(index < DMAR_FRCD_REG_NR);
+
+    lo = VTD_FRCD_FI(addr);
+    hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault);
+    if (!is_write) {
+        hi |= VTD_FRCD_T;
+    }
+    vtd_set_quad_raw(s, frcd_reg_addr, lo);
+    vtd_set_quad_raw(s, frcd_reg_addr + 8, hi);
+    VTD_DPRINTF(FLOG, "record to FRCD_REG #%"PRIu16 ": hi 0x%"PRIx64
+                ", lo 0x%"PRIx64, index, hi, lo);
+}
+
+/* Try to collapse multiple pending faults from the same requester */
+static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id)
+{
+    uint32_t i;
+    uint64_t frcd_reg;
+    hwaddr addr = DMAR_FRCD_REG_OFFSET + 8; /* The high 64-bit half */
+
+    for (i = 0; i < DMAR_FRCD_REG_NR; i++) {
+        frcd_reg = vtd_get_quad_raw(s, addr);
+        VTD_DPRINTF(FLOG, "frcd_reg #%d 0x%"PRIx64, i, frcd_reg);
+        if ((frcd_reg & VTD_FRCD_F) &&
+            ((frcd_reg & VTD_FRCD_SID_MASK) == source_id)) {
+            return true;
+        }
+        addr += 16; /* 128-bit for each */
+    }
+    return false;
+}
+
+/* Log and report an DMAR (address translation) fault to software */
+static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id,
+                                  hwaddr addr, VTDFaultReason fault,
+                                  bool is_write)
+{
+    uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
+
+    assert(fault < VTD_FR_MAX);
+
+    if (fault == VTD_FR_RESERVED_ERR) {
+        /* This is not a normal fault reason case. Drop it. */
+        return;
+    }
+    VTD_DPRINTF(FLOG, "sid 0x%"PRIx16 ", fault %d, addr 0x%"PRIx64
+                ", is_write %d", source_id, fault, addr, is_write);
+    if (fsts_reg & VTD_FSTS_PFO) {
+        VTD_DPRINTF(FLOG, "new fault is not recorded due to "
+                    "Primary Fault Overflow");
+        return;
+    }
+    if (vtd_try_collapse_fault(s, source_id)) {
+        VTD_DPRINTF(FLOG, "new fault is not recorded due to "
+                    "compression of faults");
+        return;
+    }
+    if (vtd_is_frcd_set(s, s->next_frcd_reg)) {
+        VTD_DPRINTF(FLOG, "Primary Fault Overflow and "
+                    "new fault is not recorded, set PFO field");
+        vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_PFO);
+        return;
+    }
+
+    vtd_record_frcd(s, s->next_frcd_reg, source_id, addr, fault, is_write);
+
+    if (fsts_reg & VTD_FSTS_PPF) {
+        VTD_DPRINTF(FLOG, "there are pending faults already, "
+                    "fault event is not generated");
+        vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg);
+        s->next_frcd_reg++;
+        if (s->next_frcd_reg == DMAR_FRCD_REG_NR) {
+            s->next_frcd_reg = 0;
+        }
+    } else {
+        vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_FRI_MASK,
+                                VTD_FSTS_FRI(s->next_frcd_reg));
+        vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); /* Will set PPF */
+        s->next_frcd_reg++;
+        if (s->next_frcd_reg == DMAR_FRCD_REG_NR) {
+            s->next_frcd_reg = 0;
+        }
+        /* This case actually cause the PPF to be Set.
+         * So generate fault event (interrupt).
+         */
+         vtd_generate_fault_event(s, fsts_reg);
+    }
+}
+
+/* Handle Invalidation Queue Errors of queued invalidation interface error
+ * conditions.
+ */
+static void vtd_handle_inv_queue_error(IntelIOMMUState *s)
+{
+    uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
+
+    vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_IQE);
+    vtd_generate_fault_event(s, fsts_reg);
+}
+
+/* Set the IWC field and try to generate an invalidation completion interrupt */
+static void vtd_generate_completion_event(IntelIOMMUState *s)
+{
+    VTD_DPRINTF(INV, "completes an invalidation wait command with "
+                "Interrupt Flag");
+    if (vtd_get_long_raw(s, DMAR_ICS_REG) & VTD_ICS_IWC) {
+        VTD_DPRINTF(INV, "there is a previous interrupt condition to be "
+                    "serviced by software, "
+                    "new invalidation event is not generated");
+        return;
+    }
+    vtd_set_clear_mask_long(s, DMAR_ICS_REG, 0, VTD_ICS_IWC);
+    vtd_set_clear_mask_long(s, DMAR_IECTL_REG, 0, VTD_IECTL_IP);
+    if (vtd_get_long_raw(s, DMAR_IECTL_REG) & VTD_IECTL_IM) {
+        VTD_DPRINTF(INV, "IM filed in IECTL_REG is set, new invalidation "
+                    "event is not generated");
+        return;
+    } else {
+        /* Generate the interrupt event */
+        vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG);
+        vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
+    }
+}
+
+static inline bool vtd_root_entry_present(VTDRootEntry *root)
+{
+    return root->val & VTD_ROOT_ENTRY_P;
+}
+
+static int vtd_get_root_entry(IntelIOMMUState *s, uint8_t index,
+                              VTDRootEntry *re)
+{
+    dma_addr_t addr;
+
+    addr = s->root + index * sizeof(*re);
+    if (dma_memory_read(&address_space_memory, addr, re, sizeof(*re))) {
+        VTD_DPRINTF(GENERAL, "error: fail to access root-entry at 0x%"PRIx64
+                    " + %"PRIu8, s->root, index);
+        re->val = 0;
+        return -VTD_FR_ROOT_TABLE_INV;
+    }
+    re->val = le64_to_cpu(re->val);
+    return 0;
+}
+
+static inline bool vtd_context_entry_present(VTDContextEntry *context)
+{
+    return context->lo & VTD_CONTEXT_ENTRY_P;
+}
+
+static int vtd_get_context_entry_from_root(VTDRootEntry *root, uint8_t index,
+                                           VTDContextEntry *ce)
+{
+    dma_addr_t addr;
+
+    if (!vtd_root_entry_present(root)) {
+        VTD_DPRINTF(GENERAL, "error: root-entry is not present");
+        return -VTD_FR_ROOT_ENTRY_P;
+    }
+    addr = (root->val & VTD_ROOT_ENTRY_CTP) + index * sizeof(*ce);
+    if (dma_memory_read(&address_space_memory, addr, ce, sizeof(*ce))) {
+        VTD_DPRINTF(GENERAL, "error: fail to access context-entry at 0x%"PRIx64
+                    " + %"PRIu8,
+                    (uint64_t)(root->val & VTD_ROOT_ENTRY_CTP), index);
+        return -VTD_FR_CONTEXT_TABLE_INV;
+    }
+    ce->lo = le64_to_cpu(ce->lo);
+    ce->hi = le64_to_cpu(ce->hi);
+    return 0;
+}
+
+static inline dma_addr_t vtd_get_slpt_base_from_context(VTDContextEntry *ce)
+{
+    return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR;
+}
+
+/* The shift of an addr for a certain level of paging structure */
+static inline uint32_t vtd_slpt_level_shift(uint32_t level)
+{
+    return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS;
+}
+
+static inline uint64_t vtd_get_slpte_addr(uint64_t slpte)
+{
+    return slpte & VTD_SL_PT_BASE_ADDR_MASK;
+}
+
+/* Whether the pte indicates the address of the page frame */
+static inline bool vtd_is_last_slpte(uint64_t slpte, uint32_t level)
+{
+    return level == VTD_SL_PT_LEVEL || (slpte & VTD_SL_PT_PAGE_SIZE_MASK);
+}
+
+/* Get the content of a spte located in @base_addr[@index] */
+static uint64_t vtd_get_slpte(dma_addr_t base_addr, uint32_t index)
+{
+    uint64_t slpte;
+
+    assert(index < VTD_SL_PT_ENTRY_NR);
+
+    if (dma_memory_read(&address_space_memory,
+                        base_addr + index * sizeof(slpte), &slpte,
+                        sizeof(slpte))) {
+        slpte = (uint64_t)-1;
+        return slpte;
+    }
+    slpte = le64_to_cpu(slpte);
+    return slpte;
+}
+
+/* Given a gpa and the level of paging structure, return the offset of current
+ * level.
+ */
+static inline uint32_t vtd_gpa_level_offset(uint64_t gpa, uint32_t level)
+{
+    return (gpa >> vtd_slpt_level_shift(level)) &
+            ((1ULL << VTD_SL_LEVEL_BITS) - 1);
+}
+
+/* Check Capability Register to see if the @level of page-table is supported */
+static inline bool vtd_is_level_supported(IntelIOMMUState *s, uint32_t level)
+{
+    return VTD_CAP_SAGAW_MASK & s->cap &
+           (1ULL << (level - 2 + VTD_CAP_SAGAW_SHIFT));
+}
+
+/* Get the page-table level that hardware should use for the second-level
+ * page-table walk from the Address Width field of context-entry.
+ */
+static inline uint32_t vtd_get_level_from_context_entry(VTDContextEntry *ce)
+{
+    return 2 + (ce->hi & VTD_CONTEXT_ENTRY_AW);
+}
+
+static inline uint32_t vtd_get_agaw_from_context_entry(VTDContextEntry *ce)
+{
+    return 30 + (ce->hi & VTD_CONTEXT_ENTRY_AW) * 9;
+}
+
+static const uint64_t vtd_paging_entry_rsvd_field[] = {
+    [0] = ~0ULL,
+    /* For not large page */
+    [1] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
+    [2] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
+    [3] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
+    [4] = 0x880ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
+    /* For large page */
+    [5] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
+    [6] = 0x1ff800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
+    [7] = 0x3ffff800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
+    [8] = 0x880ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
+};
+
+static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level)
+{
+    if (slpte & VTD_SL_PT_PAGE_SIZE_MASK) {
+        /* Maybe large page */
+        return slpte & vtd_paging_entry_rsvd_field[level + 4];
+    } else {
+        return slpte & vtd_paging_entry_rsvd_field[level];
+    }
+}
+
+/* Given the @gpa, get relevant @slptep. @slpte_level will be the last level
+ * of the translation, can be used for deciding the size of large page.
+ */
+static int vtd_gpa_to_slpte(VTDContextEntry *ce, uint64_t gpa, bool is_write,
+                            uint64_t *slptep, uint32_t *slpte_level,
+                            bool *reads, bool *writes)
+{
+    dma_addr_t addr = vtd_get_slpt_base_from_context(ce);
+    uint32_t level = vtd_get_level_from_context_entry(ce);
+    uint32_t offset;
+    uint64_t slpte;
+    uint32_t ce_agaw = vtd_get_agaw_from_context_entry(ce);
+    uint64_t access_right_check;
+
+    /* Check if @gpa is above 2^X-1, where X is the minimum of MGAW in CAP_REG
+     * and AW in context-entry.
+     */
+    if (gpa & ~((1ULL << MIN(ce_agaw, VTD_MGAW)) - 1)) {
+        VTD_DPRINTF(GENERAL, "error: gpa 0x%"PRIx64 " exceeds limits", gpa);
+        return -VTD_FR_ADDR_BEYOND_MGAW;
+    }
+
+    /* FIXME: what is the Atomics request here? */
+    access_right_check = is_write ? VTD_SL_W : VTD_SL_R;
+
+    while (true) {
+        offset = vtd_gpa_level_offset(gpa, level);
+        slpte = vtd_get_slpte(addr, offset);
+
+        if (slpte == (uint64_t)-1) {
+            VTD_DPRINTF(GENERAL, "error: fail to access second-level paging "
+                        "entry at level %"PRIu32 " for gpa 0x%"PRIx64,
+                        level, gpa);
+            if (level == vtd_get_level_from_context_entry(ce)) {
+                /* Invalid programming of context-entry */
+                return -VTD_FR_CONTEXT_ENTRY_INV;
+            } else {
+                return -VTD_FR_PAGING_ENTRY_INV;
+            }
+        }
+        *reads = (*reads) && (slpte & VTD_SL_R);
+        *writes = (*writes) && (slpte & VTD_SL_W);
+        if (!(slpte & access_right_check)) {
+            VTD_DPRINTF(GENERAL, "error: lack of %s permission for "
+                        "gpa 0x%"PRIx64 " slpte 0x%"PRIx64,
+                        (is_write ? "write" : "read"), gpa, slpte);
+            return is_write ? -VTD_FR_WRITE : -VTD_FR_READ;
+        }
+        if (vtd_slpte_nonzero_rsvd(slpte, level)) {
+            VTD_DPRINTF(GENERAL, "error: non-zero reserved field in second "
+                        "level paging entry level %"PRIu32 " slpte 0x%"PRIx64,
+                        level, slpte);
+            return -VTD_FR_PAGING_ENTRY_RSVD;
+        }
+
+        if (vtd_is_last_slpte(slpte, level)) {
+            *slptep = slpte;
+            *slpte_level = level;
+            return 0;
+        }
+        addr = vtd_get_slpte_addr(slpte);
+        level--;
+    }
+}
+
+/* Map a device to its corresponding domain (context-entry) */
+static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num,
+                                    uint8_t devfn, VTDContextEntry *ce)
+{
+    VTDRootEntry re;
+    int ret_fr;
+
+    ret_fr = vtd_get_root_entry(s, bus_num, &re);
+    if (ret_fr) {
+        return ret_fr;
+    }
+
+    if (!vtd_root_entry_present(&re)) {
+        VTD_DPRINTF(GENERAL, "error: root-entry #%"PRIu8 " is not present",
+                    bus_num);
+        return -VTD_FR_ROOT_ENTRY_P;
+    } else if (re.rsvd || (re.val & VTD_ROOT_ENTRY_RSVD)) {
+        VTD_DPRINTF(GENERAL, "error: non-zero reserved field in root-entry "
+                    "hi 0x%"PRIx64 " lo 0x%"PRIx64, re.rsvd, re.val);
+        return -VTD_FR_ROOT_ENTRY_RSVD;
+    }
+
+    ret_fr = vtd_get_context_entry_from_root(&re, devfn, ce);
+    if (ret_fr) {
+        return ret_fr;
+    }
+
+    if (!vtd_context_entry_present(ce)) {
+        VTD_DPRINTF(GENERAL,
+                    "error: context-entry #%"PRIu8 "(bus #%"PRIu8 ") "
+                    "is not present", devfn, bus_num);
+        return -VTD_FR_CONTEXT_ENTRY_P;
+    } else if ((ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI) ||
+               (ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO)) {
+        VTD_DPRINTF(GENERAL,
+                    "error: non-zero reserved field in context-entry "
+                    "hi 0x%"PRIx64 " lo 0x%"PRIx64, ce->hi, ce->lo);
+        return -VTD_FR_CONTEXT_ENTRY_RSVD;
+    }
+    /* Check if the programming of context-entry is valid */
+    if (!vtd_is_level_supported(s, vtd_get_level_from_context_entry(ce))) {
+        VTD_DPRINTF(GENERAL, "error: unsupported Address Width value in "
+                    "context-entry hi 0x%"PRIx64 " lo 0x%"PRIx64,
+                    ce->hi, ce->lo);
+        return -VTD_FR_CONTEXT_ENTRY_INV;
+    } else if (ce->lo & VTD_CONTEXT_ENTRY_TT) {
+        VTD_DPRINTF(GENERAL, "error: unsupported Translation Type in "
+                    "context-entry hi 0x%"PRIx64 " lo 0x%"PRIx64,
+                    ce->hi, ce->lo);
+        return -VTD_FR_CONTEXT_ENTRY_INV;
+    }
+    return 0;
+}
+
+static inline uint16_t vtd_make_source_id(uint8_t bus_num, uint8_t devfn)
+{
+    return ((bus_num & 0xffUL) << 8) | (devfn & 0xffUL);
+}
+
+static const bool vtd_qualified_faults[] = {
+    [VTD_FR_RESERVED] = false,
+    [VTD_FR_ROOT_ENTRY_P] = false,
+    [VTD_FR_CONTEXT_ENTRY_P] = true,
+    [VTD_FR_CONTEXT_ENTRY_INV] = true,
+    [VTD_FR_ADDR_BEYOND_MGAW] = true,
+    [VTD_FR_WRITE] = true,
+    [VTD_FR_READ] = true,
+    [VTD_FR_PAGING_ENTRY_INV] = true,
+    [VTD_FR_ROOT_TABLE_INV] = false,
+    [VTD_FR_CONTEXT_TABLE_INV] = false,
+    [VTD_FR_ROOT_ENTRY_RSVD] = false,
+    [VTD_FR_PAGING_ENTRY_RSVD] = true,
+    [VTD_FR_CONTEXT_ENTRY_TT] = true,
+    [VTD_FR_RESERVED_ERR] = false,
+    [VTD_FR_MAX] = false,
+};
+
+/* To see if a fault condition is "qualified", which is reported to software
+ * only if the FPD field in the context-entry used to process the faulting
+ * request is 0.
+ */
+static inline bool vtd_is_qualified_fault(VTDFaultReason fault)
+{
+    return vtd_qualified_faults[fault];
+}
+
+static inline bool vtd_is_interrupt_addr(hwaddr addr)
+{
+    return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST;
+}
+
+/* Map dev to context-entry then do a paging-structures walk to do a iommu
+ * translation.
+ * @bus_num: The bus number
+ * @devfn: The devfn, which is the  combined of device and function number
+ * @is_write: The access is a write operation
+ * @entry: IOMMUTLBEntry that contain the addr to be translated and result
+ */
+static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, uint8_t bus_num,
+                                   uint8_t devfn, hwaddr addr, bool is_write,
+                                   IOMMUTLBEntry *entry)
+{
+    IntelIOMMUState *s = vtd_as->iommu_state;
+    VTDContextEntry ce;
+    VTDContextCacheEntry *cc_entry = &vtd_as->context_cache_entry;
+    uint64_t slpte;
+    uint32_t level;
+    uint16_t source_id = vtd_make_source_id(bus_num, devfn);
+    int ret_fr;
+    bool is_fpd_set = false;
+    bool reads = true;
+    bool writes = true;
+    VTDIOTLBEntry *iotlb_entry;
+
+    /* Check if the request is in interrupt address range */
+    if (vtd_is_interrupt_addr(addr)) {
+        if (is_write) {
+            /* FIXME: since we don't know the length of the access here, we
+             * treat Non-DWORD length write requests without PASID as
+             * interrupt requests, too. Withoud interrupt remapping support,
+             * we just use 1:1 mapping.
+             */
+            VTD_DPRINTF(MMU, "write request to interrupt address "
+                        "gpa 0x%"PRIx64, addr);
+            entry->iova = addr & VTD_PAGE_MASK_4K;
+            entry->translated_addr = addr & VTD_PAGE_MASK_4K;
+            entry->addr_mask = ~VTD_PAGE_MASK_4K;
+            entry->perm = IOMMU_WO;
+            return;
+        } else {
+            VTD_DPRINTF(GENERAL, "error: read request from interrupt address "
+                        "gpa 0x%"PRIx64, addr);
+            vtd_report_dmar_fault(s, source_id, addr, VTD_FR_READ, is_write);
+            return;
+        }
+    }
+    /* Try to fetch slpte form IOTLB */
+    iotlb_entry = vtd_lookup_iotlb(s, source_id, addr);
+    if (iotlb_entry) {
+        VTD_DPRINTF(CACHE, "hit iotlb sid 0x%"PRIx16 " gpa 0x%"PRIx64
+                    " slpte 0x%"PRIx64 " did 0x%"PRIx16, source_id, addr,
+                    iotlb_entry->slpte, iotlb_entry->domain_id);
+        slpte = iotlb_entry->slpte;
+        reads = iotlb_entry->read_flags;
+        writes = iotlb_entry->write_flags;
+        goto out;
+    }
+    /* Try to fetch context-entry from cache first */
+    if (cc_entry->context_cache_gen == s->context_cache_gen) {
+        VTD_DPRINTF(CACHE, "hit context-cache bus %d devfn %d "
+                    "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 ")",
+                    bus_num, devfn, cc_entry->context_entry.hi,
+                    cc_entry->context_entry.lo, cc_entry->context_cache_gen);
+        ce = cc_entry->context_entry;
+        is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
+    } else {
+        ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce);
+        is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
+        if (ret_fr) {
+            ret_fr = -ret_fr;
+            if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) {
+                VTD_DPRINTF(FLOG, "fault processing is disabled for DMA "
+                            "requests through this context-entry "
+                            "(with FPD Set)");
+            } else {
+                vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write);
+            }
+            return;
+        }
+        /* Update context-cache */
+        VTD_DPRINTF(CACHE, "update context-cache bus %d devfn %d "
+                    "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 "->%"PRIu32 ")",
+                    bus_num, devfn, ce.hi, ce.lo,
+                    cc_entry->context_cache_gen, s->context_cache_gen);
+        cc_entry->context_entry = ce;
+        cc_entry->context_cache_gen = s->context_cache_gen;
+    }
+
+    ret_fr = vtd_gpa_to_slpte(&ce, addr, is_write, &slpte, &level,
+                              &reads, &writes);
+    if (ret_fr) {
+        ret_fr = -ret_fr;
+        if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) {
+            VTD_DPRINTF(FLOG, "fault processing is disabled for DMA requests "
+                        "through this context-entry (with FPD Set)");
+        } else {
+            vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write);
+        }
+        return;
+    }
+
+    vtd_update_iotlb(s, source_id, VTD_CONTEXT_ENTRY_DID(ce.hi), addr, slpte,
+                     reads, writes);
+out:
+    entry->iova = addr & VTD_PAGE_MASK_4K;
+    entry->translated_addr = vtd_get_slpte_addr(slpte) & VTD_PAGE_MASK_4K;
+    entry->addr_mask = ~VTD_PAGE_MASK_4K;
+    entry->perm = (writes ? 2 : 0) + (reads ? 1 : 0);
+}
+
+static void vtd_root_table_setup(IntelIOMMUState *s)
+{
+    s->root = vtd_get_quad_raw(s, DMAR_RTADDR_REG);
+    s->root_extended = s->root & VTD_RTADDR_RTT;
+    s->root &= VTD_RTADDR_ADDR_MASK;
+
+    VTD_DPRINTF(CSR, "root_table addr 0x%"PRIx64 " %s", s->root,
+                (s->root_extended ? "(extended)" : ""));
+}
+
+static void vtd_context_global_invalidate(IntelIOMMUState *s)
+{
+    s->context_cache_gen++;
+    if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) {
+        vtd_reset_context_cache(s);
+    }
+}
+
+/* Do a context-cache device-selective invalidation.
+ * @func_mask: FM field after shifting
+ */
+static void vtd_context_device_invalidate(IntelIOMMUState *s,
+                                          uint16_t source_id,
+                                          uint16_t func_mask)
+{
+    uint16_t mask;
+    VTDAddressSpace **pvtd_as;
+    VTDAddressSpace *vtd_as;
+    uint16_t devfn;
+    uint16_t devfn_it;
+
+    switch (func_mask & 3) {
+    case 0:
+        mask = 0;   /* No bits in the SID field masked */
+        break;
+    case 1:
+        mask = 4;   /* Mask bit 2 in the SID field */
+        break;
+    case 2:
+        mask = 6;   /* Mask bit 2:1 in the SID field */
+        break;
+    case 3:
+        mask = 7;   /* Mask bit 2:0 in the SID field */
+        break;
+    }
+    VTD_DPRINTF(INV, "device-selective invalidation source 0x%"PRIx16
+                    " mask %"PRIu16, source_id, mask);
+    pvtd_as = s->address_spaces[VTD_SID_TO_BUS(source_id)];
+    if (pvtd_as) {
+        devfn = VTD_SID_TO_DEVFN(source_id);
+        for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) {
+            vtd_as = pvtd_as[devfn_it];
+            if (vtd_as && ((devfn_it & mask) == (devfn & mask))) {
+                VTD_DPRINTF(INV, "invalidate context-cahce of devfn 0x%"PRIx16,
+                            devfn_it);
+                vtd_as->context_cache_entry.context_cache_gen = 0;
+            }
+        }
+    }
+}
+
+/* Context-cache invalidation
+ * Returns the Context Actual Invalidation Granularity.
+ * @val: the content of the CCMD_REG
+ */
+static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val)
+{
+    uint64_t caig;
+    uint64_t type = val & VTD_CCMD_CIRG_MASK;
+
+    switch (type) {
+    case VTD_CCMD_DOMAIN_INVL:
+        VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
+                    (uint16_t)VTD_CCMD_DID(val));
+        /* Fall through */
+    case VTD_CCMD_GLOBAL_INVL:
+        VTD_DPRINTF(INV, "global invalidation");
+        caig = VTD_CCMD_GLOBAL_INVL_A;
+        vtd_context_global_invalidate(s);
+        break;
+
+    case VTD_CCMD_DEVICE_INVL:
+        caig = VTD_CCMD_DEVICE_INVL_A;
+        vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val));
+        break;
+
+    default:
+        VTD_DPRINTF(GENERAL, "error: invalid granularity");
+        caig = 0;
+    }
+    return caig;
+}
+
+static void vtd_iotlb_global_invalidate(IntelIOMMUState *s)
+{
+    vtd_reset_iotlb(s);
+}
+
+static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id)
+{
+    g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_domain,
+                                &domain_id);
+}
+
+static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id,
+                                      hwaddr addr, uint8_t am)
+{
+    VTDIOTLBPageInvInfo info;
+
+    assert(am <= VTD_MAMV);
+    info.domain_id = domain_id;
+    info.gfn = addr >> VTD_PAGE_SHIFT_4K;
+    info.mask = ~((1 << am) - 1);
+    g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info);
+}
+
+/* Flush IOTLB
+ * Returns the IOTLB Actual Invalidation Granularity.
+ * @val: the content of the IOTLB_REG
+ */
+static uint64_t vtd_iotlb_flush(IntelIOMMUState *s, uint64_t val)
+{
+    uint64_t iaig;
+    uint64_t type = val & VTD_TLB_FLUSH_GRANU_MASK;
+    uint16_t domain_id;
+    hwaddr addr;
+    uint8_t am;
+
+    switch (type) {
+    case VTD_TLB_GLOBAL_FLUSH:
+        VTD_DPRINTF(INV, "global invalidation");
+        iaig = VTD_TLB_GLOBAL_FLUSH_A;
+        vtd_iotlb_global_invalidate(s);
+        break;
+
+    case VTD_TLB_DSI_FLUSH:
+        domain_id = VTD_TLB_DID(val);
+        VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
+                    domain_id);
+        iaig = VTD_TLB_DSI_FLUSH_A;
+        vtd_iotlb_domain_invalidate(s, domain_id);
+        break;
+
+    case VTD_TLB_PSI_FLUSH:
+        domain_id = VTD_TLB_DID(val);
+        addr = vtd_get_quad_raw(s, DMAR_IVA_REG);
+        am = VTD_IVA_AM(addr);
+        addr = VTD_IVA_ADDR(addr);
+        VTD_DPRINTF(INV, "page-selective invalidation domain 0x%"PRIx16
+                    " addr 0x%"PRIx64 " mask %"PRIu8, domain_id, addr, am);
+        if (am > VTD_MAMV) {
+            VTD_DPRINTF(GENERAL, "error: supported max address mask value is "
+                        "%"PRIu8, (uint8_t)VTD_MAMV);
+            iaig = 0;
+            break;
+        }
+        iaig = VTD_TLB_PSI_FLUSH_A;
+        vtd_iotlb_page_invalidate(s, domain_id, addr, am);
+        break;
+
+    default:
+        VTD_DPRINTF(GENERAL, "error: invalid granularity");
+        iaig = 0;
+    }
+    return iaig;
+}
+
+static inline bool vtd_queued_inv_enable_check(IntelIOMMUState *s)
+{
+    return s->iq_tail == 0;
+}
+
+static inline bool vtd_queued_inv_disable_check(IntelIOMMUState *s)
+{
+    return s->qi_enabled && (s->iq_tail == s->iq_head) &&
+           (s->iq_last_desc_type == VTD_INV_DESC_WAIT);
+}
+
+static void vtd_handle_gcmd_qie(IntelIOMMUState *s, bool en)
+{
+    uint64_t iqa_val = vtd_get_quad_raw(s, DMAR_IQA_REG);
+
+    VTD_DPRINTF(INV, "Queued Invalidation Enable %s", (en ? "on" : "off"));
+    if (en) {
+        if (vtd_queued_inv_enable_check(s)) {
+            s->iq = iqa_val & VTD_IQA_IQA_MASK;
+            /* 2^(x+8) entries */
+            s->iq_size = 1UL << ((iqa_val & VTD_IQA_QS) + 8);
+            s->qi_enabled = true;
+            VTD_DPRINTF(INV, "DMAR_IQA_REG 0x%"PRIx64, iqa_val);
+            VTD_DPRINTF(INV, "Invalidation Queue addr 0x%"PRIx64 " size %d",
+                        s->iq, s->iq_size);
+            /* Ok - report back to driver */
+            vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_QIES);
+        } else {
+            VTD_DPRINTF(GENERAL, "error: can't enable Queued Invalidation: "
+                        "tail %"PRIu16, s->iq_tail);
+        }
+    } else {
+        if (vtd_queued_inv_disable_check(s)) {
+            /* disable Queued Invalidation */
+            vtd_set_quad_raw(s, DMAR_IQH_REG, 0);
+            s->iq_head = 0;
+            s->qi_enabled = false;
+            /* Ok - report back to driver */
+            vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_QIES, 0);
+        } else {
+            VTD_DPRINTF(GENERAL, "error: can't disable Queued Invalidation: "
+                        "head %"PRIu16 ", tail %"PRIu16
+                        ", last_descriptor %"PRIu8,
+                        s->iq_head, s->iq_tail, s->iq_last_desc_type);
+        }
+    }
+}
+
+/* Set Root Table Pointer */
+static void vtd_handle_gcmd_srtp(IntelIOMMUState *s)
+{
+    VTD_DPRINTF(CSR, "set Root Table Pointer");
+
+    vtd_root_table_setup(s);
+    /* Ok - report back to driver */
+    vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS);
+}
+
+/* Handle Translation Enable/Disable */
+static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en)
+{
+    VTD_DPRINTF(CSR, "Translation Enable %s", (en ? "on" : "off"));
+
+    if (en) {
+        s->dmar_enabled = true;
+        /* Ok - report back to driver */
+        vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_TES);
+    } else {
+        s->dmar_enabled = false;
+
+        /* Clear the index of Fault Recording Register */
+        s->next_frcd_reg = 0;
+        /* Ok - report back to driver */
+        vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_TES, 0);
+    }
+}
+
+/* Handle write to Global Command Register */
+static void vtd_handle_gcmd_write(IntelIOMMUState *s)
+{
+    uint32_t status = vtd_get_long_raw(s, DMAR_GSTS_REG);
+    uint32_t val = vtd_get_long_raw(s, DMAR_GCMD_REG);
+    uint32_t changed = status ^ val;
+
+    VTD_DPRINTF(CSR, "value 0x%"PRIx32 " status 0x%"PRIx32, val, status);
+    if (changed & VTD_GCMD_TE) {
+        /* Translation enable/disable */
+        vtd_handle_gcmd_te(s, val & VTD_GCMD_TE);
+    }
+    if (val & VTD_GCMD_SRTP) {
+        /* Set/update the root-table pointer */
+        vtd_handle_gcmd_srtp(s);
+    }
+    if (changed & VTD_GCMD_QIE) {
+        /* Queued Invalidation Enable */
+        vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE);
+    }
+}
+
+/* Handle write to Context Command Register */
+static void vtd_handle_ccmd_write(IntelIOMMUState *s)
+{
+    uint64_t ret;
+    uint64_t val = vtd_get_quad_raw(s, DMAR_CCMD_REG);
+
+    /* Context-cache invalidation request */
+    if (val & VTD_CCMD_ICC) {
+        if (s->qi_enabled) {
+            VTD_DPRINTF(GENERAL, "error: Queued Invalidation enabled, "
+                        "should not use register-based invalidation");
+            return;
+        }
+        ret = vtd_context_cache_invalidate(s, val);
+        /* Invalidation completed. Change something to show */
+        vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_ICC, 0ULL);
+        ret = vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_CAIG_MASK,
+                                      ret);
+        VTD_DPRINTF(INV, "CCMD_REG write-back val: 0x%"PRIx64, ret);
+    }
+}
+
+/* Handle write to IOTLB Invalidation Register */
+static void vtd_handle_iotlb_write(IntelIOMMUState *s)
+{
+    uint64_t ret;
+    uint64_t val = vtd_get_quad_raw(s, DMAR_IOTLB_REG);
+
+    /* IOTLB invalidation request */
+    if (val & VTD_TLB_IVT) {
+        if (s->qi_enabled) {
+            VTD_DPRINTF(GENERAL, "error: Queued Invalidation enabled, "
+                        "should not use register-based invalidation");
+            return;
+        }
+        ret = vtd_iotlb_flush(s, val);
+        /* Invalidation completed. Change something to show */
+        vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, VTD_TLB_IVT, 0ULL);
+        ret = vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG,
+                                      VTD_TLB_FLUSH_GRANU_MASK_A, ret);
+        VTD_DPRINTF(INV, "IOTLB_REG write-back val: 0x%"PRIx64, ret);
+    }
+}
+
+/* Fetch an Invalidation Descriptor from the Invalidation Queue */
+static bool vtd_get_inv_desc(dma_addr_t base_addr, uint32_t offset,
+                             VTDInvDesc *inv_desc)
+{
+    dma_addr_t addr = base_addr + offset * sizeof(*inv_desc);
+    if (dma_memory_read(&address_space_memory, addr, inv_desc,
+        sizeof(*inv_desc))) {
+        VTD_DPRINTF(GENERAL, "error: fail to fetch Invalidation Descriptor "
+                    "base_addr 0x%"PRIx64 " offset %"PRIu32, base_addr, offset);
+        inv_desc->lo = 0;
+        inv_desc->hi = 0;
+
+        return false;
+    }
+    inv_desc->lo = le64_to_cpu(inv_desc->lo);
+    inv_desc->hi = le64_to_cpu(inv_desc->hi);
+    return true;
+}
+
+static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc)
+{
+    if ((inv_desc->hi & VTD_INV_DESC_WAIT_RSVD_HI) ||
+        (inv_desc->lo & VTD_INV_DESC_WAIT_RSVD_LO)) {
+        VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Invalidation "
+                    "Wait Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
+                    inv_desc->hi, inv_desc->lo);
+        return false;
+    }
+    if (inv_desc->lo & VTD_INV_DESC_WAIT_SW) {
+        /* Status Write */
+        uint32_t status_data = (uint32_t)(inv_desc->lo >>
+                               VTD_INV_DESC_WAIT_DATA_SHIFT);
+
+        assert(!(inv_desc->lo & VTD_INV_DESC_WAIT_IF));
+
+        /* FIXME: need to be masked with HAW? */
+        dma_addr_t status_addr = inv_desc->hi;
+        VTD_DPRINTF(INV, "status data 0x%x, status addr 0x%"PRIx64,
+                    status_data, status_addr);
+        status_data = cpu_to_le32(status_data);
+        if (dma_memory_write(&address_space_memory, status_addr, &status_data,
+                             sizeof(status_data))) {
+            VTD_DPRINTF(GENERAL, "error: fail to perform a coherent write");
+            return false;
+        }
+    } else if (inv_desc->lo & VTD_INV_DESC_WAIT_IF) {
+        /* Interrupt flag */
+        VTD_DPRINTF(INV, "Invalidation Wait Descriptor interrupt completion");
+        vtd_generate_completion_event(s);
+    } else {
+        VTD_DPRINTF(GENERAL, "error: invalid Invalidation Wait Descriptor: "
+                    "hi 0x%"PRIx64 " lo 0x%"PRIx64, inv_desc->hi, inv_desc->lo);
+        return false;
+    }
+    return true;
+}
+
+static bool vtd_process_context_cache_desc(IntelIOMMUState *s,
+                                           VTDInvDesc *inv_desc)
+{
+    if ((inv_desc->lo & VTD_INV_DESC_CC_RSVD) || inv_desc->hi) {
+        VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Context-cache "
+                    "Invalidate Descriptor");
+        return false;
+    }
+    switch (inv_desc->lo & VTD_INV_DESC_CC_G) {
+    case VTD_INV_DESC_CC_DOMAIN:
+        VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
+                    (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo));
+        /* Fall through */
+    case VTD_INV_DESC_CC_GLOBAL:
+        VTD_DPRINTF(INV, "global invalidation");
+        vtd_context_global_invalidate(s);
+        break;
+
+    case VTD_INV_DESC_CC_DEVICE:
+        vtd_context_device_invalidate(s, VTD_INV_DESC_CC_SID(inv_desc->lo),
+                                      VTD_INV_DESC_CC_FM(inv_desc->lo));
+        break;
+
+    default:
+        VTD_DPRINTF(GENERAL, "error: invalid granularity in Context-cache "
+                    "Invalidate Descriptor hi 0x%"PRIx64  " lo 0x%"PRIx64,
+                    inv_desc->hi, inv_desc->lo);
+        return false;
+    }
+    return true;
+}
+
+static bool vtd_process_iotlb_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc)
+{
+    uint16_t domain_id;
+    uint8_t am;
+    hwaddr addr;
+
+    if ((inv_desc->lo & VTD_INV_DESC_IOTLB_RSVD_LO) ||
+        (inv_desc->hi & VTD_INV_DESC_IOTLB_RSVD_HI)) {
+        VTD_DPRINTF(GENERAL, "error: non-zero reserved field in IOTLB "
+                    "Invalidate Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
+                    inv_desc->hi, inv_desc->lo);
+        return false;
+    }
+
+    switch (inv_desc->lo & VTD_INV_DESC_IOTLB_G) {
+    case VTD_INV_DESC_IOTLB_GLOBAL:
+        VTD_DPRINTF(INV, "global invalidation");
+        vtd_iotlb_global_invalidate(s);
+        break;
+
+    case VTD_INV_DESC_IOTLB_DOMAIN:
+        domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo);
+        VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
+                    domain_id);
+        vtd_iotlb_domain_invalidate(s, domain_id);
+        break;
+
+    case VTD_INV_DESC_IOTLB_PAGE:
+        domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo);
+        addr = VTD_INV_DESC_IOTLB_ADDR(inv_desc->hi);
+        am = VTD_INV_DESC_IOTLB_AM(inv_desc->hi);
+        VTD_DPRINTF(INV, "page-selective invalidation domain 0x%"PRIx16
+                    " addr 0x%"PRIx64 " mask %"PRIu8, domain_id, addr, am);
+        if (am > VTD_MAMV) {
+            VTD_DPRINTF(GENERAL, "error: supported max address mask value is "
+                        "%"PRIu8, (uint8_t)VTD_MAMV);
+            return false;
+        }
+        vtd_iotlb_page_invalidate(s, domain_id, addr, am);
+        break;
+
+    default:
+        VTD_DPRINTF(GENERAL, "error: invalid granularity in IOTLB Invalidate "
+                    "Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
+                    inv_desc->hi, inv_desc->lo);
+        return false;
+    }
+    return true;
+}
+
+static bool vtd_process_inv_desc(IntelIOMMUState *s)
+{
+    VTDInvDesc inv_desc;
+    uint8_t desc_type;
+
+    VTD_DPRINTF(INV, "iq head %"PRIu16, s->iq_head);
+    if (!vtd_get_inv_desc(s->iq, s->iq_head, &inv_desc)) {
+        s->iq_last_desc_type = VTD_INV_DESC_NONE;
+        return false;
+    }
+    desc_type = inv_desc.lo & VTD_INV_DESC_TYPE;
+    /* FIXME: should update at first or at last? */
+    s->iq_last_desc_type = desc_type;
+
+    switch (desc_type) {
+    case VTD_INV_DESC_CC:
+        VTD_DPRINTF(INV, "Context-cache Invalidate Descriptor hi 0x%"PRIx64
+                    " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo);
+        if (!vtd_process_context_cache_desc(s, &inv_desc)) {
+            return false;
+        }
+        break;
+
+    case VTD_INV_DESC_IOTLB:
+        VTD_DPRINTF(INV, "IOTLB Invalidate Descriptor hi 0x%"PRIx64
+                    " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo);
+        if (!vtd_process_iotlb_desc(s, &inv_desc)) {
+            return false;
+        }
+        break;
+
+    case VTD_INV_DESC_WAIT:
+        VTD_DPRINTF(INV, "Invalidation Wait Descriptor hi 0x%"PRIx64
+                    " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo);
+        if (!vtd_process_wait_desc(s, &inv_desc)) {
+            return false;
+        }
+        break;
+
+    default:
+        VTD_DPRINTF(GENERAL, "error: unkonw Invalidation Descriptor type "
+                    "hi 0x%"PRIx64 " lo 0x%"PRIx64 " type %"PRIu8,
+                    inv_desc.hi, inv_desc.lo, desc_type);
+        return false;
+    }
+    s->iq_head++;
+    if (s->iq_head == s->iq_size) {
+        s->iq_head = 0;
+    }
+    return true;
+}
+
+/* Try to fetch and process more Invalidation Descriptors */
+static void vtd_fetch_inv_desc(IntelIOMMUState *s)
+{
+    VTD_DPRINTF(INV, "fetch Invalidation Descriptors");
+    if (s->iq_tail >= s->iq_size) {
+        /* Detects an invalid Tail pointer */
+        VTD_DPRINTF(GENERAL, "error: iq_tail is %"PRIu16
+                    " while iq_size is %"PRIu16, s->iq_tail, s->iq_size);
+        vtd_handle_inv_queue_error(s);
+        return;
+    }
+    while (s->iq_head != s->iq_tail) {
+        if (!vtd_process_inv_desc(s)) {
+            /* Invalidation Queue Errors */
+            vtd_handle_inv_queue_error(s);
+            break;
+        }
+        /* Must update the IQH_REG in time */
+        vtd_set_quad_raw(s, DMAR_IQH_REG,
+                         (((uint64_t)(s->iq_head)) << VTD_IQH_QH_SHIFT) &
+                         VTD_IQH_QH_MASK);
+    }
+}
+
+/* Handle write to Invalidation Queue Tail Register */
+static void vtd_handle_iqt_write(IntelIOMMUState *s)
+{
+    uint64_t val = vtd_get_quad_raw(s, DMAR_IQT_REG);
+
+    s->iq_tail = VTD_IQT_QT(val);
+    VTD_DPRINTF(INV, "set iq tail %"PRIu16, s->iq_tail);
+    if (s->qi_enabled && !(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) {
+        /* Process Invalidation Queue here */
+        vtd_fetch_inv_desc(s);
+    }
+}
+
+static void vtd_handle_fsts_write(IntelIOMMUState *s)
+{
+    uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
+    uint32_t fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG);
+    uint32_t status_fields = VTD_FSTS_PFO | VTD_FSTS_PPF | VTD_FSTS_IQE;
+
+    if ((fectl_reg & VTD_FECTL_IP) && !(fsts_reg & status_fields)) {
+        vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
+        VTD_DPRINTF(FLOG, "all pending interrupt conditions serviced, clear "
+                    "IP field of FECTL_REG");
+    }
+    /* FIXME: when IQE is Clear, should we try to fetch some Invalidation
+     * Descriptors if there are any when Queued Invalidation is enabled?
+     */
+}
+
+static void vtd_handle_fectl_write(IntelIOMMUState *s)
+{
+    uint32_t fectl_reg;
+    /* FIXME: when software clears the IM field, check the IP field. But do we
+     * need to compare the old value and the new value to conclude that
+     * software clears the IM field? Or just check if the IM field is zero?
+     */
+    fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG);
+    if ((fectl_reg & VTD_FECTL_IP) && !(fectl_reg & VTD_FECTL_IM)) {
+        vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG);
+        vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
+        VTD_DPRINTF(FLOG, "IM field is cleared, generate "
+                    "fault event interrupt");
+    }
+}
+
+static void vtd_handle_ics_write(IntelIOMMUState *s)
+{
+    uint32_t ics_reg = vtd_get_long_raw(s, DMAR_ICS_REG);
+    uint32_t iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG);
+
+    if ((iectl_reg & VTD_IECTL_IP) && !(ics_reg & VTD_ICS_IWC)) {
+        vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
+        VTD_DPRINTF(INV, "pending completion interrupt condition serviced, "
+                    "clear IP field of IECTL_REG");
+    }
+}
+
+static void vtd_handle_iectl_write(IntelIOMMUState *s)
+{
+    uint32_t iectl_reg;
+    /* FIXME: when software clears the IM field, check the IP field. But do we
+     * need to compare the old value and the new value to conclude that
+     * software clears the IM field? Or just check if the IM field is zero?
+     */
+    iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG);
+    if ((iectl_reg & VTD_IECTL_IP) && !(iectl_reg & VTD_IECTL_IM)) {
+        vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG);
+        vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
+        VTD_DPRINTF(INV, "IM field is cleared, generate "
+                    "invalidation event interrupt");
+    }
+}
+
+static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size)
+{
+    IntelIOMMUState *s = opaque;
+    uint64_t val;
+
+    if (addr + size > DMAR_REG_SIZE) {
+        VTD_DPRINTF(GENERAL, "error: addr outside region: max 0x%"PRIx64
+                    ", got 0x%"PRIx64 " %d",
+                    (uint64_t)DMAR_REG_SIZE, addr, size);
+        return (uint64_t)-1;
+    }
+
+    switch (addr) {
+    /* Root Table Address Register, 64-bit */
+    case DMAR_RTADDR_REG:
+        if (size == 4) {
+            val = s->root & ((1ULL << 32) - 1);
+        } else {
+            val = s->root;
+        }
+        break;
+
+    case DMAR_RTADDR_REG_HI:
+        assert(size == 4);
+        val = s->root >> 32;
+        break;
+
+    /* Invalidation Queue Address Register, 64-bit */
+    case DMAR_IQA_REG:
+        val = s->iq | (vtd_get_quad(s, DMAR_IQA_REG) & VTD_IQA_QS);
+        if (size == 4) {
+            val = val & ((1ULL << 32) - 1);
+        }
+        break;
+
+    case DMAR_IQA_REG_HI:
+        assert(size == 4);
+        val = s->iq >> 32;
+        break;
+
+    default:
+        if (size == 4) {
+            val = vtd_get_long(s, addr);
+        } else {
+            val = vtd_get_quad(s, addr);
+        }
+    }
+    VTD_DPRINTF(CSR, "addr 0x%"PRIx64 " size %d val 0x%"PRIx64,
+                addr, size, val);
+    return val;
+}
+
+static void vtd_mem_write(void *opaque, hwaddr addr,
+                          uint64_t val, unsigned size)
+{
+    IntelIOMMUState *s = opaque;
+
+    if (addr + size > DMAR_REG_SIZE) {
+        VTD_DPRINTF(GENERAL, "error: addr outside region: max 0x%"PRIx64
+                    ", got 0x%"PRIx64 " %d",
+                    (uint64_t)DMAR_REG_SIZE, addr, size);
+        return;
+    }
+
+    switch (addr) {
+    /* Global Command Register, 32-bit */
+    case DMAR_GCMD_REG:
+        VTD_DPRINTF(CSR, "DMAR_GCMD_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        vtd_set_long(s, addr, val);
+        vtd_handle_gcmd_write(s);
+        break;
+
+    /* Context Command Register, 64-bit */
+    case DMAR_CCMD_REG:
+        VTD_DPRINTF(CSR, "DMAR_CCMD_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        if (size == 4) {
+            vtd_set_long(s, addr, val);
+        } else {
+            vtd_set_quad(s, addr, val);
+            vtd_handle_ccmd_write(s);
+        }
+        break;
+
+    case DMAR_CCMD_REG_HI:
+        VTD_DPRINTF(CSR, "DMAR_CCMD_REG_HI write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        vtd_handle_ccmd_write(s);
+        break;
+
+    /* IOTLB Invalidation Register, 64-bit */
+    case DMAR_IOTLB_REG:
+        VTD_DPRINTF(INV, "DMAR_IOTLB_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        if (size == 4) {
+            vtd_set_long(s, addr, val);
+        } else {
+            vtd_set_quad(s, addr, val);
+            vtd_handle_iotlb_write(s);
+        }
+        break;
+
+    case DMAR_IOTLB_REG_HI:
+        VTD_DPRINTF(INV, "DMAR_IOTLB_REG_HI write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        vtd_handle_iotlb_write(s);
+        break;
+
+    /* Invalidate Address Register, 64-bit */
+    case DMAR_IVA_REG:
+        VTD_DPRINTF(INV, "DMAR_IVA_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        if (size == 4) {
+            vtd_set_long(s, addr, val);
+        } else {
+            vtd_set_quad(s, addr, val);
+        }
+        break;
+
+    case DMAR_IVA_REG_HI:
+        VTD_DPRINTF(INV, "DMAR_IVA_REG_HI write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        break;
+
+    /* Fault Status Register, 32-bit */
+    case DMAR_FSTS_REG:
+        VTD_DPRINTF(FLOG, "DMAR_FSTS_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        vtd_handle_fsts_write(s);
+        break;
+
+    /* Fault Event Control Register, 32-bit */
+    case DMAR_FECTL_REG:
+        VTD_DPRINTF(FLOG, "DMAR_FECTL_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        vtd_handle_fectl_write(s);
+        break;
+
+    /* Fault Event Data Register, 32-bit */
+    case DMAR_FEDATA_REG:
+        VTD_DPRINTF(FLOG, "DMAR_FEDATA_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        break;
+
+    /* Fault Event Address Register, 32-bit */
+    case DMAR_FEADDR_REG:
+        VTD_DPRINTF(FLOG, "DMAR_FEADDR_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        break;
+
+    /* Fault Event Upper Address Register, 32-bit */
+    case DMAR_FEUADDR_REG:
+        VTD_DPRINTF(FLOG, "DMAR_FEUADDR_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        break;
+
+    /* Protected Memory Enable Register, 32-bit */
+    case DMAR_PMEN_REG:
+        VTD_DPRINTF(CSR, "DMAR_PMEN_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        break;
+
+    /* Root Table Address Register, 64-bit */
+    case DMAR_RTADDR_REG:
+        VTD_DPRINTF(CSR, "DMAR_RTADDR_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        if (size == 4) {
+            vtd_set_long(s, addr, val);
+        } else {
+            vtd_set_quad(s, addr, val);
+        }
+        break;
+
+    case DMAR_RTADDR_REG_HI:
+        VTD_DPRINTF(CSR, "DMAR_RTADDR_REG_HI write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        break;
+
+    /* Invalidation Queue Tail Register, 64-bit */
+    case DMAR_IQT_REG:
+        VTD_DPRINTF(INV, "DMAR_IQT_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        if (size == 4) {
+            vtd_set_long(s, addr, val);
+        } else {
+            vtd_set_quad(s, addr, val);
+        }
+        vtd_handle_iqt_write(s);
+        break;
+
+    case DMAR_IQT_REG_HI:
+        VTD_DPRINTF(INV, "DMAR_IQT_REG_HI write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        /* 19:63 of IQT_REG is RsvdZ, do nothing here */
+        break;
+
+    /* Invalidation Queue Address Register, 64-bit */
+    case DMAR_IQA_REG:
+        VTD_DPRINTF(INV, "DMAR_IQA_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        if (size == 4) {
+            vtd_set_long(s, addr, val);
+        } else {
+            vtd_set_quad(s, addr, val);
+        }
+        break;
+
+    case DMAR_IQA_REG_HI:
+        VTD_DPRINTF(INV, "DMAR_IQA_REG_HI write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        break;
+
+    /* Invalidation Completion Status Register, 32-bit */
+    case DMAR_ICS_REG:
+        VTD_DPRINTF(INV, "DMAR_ICS_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        vtd_handle_ics_write(s);
+        break;
+
+    /* Invalidation Event Control Register, 32-bit */
+    case DMAR_IECTL_REG:
+        VTD_DPRINTF(INV, "DMAR_IECTL_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        vtd_handle_iectl_write(s);
+        break;
+
+    /* Invalidation Event Data Register, 32-bit */
+    case DMAR_IEDATA_REG:
+        VTD_DPRINTF(INV, "DMAR_IEDATA_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        break;
+
+    /* Invalidation Event Address Register, 32-bit */
+    case DMAR_IEADDR_REG:
+        VTD_DPRINTF(INV, "DMAR_IEADDR_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        break;
+
+    /* Invalidation Event Upper Address Register, 32-bit */
+    case DMAR_IEUADDR_REG:
+        VTD_DPRINTF(INV, "DMAR_IEUADDR_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        break;
+
+    /* Fault Recording Registers, 128-bit */
+    case DMAR_FRCD_REG_0_0:
+        VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_0 write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        if (size == 4) {
+            vtd_set_long(s, addr, val);
+        } else {
+            vtd_set_quad(s, addr, val);
+        }
+        break;
+
+    case DMAR_FRCD_REG_0_1:
+        VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_1 write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        break;
+
+    case DMAR_FRCD_REG_0_2:
+        VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_2 write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        if (size == 4) {
+            vtd_set_long(s, addr, val);
+        } else {
+            vtd_set_quad(s, addr, val);
+            /* May clear bit 127 (Fault), update PPF */
+            vtd_update_fsts_ppf(s);
+        }
+        break;
+
+    case DMAR_FRCD_REG_0_3:
+        VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_3 write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        /* May clear bit 127 (Fault), update PPF */
+        vtd_update_fsts_ppf(s);
+        break;
+
+    default:
+        VTD_DPRINTF(GENERAL, "error: unhandled reg write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        if (size == 4) {
+            vtd_set_long(s, addr, val);
+        } else {
+            vtd_set_quad(s, addr, val);
+        }
+    }
+}
+
+static IOMMUTLBEntry vtd_iommu_translate(MemoryRegion *iommu, hwaddr addr,
+                                         bool is_write)
+{
+    VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
+    IntelIOMMUState *s = vtd_as->iommu_state;
+    IOMMUTLBEntry ret = {
+        .target_as = &address_space_memory,
+        .iova = addr,
+        .translated_addr = 0,
+        .addr_mask = ~(hwaddr)0,
+        .perm = IOMMU_NONE,
+    };
+
+    if (!s->dmar_enabled) {
+        /* DMAR disabled, passthrough, use 4k-page*/
+        ret.iova = addr & VTD_PAGE_MASK_4K;
+        ret.translated_addr = addr & VTD_PAGE_MASK_4K;
+        ret.addr_mask = ~VTD_PAGE_MASK_4K;
+        ret.perm = IOMMU_RW;
+        return ret;
+    }
+
+    vtd_do_iommu_translate(vtd_as, vtd_as->bus_num, vtd_as->devfn, addr,
+                           is_write, &ret);
+    VTD_DPRINTF(MMU,
+                "bus %"PRIu8 " slot %"PRIu8 " func %"PRIu8 " devfn %"PRIu8
+                " gpa 0x%"PRIx64 " hpa 0x%"PRIx64, vtd_as->bus_num,
+                VTD_PCI_SLOT(vtd_as->devfn), VTD_PCI_FUNC(vtd_as->devfn),
+                vtd_as->devfn, addr, ret.translated_addr);
+    return ret;
+}
+
+static const VMStateDescription vtd_vmstate = {
+    .name = "iommu-intel",
+    .unmigratable = 1,
+};
+
+static const MemoryRegionOps vtd_mem_ops = {
+    .read = vtd_mem_read,
+    .write = vtd_mem_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 8,
+    },
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 8,
+    },
+};
+
+static Property vtd_properties[] = {
+    DEFINE_PROP_UINT32("version", IntelIOMMUState, version, 0),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+/* Do the initialization. It will also be called when reset, so pay
+ * attention when adding new initialization stuff.
+ */
+static void vtd_init(IntelIOMMUState *s)
+{
+    memset(s->csr, 0, DMAR_REG_SIZE);
+    memset(s->wmask, 0, DMAR_REG_SIZE);
+    memset(s->w1cmask, 0, DMAR_REG_SIZE);
+    memset(s->womask, 0, DMAR_REG_SIZE);
+
+    s->iommu_ops.translate = vtd_iommu_translate;
+    s->root = 0;
+    s->root_extended = false;
+    s->dmar_enabled = false;
+    s->iq_head = 0;
+    s->iq_tail = 0;
+    s->iq = 0;
+    s->iq_size = 0;
+    s->qi_enabled = false;
+    s->iq_last_desc_type = VTD_INV_DESC_NONE;
+    s->next_frcd_reg = 0;
+    s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND | VTD_CAP_MGAW |
+             VTD_CAP_SAGAW | VTD_CAP_MAMV | VTD_CAP_PSI;
+    s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO;
+
+    vtd_reset_context_cache(s);
+    vtd_reset_iotlb(s);
+
+    /* Define registers with default values and bit semantics */
+    vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0);
+    vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0);
+    vtd_define_quad(s, DMAR_ECAP_REG, s->ecap, 0, 0);
+    vtd_define_long(s, DMAR_GCMD_REG, 0, 0xff800000UL, 0);
+    vtd_define_long_wo(s, DMAR_GCMD_REG, 0xff800000UL);
+    vtd_define_long(s, DMAR_GSTS_REG, 0, 0, 0);
+    vtd_define_quad(s, DMAR_RTADDR_REG, 0, 0xfffffffffffff000ULL, 0);
+    vtd_define_quad(s, DMAR_CCMD_REG, 0, 0xe0000003ffffffffULL, 0);
+    vtd_define_quad_wo(s, DMAR_CCMD_REG, 0x3ffff0000ULL);
+
+    /* Advanced Fault Logging not supported */
+    vtd_define_long(s, DMAR_FSTS_REG, 0, 0, 0x11UL);
+    vtd_define_long(s, DMAR_FECTL_REG, 0x80000000UL, 0x80000000UL, 0);
+    vtd_define_long(s, DMAR_FEDATA_REG, 0, 0x0000ffffUL, 0);
+    vtd_define_long(s, DMAR_FEADDR_REG, 0, 0xfffffffcUL, 0);
+
+    /* Treated as RsvdZ when EIM in ECAP_REG is not supported
+     * vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0xffffffffUL, 0);
+     */
+    vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0, 0);
+
+    /* Treated as RO for implementations that PLMR and PHMR fields reported
+     * as Clear in the CAP_REG.
+     * vtd_define_long(s, DMAR_PMEN_REG, 0, 0x80000000UL, 0);
+     */
+    vtd_define_long(s, DMAR_PMEN_REG, 0, 0, 0);
+
+    vtd_define_quad(s, DMAR_IQH_REG, 0, 0, 0);
+    vtd_define_quad(s, DMAR_IQT_REG, 0, 0x7fff0ULL, 0);
+    vtd_define_quad(s, DMAR_IQA_REG, 0, 0xfffffffffffff007ULL, 0);
+    vtd_define_long(s, DMAR_ICS_REG, 0, 0, 0x1UL);
+    vtd_define_long(s, DMAR_IECTL_REG, 0x80000000UL, 0x80000000UL, 0);
+    vtd_define_long(s, DMAR_IEDATA_REG, 0, 0xffffffffUL, 0);
+    vtd_define_long(s, DMAR_IEADDR_REG, 0, 0xfffffffcUL, 0);
+    /* Treadted as RsvdZ when EIM in ECAP_REG is not supported */
+    vtd_define_long(s, DMAR_IEUADDR_REG, 0, 0, 0);
+
+    /* IOTLB registers */
+    vtd_define_quad(s, DMAR_IOTLB_REG, 0, 0Xb003ffff00000000ULL, 0);
+    vtd_define_quad(s, DMAR_IVA_REG, 0, 0xfffffffffffff07fULL, 0);
+    vtd_define_quad_wo(s, DMAR_IVA_REG, 0xfffffffffffff07fULL);
+
+    /* Fault Recording Registers, 128-bit */
+    vtd_define_quad(s, DMAR_FRCD_REG_0_0, 0, 0, 0);
+    vtd_define_quad(s, DMAR_FRCD_REG_0_2, 0, 0, 0x8000000000000000ULL);
+}
+
+/* Should not reset address_spaces when reset because devices will still use
+ * the address space they got at first (won't ask the bus again).
+ */
+static void vtd_reset(DeviceState *dev)
+{
+    IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev);
+
+    VTD_DPRINTF(GENERAL, "");
+    vtd_init(s);
+}
+
+static void vtd_realize(DeviceState *dev, Error **errp)
+{
+    IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev);
+
+    VTD_DPRINTF(GENERAL, "");
+    memset(s->address_spaces, 0, sizeof(s->address_spaces));
+    memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s,
+                          "intel_iommu", DMAR_REG_SIZE);
+    sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->csrmem);
+    /* No corresponding destroy */
+    s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
+                                     g_free, g_free);
+    vtd_init(s);
+}
+
+static void vtd_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->reset = vtd_reset;
+    dc->realize = vtd_realize;
+    dc->vmsd = &vtd_vmstate;
+    dc->props = vtd_properties;
+}
+
+static const TypeInfo vtd_info = {
+    .name          = TYPE_INTEL_IOMMU_DEVICE,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(IntelIOMMUState),
+    .class_init    = vtd_class_init,
+};
+
+static void vtd_register_types(void)
+{
+    VTD_DPRINTF(GENERAL, "");
+    type_register_static(&vtd_info);
+}
+
+type_init(vtd_register_types)
diff --git a/hw/i386/intel_iommu_internal.h b/hw/i386/intel_iommu_internal.h
new file mode 100644
index 000000000..ba288ab1d
--- /dev/null
+++ b/hw/i386/intel_iommu_internal.h
@@ -0,0 +1,389 @@
+/*
+ * QEMU emulation of an Intel IOMMU (VT-d)
+ *   (DMA Remapping device)
+ *
+ * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com>
+ * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * Lots of defines copied from kernel/include/linux/intel-iommu.h:
+ *   Copyright (C) 2006-2008 Intel Corporation
+ *   Author: Ashok Raj <ashok.raj@intel.com>
+ *   Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
+ *
+ */
+
+#ifndef HW_I386_INTEL_IOMMU_INTERNAL_H
+#define HW_I386_INTEL_IOMMU_INTERNAL_H
+#include "hw/i386/intel_iommu.h"
+
+/*
+ * Intel IOMMU register specification
+ */
+#define DMAR_VER_REG            0x0  /* Arch version supported by this IOMMU */
+#define DMAR_CAP_REG            0x8  /* Hardware supported capabilities */
+#define DMAR_CAP_REG_HI         0xc  /* High 32-bit of DMAR_CAP_REG */
+#define DMAR_ECAP_REG           0x10 /* Extended capabilities supported */
+#define DMAR_ECAP_REG_HI        0X14
+#define DMAR_GCMD_REG           0x18 /* Global command */
+#define DMAR_GSTS_REG           0x1c /* Global status */
+#define DMAR_RTADDR_REG         0x20 /* Root entry table */
+#define DMAR_RTADDR_REG_HI      0X24
+#define DMAR_CCMD_REG           0x28 /* Context command */
+#define DMAR_CCMD_REG_HI        0x2c
+#define DMAR_FSTS_REG           0x34 /* Fault status */
+#define DMAR_FECTL_REG          0x38 /* Fault control */
+#define DMAR_FEDATA_REG         0x3c /* Fault event interrupt data */
+#define DMAR_FEADDR_REG         0x40 /* Fault event interrupt addr */
+#define DMAR_FEUADDR_REG        0x44 /* Upper address */
+#define DMAR_AFLOG_REG          0x58 /* Advanced fault control */
+#define DMAR_AFLOG_REG_HI       0X5c
+#define DMAR_PMEN_REG           0x64 /* Enable protected memory region */
+#define DMAR_PLMBASE_REG        0x68 /* PMRR low addr */
+#define DMAR_PLMLIMIT_REG       0x6c /* PMRR low limit */
+#define DMAR_PHMBASE_REG        0x70 /* PMRR high base addr */
+#define DMAR_PHMBASE_REG_HI     0X74
+#define DMAR_PHMLIMIT_REG       0x78 /* PMRR high limit */
+#define DMAR_PHMLIMIT_REG_HI    0x7c
+#define DMAR_IQH_REG            0x80 /* Invalidation queue head */
+#define DMAR_IQH_REG_HI         0X84
+#define DMAR_IQT_REG            0x88 /* Invalidation queue tail */
+#define DMAR_IQT_REG_HI         0X8c
+#define DMAR_IQA_REG            0x90 /* Invalidation queue addr */
+#define DMAR_IQA_REG_HI         0x94
+#define DMAR_ICS_REG            0x9c /* Invalidation complete status */
+#define DMAR_IRTA_REG           0xb8 /* Interrupt remapping table addr */
+#define DMAR_IRTA_REG_HI        0xbc
+#define DMAR_IECTL_REG          0xa0 /* Invalidation event control */
+#define DMAR_IEDATA_REG         0xa4 /* Invalidation event data */
+#define DMAR_IEADDR_REG         0xa8 /* Invalidation event address */
+#define DMAR_IEUADDR_REG        0xac /* Invalidation event address */
+#define DMAR_PQH_REG            0xc0 /* Page request queue head */
+#define DMAR_PQH_REG_HI         0xc4
+#define DMAR_PQT_REG            0xc8 /* Page request queue tail*/
+#define DMAR_PQT_REG_HI         0xcc
+#define DMAR_PQA_REG            0xd0 /* Page request queue address */
+#define DMAR_PQA_REG_HI         0xd4
+#define DMAR_PRS_REG            0xdc /* Page request status */
+#define DMAR_PECTL_REG          0xe0 /* Page request event control */
+#define DMAR_PEDATA_REG         0xe4 /* Page request event data */
+#define DMAR_PEADDR_REG         0xe8 /* Page request event address */
+#define DMAR_PEUADDR_REG        0xec /* Page event upper address */
+#define DMAR_MTRRCAP_REG        0x100 /* MTRR capability */
+#define DMAR_MTRRCAP_REG_HI     0x104
+#define DMAR_MTRRDEF_REG        0x108 /* MTRR default type */
+#define DMAR_MTRRDEF_REG_HI     0x10c
+
+/* IOTLB registers */
+#define DMAR_IOTLB_REG_OFFSET   0xf0 /* Offset to the IOTLB registers */
+#define DMAR_IVA_REG            DMAR_IOTLB_REG_OFFSET /* Invalidate address */
+#define DMAR_IVA_REG_HI         (DMAR_IVA_REG + 4)
+/* IOTLB invalidate register */
+#define DMAR_IOTLB_REG          (DMAR_IOTLB_REG_OFFSET + 0x8)
+#define DMAR_IOTLB_REG_HI       (DMAR_IOTLB_REG + 4)
+
+/* FRCD */
+#define DMAR_FRCD_REG_OFFSET    0x220 /* Offset to the fault recording regs */
+/* NOTICE: If you change the DMAR_FRCD_REG_NR, please remember to change the
+ * DMAR_REG_SIZE in include/hw/i386/intel_iommu.h.
+ * #define DMAR_REG_SIZE   (DMAR_FRCD_REG_OFFSET + 16 * DMAR_FRCD_REG_NR)
+ */
+#define DMAR_FRCD_REG_NR        1ULL /* Num of fault recording regs */
+
+#define DMAR_FRCD_REG_0_0       0x220 /* The 0th fault recording regs */
+#define DMAR_FRCD_REG_0_1       0x224
+#define DMAR_FRCD_REG_0_2       0x228
+#define DMAR_FRCD_REG_0_3       0x22c
+
+/* Interrupt Address Range */
+#define VTD_INTERRUPT_ADDR_FIRST    0xfee00000ULL
+#define VTD_INTERRUPT_ADDR_LAST     0xfeefffffULL
+
+/* The shift of source_id in the key of IOTLB hash table */
+#define VTD_IOTLB_SID_SHIFT         36
+#define VTD_IOTLB_MAX_SIZE          1024    /* Max size of the hash table */
+
+/* IOTLB_REG */
+#define VTD_TLB_GLOBAL_FLUSH        (1ULL << 60) /* Global invalidation */
+#define VTD_TLB_DSI_FLUSH           (2ULL << 60) /* Domain-selective */
+#define VTD_TLB_PSI_FLUSH           (3ULL << 60) /* Page-selective */
+#define VTD_TLB_FLUSH_GRANU_MASK    (3ULL << 60)
+#define VTD_TLB_GLOBAL_FLUSH_A      (1ULL << 57)
+#define VTD_TLB_DSI_FLUSH_A         (2ULL << 57)
+#define VTD_TLB_PSI_FLUSH_A         (3ULL << 57)
+#define VTD_TLB_FLUSH_GRANU_MASK_A  (3ULL << 57)
+#define VTD_TLB_IVT                 (1ULL << 63)
+#define VTD_TLB_DID(val)            (((val) >> 32) & VTD_DOMAIN_ID_MASK)
+
+/* IVA_REG */
+#define VTD_IVA_ADDR(val)       ((val) & ~0xfffULL & ((1ULL << VTD_MGAW) - 1))
+#define VTD_IVA_AM(val)         ((val) & 0x3fULL)
+
+/* GCMD_REG */
+#define VTD_GCMD_TE                 (1UL << 31)
+#define VTD_GCMD_SRTP               (1UL << 30)
+#define VTD_GCMD_SFL                (1UL << 29)
+#define VTD_GCMD_EAFL               (1UL << 28)
+#define VTD_GCMD_WBF                (1UL << 27)
+#define VTD_GCMD_QIE                (1UL << 26)
+#define VTD_GCMD_IRE                (1UL << 25)
+#define VTD_GCMD_SIRTP              (1UL << 24)
+#define VTD_GCMD_CFI                (1UL << 23)
+
+/* GSTS_REG */
+#define VTD_GSTS_TES                (1UL << 31)
+#define VTD_GSTS_RTPS               (1UL << 30)
+#define VTD_GSTS_FLS                (1UL << 29)
+#define VTD_GSTS_AFLS               (1UL << 28)
+#define VTD_GSTS_WBFS               (1UL << 27)
+#define VTD_GSTS_QIES               (1UL << 26)
+#define VTD_GSTS_IRES               (1UL << 25)
+#define VTD_GSTS_IRTPS              (1UL << 24)
+#define VTD_GSTS_CFIS               (1UL << 23)
+
+/* CCMD_REG */
+#define VTD_CCMD_ICC                (1ULL << 63)
+#define VTD_CCMD_GLOBAL_INVL        (1ULL << 61)
+#define VTD_CCMD_DOMAIN_INVL        (2ULL << 61)
+#define VTD_CCMD_DEVICE_INVL        (3ULL << 61)
+#define VTD_CCMD_CIRG_MASK          (3ULL << 61)
+#define VTD_CCMD_GLOBAL_INVL_A      (1ULL << 59)
+#define VTD_CCMD_DOMAIN_INVL_A      (2ULL << 59)
+#define VTD_CCMD_DEVICE_INVL_A      (3ULL << 59)
+#define VTD_CCMD_CAIG_MASK          (3ULL << 59)
+#define VTD_CCMD_DID(val)           ((val) & VTD_DOMAIN_ID_MASK)
+#define VTD_CCMD_SID(val)           (((val) >> 16) & 0xffffULL)
+#define VTD_CCMD_FM(val)            (((val) >> 32) & 3ULL)
+
+/* RTADDR_REG */
+#define VTD_RTADDR_RTT              (1ULL << 11)
+#define VTD_RTADDR_ADDR_MASK        (VTD_HAW_MASK ^ 0xfffULL)
+
+/* ECAP_REG */
+/* (offset >> 4) << 8 */
+#define VTD_ECAP_IRO                (DMAR_IOTLB_REG_OFFSET << 4)
+#define VTD_ECAP_QI                 (1ULL << 1)
+
+/* CAP_REG */
+/* (offset >> 4) << 24 */
+#define VTD_CAP_FRO                 (DMAR_FRCD_REG_OFFSET << 20)
+#define VTD_CAP_NFR                 ((DMAR_FRCD_REG_NR - 1) << 40)
+#define VTD_DOMAIN_ID_SHIFT         16  /* 16-bit domain id for 64K domains */
+#define VTD_DOMAIN_ID_MASK          ((1UL << VTD_DOMAIN_ID_SHIFT) - 1)
+#define VTD_CAP_ND                  (((VTD_DOMAIN_ID_SHIFT - 4) / 2) & 7ULL)
+#define VTD_MGAW                    39  /* Maximum Guest Address Width */
+#define VTD_CAP_MGAW                (((VTD_MGAW - 1) & 0x3fULL) << 16)
+#define VTD_MAMV                    9ULL
+#define VTD_CAP_MAMV                (VTD_MAMV << 48)
+#define VTD_CAP_PSI                 (1ULL << 39)
+
+/* Supported Adjusted Guest Address Widths */
+#define VTD_CAP_SAGAW_SHIFT         8
+#define VTD_CAP_SAGAW_MASK          (0x1fULL << VTD_CAP_SAGAW_SHIFT)
+ /* 39-bit AGAW, 3-level page-table */
+#define VTD_CAP_SAGAW_39bit         (0x2ULL << VTD_CAP_SAGAW_SHIFT)
+ /* 48-bit AGAW, 4-level page-table */
+#define VTD_CAP_SAGAW_48bit         (0x4ULL << VTD_CAP_SAGAW_SHIFT)
+#define VTD_CAP_SAGAW               VTD_CAP_SAGAW_39bit
+
+/* IQT_REG */
+#define VTD_IQT_QT(val)             (((val) >> 4) & 0x7fffULL)
+
+/* IQA_REG */
+#define VTD_IQA_IQA_MASK            (VTD_HAW_MASK ^ 0xfffULL)
+#define VTD_IQA_QS                  0x7ULL
+
+/* IQH_REG */
+#define VTD_IQH_QH_SHIFT            4
+#define VTD_IQH_QH_MASK             0x7fff0ULL
+
+/* ICS_REG */
+#define VTD_ICS_IWC                 1UL
+
+/* IECTL_REG */
+#define VTD_IECTL_IM                (1UL << 31)
+#define VTD_IECTL_IP                (1UL << 30)
+
+/* FSTS_REG */
+#define VTD_FSTS_FRI_MASK       0xff00UL
+#define VTD_FSTS_FRI(val)       ((((uint32_t)(val)) << 8) & VTD_FSTS_FRI_MASK)
+#define VTD_FSTS_IQE            (1UL << 4)
+#define VTD_FSTS_PPF            (1UL << 1)
+#define VTD_FSTS_PFO            1UL
+
+/* FECTL_REG */
+#define VTD_FECTL_IM            (1UL << 31)
+#define VTD_FECTL_IP            (1UL << 30)
+
+/* Fault Recording Register */
+/* For the high 64-bit of 128-bit */
+#define VTD_FRCD_F              (1ULL << 63)
+#define VTD_FRCD_T              (1ULL << 62)
+#define VTD_FRCD_FR(val)        (((val) & 0xffULL) << 32)
+#define VTD_FRCD_SID_MASK       0xffffULL
+#define VTD_FRCD_SID(val)       ((val) & VTD_FRCD_SID_MASK)
+/* For the low 64-bit of 128-bit */
+#define VTD_FRCD_FI(val)        ((val) & (((1ULL << VTD_MGAW) - 1) ^ 0xfffULL))
+
+/* DMA Remapping Fault Conditions */
+typedef enum VTDFaultReason {
+    VTD_FR_RESERVED = 0,        /* Reserved for Advanced Fault logging */
+    VTD_FR_ROOT_ENTRY_P = 1,    /* The Present(P) field of root-entry is 0 */
+    VTD_FR_CONTEXT_ENTRY_P,     /* The Present(P) field of context-entry is 0 */
+    VTD_FR_CONTEXT_ENTRY_INV,   /* Invalid programming of a context-entry */
+    VTD_FR_ADDR_BEYOND_MGAW,    /* Input-address above (2^x-1) */
+    VTD_FR_WRITE,               /* No write permission */
+    VTD_FR_READ,                /* No read permission */
+    /* Fail to access a second-level paging entry (not SL_PML4E) */
+    VTD_FR_PAGING_ENTRY_INV,
+    VTD_FR_ROOT_TABLE_INV,      /* Fail to access a root-entry */
+    VTD_FR_CONTEXT_TABLE_INV,   /* Fail to access a context-entry */
+    /* Non-zero reserved field in a present root-entry */
+    VTD_FR_ROOT_ENTRY_RSVD,
+    /* Non-zero reserved field in a present context-entry */
+    VTD_FR_CONTEXT_ENTRY_RSVD,
+    /* Non-zero reserved field in a second-level paging entry with at lease one
+     * Read(R) and Write(W) or Execute(E) field is Set.
+     */
+    VTD_FR_PAGING_ENTRY_RSVD,
+    /* Translation request or translated request explicitly blocked dut to the
+     * programming of the Translation Type (T) field in the present
+     * context-entry.
+     */
+    VTD_FR_CONTEXT_ENTRY_TT,
+    /* This is not a normal fault reason. We use this to indicate some faults
+     * that are not referenced by the VT-d specification.
+     * Fault event with such reason should not be recorded.
+     */
+    VTD_FR_RESERVED_ERR,
+    VTD_FR_MAX,                 /* Guard */
+} VTDFaultReason;
+
+#define VTD_CONTEXT_CACHE_GEN_MAX       0xffffffffUL
+
+/* Queued Invalidation Descriptor */
+struct VTDInvDesc {
+    uint64_t lo;
+    uint64_t hi;
+};
+typedef struct VTDInvDesc VTDInvDesc;
+
+/* Masks for struct VTDInvDesc */
+#define VTD_INV_DESC_TYPE               0xf
+#define VTD_INV_DESC_CC                 0x1 /* Context-cache Invalidate Desc */
+#define VTD_INV_DESC_IOTLB              0x2
+#define VTD_INV_DESC_WAIT               0x5 /* Invalidation Wait Descriptor */
+#define VTD_INV_DESC_NONE               0   /* Not an Invalidate Descriptor */
+
+/* Masks for Invalidation Wait Descriptor*/
+#define VTD_INV_DESC_WAIT_SW            (1ULL << 5)
+#define VTD_INV_DESC_WAIT_IF            (1ULL << 4)
+#define VTD_INV_DESC_WAIT_FN            (1ULL << 6)
+#define VTD_INV_DESC_WAIT_DATA_SHIFT    32
+#define VTD_INV_DESC_WAIT_RSVD_LO       0Xffffff80ULL
+#define VTD_INV_DESC_WAIT_RSVD_HI       3ULL
+
+/* Masks for Context-cache Invalidation Descriptor */
+#define VTD_INV_DESC_CC_G               (3ULL << 4)
+#define VTD_INV_DESC_CC_GLOBAL          (1ULL << 4)
+#define VTD_INV_DESC_CC_DOMAIN          (2ULL << 4)
+#define VTD_INV_DESC_CC_DEVICE          (3ULL << 4)
+#define VTD_INV_DESC_CC_DID(val)        (((val) >> 16) & VTD_DOMAIN_ID_MASK)
+#define VTD_INV_DESC_CC_SID(val)        (((val) >> 32) & 0xffffUL)
+#define VTD_INV_DESC_CC_FM(val)         (((val) >> 48) & 3UL)
+#define VTD_INV_DESC_CC_RSVD            0xfffc00000000ffc0ULL
+
+/* Masks for IOTLB Invalidate Descriptor */
+#define VTD_INV_DESC_IOTLB_G            (3ULL << 4)
+#define VTD_INV_DESC_IOTLB_GLOBAL       (1ULL << 4)
+#define VTD_INV_DESC_IOTLB_DOMAIN       (2ULL << 4)
+#define VTD_INV_DESC_IOTLB_PAGE         (3ULL << 4)
+#define VTD_INV_DESC_IOTLB_DID(val)     (((val) >> 16) & VTD_DOMAIN_ID_MASK)
+#define VTD_INV_DESC_IOTLB_ADDR(val)    ((val) & ~0xfffULL & \
+                                         ((1ULL << VTD_MGAW) - 1))
+#define VTD_INV_DESC_IOTLB_AM(val)      ((val) & 0x3fULL)
+#define VTD_INV_DESC_IOTLB_RSVD_LO      0xffffffff0000ff00ULL
+#define VTD_INV_DESC_IOTLB_RSVD_HI      0xf80ULL
+
+/* Information about page-selective IOTLB invalidate */
+struct VTDIOTLBPageInvInfo {
+    uint16_t domain_id;
+    uint64_t gfn;
+    uint8_t mask;
+};
+typedef struct VTDIOTLBPageInvInfo VTDIOTLBPageInvInfo;
+
+/* Pagesize of VTD paging structures, including root and context tables */
+#define VTD_PAGE_SHIFT              12
+#define VTD_PAGE_SIZE               (1ULL << VTD_PAGE_SHIFT)
+
+#define VTD_PAGE_SHIFT_4K           12
+#define VTD_PAGE_MASK_4K            (~((1ULL << VTD_PAGE_SHIFT_4K) - 1))
+#define VTD_PAGE_SHIFT_2M           21
+#define VTD_PAGE_MASK_2M            (~((1ULL << VTD_PAGE_SHIFT_2M) - 1))
+#define VTD_PAGE_SHIFT_1G           30
+#define VTD_PAGE_MASK_1G            (~((1ULL << VTD_PAGE_SHIFT_1G) - 1))
+
+struct VTDRootEntry {
+    uint64_t val;
+    uint64_t rsvd;
+};
+typedef struct VTDRootEntry VTDRootEntry;
+
+/* Masks for struct VTDRootEntry */
+#define VTD_ROOT_ENTRY_P            1ULL
+#define VTD_ROOT_ENTRY_CTP          (~0xfffULL)
+
+#define VTD_ROOT_ENTRY_NR           (VTD_PAGE_SIZE / sizeof(VTDRootEntry))
+#define VTD_ROOT_ENTRY_RSVD         (0xffeULL | ~VTD_HAW_MASK)
+
+/* Masks for struct VTDContextEntry */
+/* lo */
+#define VTD_CONTEXT_ENTRY_P         (1ULL << 0)
+#define VTD_CONTEXT_ENTRY_FPD       (1ULL << 1) /* Fault Processing Disable */
+#define VTD_CONTEXT_ENTRY_TT        (3ULL << 2) /* Translation Type */
+#define VTD_CONTEXT_TT_MULTI_LEVEL  0
+#define VTD_CONTEXT_TT_DEV_IOTLB    1
+#define VTD_CONTEXT_TT_PASS_THROUGH 2
+/* Second Level Page Translation Pointer*/
+#define VTD_CONTEXT_ENTRY_SLPTPTR   (~0xfffULL)
+#define VTD_CONTEXT_ENTRY_RSVD_LO   (0xff0ULL | ~VTD_HAW_MASK)
+/* hi */
+#define VTD_CONTEXT_ENTRY_AW        7ULL /* Adjusted guest-address-width */
+#define VTD_CONTEXT_ENTRY_DID(val)  (((val) >> 8) & VTD_DOMAIN_ID_MASK)
+#define VTD_CONTEXT_ENTRY_RSVD_HI   0xffffffffff000080ULL
+
+#define VTD_CONTEXT_ENTRY_NR        (VTD_PAGE_SIZE / sizeof(VTDContextEntry))
+
+/* Paging Structure common */
+#define VTD_SL_PT_PAGE_SIZE_MASK    (1ULL << 7)
+/* Bits to decide the offset for each level */
+#define VTD_SL_LEVEL_BITS           9
+
+/* Second Level Paging Structure */
+#define VTD_SL_PML4_LEVEL           4
+#define VTD_SL_PDP_LEVEL            3
+#define VTD_SL_PD_LEVEL             2
+#define VTD_SL_PT_LEVEL             1
+#define VTD_SL_PT_ENTRY_NR          512
+
+/* Masks for Second Level Paging Entry */
+#define VTD_SL_RW_MASK              3ULL
+#define VTD_SL_R                    1ULL
+#define VTD_SL_W                    (1ULL << 1)
+#define VTD_SL_PT_BASE_ADDR_MASK    (~(VTD_PAGE_SIZE - 1) & VTD_HAW_MASK)
+#define VTD_SL_IGN_COM              0xbff0000000000000ULL
+
+#endif
diff --git a/hw/i386/kvm/apic.c b/hw/i386/kvm/apic.c
index e873b509a..271e97f86 100644
--- a/hw/i386/kvm/apic.c
+++ b/hw/i386/kvm/apic.c
@@ -175,6 +175,9 @@ static void kvm_apic_realize(DeviceState *dev, Error **errp)
 {
     APICCommonState *s = APIC_COMMON(dev);
 
+    /* Not used by KVM, which uses the CPU mp_state instead.  */
+    s->wait_for_sipi = 0;
+
     memory_region_init_io(&s->io_memory, NULL, &kvm_apic_io_ops, s, "kvm-apic-msi",
                           APIC_SPACE_SIZE);
 
diff --git a/hw/i386/kvm/clock.c b/hw/i386/kvm/clock.c
index 07b9c0e58..58be2bda2 100644
--- a/hw/i386/kvm/clock.c
+++ b/hw/i386/kvm/clock.c
@@ -14,8 +14,10 @@
  */
 
 #include "qemu-common.h"
+#include "qemu/host-utils.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
+#include "sysemu/cpus.h"
 #include "hw/sysbus.h"
 #include "hw/kvm/clock.h"
 
@@ -34,6 +36,48 @@ typedef struct KVMClockState {
     bool clock_valid;
 } KVMClockState;
 
+struct pvclock_vcpu_time_info {
+    uint32_t   version;
+    uint32_t   pad0;
+    uint64_t   tsc_timestamp;
+    uint64_t   system_time;
+    uint32_t   tsc_to_system_mul;
+    int8_t     tsc_shift;
+    uint8_t    flags;
+    uint8_t    pad[2];
+} __attribute__((__packed__)); /* 32 bytes */
+
+static uint64_t kvmclock_current_nsec(KVMClockState *s)
+{
+    CPUState *cpu = first_cpu;
+    CPUX86State *env = cpu->env_ptr;
+    hwaddr kvmclock_struct_pa = env->system_time_msr & ~1ULL;
+    uint64_t migration_tsc = env->tsc;
+    struct pvclock_vcpu_time_info time;
+    uint64_t delta;
+    uint64_t nsec_lo;
+    uint64_t nsec_hi;
+    uint64_t nsec;
+
+    if (!(env->system_time_msr & 1ULL)) {
+        /* KVM clock not active */
+        return 0;
+    }
+
+    cpu_physical_memory_read(kvmclock_struct_pa, &time, sizeof(time));
+
+    assert(time.tsc_timestamp <= migration_tsc);
+    delta = migration_tsc - time.tsc_timestamp;
+    if (time.tsc_shift < 0) {
+        delta >>= -time.tsc_shift;
+    } else {
+        delta <<= time.tsc_shift;
+    }
+
+    mulu64(&nsec_lo, &nsec_hi, delta, time.tsc_to_system_mul);
+    nsec = (nsec_lo >> 32) | (nsec_hi << 32);
+    return nsec + time.system_time;
+}
 
 static void kvmclock_vm_state_change(void *opaque, int running,
                                      RunState state)
@@ -45,9 +89,15 @@ static void kvmclock_vm_state_change(void *opaque, int running,
 
     if (running) {
         struct kvm_clock_data data;
+        uint64_t time_at_migration = kvmclock_current_nsec(s);
 
         s->clock_valid = false;
 
+        /* We can't rely on the migrated clock value, just discard it */
+        if (time_at_migration) {
+            s->clock = time_at_migration;
+        }
+
         data.clock = s->clock;
         data.flags = 0;
         ret = kvm_vm_ioctl(kvm_state, KVM_SET_CLOCK, &data);
@@ -75,6 +125,23 @@ static void kvmclock_vm_state_change(void *opaque, int running,
         if (s->clock_valid) {
             return;
         }
+
+        cpu_synchronize_all_states();
+        /* In theory, the cpu_synchronize_all_states() call above wouldn't
+         * affect the rest of the code, as the VCPU state inside CPUState
+         * is supposed to always match the VCPU state on the kernel side.
+         *
+         * In practice, calling cpu_synchronize_state() too soon will load the
+         * kernel-side APIC state into X86CPU.apic_state too early, APIC state
+         * won't be reloaded later because CPUState.vcpu_dirty==true, and
+         * outdated APIC state may be migrated to another host.
+         *
+         * The real fix would be to make sure outdated APIC state is read
+         * from the kernel again when necessary. While this is not fixed, we
+         * need the cpu_clean_all_dirty() call below.
+         */
+        cpu_clean_all_dirty();
+
         ret = kvm_vm_ioctl(kvm_state, KVM_GET_CLOCK, &data);
         if (ret < 0) {
             fprintf(stderr, "KVM_GET_CLOCK failed: %s\n", strerror(ret));
diff --git a/hw/i386/kvm/i8254.c b/hw/i386/kvm/i8254.c
index 59373aaad..472af811c 100644
--- a/hw/i386/kvm/i8254.c
+++ b/hw/i386/kvm/i8254.c
@@ -239,6 +239,7 @@ static void kvm_pit_vm_state_change(void *opaque, int running,
 
     if (running) {
         kvm_pit_update_clock_offset(s);
+        kvm_pit_put(PIT_COMMON(s));
         s->vm_stopped = false;
     } else {
         kvm_pit_update_clock_offset(s);
@@ -314,8 +315,6 @@ static void kvm_pit_class_init(ObjectClass *klass, void *data)
     dc->realize = kvm_pit_realizefn;
     k->set_channel_gate = kvm_pit_set_gate;
     k->get_channel_info = kvm_pit_get_channel_info;
-    k->pre_save = kvm_pit_get;
-    k->post_load = kvm_pit_put;
     dc->reset = kvm_pit_reset;
     dc->props = kvm_pit_properties;
 }
diff --git a/hw/i386/kvm/pci-assign.c b/hw/i386/kvm/pci-assign.c
index de3365756..bb206da05 100644
--- a/hw/i386/kvm/pci-assign.c
+++ b/hw/i386/kvm/pci-assign.c
@@ -697,8 +697,6 @@ static void free_assigned_device(AssignedDevice *dev)
             if (region->u.r_baseport) {
                 memory_region_del_subregion(&region->container,
                                             &region->real_iomem);
-                memory_region_destroy(&region->real_iomem);
-                memory_region_destroy(&region->container);
             }
         } else if (pci_region->type & IORESOURCE_MEM) {
             if (region->u.r_virtbase) {
@@ -712,9 +710,6 @@ static void free_assigned_device(AssignedDevice *dev)
                     memory_region_del_subregion(&region->container,
                                                 &dev->mmio);
                 }
-
-                memory_region_destroy(&region->real_iomem);
-                memory_region_destroy(&region->container);
                 if (munmap(region->u.r_virtbase,
                            (pci_region->size + 0xFFF) & 0xFFFFF000)) {
                     error_report("Failed to unmap assigned device region: %s",
@@ -1680,8 +1675,6 @@ static void assigned_dev_unregister_msix_mmio(AssignedDevice *dev)
         return;
     }
 
-    memory_region_destroy(&dev->mmio);
-
     if (munmap(dev->msix_table, MSIX_PAGE_SIZE) == -1) {
         error_report("error unmapping msix_table! %s", strerror(errno));
     }
@@ -1832,8 +1825,6 @@ static int assigned_initfn(struct PCIDevice *pci_dev)
 
     assigned_dev_load_option_rom(dev);
 
-    add_boot_device_path(dev->bootindex, &pci_dev->qdev, NULL);
-
     return 0;
 
 assigned_out:
@@ -1857,13 +1848,22 @@ static void assigned_exitfn(struct PCIDevice *pci_dev)
     free_assigned_device(dev);
 }
 
+static void assigned_dev_instance_init(Object *obj)
+{
+    PCIDevice *pci_dev = PCI_DEVICE(obj);
+    AssignedDevice *d = DO_UPCAST(AssignedDevice, dev, PCI_DEVICE(obj));
+
+    device_add_bootindex_property(obj, &d->bootindex,
+                                  "bootindex", NULL,
+                                  &pci_dev->qdev, NULL);
+}
+
 static Property assigned_dev_properties[] = {
     DEFINE_PROP_PCI_HOST_DEVADDR("host", AssignedDevice, host),
     DEFINE_PROP_BIT("prefer_msi", AssignedDevice, features,
                     ASSIGNED_DEVICE_PREFER_MSI_BIT, false),
     DEFINE_PROP_BIT("share_intx", AssignedDevice, features,
                     ASSIGNED_DEVICE_SHARE_INTX_BIT, true),
-    DEFINE_PROP_INT32("bootindex", AssignedDevice, bootindex, -1),
     DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name),
     DEFINE_PROP_END_OF_LIST(),
 };
@@ -1889,6 +1889,7 @@ static const TypeInfo assign_info = {
     .parent             = TYPE_PCI_DEVICE,
     .instance_size      = sizeof(AssignedDevice),
     .class_init         = assign_class_init,
+    .instance_init      = assigned_dev_instance_init,
 };
 
 static void assign_register_types(void)
@@ -1943,7 +1944,8 @@ static void assigned_dev_load_option_rom(AssignedDevice *dev)
 
     snprintf(name, sizeof(name), "%s.rom",
             object_get_typename(OBJECT(dev)));
-    memory_region_init_ram(&dev->dev.rom, OBJECT(dev), name, st.st_size);
+    memory_region_init_ram(&dev->dev.rom, OBJECT(dev), name, st.st_size,
+                           &error_abort);
     vmstate_register_ram(&dev->dev.rom, &dev->dev.qdev);
     ptr = memory_region_get_ram_ptr(&dev->dev.rom);
     memset(ptr, 0xff, st.st_size);
@@ -1953,7 +1955,6 @@ static void assigned_dev_load_option_rom(AssignedDevice *dev)
         error_printf("Device option ROM contents are probably invalid "
                      "(check dmesg).\nSkip option ROM probe with rombar=0, "
                      "or load from file with romfile=\n");
-        memory_region_destroy(&dev->dev.rom);
         goto close_rom;
     }
 
diff --git a/hw/i386/kvmvapic.c b/hw/i386/kvmvapic.c
index cb855c7b7..c6d34b254 100644
--- a/hw/i386/kvmvapic.c
+++ b/hw/i386/kvmvapic.c
@@ -59,6 +59,7 @@ typedef struct VAPICROMState {
     GuestROMState rom_state;
     size_t rom_size;
     bool rom_mapped_writable;
+    VMChangeStateEntry *vmsentry;
 } VAPICROMState;
 
 #define TYPE_VAPIC "kvmvapic"
@@ -404,7 +405,6 @@ static void patch_instruction(VAPICROMState *s, X86CPU *cpu, target_ulong ip)
     }
 
     if (!kvm_enabled()) {
-        cpu_restore_state(cs, cs->mem_io_pc);
         cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
                              &current_flags);
     }
@@ -584,7 +584,7 @@ static int vapic_map_rom_writable(VAPICROMState *s)
 
     if (s->rom_mapped_writable) {
         memory_region_del_subregion(as, &s->rom);
-        memory_region_destroy(&s->rom);
+        object_unparent(OBJECT(&s->rom));
     }
 
     /* grab RAM memory region (region @rom_paddr may still be pc.rom) */
@@ -731,14 +731,41 @@ static void do_vapic_enable(void *data)
     VAPICROMState *s = data;
     X86CPU *cpu = X86_CPU(first_cpu);
 
-    vapic_enable(s, cpu);
+    static const uint8_t enabled = 1;
+    cpu_physical_memory_write(s->vapic_paddr + offsetof(VAPICState, enabled),
+                              &enabled, sizeof(enabled));
+    apic_enable_vapic(cpu->apic_state, s->vapic_paddr);
+    s->state = VAPIC_ACTIVE;
 }
 
-static int vapic_post_load(void *opaque, int version_id)
+static void kvmvapic_vm_state_change(void *opaque, int running,
+                                     RunState state)
 {
     VAPICROMState *s = opaque;
     uint8_t *zero;
 
+    if (!running) {
+        return;
+    }
+
+    if (s->state == VAPIC_ACTIVE) {
+        if (smp_cpus == 1) {
+            run_on_cpu(first_cpu, do_vapic_enable, s);
+        } else {
+            zero = g_malloc0(s->rom_state.vapic_size);
+            cpu_physical_memory_write(s->vapic_paddr, zero,
+                                      s->rom_state.vapic_size);
+            g_free(zero);
+        }
+    }
+
+    qemu_del_vm_change_state_handler(s->vmsentry);
+}
+
+static int vapic_post_load(void *opaque, int version_id)
+{
+    VAPICROMState *s = opaque;
+
     /*
      * The old implementation of qemu-kvm did not provide the state
      * VAPIC_STANDBY. Reconstruct it.
@@ -752,17 +779,11 @@ static int vapic_post_load(void *opaque, int version_id)
             return -1;
         }
     }
-    if (s->state == VAPIC_ACTIVE) {
-        if (smp_cpus == 1) {
-            run_on_cpu(first_cpu, do_vapic_enable, s);
-        } else {
-            zero = g_malloc0(s->rom_state.vapic_size);
-            cpu_physical_memory_write(s->vapic_paddr, zero,
-                                      s->rom_state.vapic_size);
-            g_free(zero);
-        }
-    }
 
+    if (!s->vmsentry) {
+        s->vmsentry =
+            qemu_add_vm_change_state_handler(kvmvapic_vm_state_change, s);
+    }
     return 0;
 }
 
diff --git a/hw/i386/pc.c b/hw/i386/pc.c
index 2cf22b129..f31d55e7e 100644
--- a/hw/i386/pc.c
+++ b/hw/i386/pc.c
@@ -44,7 +44,7 @@
 #include "sysemu/kvm.h"
 #include "kvm_i386.h"
 #include "hw/xen/xen.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "hw/block/block.h"
 #include "ui/qemu-spice.h"
 #include "exec/memory.h"
@@ -61,6 +61,7 @@
 #include "hw/mem/pc-dimm.h"
 #include "trace.h"
 #include "qapi/visitor.h"
+#include "qapi-visit.h"
 
 /* debug PC/ISA interrupts */
 //#define DEBUG_IRQ
@@ -72,8 +73,15 @@
 #define DPRINTF(fmt, ...)
 #endif
 
-/* Leave a chunk of memory at the top of RAM for the BIOS ACPI tables.  */
-#define ACPI_DATA_SIZE       0x10000
+/* Leave a chunk of memory at the top of RAM for the BIOS ACPI tables
+ * (128K) and other BIOS datastructures (less than 4K reported to be used at
+ * the moment, 32K should be enough for a while).  */
+static unsigned acpi_data_size = 0x20000 + 0x8000;
+void pc_set_legacy_acpi_data_size(void)
+{
+    acpi_data_size = 0x10000;
+}
+
 #define BIOS_CFG_IOPORT 0x510
 #define FW_CFG_ACPI_TABLES (FW_CFG_ARCH_LOCAL + 0)
 #define FW_CFG_SMBIOS_ENTRIES (FW_CFG_ARCH_LOCAL + 1)
@@ -348,30 +356,15 @@ static void pc_cmos_init_late(void *opaque)
     qemu_unregister_reset(pc_cmos_init_late, opaque);
 }
 
-typedef struct RTCCPUHotplugArg {
-    Notifier cpu_added_notifier;
-    ISADevice *rtc_state;
-} RTCCPUHotplugArg;
-
-static void rtc_notify_cpu_added(Notifier *notifier, void *data)
-{
-    RTCCPUHotplugArg *arg = container_of(notifier, RTCCPUHotplugArg,
-                                         cpu_added_notifier);
-    ISADevice *s = arg->rtc_state;
-
-    /* increment the number of CPUs */
-    rtc_set_memory(s, 0x5f, rtc_get_memory(s, 0x5f) + 1);
-}
-
 void pc_cmos_init(ram_addr_t ram_size, ram_addr_t above_4g_mem_size,
-                  const char *boot_device,
+                  const char *boot_device, MachineState *machine,
                   ISADevice *floppy, BusState *idebus0, BusState *idebus1,
                   ISADevice *s)
 {
     int val, nb, i;
     FDriveType fd_type[2] = { FDRIVE_DRV_NONE, FDRIVE_DRV_NONE };
     static pc_cmos_init_late_arg arg;
-    static RTCCPUHotplugArg cpu_hotplug_cb;
+    PCMachineState *pc_machine = PC_MACHINE(machine);
 
     /* various important CMOS locations needed by PC/Bochs bios */
 
@@ -410,10 +403,14 @@ void pc_cmos_init(ram_addr_t ram_size, ram_addr_t above_4g_mem_size,
 
     /* set the number of CPU */
     rtc_set_memory(s, 0x5f, smp_cpus - 1);
-    /* init CPU hotplug notifier */
-    cpu_hotplug_cb.rtc_state = s;
-    cpu_hotplug_cb.cpu_added_notifier.notify = rtc_notify_cpu_added;
-    qemu_register_cpu_added_notifier(&cpu_hotplug_cb.cpu_added_notifier);
+
+    object_property_add_link(OBJECT(machine), "rtc_state",
+                             TYPE_ISA_DEVICE,
+                             (Object **)&pc_machine->rtc,
+                             object_property_allow_set_link,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE, &error_abort);
+    object_property_set_link(OBJECT(machine), OBJECT(s),
+                             "rtc_state", &error_abort);
 
     if (set_boot_dev(s, boot_device)) {
         exit(1);
@@ -476,7 +473,7 @@ static void port92_write(void *opaque, hwaddr addr, uint64_t val,
     Port92State *s = opaque;
     int oldval = s->outport;
 
-    DPRINTF("port92: write 0x%02x\n", val);
+    DPRINTF("port92: write 0x%02" PRIx64 "\n", val);
     s->outport = val;
     qemu_set_irq(*s->a20_out, (val >> 1) & 1);
     if ((val & 1) && !(oldval & 1)) {
@@ -811,8 +808,9 @@ static void load_linux(FWCfgState *fw_cfg,
         initrd_max = 0x37ffffff;
     }
 
-    if (initrd_max >= max_ram_size-ACPI_DATA_SIZE)
-    	initrd_max = max_ram_size-ACPI_DATA_SIZE-1;
+    if (initrd_max >= max_ram_size - acpi_data_size) {
+        initrd_max = max_ram_size - acpi_data_size - 1;
+    }
 
     fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_ADDR, cmdline_addr);
     fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_SIZE, strlen(kernel_cmdline)+1);
@@ -1066,35 +1064,6 @@ typedef struct PcRomPciInfo {
     uint64_t w64_max;
 } PcRomPciInfo;
 
-static void pc_fw_cfg_guest_info(PcGuestInfo *guest_info)
-{
-    PcRomPciInfo *info;
-    Object *pci_info;
-    bool ambiguous = false;
-
-    if (!guest_info->has_pci_info || !guest_info->fw_cfg) {
-        return;
-    }
-    pci_info = object_resolve_path_type("", TYPE_PCI_HOST_BRIDGE, &ambiguous);
-    g_assert(!ambiguous);
-    if (!pci_info) {
-        return;
-    }
-
-    info = g_malloc(sizeof *info);
-    info->w32_min = cpu_to_le64(object_property_get_int(pci_info,
-                                PCI_HOST_PROP_PCI_HOLE_START, NULL));
-    info->w32_max = cpu_to_le64(object_property_get_int(pci_info,
-                                PCI_HOST_PROP_PCI_HOLE_END, NULL));
-    info->w64_min = cpu_to_le64(object_property_get_int(pci_info,
-                                PCI_HOST_PROP_PCI_HOLE64_START, NULL));
-    info->w64_max = cpu_to_le64(object_property_get_int(pci_info,
-                                PCI_HOST_PROP_PCI_HOLE64_END, NULL));
-    /* Pass PCI hole info to guest via a side channel.
-     * Required so guest PCI enumeration does the right thing. */
-    fw_cfg_add_file(guest_info->fw_cfg, "etc/pci-info", info, sizeof *info);
-}
-
 typedef struct PcGuestInfoState {
     PcGuestInfo info;
     Notifier machine_done;
@@ -1106,7 +1075,6 @@ void pc_guest_info_machine_done(Notifier *notifier, void *data)
     PcGuestInfoState *guest_info_state = container_of(notifier,
                                                       PcGuestInfoState,
                                                       machine_done);
-    pc_fw_cfg_guest_info(&guest_info_state->info);
     acpi_setup(&guest_info_state->info);
 }
 
@@ -1190,6 +1158,31 @@ void pc_acpi_init(const char *default_dsdt)
     }
 }
 
+FWCfgState *xen_load_linux(const char *kernel_filename,
+                           const char *kernel_cmdline,
+                           const char *initrd_filename,
+                           ram_addr_t below_4g_mem_size,
+                           PcGuestInfo *guest_info)
+{
+    int i;
+    FWCfgState *fw_cfg;
+
+    assert(kernel_filename != NULL);
+
+    fw_cfg = fw_cfg_init(BIOS_CFG_IOPORT, BIOS_CFG_IOPORT + 1, 0, 0);
+    rom_set_fw(fw_cfg);
+
+    load_linux(fw_cfg, kernel_filename, initrd_filename,
+               kernel_cmdline, below_4g_mem_size);
+    for (i = 0; i < nb_option_roms; i++) {
+        assert(!strcmp(option_rom[i].name, "linuxboot.bin") ||
+               !strcmp(option_rom[i].name, "multiboot.bin"));
+        rom_add_option(option_rom[i].name, option_rom[i].bootindex);
+    }
+    guest_info->fw_cfg = fw_cfg;
+    return fw_cfg;
+}
+
 FWCfgState *pc_memory_init(MachineState *machine,
                            MemoryRegion *system_memory,
                            ram_addr_t below_4g_mem_size,
@@ -1255,6 +1248,11 @@ FWCfgState *pc_memory_init(MachineState *machine,
         pcms->hotplug_memory_base =
             ROUND_UP(0x100000000ULL + above_4g_mem_size, 1ULL << 30);
 
+        if (pcms->enforce_aligned_dimm) {
+            /* size hotplug region assuming 1G page max alignment per slot */
+            hotplug_mem_size += (1ULL << 30) * machine->ram_slots;
+        }
+
         if ((pcms->hotplug_memory_base + hotplug_mem_size) <
             hotplug_mem_size) {
             error_report("unsupported amount of maximum memory: " RAM_ADDR_FMT,
@@ -1272,7 +1270,8 @@ FWCfgState *pc_memory_init(MachineState *machine,
     pc_system_firmware_init(rom_memory, guest_info->isapc_ram_fw);
 
     option_rom_mr = g_malloc(sizeof(*option_rom_mr));
-    memory_region_init_ram(option_rom_mr, NULL, "pc.rom", PC_ROM_SIZE);
+    memory_region_init_ram(option_rom_mr, NULL, "pc.rom", PC_ROM_SIZE,
+                           &error_abort);
     vmstate_register_ram_global(option_rom_mr);
     memory_region_add_subregion_overlap(rom_memory,
                                         PC_ROM_MIN_VGA,
@@ -1512,6 +1511,7 @@ static void pc_generic_machine_class_init(ObjectClass *oc, void *data)
     MachineClass *mc = MACHINE_CLASS(oc);
     QEMUMachine *qm = data;
 
+    mc->family = qm->family;
     mc->name = qm->name;
     mc->alias = qm->alias;
     mc->desc = qm->desc;
@@ -1520,6 +1520,7 @@ static void pc_generic_machine_class_init(ObjectClass *oc, void *data)
     mc->hot_add_cpu = qm->hot_add_cpu;
     mc->kvm_type = qm->kvm_type;
     mc->block_default_type = qm->block_default_type;
+    mc->units_per_default_bus = qm->units_per_default_bus;
     mc->max_cpus = qm->max_cpus;
     mc->no_serial = qm->no_serial;
     mc->no_parallel = qm->no_parallel;
@@ -1531,6 +1532,7 @@ static void pc_generic_machine_class_init(ObjectClass *oc, void *data)
     mc->is_default = qm->is_default;
     mc->default_machine_opts = qm->default_machine_opts;
     mc->default_boot_order = qm->default_boot_order;
+    mc->default_display = qm->default_display;
     mc->compat_props = qm->compat_props;
     mc->hw_version = qm->hw_version;
 }
@@ -1549,6 +1551,37 @@ void qemu_register_pc_machine(QEMUMachine *m)
     g_free(name);
 }
 
+static int pc_dimm_count(Object *obj, void *opaque)
+{
+    int *count = opaque;
+
+    if (object_dynamic_cast(obj, TYPE_PC_DIMM)) {
+        (*count)++;
+    }
+
+    object_child_foreach(obj, pc_dimm_count, opaque);
+    return 0;
+}
+
+static int pc_existing_dimms_capacity(Object *obj, void *opaque)
+{
+    Error *local_err = NULL;
+    uint64_t *size = opaque;
+
+    if (object_dynamic_cast(obj, TYPE_PC_DIMM)) {
+        (*size) += object_property_get_int(obj, PC_DIMM_SIZE_PROP, &local_err);
+
+        if (local_err) {
+            qerror_report_err(local_err);
+            error_free(local_err);
+            return 1;
+        }
+    }
+
+    object_child_foreach(obj, pc_dimm_count, opaque);
+    return 0;
+}
+
 static void pc_dimm_plug(HotplugHandler *hotplug_dev,
                          DeviceState *dev, Error **errp)
 {
@@ -1560,20 +1593,40 @@ static void pc_dimm_plug(HotplugHandler *hotplug_dev,
     PCDIMMDevice *dimm = PC_DIMM(dev);
     PCDIMMDeviceClass *ddc = PC_DIMM_GET_CLASS(dimm);
     MemoryRegion *mr = ddc->get_memory_region(dimm);
-    uint64_t addr = object_property_get_int(OBJECT(dimm), PC_DIMM_ADDR_PROP,
-                                            &local_err);
+    uint64_t existing_dimms_capacity = 0;
+    uint64_t align = TARGET_PAGE_SIZE;
+    uint64_t addr;
+
+    addr = object_property_get_int(OBJECT(dimm), PC_DIMM_ADDR_PROP, &local_err);
     if (local_err) {
         goto out;
     }
 
+    if (memory_region_get_alignment(mr) && pcms->enforce_aligned_dimm) {
+        align = memory_region_get_alignment(mr);
+    }
+
     addr = pc_dimm_get_free_addr(pcms->hotplug_memory_base,
                                  memory_region_size(&pcms->hotplug_memory),
-                                 !addr ? NULL : &addr,
+                                 !addr ? NULL : &addr, align,
                                  memory_region_size(mr), &local_err);
     if (local_err) {
         goto out;
     }
 
+    if (pc_existing_dimms_capacity(OBJECT(machine), &existing_dimms_capacity)) {
+        error_setg(&local_err, "failed to get total size of existing DIMMs");
+        goto out;
+    }
+
+    if (existing_dimms_capacity + memory_region_size(mr) >
+        machine->maxram_size - machine->ram_size) {
+        error_setg(&local_err, "not enough space, currently 0x%" PRIx64
+                   " in use of total 0x" RAM_ADDR_FMT,
+                   existing_dimms_capacity, machine->maxram_size);
+        goto out;
+    }
+
     object_property_set_int(OBJECT(dev), addr, PC_DIMM_ADDR_PROP, &local_err);
     if (local_err) {
         goto out;
@@ -1602,6 +1655,11 @@ static void pc_dimm_plug(HotplugHandler *hotplug_dev,
         goto out;
     }
 
+    if (kvm_enabled() && !kvm_has_free_slot(machine)) {
+        error_setg(&local_err, "hypervisor has no free memory slots left");
+        goto out;
+    }
+
     memory_region_add_subregion(&pcms->hotplug_memory,
                                 addr - pcms->hotplug_memory_base, mr);
     vmstate_register_ram(mr, dev);
@@ -1612,11 +1670,42 @@ out:
     error_propagate(errp, local_err);
 }
 
+static void pc_cpu_plug(HotplugHandler *hotplug_dev,
+                        DeviceState *dev, Error **errp)
+{
+    HotplugHandlerClass *hhc;
+    Error *local_err = NULL;
+    PCMachineState *pcms = PC_MACHINE(hotplug_dev);
+
+    if (!dev->hotplugged) {
+        goto out;
+    }
+
+    if (!pcms->acpi_dev) {
+        error_setg(&local_err,
+                   "cpu hotplug is not enabled: missing acpi device");
+        goto out;
+    }
+
+    hhc = HOTPLUG_HANDLER_GET_CLASS(pcms->acpi_dev);
+    hhc->plug(HOTPLUG_HANDLER(pcms->acpi_dev), dev, &local_err);
+    if (local_err) {
+        goto out;
+    }
+
+    /* increment the number of CPUs */
+    rtc_set_memory(pcms->rtc, 0x5f, rtc_get_memory(pcms->rtc, 0x5f) + 1);
+out:
+    error_propagate(errp, local_err);
+}
+
 static void pc_machine_device_plug_cb(HotplugHandler *hotplug_dev,
                                       DeviceState *dev, Error **errp)
 {
     if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
         pc_dimm_plug(hotplug_dev, dev, errp);
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
+        pc_cpu_plug(hotplug_dev, dev, errp);
     }
 }
 
@@ -1625,7 +1714,8 @@ static HotplugHandler *pc_get_hotpug_handler(MachineState *machine,
 {
     PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(machine);
 
-    if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM) ||
+        object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
         return HOTPLUG_HANDLER(machine);
     }
 
@@ -1683,6 +1773,30 @@ static void pc_machine_set_max_ram_below_4g(Object *obj, Visitor *v,
     pcms->max_ram_below_4g = value;
 }
 
+static void pc_machine_get_vmport(Object *obj, Visitor *v, void *opaque,
+                                  const char *name, Error **errp)
+{
+    PCMachineState *pcms = PC_MACHINE(obj);
+    OnOffAuto vmport = pcms->vmport;
+
+    visit_type_OnOffAuto(v, &vmport, name, errp);
+}
+
+static void pc_machine_set_vmport(Object *obj, Visitor *v, void *opaque,
+                                  const char *name, Error **errp)
+{
+    PCMachineState *pcms = PC_MACHINE(obj);
+
+    visit_type_OnOffAuto(v, &pcms->vmport, name, errp);
+}
+
+static bool pc_machine_get_aligned_dimm(Object *obj, Error **errp)
+{
+    PCMachineState *pcms = PC_MACHINE(obj);
+
+    return pcms->enforce_aligned_dimm;
+}
+
 static void pc_machine_initfn(Object *obj)
 {
     PCMachineState *pcms = PC_MACHINE(obj);
@@ -1695,6 +1809,17 @@ static void pc_machine_initfn(Object *obj)
                         pc_machine_get_max_ram_below_4g,
                         pc_machine_set_max_ram_below_4g,
                         NULL, NULL, NULL);
+
+    pcms->vmport = ON_OFF_AUTO_AUTO;
+    object_property_add(obj, PC_MACHINE_VMPORT, "OnOffAuto",
+                        pc_machine_get_vmport,
+                        pc_machine_set_vmport,
+                        NULL, NULL, NULL);
+
+    pcms->enforce_aligned_dimm = true;
+    object_property_add_bool(obj, PC_MACHINE_ENFORCE_ALIGNED_DIMM,
+                             pc_machine_get_aligned_dimm,
+                             NULL, NULL);
 }
 
 static void pc_machine_class_init(ObjectClass *oc, void *data)
diff --git a/hw/i386/pc_piix.c b/hw/i386/pc_piix.c
index 9694f8805..85ed3c878 100644
--- a/hw/i386/pc_piix.c
+++ b/hw/i386/pc_piix.c
@@ -41,7 +41,7 @@
 #include "hw/sysbus.h"
 #include "hw/cpu/icc_bus.h"
 #include "sysemu/arch_init.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "hw/i2c/smbus.h"
 #include "hw/xen/xen.h"
 #include "exec/memory.h"
@@ -59,11 +59,11 @@ static const int ide_iobase[MAX_IDE_BUS] = { 0x1f0, 0x170 };
 static const int ide_iobase2[MAX_IDE_BUS] = { 0x3f6, 0x376 };
 static const int ide_irq[MAX_IDE_BUS] = { 14, 15 };
 
-static bool has_pci_info;
 static bool has_acpi_build = true;
 static int legacy_acpi_table_size;
 static bool smbios_defaults = true;
 static bool smbios_legacy_mode;
+static bool smbios_uuid_encoded = true;
 /* Make sure that guest addresses aligned at 1Gbyte boundaries get mapped to
  * host addresses aligned at 1Gbyte boundaries.  This way we can use 1GByte
  * pages in the host.
@@ -166,7 +166,6 @@ static void pc_init1(MachineState *machine,
     guest_info->has_acpi_build = has_acpi_build;
     guest_info->legacy_acpi_table_size = legacy_acpi_table_size;
 
-    guest_info->has_pci_info = has_pci_info;
     guest_info->isapc_ram_fw = !pci_enabled;
     guest_info->has_reserved_memory = has_reserved_memory;
 
@@ -174,7 +173,7 @@ static void pc_init1(MachineState *machine,
         MachineClass *mc = MACHINE_GET_CLASS(machine);
         /* These values are guest ABI, do not change */
         smbios_set_defaults("QEMU", "Standard PC (i440FX + PIIX, 1996)",
-                            mc->name, smbios_legacy_mode);
+                            mc->name, smbios_legacy_mode, smbios_uuid_encoded);
     }
 
     /* allocate ram and load rom/bios */
@@ -182,6 +181,13 @@ static void pc_init1(MachineState *machine,
         fw_cfg = pc_memory_init(machine, system_memory,
                                 below_4g_mem_size, above_4g_mem_size,
                                 rom_memory, &ram_memory, guest_info);
+    } else if (machine->kernel_filename != NULL) {
+        /* For xen HVM direct kernel boot, load linux here */
+        fw_cfg = xen_load_linux(machine->kernel_filename,
+                                machine->kernel_cmdline,
+                                machine->initrd_filename,
+                                below_4g_mem_size,
+                                guest_info);
     }
 
     gsi_state = g_malloc0(sizeof(*gsi_state));
@@ -228,13 +234,18 @@ static void pc_init1(MachineState *machine,
 
     pc_vga_init(isa_bus, pci_enabled ? pci_bus : NULL);
 
+    assert(pc_machine->vmport != ON_OFF_AUTO_MAX);
+    if (pc_machine->vmport == ON_OFF_AUTO_AUTO) {
+        pc_machine->vmport = xen_enabled() ? ON_OFF_AUTO_OFF : ON_OFF_AUTO_ON;
+    }
+
     /* init basic PC hardware */
-    pc_basic_device_init(isa_bus, gsi, &rtc_state, &floppy, xen_enabled(),
-        0x4);
+    pc_basic_device_init(isa_bus, gsi, &rtc_state, &floppy,
+                         (pc_machine->vmport != ON_OFF_AUTO_ON), 0x4);
 
     pc_nic_init(isa_bus, pci_bus);
 
-    ide_drive_get(hd, MAX_IDE_BUS);
+    ide_drive_get(hd, ARRAY_SIZE(hd));
     if (pci_enabled) {
         PCIDevice *dev;
         if (xen_enabled()) {
@@ -261,7 +272,7 @@ static void pc_init1(MachineState *machine,
     }
 
     pc_cmos_init(below_4g_mem_size, above_4g_mem_size, machine->boot_order,
-                 floppy, idebus[0], idebus[1], rtc_state);
+                 machine, floppy, idebus[0], idebus[1], rtc_state);
 
     if (pci_enabled && usb_enabled(false)) {
         pci_create_simple(pci_bus, piix3_devfn + 2, "piix3-usb-uhci");
@@ -297,8 +308,19 @@ static void pc_init_pci(MachineState *machine)
     pc_init1(machine, 1, 1);
 }
 
+static void pc_compat_2_1(MachineState *machine)
+{
+    PCMachineState *pcms = PC_MACHINE(machine);
+    smbios_uuid_encoded = false;
+    x86_cpu_compat_set_features("coreduo", FEAT_1_ECX, CPUID_EXT_VMX, 0);
+    x86_cpu_compat_set_features("core2duo", FEAT_1_ECX, CPUID_EXT_VMX, 0);
+    x86_cpu_compat_kvm_no_autodisable(FEAT_8000_0001_ECX, CPUID_EXT3_SVM);
+    pcms->enforce_aligned_dimm = false;
+}
+
 static void pc_compat_2_0(MachineState *machine)
 {
+    pc_compat_2_1(machine);
     /* This value depends on the actual DSDT and SSDT compiled into
      * the source QEMU; unfortunately it depends on the binary and
      * not on the machine type, so we cannot make pc-i440fx-1.7 work on
@@ -318,6 +340,7 @@ static void pc_compat_2_0(MachineState *machine)
     legacy_acpi_table_size = 6652;
     smbios_legacy_mode = true;
     has_reserved_memory = false;
+    pc_set_legacy_acpi_data_size();
 }
 
 static void pc_compat_1_7(MachineState *machine)
@@ -327,13 +350,12 @@ static void pc_compat_1_7(MachineState *machine)
     gigabyte_align = false;
     option_rom_has_mr = true;
     legacy_acpi_table_size = 6414;
-    x86_cpu_compat_disable_kvm_features(FEAT_1_ECX, CPUID_EXT_X2APIC);
+    x86_cpu_compat_kvm_no_autoenable(FEAT_1_ECX, CPUID_EXT_X2APIC);
 }
 
 static void pc_compat_1_6(MachineState *machine)
 {
     pc_compat_1_7(machine);
-    has_pci_info = false;
     rom_file_has_mr = false;
     has_acpi_build = false;
 }
@@ -360,7 +382,13 @@ static void pc_compat_1_3(MachineState *machine)
 static void pc_compat_1_2(MachineState *machine)
 {
     pc_compat_1_3(machine);
-    x86_cpu_compat_disable_kvm_features(FEAT_KVM, KVM_FEATURE_PV_EOI);
+    x86_cpu_compat_kvm_no_autoenable(FEAT_KVM, KVM_FEATURE_PV_EOI);
+}
+
+static void pc_init_pci_2_1(MachineState *machine)
+{
+    pc_compat_2_1(machine);
+    pc_init_pci(machine);
 }
 
 static void pc_init_pci_2_0(MachineState *machine)
@@ -415,7 +443,6 @@ static void pc_init_pci_no_kvmclock(MachineState *machine)
 
 static void pc_init_isa(MachineState *machine)
 {
-    has_pci_info = false;
     has_acpi_build = false;
     smbios_defaults = false;
     gigabyte_align = false;
@@ -426,7 +453,7 @@ static void pc_init_isa(MachineState *machine)
     if (!machine->cpu_model) {
         machine->cpu_model = "486";
     }
-    x86_cpu_compat_disable_kvm_features(FEAT_KVM, KVM_FEATURE_PV_EOI);
+    x86_cpu_compat_kvm_no_autoenable(FEAT_KVM, KVM_FEATURE_PV_EOI);
     enable_compat_apic_id_mode();
     pc_init1(machine, 0, 1);
 }
@@ -447,9 +474,23 @@ static void pc_xen_hvm_init(MachineState *machine)
 
 #define PC_I440FX_MACHINE_OPTIONS \
     PC_DEFAULT_MACHINE_OPTIONS, \
+    .family = "pc_piix", \
     .desc = "Standard PC (i440FX + PIIX, 1996)", \
     .hot_add_cpu = pc_hot_add_cpu
 
+#define PC_I440FX_2_2_MACHINE_OPTIONS                           \
+    PC_I440FX_MACHINE_OPTIONS,                                  \
+    .default_machine_opts = "firmware=bios-256k.bin",           \
+    .default_display = "std"
+
+static QEMUMachine pc_i440fx_machine_v2_2 = {
+    PC_I440FX_2_2_MACHINE_OPTIONS,
+    .name = "pc-i440fx-2.2",
+    .alias = "pc",
+    .init = pc_init_pci,
+    .is_default = 1,
+};
+
 #define PC_I440FX_2_1_MACHINE_OPTIONS                           \
     PC_I440FX_MACHINE_OPTIONS,                                  \
     .default_machine_opts = "firmware=bios-256k.bin"
@@ -457,9 +498,11 @@ static void pc_xen_hvm_init(MachineState *machine)
 static QEMUMachine pc_i440fx_machine_v2_1 = {
     PC_I440FX_2_1_MACHINE_OPTIONS,
     .name = "pc-i440fx-2.1",
-    .alias = "pc",
-    .init = pc_init_pci,
-    .is_default = 1,
+    .init = pc_init_pci_2_1,
+    .compat_props = (GlobalProperty[]) {
+        HW_COMPAT_2_1,
+        { /* end of list */ }
+    },
 };
 
 #define PC_I440FX_2_0_MACHINE_OPTIONS PC_I440FX_2_1_MACHINE_OPTIONS
@@ -645,7 +688,7 @@ static QEMUMachine pc_machine_v1_1 = {
             .property = "class",\
             .value    = stringify(PCI_CLASS_MEMORY_RAM),\
         },{\
-            .driver   = "apic",\
+            .driver   = "apic-common",\
             .property = "vapic",\
             .value    = "off",\
         },{\
@@ -880,22 +923,12 @@ static QEMUMachine xenfv_machine = {
     .max_cpus = HVM_MAX_VCPUS,
     .default_machine_opts = "accel=xen",
     .hot_add_cpu = pc_hot_add_cpu,
-    .compat_props = (GlobalProperty[]) {
-        /* xenfv has no fwcfg and so does not load acpi from QEMU.
-         * as such new acpi features don't work.
-         */
-        {
-            .driver   = "PIIX4_PM",
-            .property = "acpi-pci-hotplug-with-bridge-support",
-            .value    = "off",
-        },
-        { /* end of list */ }
-    },
 };
 #endif
 
 static void pc_machine_init(void)
 {
+    qemu_register_pc_machine(&pc_i440fx_machine_v2_2);
     qemu_register_pc_machine(&pc_i440fx_machine_v2_1);
     qemu_register_pc_machine(&pc_i440fx_machine_v2_0);
     qemu_register_pc_machine(&pc_i440fx_machine_v1_7);
diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c
index c39ee9893..0262b5ef1 100644
--- a/hw/i386/pc_q35.c
+++ b/hw/i386/pc_q35.c
@@ -49,10 +49,10 @@
 /* ICH9 AHCI has 6 ports */
 #define MAX_SATA_PORTS     6
 
-static bool has_pci_info;
 static bool has_acpi_build = true;
 static bool smbios_defaults = true;
 static bool smbios_legacy_mode;
+static bool smbios_uuid_encoded = true;
 /* Make sure that guest addresses aligned at 1Gbyte boundaries get mapped to
  * host addresses aligned at 1Gbyte boundaries.  This way we can use 1GByte
  * pages in the host.
@@ -87,6 +87,7 @@ static void pc_q35_init(MachineState *machine)
     DeviceState *icc_bridge;
     PcGuestInfo *guest_info;
     ram_addr_t lowmem;
+    DriveInfo *hd[MAX_SATA_PORTS];
 
     /* Check whether RAM fits below 4G (leaving 1/2 GByte for IO memory
      * and 256 Mbytes for PCI Express Enhanced Configuration Access Mapping
@@ -150,7 +151,6 @@ static void pc_q35_init(MachineState *machine)
     }
 
     guest_info = pc_guest_info_init(below_4g_mem_size, above_4g_mem_size);
-    guest_info->has_pci_info = has_pci_info;
     guest_info->isapc_ram_fw = false;
     guest_info->has_acpi_build = has_acpi_build;
     guest_info->has_reserved_memory = has_reserved_memory;
@@ -164,7 +164,7 @@ static void pc_q35_init(MachineState *machine)
         MachineClass *mc = MACHINE_GET_CLASS(machine);
         /* These values are guest ABI, do not change */
         smbios_set_defaults("QEMU", "Standard PC (Q35 + ICH9, 2009)",
-                            mc->name, smbios_legacy_mode);
+                            mc->name, smbios_legacy_mode, smbios_uuid_encoded);
     }
 
     /* allocate ram and load rom/bios */
@@ -236,14 +236,20 @@ static void pc_q35_init(MachineState *machine)
         gsi_state->i8259_irq[i] = i8259[i];
     }
     if (pci_enabled) {
-        ioapic_init_gsi(gsi_state, NULL);
+        ioapic_init_gsi(gsi_state, "q35");
     }
     qdev_init_nofail(icc_bridge);
 
     pc_register_ferr_irq(gsi[13]);
 
+    assert(pc_machine->vmport != ON_OFF_AUTO_MAX);
+    if (pc_machine->vmport == ON_OFF_AUTO_AUTO) {
+        pc_machine->vmport = xen_enabled() ? ON_OFF_AUTO_OFF : ON_OFF_AUTO_ON;
+    }
+
     /* init basic PC hardware */
-    pc_basic_device_init(isa_bus, gsi, &rtc_state, &floppy, false, 0xff0104);
+    pc_basic_device_init(isa_bus, gsi, &rtc_state, &floppy,
+                         (pc_machine->vmport != ON_OFF_AUTO_ON), 0xff0104);
 
     /* connect pm stuff to lpc */
     ich9_lpc_pm_init(lpc);
@@ -255,6 +261,9 @@ static void pc_q35_init(MachineState *machine)
                                            true, "ich9-ahci");
     idebus[0] = qdev_get_child_bus(&ahci->qdev, "ide.0");
     idebus[1] = qdev_get_child_bus(&ahci->qdev, "ide.1");
+    g_assert(MAX_SATA_PORTS == ICH_AHCI(ahci)->ahci.ports);
+    ide_drive_get(hd, ICH_AHCI(ahci)->ahci.ports);
+    ahci_ide_create_devs(ahci, hd);
 
     if (usb_enabled(false)) {
         /* Should we create 6 UHCI according to ich9 spec? */
@@ -268,7 +277,7 @@ static void pc_q35_init(MachineState *machine)
                       8, NULL, 0);
 
     pc_cmos_init(below_4g_mem_size, above_4g_mem_size, machine->boot_order,
-                 floppy, idebus[0], idebus[1], rtc_state);
+                 machine, floppy, idebus[0], idebus[1], rtc_state);
 
     /* the rest devices to which pci devfn is automatically assigned */
     pc_vga_init(isa_bus, host_bus);
@@ -278,10 +287,23 @@ static void pc_q35_init(MachineState *machine)
     }
 }
 
+static void pc_compat_2_1(MachineState *machine)
+{
+    PCMachineState *pcms = PC_MACHINE(machine);
+
+    pcms->enforce_aligned_dimm = false;
+    smbios_uuid_encoded = false;
+    x86_cpu_compat_set_features("coreduo", FEAT_1_ECX, CPUID_EXT_VMX, 0);
+    x86_cpu_compat_set_features("core2duo", FEAT_1_ECX, CPUID_EXT_VMX, 0);
+    x86_cpu_compat_kvm_no_autodisable(FEAT_8000_0001_ECX, CPUID_EXT3_SVM);
+}
+
 static void pc_compat_2_0(MachineState *machine)
 {
+    pc_compat_2_1(machine);
     smbios_legacy_mode = true;
     has_reserved_memory = false;
+    pc_set_legacy_acpi_data_size();
 }
 
 static void pc_compat_1_7(MachineState *machine)
@@ -290,13 +312,12 @@ static void pc_compat_1_7(MachineState *machine)
     smbios_defaults = false;
     gigabyte_align = false;
     option_rom_has_mr = true;
-    x86_cpu_compat_disable_kvm_features(FEAT_1_ECX, CPUID_EXT_X2APIC);
+    x86_cpu_compat_kvm_no_autoenable(FEAT_1_ECX, CPUID_EXT_X2APIC);
 }
 
 static void pc_compat_1_6(MachineState *machine)
 {
     pc_compat_1_7(machine);
-    has_pci_info = false;
     rom_file_has_mr = false;
     has_acpi_build = false;
 }
@@ -313,6 +334,12 @@ static void pc_compat_1_4(MachineState *machine)
     x86_cpu_compat_set_features("Westmere", FEAT_1_ECX, 0, CPUID_EXT_PCLMULQDQ);
 }
 
+static void pc_q35_init_2_1(MachineState *machine)
+{
+    pc_compat_2_1(machine);
+    pc_q35_init(machine);
+}
+
 static void pc_q35_init_2_0(MachineState *machine)
 {
     pc_compat_2_0(machine);
@@ -345,8 +372,22 @@ static void pc_q35_init_1_4(MachineState *machine)
 
 #define PC_Q35_MACHINE_OPTIONS \
     PC_DEFAULT_MACHINE_OPTIONS, \
+    .family = "pc_q35", \
     .desc = "Standard PC (Q35 + ICH9, 2009)", \
-    .hot_add_cpu = pc_hot_add_cpu
+    .hot_add_cpu = pc_hot_add_cpu, \
+    .units_per_default_bus = 1
+
+#define PC_Q35_2_2_MACHINE_OPTIONS                      \
+    PC_Q35_MACHINE_OPTIONS,                             \
+    .default_machine_opts = "firmware=bios-256k.bin",   \
+    .default_display = "std"
+
+static QEMUMachine pc_q35_machine_v2_2 = {
+    PC_Q35_2_2_MACHINE_OPTIONS,
+    .name = "pc-q35-2.2",
+    .alias = "q35",
+    .init = pc_q35_init,
+};
 
 #define PC_Q35_2_1_MACHINE_OPTIONS                      \
     PC_Q35_MACHINE_OPTIONS,                             \
@@ -355,8 +396,11 @@ static void pc_q35_init_1_4(MachineState *machine)
 static QEMUMachine pc_q35_machine_v2_1 = {
     PC_Q35_2_1_MACHINE_OPTIONS,
     .name = "pc-q35-2.1",
-    .alias = "q35",
-    .init = pc_q35_init,
+    .init = pc_q35_init_2_1,
+    .compat_props = (GlobalProperty[]) {
+        HW_COMPAT_2_1,
+        { /* end of list */ }
+    },
 };
 
 #define PC_Q35_2_0_MACHINE_OPTIONS PC_Q35_2_1_MACHINE_OPTIONS
@@ -421,6 +465,7 @@ static QEMUMachine pc_q35_machine_v1_4 = {
 
 static void pc_q35_machine_init(void)
 {
+    qemu_register_pc_machine(&pc_q35_machine_v2_2);
     qemu_register_pc_machine(&pc_q35_machine_v2_1);
     qemu_register_pc_machine(&pc_q35_machine_v2_0);
     qemu_register_pc_machine(&pc_q35_machine_v1_7);
diff --git a/hw/i386/pc_sysfw.c b/hw/i386/pc_sysfw.c
index 75a7ebbaa..75913c5b2 100644
--- a/hw/i386/pc_sysfw.c
+++ b/hw/i386/pc_sysfw.c
@@ -23,7 +23,7 @@
  * THE SOFTWARE.
  */
 
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "qemu/error-report.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
@@ -55,7 +55,8 @@ static void pc_isa_bios_init(MemoryRegion *rom_memory,
     /* map the last 128KB of the BIOS in ISA space */
     isa_bios_size = MIN(flash_size, 128 * 1024);
     isa_bios = g_malloc(sizeof(*isa_bios));
-    memory_region_init_ram(isa_bios, NULL, "isa-bios", isa_bios_size);
+    memory_region_init_ram(isa_bios, NULL, "isa-bios", isa_bios_size,
+                           &error_abort);
     vmstate_register_ram_global(isa_bios);
     memory_region_add_subregion_overlap(rom_memory,
                                         0x100000 - isa_bios_size,
@@ -102,7 +103,7 @@ static void pc_system_flash_init(MemoryRegion *rom_memory)
 {
     int unit;
     DriveInfo *pflash_drv;
-    BlockDriverState *bdrv;
+    BlockBackend *blk;
     int64_t size;
     char *fatal_errmsg = NULL;
     hwaddr phys_addr = 0x100000000ULL;
@@ -118,8 +119,8 @@ static void pc_system_flash_init(MemoryRegion *rom_memory)
          (unit < FLASH_MAP_UNIT_MAX &&
           (pflash_drv = drive_get(IF_PFLASH, 0, unit)) != NULL);
          ++unit) {
-        bdrv = pflash_drv->bdrv;
-        size = bdrv_getlength(bdrv);
+        blk = blk_by_legacy_dinfo(pflash_drv);
+        size = blk_getlength(blk);
         if (size < 0) {
             fatal_errmsg = g_strdup_printf("failed to get backing file size");
         } else if (size == 0) {
@@ -155,7 +156,7 @@ static void pc_system_flash_init(MemoryRegion *rom_memory)
         /* pflash_cfi01_register() creates a deep copy of the name */
         snprintf(name, sizeof name, "system.flash%d", unit);
         system_flash = pflash_cfi01_register(phys_addr, NULL /* qdev */, name,
-                                             size, bdrv, sector_size,
+                                             size, blk, sector_size,
                                              size >> sector_bits,
                                              1      /* width */,
                                              0x0000 /* id0 */,
@@ -192,7 +193,7 @@ static void old_pc_system_rom_init(MemoryRegion *rom_memory, bool isapc_ram_fw)
         goto bios_error;
     }
     bios = g_malloc(sizeof(*bios));
-    memory_region_init_ram(bios, NULL, "pc.bios", bios_size);
+    memory_region_init_ram(bios, NULL, "pc.bios", bios_size, &error_abort);
     vmstate_register_ram_global(bios);
     if (!isapc_ram_fw) {
         memory_region_set_readonly(bios, true);
diff --git a/hw/i386/q35-acpi-dsdt.dsl b/hw/i386/q35-acpi-dsdt.dsl
index 8c3eae73b..e1cee5d89 100644
--- a/hw/i386/q35-acpi-dsdt.dsl
+++ b/hw/i386/q35-acpi-dsdt.dsl
@@ -405,13 +405,12 @@ DefinitionBlock (
 #include "hw/acpi/pc-hotplug.h"
 #define CPU_STATUS_BASE ICH9_CPU_HOTPLUG_IO_BASE
 #include "acpi-dsdt-cpu-hotplug.dsl"
+#include "acpi-dsdt-mem-hotplug.dsl"
 
 
 /****************************************************************
  * General purpose events
  ****************************************************************/
-    External(\_SB.PCI0.MEMORY_HOPTLUG_DEVICE.MEMORY_SLOT_SCAN_METHOD, MethodObj)
-
     Scope(\_GPE) {
         Name(_HID, "ACPI0006")
 
@@ -425,7 +424,7 @@ DefinitionBlock (
         }
         Method(_E03) {
             // Memory hotplug event
-            \_SB.PCI0.MEMORY_HOPTLUG_DEVICE.MEMORY_SLOT_SCAN_METHOD()
+            \_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_SCAN_METHOD()
         }
         Method(_L04) {
         }
diff --git a/hw/i386/q35-acpi-dsdt.hex.generated b/hw/i386/q35-acpi-dsdt.hex.generated
index c9eb4ac6a..4807bdf48 100644
--- a/hw/i386/q35-acpi-dsdt.hex.generated
+++ b/hw/i386/q35-acpi-dsdt.hex.generated
@@ -3,12 +3,12 @@ static unsigned char Q35AcpiDsdtAmlCode[] = {
 0x53,
 0x44,
 0x54,
-0xe5,
-0x1c,
+0xf6,
+0x1f,
 0x0,
 0x0,
 0x1,
-0xb7,
+0x91,
 0x42,
 0x58,
 0x50,
@@ -31,9 +31,9 @@ static unsigned char Q35AcpiDsdtAmlCode[] = {
 0x4e,
 0x54,
 0x4c,
-0x15,
-0x11,
-0x13,
+0x28,
+0x8,
+0x14,
 0x20,
 0x10,
 0x49,
@@ -7234,6 +7234,791 @@ static unsigned char Q35AcpiDsdtAmlCode[] = {
 0xa,
 0xb,
 0x10,
+0x40,
+0x31,
+0x2e,
+0x5f,
+0x53,
+0x42,
+0x5f,
+0x50,
+0x43,
+0x49,
+0x30,
+0x5b,
+0x82,
+0x43,
+0x30,
+0x4d,
+0x48,
+0x50,
+0x44,
+0x8,
+0x5f,
+0x48,
+0x49,
+0x44,
+0xd,
+0x50,
+0x4e,
+0x50,
+0x30,
+0x41,
+0x30,
+0x36,
+0x0,
+0x8,
+0x5f,
+0x55,
+0x49,
+0x44,
+0xd,
+0x4d,
+0x65,
+0x6d,
+0x6f,
+0x72,
+0x79,
+0x20,
+0x68,
+0x6f,
+0x74,
+0x70,
+0x6c,
+0x75,
+0x67,
+0x20,
+0x72,
+0x65,
+0x73,
+0x6f,
+0x75,
+0x72,
+0x63,
+0x65,
+0x73,
+0x0,
+0x5b,
+0x80,
+0x48,
+0x50,
+0x4d,
+0x52,
+0x1,
+0xb,
+0x0,
+0xa,
+0xa,
+0x18,
+0x8,
+0x5f,
+0x43,
+0x52,
+0x53,
+0x11,
+0xd,
+0xa,
+0xa,
+0x47,
+0x1,
+0x0,
+0xa,
+0x0,
+0xa,
+0x0,
+0x18,
+0x79,
+0x0,
+0x14,
+0x13,
+0x5f,
+0x53,
+0x54,
+0x41,
+0x0,
+0xa0,
+0x9,
+0x93,
+0x4d,
+0x44,
+0x4e,
+0x52,
+0x0,
+0xa4,
+0x0,
+0xa4,
+0xa,
+0xb,
+0x5b,
+0x81,
+0x1f,
+0x48,
+0x50,
+0x4d,
+0x52,
+0x3,
+0x4d,
+0x52,
+0x42,
+0x4c,
+0x20,
+0x4d,
+0x52,
+0x42,
+0x48,
+0x20,
+0x4d,
+0x52,
+0x4c,
+0x4c,
+0x20,
+0x4d,
+0x52,
+0x4c,
+0x48,
+0x20,
+0x4d,
+0x50,
+0x58,
+0x5f,
+0x20,
+0x5b,
+0x81,
+0x13,
+0x48,
+0x50,
+0x4d,
+0x52,
+0x1,
+0x0,
+0x40,
+0xa,
+0x4d,
+0x45,
+0x53,
+0x5f,
+0x1,
+0x4d,
+0x49,
+0x4e,
+0x53,
+0x1,
+0x5b,
+0x1,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0x0,
+0x5b,
+0x81,
+0x15,
+0x48,
+0x50,
+0x4d,
+0x52,
+0x3,
+0x4d,
+0x53,
+0x45,
+0x4c,
+0x20,
+0x4d,
+0x4f,
+0x45,
+0x56,
+0x20,
+0x4d,
+0x4f,
+0x53,
+0x43,
+0x20,
+0x14,
+0x4a,
+0x4,
+0x4d,
+0x53,
+0x43,
+0x4e,
+0x0,
+0xa0,
+0x9,
+0x93,
+0x4d,
+0x44,
+0x4e,
+0x52,
+0x0,
+0xa4,
+0x0,
+0x70,
+0x0,
+0x60,
+0x5b,
+0x23,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xff,
+0xff,
+0xa2,
+0x25,
+0x95,
+0x60,
+0x4d,
+0x44,
+0x4e,
+0x52,
+0x70,
+0x60,
+0x4d,
+0x53,
+0x45,
+0x4c,
+0xa0,
+0x13,
+0x93,
+0x4d,
+0x49,
+0x4e,
+0x53,
+0x1,
+0x4d,
+0x54,
+0x46,
+0x59,
+0x60,
+0x1,
+0x70,
+0x1,
+0x4d,
+0x49,
+0x4e,
+0x53,
+0x72,
+0x60,
+0x1,
+0x60,
+0x5b,
+0x27,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xa4,
+0x1,
+0x14,
+0x2d,
+0x4d,
+0x52,
+0x53,
+0x54,
+0x1,
+0x70,
+0x0,
+0x60,
+0x5b,
+0x23,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xff,
+0xff,
+0x70,
+0x99,
+0x68,
+0x0,
+0x4d,
+0x53,
+0x45,
+0x4c,
+0xa0,
+0xb,
+0x93,
+0x4d,
+0x45,
+0x53,
+0x5f,
+0x1,
+0x70,
+0xa,
+0xf,
+0x60,
+0x5b,
+0x27,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xa4,
+0x60,
+0x14,
+0x41,
+0x18,
+0x4d,
+0x43,
+0x52,
+0x53,
+0x9,
+0x5b,
+0x23,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xff,
+0xff,
+0x70,
+0x99,
+0x68,
+0x0,
+0x4d,
+0x53,
+0x45,
+0x4c,
+0x8,
+0x4d,
+0x52,
+0x36,
+0x34,
+0x11,
+0x33,
+0xa,
+0x30,
+0x8a,
+0x2b,
+0x0,
+0x0,
+0xc,
+0x3,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0xfe,
+0xff,
+0xff,
+0xff,
+0xff,
+0xff,
+0xff,
+0xff,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0xff,
+0xff,
+0xff,
+0xff,
+0xff,
+0xff,
+0xff,
+0xff,
+0x79,
+0x0,
+0x8a,
+0x4d,
+0x52,
+0x36,
+0x34,
+0xa,
+0xe,
+0x4d,
+0x49,
+0x4e,
+0x4c,
+0x8a,
+0x4d,
+0x52,
+0x36,
+0x34,
+0xa,
+0x12,
+0x4d,
+0x49,
+0x4e,
+0x48,
+0x8a,
+0x4d,
+0x52,
+0x36,
+0x34,
+0xa,
+0x26,
+0x4c,
+0x45,
+0x4e,
+0x4c,
+0x8a,
+0x4d,
+0x52,
+0x36,
+0x34,
+0xa,
+0x2a,
+0x4c,
+0x45,
+0x4e,
+0x48,
+0x8a,
+0x4d,
+0x52,
+0x36,
+0x34,
+0xa,
+0x16,
+0x4d,
+0x41,
+0x58,
+0x4c,
+0x8a,
+0x4d,
+0x52,
+0x36,
+0x34,
+0xa,
+0x1a,
+0x4d,
+0x41,
+0x58,
+0x48,
+0x70,
+0x4d,
+0x52,
+0x42,
+0x48,
+0x4d,
+0x49,
+0x4e,
+0x48,
+0x70,
+0x4d,
+0x52,
+0x42,
+0x4c,
+0x4d,
+0x49,
+0x4e,
+0x4c,
+0x70,
+0x4d,
+0x52,
+0x4c,
+0x48,
+0x4c,
+0x45,
+0x4e,
+0x48,
+0x70,
+0x4d,
+0x52,
+0x4c,
+0x4c,
+0x4c,
+0x45,
+0x4e,
+0x4c,
+0x72,
+0x4d,
+0x49,
+0x4e,
+0x4c,
+0x4c,
+0x45,
+0x4e,
+0x4c,
+0x4d,
+0x41,
+0x58,
+0x4c,
+0x72,
+0x4d,
+0x49,
+0x4e,
+0x48,
+0x4c,
+0x45,
+0x4e,
+0x48,
+0x4d,
+0x41,
+0x58,
+0x48,
+0xa0,
+0x14,
+0x95,
+0x4d,
+0x41,
+0x58,
+0x4c,
+0x4d,
+0x49,
+0x4e,
+0x4c,
+0x72,
+0x4d,
+0x41,
+0x58,
+0x48,
+0x1,
+0x4d,
+0x41,
+0x58,
+0x48,
+0xa0,
+0x11,
+0x95,
+0x4d,
+0x41,
+0x58,
+0x4c,
+0x1,
+0x74,
+0x4d,
+0x41,
+0x58,
+0x48,
+0x1,
+0x4d,
+0x41,
+0x58,
+0x48,
+0x74,
+0x4d,
+0x41,
+0x58,
+0x4c,
+0x1,
+0x4d,
+0x41,
+0x58,
+0x4c,
+0xa0,
+0x44,
+0x7,
+0x93,
+0x4d,
+0x41,
+0x58,
+0x48,
+0x0,
+0x8,
+0x4d,
+0x52,
+0x33,
+0x32,
+0x11,
+0x1f,
+0xa,
+0x1c,
+0x87,
+0x17,
+0x0,
+0x0,
+0xc,
+0x3,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0x0,
+0xfe,
+0xff,
+0xff,
+0xff,
+0x0,
+0x0,
+0x0,
+0x0,
+0xff,
+0xff,
+0xff,
+0xff,
+0x79,
+0x0,
+0x8a,
+0x4d,
+0x52,
+0x33,
+0x32,
+0xa,
+0xa,
+0x4d,
+0x49,
+0x4e,
+0x5f,
+0x8a,
+0x4d,
+0x52,
+0x33,
+0x32,
+0xa,
+0xe,
+0x4d,
+0x41,
+0x58,
+0x5f,
+0x8a,
+0x4d,
+0x52,
+0x33,
+0x32,
+0xa,
+0x16,
+0x4c,
+0x45,
+0x4e,
+0x5f,
+0x70,
+0x4d,
+0x49,
+0x4e,
+0x4c,
+0x4d,
+0x49,
+0x4e,
+0x5f,
+0x70,
+0x4d,
+0x41,
+0x58,
+0x4c,
+0x4d,
+0x41,
+0x58,
+0x5f,
+0x70,
+0x4c,
+0x45,
+0x4e,
+0x4c,
+0x4c,
+0x45,
+0x4e,
+0x5f,
+0x5b,
+0x27,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xa4,
+0x4d,
+0x52,
+0x33,
+0x32,
+0x5b,
+0x27,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xa4,
+0x4d,
+0x52,
+0x36,
+0x34,
+0x14,
+0x24,
+0x4d,
+0x50,
+0x58,
+0x4d,
+0x1,
+0x5b,
+0x23,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xff,
+0xff,
+0x70,
+0x99,
+0x68,
+0x0,
+0x4d,
+0x53,
+0x45,
+0x4c,
+0x70,
+0x4d,
+0x50,
+0x58,
+0x5f,
+0x60,
+0x5b,
+0x27,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xa4,
+0x60,
+0x14,
+0x28,
+0x4d,
+0x4f,
+0x53,
+0x54,
+0x4,
+0x5b,
+0x23,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0xff,
+0xff,
+0x70,
+0x99,
+0x68,
+0x0,
+0x4d,
+0x53,
+0x45,
+0x4c,
+0x70,
+0x69,
+0x4d,
+0x4f,
+0x45,
+0x56,
+0x70,
+0x6a,
+0x4d,
+0x4f,
+0x53,
+0x43,
+0x5b,
+0x27,
+0x4d,
+0x4c,
+0x43,
+0x4b,
+0x10,
 0x42,
 0xa,
 0x5f,
diff --git a/hw/i386/smbios.c b/hw/i386/smbios.c
index e3fa1b2fc..024e59445 100644
--- a/hw/i386/smbios.c
+++ b/hw/i386/smbios.c
@@ -48,6 +48,7 @@ struct smbios_table {
 static uint8_t *smbios_entries;
 static size_t smbios_entries_len;
 static bool smbios_legacy = true;
+static bool smbios_uuid_encoded = true;
 /* end: legacy structures & constants for <= 2.0 machines */
 
 
@@ -391,6 +392,11 @@ static void smbios_build_type_1_fields(void)
     smbios_maybe_add_str(1, offsetof(struct smbios_type_1, family_str),
                          type1.family);
     if (qemu_uuid_set) {
+        /* We don't encode the UUID in the "wire format" here because this
+         * function is for legacy mode and needs to keep the guest ABI, and
+         * because we don't know what's the SMBIOS version advertised by the
+         * BIOS.
+         */
         smbios_add_field(1, offsetof(struct smbios_type_1, uuid),
                          qemu_uuid, 16);
     }
@@ -523,6 +529,19 @@ static void smbios_build_type_0_table(void)
     SMBIOS_BUILD_TABLE_POST;
 }
 
+/* Encode UUID from the big endian encoding described on RFC4122 to the wire
+ * format specified by SMBIOS version 2.6.
+ */
+static void smbios_encode_uuid(struct smbios_uuid *uuid, const uint8_t *buf)
+{
+    memcpy(uuid, buf, 16);
+    if (smbios_uuid_encoded) {
+        uuid->time_low = bswap32(uuid->time_low);
+        uuid->time_mid = bswap16(uuid->time_mid);
+        uuid->time_hi_and_version = bswap16(uuid->time_hi_and_version);
+    }
+}
+
 static void smbios_build_type_1_table(void)
 {
     SMBIOS_BUILD_TABLE_PRE(1, 0x100, true); /* required */
@@ -532,9 +551,9 @@ static void smbios_build_type_1_table(void)
     SMBIOS_TABLE_SET_STR(1, version_str, type1.version);
     SMBIOS_TABLE_SET_STR(1, serial_number_str, type1.serial);
     if (qemu_uuid_set) {
-        memcpy(t->uuid, qemu_uuid, 16);
+        smbios_encode_uuid(&t->uuid, qemu_uuid);
     } else {
-        memset(t->uuid, 0, 16);
+        memset(&t->uuid, 0, 16);
     }
     t->wake_up_type = 0x06; /* power switch */
     SMBIOS_TABLE_SET_STR(1, sku_number_str, type1.sku);
@@ -626,7 +645,7 @@ static void smbios_build_type_4_table(unsigned instance)
 
 static void smbios_build_type_16_table(unsigned dimm_cnt)
 {
-    ram_addr_t size_kb;
+    uint64_t size_kb;
 
     SMBIOS_BUILD_TABLE_PRE(16, 0x1000, true); /* required */
 
@@ -650,10 +669,10 @@ static void smbios_build_type_16_table(unsigned dimm_cnt)
 #define MAX_T17_STD_SZ 0x7FFF /* (32G - 1M), in Megabytes */
 #define MAX_T17_EXT_SZ 0x80000000 /* 2P, in Megabytes */
 
-static void smbios_build_type_17_table(unsigned instance, ram_addr_t size)
+static void smbios_build_type_17_table(unsigned instance, uint64_t size)
 {
     char loc_str[128];
-    ram_addr_t size_mb;
+    uint64_t size_mb;
 
     SMBIOS_BUILD_TABLE_PRE(17, 0x1100 + instance, true); /* required */
 
@@ -692,9 +711,9 @@ static void smbios_build_type_17_table(unsigned instance, ram_addr_t size)
 }
 
 static void smbios_build_type_19_table(unsigned instance,
-                                       ram_addr_t start, ram_addr_t size)
+                                       uint64_t start, uint64_t size)
 {
-    ram_addr_t end, start_kb, end_kb;
+    uint64_t end, start_kb, end_kb;
 
     SMBIOS_BUILD_TABLE_PRE(19, 0x1300 + instance, true); /* required */
 
@@ -746,10 +765,12 @@ void smbios_set_cpuid(uint32_t version, uint32_t features)
     }
 
 void smbios_set_defaults(const char *manufacturer, const char *product,
-                         const char *version, bool legacy_mode)
+                         const char *version, bool legacy_mode,
+                         bool uuid_encoded)
 {
     smbios_have_defaults = true;
     smbios_legacy = legacy_mode;
+    smbios_uuid_encoded = uuid_encoded;
 
     /* drop unwanted version of command-line file blob(s) */
     if (smbios_legacy) {
@@ -821,7 +842,7 @@ void smbios_get_tables(uint8_t **tables, size_t *tables_len,
         smbios_build_type_2_table();
         smbios_build_type_3_table();
 
-        smbios_smp_sockets = smp_cpus / (smp_cores * smp_threads);
+        smbios_smp_sockets = DIV_ROUND_UP(smp_cpus, smp_cores * smp_threads);
         assert(smbios_smp_sockets >= 1);
 
         for (i = 0; i < smbios_smp_sockets; i++) {
diff --git a/hw/i386/ssdt-mem.dsl b/hw/i386/ssdt-mem.dsl
index 8e17bd1f9..22ff5ddfc 100644
--- a/hw/i386/ssdt-mem.dsl
+++ b/hw/i386/ssdt-mem.dsl
@@ -39,10 +39,10 @@ ACPI_EXTRACT_ALL_CODE ssdm_mem_aml
 DefinitionBlock ("ssdt-mem.aml", "SSDT", 0x02, "BXPC", "CSSDT", 0x1)
 {
 
-    External(\_SB.PCI0.MEMORY_HOPTLUG_DEVICE.MEMORY_SLOT_CRS_METHOD, MethodObj)
-    External(\_SB.PCI0.MEMORY_HOPTLUG_DEVICE.MEMORY_SLOT_STATUS_METHOD, MethodObj)
-    External(\_SB.PCI0.MEMORY_HOPTLUG_DEVICE.MEMORY_SLOT_OST_METHOD, MethodObj)
-    External(\_SB.PCI0.MEMORY_HOPTLUG_DEVICE.MEMORY_SLOT_PROXIMITY_METHOD, MethodObj)
+    External(\_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_CRS_METHOD, MethodObj)
+    External(\_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_STATUS_METHOD, MethodObj)
+    External(\_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_OST_METHOD, MethodObj)
+    External(\_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_PROXIMITY_METHOD, MethodObj)
 
     Scope(\_SB) {
 /*  v------------------ DO NOT EDIT ------------------v */
@@ -58,19 +58,19 @@ DefinitionBlock ("ssdt-mem.aml", "SSDT", 0x02, "BXPC", "CSSDT", 0x1)
             Name(_HID, EISAID("PNP0C80"))
 
             Method(_CRS, 0) {
-                Return(\_SB.PCI0.MEMORY_HOPTLUG_DEVICE.MEMORY_SLOT_CRS_METHOD(_UID))
+                Return(\_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_CRS_METHOD(_UID))
             }
 
             Method(_STA, 0) {
-                Return(\_SB.PCI0.MEMORY_HOPTLUG_DEVICE.MEMORY_SLOT_STATUS_METHOD(_UID))
+                Return(\_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_STATUS_METHOD(_UID))
             }
 
             Method(_PXM, 0) {
-                Return(\_SB.PCI0.MEMORY_HOPTLUG_DEVICE.MEMORY_SLOT_PROXIMITY_METHOD(_UID))
+                Return(\_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_PROXIMITY_METHOD(_UID))
             }
 
             Method(_OST, 3) {
-                \_SB.PCI0.MEMORY_HOPTLUG_DEVICE.MEMORY_SLOT_OST_METHOD(_UID, Arg0, Arg1, Arg2)
+                \_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_OST_METHOD(_UID, Arg0, Arg1, Arg2)
             }
         }
     }
diff --git a/hw/i386/ssdt-mem.hex.generated b/hw/i386/ssdt-mem.hex.generated
index 00bd34d26..b3bfbbda0 100644
--- a/hw/i386/ssdt-mem.hex.generated
+++ b/hw/i386/ssdt-mem.hex.generated
@@ -11,7 +11,7 @@ static unsigned char ssdm_mem_aml[] = {
 0x0,
 0x0,
 0x2,
-0x71,
+0x66,
 0x42,
 0x58,
 0x50,
@@ -34,9 +34,9 @@ static unsigned char ssdm_mem_aml[] = {
 0x4e,
 0x54,
 0x4c,
-0x15,
-0x11,
-0x13,
+0x28,
+0x8,
+0x14,
 0x20,
 0x10,
 0x42,
diff --git a/hw/i386/ssdt-misc.dsl b/hw/i386/ssdt-misc.dsl
index d329b8ba5..1e3baaed3 100644
--- a/hw/i386/ssdt-misc.dsl
+++ b/hw/i386/ssdt-misc.dsl
@@ -36,6 +36,8 @@ DefinitionBlock ("ssdt-misc.aml", "SSDT", 0x01, "BXPC", "BXSSDTSUSP", 0x1)
        Name(P1E, Buffer() { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 })
        ACPI_EXTRACT_NAME_BUFFER8 acpi_pci64_length
        Name(P1L, Buffer() { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 })
+       ACPI_EXTRACT_NAME_DWORD_CONST ssdt_mctrl_nr_slots
+       Name(MEMORY_SLOTS_NUMBER, 0x12345678)
     }
 
 
@@ -117,167 +119,4 @@ DefinitionBlock ("ssdt-misc.aml", "SSDT", 0x01, "BXPC", "BXSSDTSUSP", 0x1)
             }
         }
     }
-
-    External(MEMORY_SLOT_NOTIFY_METHOD, MethodObj)
-    Scope(\_SB.PCI0) {
-        Device(MEMORY_HOPTLUG_DEVICE) {
-            Name(_HID, "PNP0A06")
-            Name(_UID, "Memory hotplug resources")
-
-            ACPI_EXTRACT_NAME_DWORD_CONST ssdt_mctrl_nr_slots
-            Name(MEMORY_SLOTS_NUMBER, 0x12345678)
-
-            /* Memory hotplug IO registers */
-            OperationRegion(MEMORY_HOTPLUG_IO_REGION, SystemIO,
-                            ACPI_MEMORY_HOTPLUG_BASE,
-                            ACPI_MEMORY_HOTPLUG_IO_LEN)
-
-            Name(_CRS, ResourceTemplate() {
-                IO(Decode16, ACPI_MEMORY_HOTPLUG_BASE, ACPI_MEMORY_HOTPLUG_BASE,
-                   0, ACPI_MEMORY_HOTPLUG_IO_LEN, IO)
-            })
-
-            Method(_STA, 0) {
-                If (LEqual(MEMORY_SLOTS_NUMBER, Zero)) {
-                    Return(0x0)
-                }
-                /* present, functioning, decoding, not shown in UI */
-                Return(0xB)
-            }
-
-            Field(MEMORY_HOTPLUG_IO_REGION, DWordAcc, NoLock, Preserve) {
-                MEMORY_SLOT_ADDR_LOW, 32,  // read only
-                MEMORY_SLOT_ADDR_HIGH, 32, // read only
-                MEMORY_SLOT_SIZE_LOW, 32,  // read only
-                MEMORY_SLOT_SIZE_HIGH, 32, // read only
-                MEMORY_SLOT_PROXIMITY, 32, // read only
-            }
-            Field(MEMORY_HOTPLUG_IO_REGION, ByteAcc, NoLock, Preserve) {
-                Offset(20),
-                MEMORY_SLOT_ENABLED,  1, // 1 if enabled, read only
-                MEMORY_SLOT_INSERT_EVENT, 1, // (read) 1 if has a insert event. (write) 1 to clear event
-            }
-
-            Mutex (MEMORY_SLOT_LOCK, 0)
-            Field (MEMORY_HOTPLUG_IO_REGION, DWordAcc, NoLock, Preserve) {
-                MEMORY_SLOT_SLECTOR, 32,  // DIMM selector, write only
-                MEMORY_SLOT_OST_EVENT, 32,  // _OST event code, write only
-                MEMORY_SLOT_OST_STATUS, 32,  // _OST status code, write only
-            }
-
-            Method(MEMORY_SLOT_SCAN_METHOD, 0) {
-                If (LEqual(MEMORY_SLOTS_NUMBER, Zero)) {
-                     Return(Zero)
-                }
-
-                Store(Zero, Local0) // Mem devs iterrator
-                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
-                while (LLess(Local0, MEMORY_SLOTS_NUMBER)) {
-                    Store(Local0, MEMORY_SLOT_SLECTOR) // select Local0 DIMM
-                    If (LEqual(MEMORY_SLOT_INSERT_EVENT, One)) { // Memory device needs check
-                        MEMORY_SLOT_NOTIFY_METHOD(Local0, 1)
-                        Store(1, MEMORY_SLOT_INSERT_EVENT)
-                    }
-                    // TODO: handle memory eject request
-                    Add(Local0, One, Local0) // goto next DIMM
-                }
-                Release(MEMORY_SLOT_LOCK)
-                Return(One)
-            }
-
-            Method(MEMORY_SLOT_STATUS_METHOD, 1) {
-                Store(Zero, Local0)
-
-                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
-                Store(ToInteger(Arg0), MEMORY_SLOT_SLECTOR) // select DIMM
-
-                If (LEqual(MEMORY_SLOT_ENABLED, One)) {
-                    Store(0xF, Local0)
-                }
-
-                Release(MEMORY_SLOT_LOCK)
-                Return(Local0)
-            }
-
-            Method(MEMORY_SLOT_CRS_METHOD, 1, Serialized) {
-                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
-                Store(ToInteger(Arg0), MEMORY_SLOT_SLECTOR) // select DIMM
-
-                Name(MR64, ResourceTemplate() {
-                    QWordMemory(ResourceProducer, PosDecode, MinFixed, MaxFixed,
-                    Cacheable, ReadWrite,
-                    0x0000000000000000,        // Address Space Granularity
-                    0x0000000000000000,        // Address Range Minimum
-                    0xFFFFFFFFFFFFFFFE,        // Address Range Maximum
-                    0x0000000000000000,        // Address Translation Offset
-                    0xFFFFFFFFFFFFFFFF,        // Address Length
-                    ,, MW64, AddressRangeMemory, TypeStatic)
-                })
-
-                CreateDWordField(MR64, 14, MINL)
-                CreateDWordField(MR64, 18, MINH)
-                CreateDWordField(MR64, 38, LENL)
-                CreateDWordField(MR64, 42, LENH)
-                CreateDWordField(MR64, 22, MAXL)
-                CreateDWordField(MR64, 26, MAXH)
-
-                Store(MEMORY_SLOT_ADDR_HIGH, MINH)
-                Store(MEMORY_SLOT_ADDR_LOW, MINL)
-                Store(MEMORY_SLOT_SIZE_HIGH, LENH)
-                Store(MEMORY_SLOT_SIZE_LOW, LENL)
-
-                // 64-bit math: MAX = MIN + LEN - 1
-                Add(MINL, LENL, MAXL)
-                Add(MINH, LENH, MAXH)
-                If (LLess(MAXL, MINL)) {
-                    Add(MAXH, One, MAXH)
-                }
-                If (LLess(MAXL, One)) {
-                    Subtract(MAXH, One, MAXH)
-                }
-                Subtract(MAXL, One, MAXL)
-
-                If (LEqual(MAXH, Zero)){
-                    Name(MR32, ResourceTemplate() {
-                        DWordMemory(ResourceProducer, PosDecode, MinFixed, MaxFixed,
-                        Cacheable, ReadWrite,
-                        0x00000000,        // Address Space Granularity
-                        0x00000000,        // Address Range Minimum
-                        0xFFFFFFFE,        // Address Range Maximum
-                        0x00000000,        // Address Translation Offset
-                        0xFFFFFFFF,        // Address Length
-                        ,, MW32, AddressRangeMemory, TypeStatic)
-                    })
-                    CreateDWordField(MR32, MW32._MIN, MIN)
-                    CreateDWordField(MR32, MW32._MAX, MAX)
-                    CreateDWordField(MR32, MW32._LEN, LEN)
-                    Store(MINL, MIN)
-                    Store(MAXL, MAX)
-                    Store(LENL, LEN)
-
-                    Release(MEMORY_SLOT_LOCK)
-                    Return(MR32)
-                }
-
-                Release(MEMORY_SLOT_LOCK)
-                Return(MR64)
-            }
-
-            Method(MEMORY_SLOT_PROXIMITY_METHOD, 1) {
-                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
-                Store(ToInteger(Arg0), MEMORY_SLOT_SLECTOR) // select DIMM
-                Store(MEMORY_SLOT_PROXIMITY, Local0)
-                Release(MEMORY_SLOT_LOCK)
-                Return(Local0)
-            }
-
-            Method(MEMORY_SLOT_OST_METHOD, 4) {
-                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
-                Store(ToInteger(Arg0), MEMORY_SLOT_SLECTOR) // select DIMM
-                Store(Arg1, MEMORY_SLOT_OST_EVENT)
-                Store(Arg2, MEMORY_SLOT_OST_STATUS)
-                Release(MEMORY_SLOT_LOCK)
-            }
-        } // Device()
-    } // Scope()
 }
diff --git a/hw/i386/ssdt-misc.hex.generated b/hw/i386/ssdt-misc.hex.generated
index ba4268a60..cbcf61dcc 100644
--- a/hw/i386/ssdt-misc.hex.generated
+++ b/hw/i386/ssdt-misc.hex.generated
@@ -2,13 +2,13 @@ static unsigned char acpi_pci64_length[] = {
 0x6f
 };
 static unsigned char acpi_s4_pkg[] = {
-0x8f
+0x99
 };
-static unsigned short ssdt_mctrl_nr_slots[] = {
-0x1aa
+static unsigned char ssdt_mctrl_nr_slots[] = {
+0x7d
 };
 static unsigned char acpi_s3_name[] = {
-0x7c
+0x86
 };
 static unsigned char acpi_pci32_start[] = {
 0x2f
@@ -21,12 +21,12 @@ static unsigned char ssdp_misc_aml[] = {
 0x53,
 0x44,
 0x54,
-0x7e,
-0x4,
+0x6c,
+0x1,
 0x0,
 0x0,
 0x1,
-0x8b,
+0x3,
 0x42,
 0x58,
 0x50,
@@ -49,12 +49,12 @@ static unsigned char ssdp_misc_aml[] = {
 0x4e,
 0x54,
 0x4c,
-0x15,
-0x11,
-0x13,
+0x28,
+0x8,
+0x14,
 0x20,
 0x10,
-0x42,
+0x4c,
 0x5,
 0x5c,
 0x0,
@@ -136,6 +136,16 @@ static unsigned char ssdp_misc_aml[] = {
 0x0,
 0x0,
 0x0,
+0x8,
+0x4d,
+0x44,
+0x4e,
+0x52,
+0xc,
+0x78,
+0x56,
+0x34,
+0x12,
 0x10,
 0x29,
 0x5c,
@@ -370,809 +380,13 @@ static unsigned char ssdp_misc_aml[] = {
 0x49,
 0x4f,
 0x4d,
-0x58,
-0x10,
-0x4b,
-0x31,
-0x5c,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x5b,
-0x82,
-0x4d,
-0x30,
-0x4d,
-0x48,
-0x50,
-0x44,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xd,
-0x50,
-0x4e,
-0x50,
-0x30,
-0x41,
-0x30,
-0x36,
-0x0,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xd,
-0x4d,
-0x65,
-0x6d,
-0x6f,
-0x72,
-0x79,
-0x20,
-0x68,
-0x6f,
-0x74,
-0x70,
-0x6c,
-0x75,
-0x67,
-0x20,
-0x72,
-0x65,
-0x73,
-0x6f,
-0x75,
-0x72,
-0x63,
-0x65,
-0x73,
-0x0,
-0x8,
-0x4d,
-0x44,
-0x4e,
-0x52,
-0xc,
-0x78,
-0x56,
-0x34,
-0x12,
-0x5b,
-0x80,
-0x48,
-0x50,
-0x4d,
-0x52,
-0x1,
-0xb,
-0x0,
-0xa,
-0xa,
-0x18,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xd,
-0xa,
-0xa,
-0x47,
-0x1,
-0x0,
-0xa,
-0x0,
-0xa,
-0x0,
-0x18,
-0x79,
-0x0,
-0x14,
-0x13,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa0,
-0x9,
-0x93,
-0x4d,
-0x44,
-0x4e,
-0x52,
-0x0,
-0xa4,
-0x0,
-0xa4,
-0xa,
-0xb,
-0x5b,
-0x81,
-0x1f,
-0x48,
-0x50,
-0x4d,
-0x52,
-0x3,
-0x4d,
-0x52,
-0x42,
-0x4c,
-0x20,
-0x4d,
-0x52,
-0x42,
-0x48,
-0x20,
-0x4d,
-0x52,
-0x4c,
-0x4c,
-0x20,
-0x4d,
-0x52,
-0x4c,
-0x48,
-0x20,
-0x4d,
-0x50,
-0x58,
-0x5f,
-0x20,
-0x5b,
-0x81,
-0x13,
-0x48,
-0x50,
-0x4d,
-0x52,
-0x1,
-0x0,
-0x40,
-0xa,
-0x4d,
-0x45,
-0x53,
-0x5f,
-0x1,
-0x4d,
-0x49,
-0x4e,
-0x53,
-0x1,
-0x5b,
-0x1,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0x0,
-0x5b,
-0x81,
-0x15,
-0x48,
-0x50,
-0x4d,
-0x52,
-0x3,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0x20,
-0x4d,
-0x4f,
-0x45,
-0x56,
-0x20,
-0x4d,
-0x4f,
-0x53,
-0x43,
-0x20,
-0x14,
-0x4a,
-0x4,
-0x4d,
-0x53,
-0x43,
-0x4e,
-0x0,
-0xa0,
-0x9,
-0x93,
-0x4d,
-0x44,
-0x4e,
-0x52,
-0x0,
-0xa4,
-0x0,
-0x70,
-0x0,
-0x60,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0xa2,
-0x25,
-0x95,
-0x60,
-0x4d,
-0x44,
-0x4e,
-0x52,
-0x70,
-0x60,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0xa0,
-0x13,
-0x93,
-0x4d,
-0x49,
-0x4e,
-0x53,
-0x1,
-0x4d,
-0x54,
-0x46,
-0x59,
-0x60,
-0x1,
-0x70,
-0x1,
-0x4d,
-0x49,
-0x4e,
-0x53,
-0x72,
-0x60,
-0x1,
-0x60,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x1,
-0x14,
-0x2d,
-0x4d,
-0x52,
-0x53,
-0x54,
-0x1,
-0x70,
-0x0,
-0x60,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0xa0,
-0xb,
-0x93,
-0x4d,
-0x45,
-0x53,
-0x5f,
-0x1,
-0x70,
-0xa,
-0xf,
-0x60,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x60,
-0x14,
-0x41,
-0x18,
-0x4d,
-0x43,
-0x52,
-0x53,
-0x9,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0x8,
-0x4d,
-0x52,
-0x36,
-0x34,
-0x11,
-0x33,
-0xa,
-0x30,
-0x8a,
-0x2b,
-0x0,
-0x0,
-0xc,
-0x3,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xfe,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0x79,
-0x0,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0xe,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x12,
-0x4d,
-0x49,
-0x4e,
-0x48,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x26,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x2a,
-0x4c,
-0x45,
-0x4e,
-0x48,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x16,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x1a,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x70,
-0x4d,
-0x52,
-0x42,
-0x48,
-0x4d,
-0x49,
-0x4e,
-0x48,
-0x70,
-0x4d,
-0x52,
-0x42,
-0x4c,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x70,
-0x4d,
-0x52,
-0x4c,
-0x48,
-0x4c,
-0x45,
-0x4e,
-0x48,
-0x70,
-0x4d,
-0x52,
-0x4c,
-0x4c,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x72,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x72,
-0x4d,
-0x49,
-0x4e,
-0x48,
-0x4c,
-0x45,
-0x4e,
-0x48,
-0x4d,
-0x41,
-0x58,
-0x48,
-0xa0,
-0x14,
-0x95,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x72,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x1,
-0x4d,
-0x41,
-0x58,
-0x48,
-0xa0,
-0x11,
-0x95,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x1,
-0x74,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x1,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x74,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x1,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0xa0,
-0x44,
-0x7,
-0x93,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x0,
-0x8,
-0x4d,
-0x52,
-0x33,
-0x32,
-0x11,
-0x1f,
-0xa,
-0x1c,
-0x87,
-0x17,
-0x0,
-0x0,
-0xc,
-0x3,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xfe,
-0xff,
-0xff,
-0xff,
-0x0,
-0x0,
-0x0,
-0x0,
-0xff,
-0xff,
-0xff,
-0xff,
-0x79,
-0x0,
-0x8a,
-0x4d,
-0x52,
-0x33,
-0x32,
-0xa,
-0xa,
-0x4d,
-0x49,
-0x4e,
-0x5f,
-0x8a,
-0x4d,
-0x52,
-0x33,
-0x32,
-0xa,
-0xe,
-0x4d,
-0x41,
-0x58,
-0x5f,
-0x8a,
-0x4d,
-0x52,
-0x33,
-0x32,
-0xa,
-0x16,
-0x4c,
-0x45,
-0x4e,
-0x5f,
-0x70,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x4d,
-0x49,
-0x4e,
-0x5f,
-0x70,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x4d,
-0x41,
-0x58,
-0x5f,
-0x70,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x4c,
-0x45,
-0x4e,
-0x5f,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x4d,
-0x52,
-0x33,
-0x32,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x4d,
-0x52,
-0x36,
-0x34,
-0x14,
-0x24,
-0x4d,
-0x50,
-0x58,
-0x4d,
-0x1,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0x70,
-0x4d,
-0x50,
-0x58,
-0x5f,
-0x60,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x60,
-0x14,
-0x28,
-0x4d,
-0x4f,
-0x53,
-0x54,
-0x4,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0x70,
-0x69,
-0x4d,
-0x4f,
-0x45,
-0x56,
-0x70,
-0x6a,
-0x4d,
-0x4f,
-0x53,
-0x43,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b
+0x58
 };
 static unsigned char ssdt_isa_pest[] = {
-0xd0
+0xda
 };
 static unsigned char acpi_s4_name[] = {
-0x88
+0x92
 };
 static unsigned char acpi_pci64_start[] = {
 0x4d
diff --git a/hw/i386/ssdt-tpm.dsl b/hw/i386/ssdt-tpm.dsl
new file mode 100644
index 000000000..75d96910b
--- /dev/null
+++ b/hw/i386/ssdt-tpm.dsl
@@ -0,0 +1,43 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+#include "hw/acpi/tpm.h"
+
+ACPI_EXTRACT_ALL_CODE ssdt_tpm_aml
+
+DefinitionBlock (
+    "ssdt-tpm.aml",     // Output Filename
+    "SSDT",             // Signature
+    0x01,               // SSDT Compliance Revision
+    "BXPC",             // OEMID
+    "BXSSDT",           // TABLE ID
+    0x1                 // OEM Revision
+    )
+{
+    Scope(\_SB) {
+        /* TPM with emulated TPM TIS interface */
+        Device (TPM) {
+            Name (_HID, EisaID ("PNP0C31"))
+            Name (_CRS, ResourceTemplate ()
+            {
+                Memory32Fixed (ReadWrite, TPM_TIS_ADDR_BASE, TPM_TIS_ADDR_SIZE)
+                // older Linux tpm_tis drivers do not work with IRQ
+                //IRQNoFlags () {TPM_TIS_IRQ}
+            })
+            Method (_STA, 0, NotSerialized) {
+                Return (0x0F)
+            }
+        }
+    }
+}
diff --git a/hw/i386/ssdt-tpm.hex.generated b/hw/i386/ssdt-tpm.hex.generated
new file mode 100644
index 000000000..4a916a839
--- /dev/null
+++ b/hw/i386/ssdt-tpm.hex.generated
@@ -0,0 +1,95 @@
+static unsigned char ssdt_tpm_aml[] = {
+0x53,
+0x53,
+0x44,
+0x54,
+0x5d,
+0x0,
+0x0,
+0x0,
+0x1,
+0xf,
+0x42,
+0x58,
+0x50,
+0x43,
+0x0,
+0x0,
+0x42,
+0x58,
+0x53,
+0x53,
+0x44,
+0x54,
+0x0,
+0x0,
+0x1,
+0x0,
+0x0,
+0x0,
+0x49,
+0x4e,
+0x54,
+0x4c,
+0x15,
+0x11,
+0x13,
+0x20,
+0x10,
+0x38,
+0x5c,
+0x5f,
+0x53,
+0x42,
+0x5f,
+0x5b,
+0x82,
+0x30,
+0x54,
+0x50,
+0x4d,
+0x5f,
+0x8,
+0x5f,
+0x48,
+0x49,
+0x44,
+0xc,
+0x41,
+0xd0,
+0xc,
+0x31,
+0x8,
+0x5f,
+0x43,
+0x52,
+0x53,
+0x11,
+0x11,
+0xa,
+0xe,
+0x86,
+0x9,
+0x0,
+0x1,
+0x0,
+0x0,
+0xd4,
+0xfe,
+0x0,
+0x50,
+0x0,
+0x0,
+0x79,
+0x0,
+0x14,
+0x9,
+0x5f,
+0x53,
+0x54,
+0x41,
+0x0,
+0xa4,
+0xa,
+0xf
+};
diff --git a/hw/i386/xen/xen_apic.c b/hw/i386/xen/xen_apic.c
index 63bb7f77c..f5acd6a09 100644
--- a/hw/i386/xen/xen_apic.c
+++ b/hw/i386/xen/xen_apic.c
@@ -40,6 +40,7 @@ static void xen_apic_realize(DeviceState *dev, Error **errp)
 {
     APICCommonState *s = APIC_COMMON(dev);
 
+    s->vapic_control = 0;
     memory_region_init_io(&s->io_memory, OBJECT(s), &xen_apic_io_ops, s,
                           "xen-apic-msi", APIC_SPACE_SIZE);
 
diff --git a/hw/i386/xen/xen_platform.c b/hw/i386/xen/xen_platform.c
index 8bb18b403..28b324a6f 100644
--- a/hw/i386/xen/xen_platform.c
+++ b/hw/i386/xen/xen_platform.c
@@ -34,6 +34,7 @@
 #include "hw/xen/xen_backend.h"
 #include "trace.h"
 #include "exec/address-spaces.h"
+#include "sysemu/block-backend.h"
 
 #include <xenguest.h>
 
@@ -132,8 +133,8 @@ static void platform_fixed_ioport_writew(void *opaque, uint32_t addr, uint32_t v
            devices, and bit 2 the non-primary-master IDE devices. */
         if (val & UNPLUG_ALL_IDE_DISKS) {
             DPRINTF("unplug disks\n");
-            bdrv_drain_all();
-            bdrv_flush_all();
+            blk_drain_all();
+            blk_flush_all();
             pci_unplug_disks(pci_dev->bus);
         }
         if (val & UNPLUG_ALL_NICS) {
diff --git a/hw/ide/ahci.c b/hw/ide/ahci.c
index 604152a82..94f28e6ba 100644
--- a/hw/ide/ahci.c
+++ b/hw/ide/ahci.c
@@ -28,6 +28,7 @@
 #include <hw/sysbus.h>
 
 #include "monitor/monitor.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/dma.h"
 #include "internal.h"
 #include <hw/ide/pci.h>
@@ -48,6 +49,9 @@ static int handle_cmd(AHCIState *s,int port,int slot);
 static void ahci_reset_port(AHCIState *s, int port);
 static void ahci_write_fis_d2h(AHCIDevice *ad, uint8_t *cmd_fis);
 static void ahci_init_d2h(AHCIDevice *ad);
+static int ahci_dma_prepare_buf(IDEDMA *dma, int is_write);
+static void ahci_commit_buf(IDEDMA *dma, uint32_t tx_bytes);
+
 
 static uint32_t  ahci_port_read(AHCIState *s, int port, int offset)
 {
@@ -78,14 +82,13 @@ static uint32_t  ahci_port_read(AHCIState *s, int port, int offset)
         val = pr->cmd;
         break;
     case PORT_TFDATA:
-        val = ((uint16_t)s->dev[port].port.ifs[0].error << 8) |
-              s->dev[port].port.ifs[0].status;
+        val = pr->tfdata;
         break;
     case PORT_SIG:
         val = pr->sig;
         break;
     case PORT_SCR_STAT:
-        if (s->dev[port].port.ifs[0].bs) {
+        if (s->dev[port].port.ifs[0].blk) {
             val = SATA_SCR_SSTATUS_DET_DEV_PRESENT_PHY_UP |
                   SATA_SCR_SSTATUS_SPD_GEN1 | SATA_SCR_SSTATUS_IPM_ACTIVE;
         } else {
@@ -251,14 +254,13 @@ static void  ahci_port_write(AHCIState *s, int port, int offset, uint32_t val)
             check_cmd(s, port);
             break;
         case PORT_TFDATA:
-            s->dev[port].port.ifs[0].error = (val >> 8) & 0xff;
-            s->dev[port].port.ifs[0].status = val & 0xff;
+            /* Read Only. */
             break;
         case PORT_SIG:
-            pr->sig = val;
+            /* Read Only */
             break;
         case PORT_SCR_STAT:
-            pr->scr_stat = val;
+            /* Read Only */
             break;
         case PORT_SCR_CTL:
             if (((pr->scr_ctl & AHCI_SCR_SCTL_DET) == 1) &&
@@ -497,11 +499,13 @@ static void ahci_reset_port(AHCIState *s, int port)
     pr->scr_stat = 0;
     pr->scr_err = 0;
     pr->scr_act = 0;
+    pr->tfdata = 0x7F;
+    pr->sig = 0xFFFFFFFF;
     d->busy_slot = -1;
     d->init_d2h_sent = false;
 
     ide_state = &s->dev[port].port.ifs[0];
-    if (!ide_state->bs) {
+    if (!ide_state->blk) {
         return;
     }
 
@@ -513,11 +517,11 @@ static void ahci_reset_port(AHCIState *s, int port)
         }
 
         if (ncq_tfs->aiocb) {
-            bdrv_aio_cancel(ncq_tfs->aiocb);
+            blk_aio_cancel(ncq_tfs->aiocb);
             ncq_tfs->aiocb = NULL;
         }
 
-        /* Maybe we just finished the request thanks to bdrv_aio_cancel() */
+        /* Maybe we just finished the request thanks to blk_aio_cancel() */
         if (!ncq_tfs->used) {
             continue;
         }
@@ -527,17 +531,17 @@ static void ahci_reset_port(AHCIState *s, int port)
     }
 
     s->dev[port].port_state = STATE_RUN;
-    if (!ide_state->bs) {
-        s->dev[port].port_regs.sig = 0;
+    if (!ide_state->blk) {
+        pr->sig = 0;
         ide_state->status = SEEK_STAT | WRERR_STAT;
     } else if (ide_state->drive_kind == IDE_CD) {
-        s->dev[port].port_regs.sig = SATA_SIGNATURE_CDROM;
+        pr->sig = SATA_SIGNATURE_CDROM;
         ide_state->lcyl = 0x14;
         ide_state->hcyl = 0xeb;
         DPRINTF(port, "set lcyl = %d\n", ide_state->lcyl);
         ide_state->status = SEEK_STAT | WRERR_STAT | READY_STAT;
     } else {
-        s->dev[port].port_regs.sig = SATA_SIGNATURE_DISK;
+        pr->sig = SATA_SIGNATURE_DISK;
         ide_state->status = SEEK_STAT | WRERR_STAT;
     }
 
@@ -563,28 +567,104 @@ static void debug_print_fis(uint8_t *fis, int cmd_len)
 
 static void ahci_write_fis_sdb(AHCIState *s, int port, uint32_t finished)
 {
-    AHCIPortRegs *pr = &s->dev[port].port_regs;
+    AHCIDevice *ad = &s->dev[port];
+    AHCIPortRegs *pr = &ad->port_regs;
     IDEState *ide_state;
-    uint8_t *sdb_fis;
+    SDBFIS *sdb_fis;
 
     if (!s->dev[port].res_fis ||
         !(pr->cmd & PORT_CMD_FIS_RX)) {
         return;
     }
 
-    sdb_fis = &s->dev[port].res_fis[RES_FIS_SDBFIS];
-    ide_state = &s->dev[port].port.ifs[0];
-
-    /* clear memory */
-    *(uint32_t*)sdb_fis = 0;
+    sdb_fis = (SDBFIS *)&ad->res_fis[RES_FIS_SDBFIS];
+    ide_state = &ad->port.ifs[0];
 
-    /* write values */
-    sdb_fis[0] = ide_state->error;
-    sdb_fis[2] = ide_state->status & 0x77;
+    sdb_fis->type = 0xA1;
+    /* Interrupt pending & Notification bit */
+    sdb_fis->flags = (ad->hba->control_regs.irqstatus ? (1 << 6) : 0);
+    sdb_fis->status = ide_state->status & 0x77;
+    sdb_fis->error = ide_state->error;
+    /* update SAct field in SDB_FIS */
     s->dev[port].finished |= finished;
-    *(uint32_t*)(sdb_fis + 4) = cpu_to_le32(s->dev[port].finished);
+    sdb_fis->payload = cpu_to_le32(ad->finished);
+
+    /* Update shadow registers (except BSY 0x80 and DRQ 0x08) */
+    pr->tfdata = (ad->port.ifs[0].error << 8) |
+        (ad->port.ifs[0].status & 0x77) |
+        (pr->tfdata & 0x88);
+
+    ahci_trigger_irq(s, ad, PORT_IRQ_SDB_FIS);
+}
+
+static void ahci_write_fis_pio(AHCIDevice *ad, uint16_t len)
+{
+    AHCIPortRegs *pr = &ad->port_regs;
+    uint8_t *pio_fis, *cmd_fis;
+    uint64_t tbl_addr;
+    dma_addr_t cmd_len = 0x80;
+    IDEState *s = &ad->port.ifs[0];
+
+    if (!ad->res_fis || !(pr->cmd & PORT_CMD_FIS_RX)) {
+        return;
+    }
+
+    /* map cmd_fis */
+    tbl_addr = le64_to_cpu(ad->cur_cmd->tbl_addr);
+    cmd_fis = dma_memory_map(ad->hba->as, tbl_addr, &cmd_len,
+                             DMA_DIRECTION_TO_DEVICE);
+
+    if (cmd_fis == NULL) {
+        DPRINTF(ad->port_no, "dma_memory_map failed in ahci_write_fis_pio");
+        ahci_trigger_irq(ad->hba, ad, PORT_IRQ_HBUS_ERR);
+        return;
+    }
+
+    if (cmd_len != 0x80) {
+        DPRINTF(ad->port_no,
+                "dma_memory_map mapped too few bytes in ahci_write_fis_pio");
+        dma_memory_unmap(ad->hba->as, cmd_fis, cmd_len,
+                         DMA_DIRECTION_TO_DEVICE, cmd_len);
+        ahci_trigger_irq(ad->hba, ad, PORT_IRQ_HBUS_ERR);
+        return;
+    }
+
+    pio_fis = &ad->res_fis[RES_FIS_PSFIS];
+
+    pio_fis[0] = 0x5f;
+    pio_fis[1] = (ad->hba->control_regs.irqstatus ? (1 << 6) : 0);
+    pio_fis[2] = s->status;
+    pio_fis[3] = s->error;
+
+    pio_fis[4] = s->sector;
+    pio_fis[5] = s->lcyl;
+    pio_fis[6] = s->hcyl;
+    pio_fis[7] = s->select;
+    pio_fis[8] = s->hob_sector;
+    pio_fis[9] = s->hob_lcyl;
+    pio_fis[10] = s->hob_hcyl;
+    pio_fis[11] = 0;
+    pio_fis[12] = cmd_fis[12];
+    pio_fis[13] = cmd_fis[13];
+    pio_fis[14] = 0;
+    pio_fis[15] = s->status;
+    pio_fis[16] = len & 255;
+    pio_fis[17] = len >> 8;
+    pio_fis[18] = 0;
+    pio_fis[19] = 0;
+
+    /* Update shadow registers: */
+    pr->tfdata = (ad->port.ifs[0].error << 8) |
+        ad->port.ifs[0].status;
+
+    if (pio_fis[2] & ERR_STAT) {
+        ahci_trigger_irq(ad->hba, ad, PORT_IRQ_TF_ERR);
+    }
 
-    ahci_trigger_irq(s, &s->dev[port], PORT_IRQ_STAT_SDBS);
+    ahci_trigger_irq(ad->hba, ad, PORT_IRQ_PIOS_FIS);
+
+    dma_memory_unmap(ad->hba->as, cmd_fis, cmd_len,
+                     DMA_DIRECTION_TO_DEVICE, cmd_len);
 }
 
 static void ahci_write_fis_d2h(AHCIDevice *ad, uint8_t *cmd_fis)
@@ -594,6 +674,7 @@ static void ahci_write_fis_d2h(AHCIDevice *ad, uint8_t *cmd_fis)
     int i;
     dma_addr_t cmd_len = 0x80;
     int cmd_mapped = 0;
+    IDEState *s = &ad->port.ifs[0];
 
     if (!ad->res_fis || !(pr->cmd & PORT_CMD_FIS_RX)) {
         return;
@@ -611,25 +692,29 @@ static void ahci_write_fis_d2h(AHCIDevice *ad, uint8_t *cmd_fis)
 
     d2h_fis[0] = 0x34;
     d2h_fis[1] = (ad->hba->control_regs.irqstatus ? (1 << 6) : 0);
-    d2h_fis[2] = ad->port.ifs[0].status;
-    d2h_fis[3] = ad->port.ifs[0].error;
-
-    d2h_fis[4] = cmd_fis[4];
-    d2h_fis[5] = cmd_fis[5];
-    d2h_fis[6] = cmd_fis[6];
-    d2h_fis[7] = cmd_fis[7];
-    d2h_fis[8] = cmd_fis[8];
-    d2h_fis[9] = cmd_fis[9];
-    d2h_fis[10] = cmd_fis[10];
-    d2h_fis[11] = cmd_fis[11];
+    d2h_fis[2] = s->status;
+    d2h_fis[3] = s->error;
+
+    d2h_fis[4] = s->sector;
+    d2h_fis[5] = s->lcyl;
+    d2h_fis[6] = s->hcyl;
+    d2h_fis[7] = s->select;
+    d2h_fis[8] = s->hob_sector;
+    d2h_fis[9] = s->hob_lcyl;
+    d2h_fis[10] = s->hob_hcyl;
+    d2h_fis[11] = 0;
     d2h_fis[12] = cmd_fis[12];
     d2h_fis[13] = cmd_fis[13];
     for (i = 14; i < 20; i++) {
         d2h_fis[i] = 0;
     }
 
+    /* Update shadow registers: */
+    pr->tfdata = (ad->port.ifs[0].error << 8) |
+        ad->port.ifs[0].status;
+
     if (d2h_fis[2] & ERR_STAT) {
-        ahci_trigger_irq(ad->hba, ad, PORT_IRQ_STAT_TFES);
+        ahci_trigger_irq(ad->hba, ad, PORT_IRQ_TF_ERR);
     }
 
     ahci_trigger_irq(ad->hba, ad, PORT_IRQ_D2H_REG_FIS);
@@ -645,7 +730,8 @@ static int prdt_tbl_entry_size(const AHCI_SG *tbl)
     return (le32_to_cpu(tbl->flags_size) & AHCI_PRDT_SIZE_MASK) + 1;
 }
 
-static int ahci_populate_sglist(AHCIDevice *ad, QEMUSGList *sglist, int offset)
+static int ahci_populate_sglist(AHCIDevice *ad, QEMUSGList *sglist,
+                                int32_t offset)
 {
     AHCICmdHdr *cmd = ad->cur_cmd;
     uint32_t opts = le32_to_cpu(cmd->opts);
@@ -656,13 +742,21 @@ static int ahci_populate_sglist(AHCIDevice *ad, QEMUSGList *sglist, int offset)
     uint8_t *prdt;
     int i;
     int r = 0;
-    int sum = 0;
+    uint64_t sum = 0;
     int off_idx = -1;
-    int off_pos = -1;
+    int64_t off_pos = -1;
     int tbl_entry_size;
     IDEBus *bus = &ad->port;
     BusState *qbus = BUS(bus);
 
+    /*
+     * Note: AHCI PRDT can describe up to 256GiB. SATA/ATA only support
+     * transactions of up to 32MiB as of ATA8-ACS3 rev 1b, assuming a
+     * 512 byte sector size. We limit the PRDT in this implementation to
+     * a reasonably large 2GiB, which can accommodate the maximum transfer
+     * request for sector sizes up to 32K.
+     */
+
     if (!sglist_alloc_hint) {
         DPRINTF(ad->port_no, "no sg list given by guest: 0x%08x\n", opts);
         return -1;
@@ -697,7 +791,7 @@ static int ahci_populate_sglist(AHCIDevice *ad, QEMUSGList *sglist, int offset)
         }
         if ((off_idx == -1) || (off_pos < 0) || (off_pos > tbl_entry_size)) {
             DPRINTF(ad->port_no, "%s: Incorrect offset! "
-                            "off_idx: %d, off_pos: %d\n",
+                            "off_idx: %d, off_pos: %"PRId64"\n",
                             __func__, off_idx, off_pos);
             r = -1;
             goto out;
@@ -712,6 +806,13 @@ static int ahci_populate_sglist(AHCIDevice *ad, QEMUSGList *sglist, int offset)
             /* flags_size is zero-based */
             qemu_sglist_add(sglist, le64_to_cpu(tbl[i].addr),
                             prdt_tbl_entry_size(&tbl[i]));
+            if (sglist->size > INT32_MAX) {
+                error_report("AHCI Physical Region Descriptor Table describes "
+                             "more than 2 GiB.\n");
+                qemu_sglist_destroy(sglist);
+                r = -1;
+                goto out;
+            }
         }
     }
 
@@ -726,6 +827,9 @@ static void ncq_cb(void *opaque, int ret)
     NCQTransferState *ncq_tfs = (NCQTransferState *)opaque;
     IDEState *ide_state = &ncq_tfs->drive->port.ifs[0];
 
+    if (ret == -ECANCELED) {
+        return;
+    }
     /* Clear bit for this tag in SActive */
     ncq_tfs->drive->port_regs.scr_act &= ~(1 << ncq_tfs->tag);
 
@@ -744,11 +848,27 @@ static void ncq_cb(void *opaque, int ret)
     DPRINTF(ncq_tfs->drive->port_no, "NCQ transfer tag %d finished\n",
             ncq_tfs->tag);
 
-    bdrv_acct_done(ncq_tfs->drive->port.ifs[0].bs, &ncq_tfs->acct);
+    block_acct_done(blk_get_stats(ncq_tfs->drive->port.ifs[0].blk),
+                    &ncq_tfs->acct);
     qemu_sglist_destroy(&ncq_tfs->sglist);
     ncq_tfs->used = 0;
 }
 
+static int is_ncq(uint8_t ata_cmd)
+{
+    /* Based on SATA 3.2 section 13.6.3.2 */
+    switch (ata_cmd) {
+    case READ_FPDMA_QUEUED:
+    case WRITE_FPDMA_QUEUED:
+    case NCQ_NON_DATA:
+    case RECEIVE_FPDMA_QUEUED:
+    case SEND_FPDMA_QUEUED:
+        return 1;
+    default:
+        return 0;
+    }
+}
+
 static void process_ncq_command(AHCIState *s, int port, uint8_t *cmd_fis,
                                 int slot)
 {
@@ -794,11 +914,11 @@ static void process_ncq_command(AHCIState *s, int port, uint8_t *cmd_fis,
             DPRINTF(port, "tag %d aio read %"PRId64"\n",
                     ncq_tfs->tag, ncq_tfs->lba);
 
-            dma_acct_start(ncq_tfs->drive->port.ifs[0].bs, &ncq_tfs->acct,
-                           &ncq_tfs->sglist, BDRV_ACCT_READ);
-            ncq_tfs->aiocb = dma_bdrv_read(ncq_tfs->drive->port.ifs[0].bs,
-                                           &ncq_tfs->sglist, ncq_tfs->lba,
-                                           ncq_cb, ncq_tfs);
+            dma_acct_start(ncq_tfs->drive->port.ifs[0].blk, &ncq_tfs->acct,
+                           &ncq_tfs->sglist, BLOCK_ACCT_READ);
+            ncq_tfs->aiocb = dma_blk_read(ncq_tfs->drive->port.ifs[0].blk,
+                                          &ncq_tfs->sglist, ncq_tfs->lba,
+                                          ncq_cb, ncq_tfs);
             break;
         case WRITE_FPDMA_QUEUED:
             DPRINTF(port, "NCQ writing %d sectors to LBA %"PRId64", tag %d\n",
@@ -807,23 +927,113 @@ static void process_ncq_command(AHCIState *s, int port, uint8_t *cmd_fis,
             DPRINTF(port, "tag %d aio write %"PRId64"\n",
                     ncq_tfs->tag, ncq_tfs->lba);
 
-            dma_acct_start(ncq_tfs->drive->port.ifs[0].bs, &ncq_tfs->acct,
-                           &ncq_tfs->sglist, BDRV_ACCT_WRITE);
-            ncq_tfs->aiocb = dma_bdrv_write(ncq_tfs->drive->port.ifs[0].bs,
-                                            &ncq_tfs->sglist, ncq_tfs->lba,
-                                            ncq_cb, ncq_tfs);
+            dma_acct_start(ncq_tfs->drive->port.ifs[0].blk, &ncq_tfs->acct,
+                           &ncq_tfs->sglist, BLOCK_ACCT_WRITE);
+            ncq_tfs->aiocb = dma_blk_write(ncq_tfs->drive->port.ifs[0].blk,
+                                           &ncq_tfs->sglist, ncq_tfs->lba,
+                                           ncq_cb, ncq_tfs);
             break;
         default:
-            DPRINTF(port, "error: tried to process non-NCQ command as NCQ\n");
+            if (is_ncq(cmd_fis[2])) {
+                DPRINTF(port,
+                        "error: unsupported NCQ command (0x%02x) received\n",
+                        cmd_fis[2]);
+            } else {
+                DPRINTF(port,
+                        "error: tried to process non-NCQ command as NCQ\n");
+            }
             qemu_sglist_destroy(&ncq_tfs->sglist);
+    }
+}
+
+static void handle_reg_h2d_fis(AHCIState *s, int port,
+                               int slot, uint8_t *cmd_fis)
+{
+    IDEState *ide_state = &s->dev[port].port.ifs[0];
+    AHCICmdHdr *cmd = s->dev[port].cur_cmd;
+    uint32_t opts = le32_to_cpu(cmd->opts);
+
+    if (cmd_fis[1] & 0x0F) {
+        DPRINTF(port, "Port Multiplier not supported."
+                " cmd_fis[0]=%02x cmd_fis[1]=%02x cmd_fis[2]=%02x\n",
+                cmd_fis[0], cmd_fis[1], cmd_fis[2]);
+        return;
+    }
+
+    if (cmd_fis[1] & 0x70) {
+        DPRINTF(port, "Reserved flags set in H2D Register FIS."
+                " cmd_fis[0]=%02x cmd_fis[1]=%02x cmd_fis[2]=%02x\n",
+                cmd_fis[0], cmd_fis[1], cmd_fis[2]);
+        return;
+    }
+
+    if (!(cmd_fis[1] & SATA_FIS_REG_H2D_UPDATE_COMMAND_REGISTER)) {
+        switch (s->dev[port].port_state) {
+        case STATE_RUN:
+            if (cmd_fis[15] & ATA_SRST) {
+                s->dev[port].port_state = STATE_RESET;
+            }
+            break;
+        case STATE_RESET:
+            if (!(cmd_fis[15] & ATA_SRST)) {
+                ahci_reset_port(s, port);
+            }
             break;
+        }
+        return;
+    }
+
+    /* Check for NCQ command */
+    if (is_ncq(cmd_fis[2])) {
+        process_ncq_command(s, port, cmd_fis, slot);
+        return;
+    }
+
+    /* Decompose the FIS:
+     * AHCI does not interpret FIS packets, it only forwards them.
+     * SATA 1.0 describes how to decode LBA28 and CHS FIS packets.
+     * Later specifications, e.g, SATA 3.2, describe LBA48 FIS packets.
+     *
+     * ATA4 describes sector number for LBA28/CHS commands.
+     * ATA6 describes sector number for LBA48 commands.
+     * ATA8 deprecates CHS fully, describing only LBA28/48.
+     *
+     * We dutifully convert the FIS into IDE registers, and allow the
+     * core layer to interpret them as needed. */
+    ide_state->feature = cmd_fis[3];
+    ide_state->sector = cmd_fis[4];      /* LBA 7:0 */
+    ide_state->lcyl = cmd_fis[5];        /* LBA 15:8  */
+    ide_state->hcyl = cmd_fis[6];        /* LBA 23:16 */
+    ide_state->select = cmd_fis[7];      /* LBA 27:24 (LBA28) */
+    ide_state->hob_sector = cmd_fis[8];  /* LBA 31:24 */
+    ide_state->hob_lcyl = cmd_fis[9];    /* LBA 39:32 */
+    ide_state->hob_hcyl = cmd_fis[10];   /* LBA 47:40 */
+    ide_state->hob_feature = cmd_fis[11];
+    ide_state->nsector = (int64_t)((cmd_fis[13] << 8) | cmd_fis[12]);
+    /* 14, 16, 17, 18, 19: Reserved (SATA 1.0) */
+    /* 15: Only valid when UPDATE_COMMAND not set. */
+
+    /* Copy the ACMD field (ATAPI packet, if any) from the AHCI command
+     * table to ide_state->io_buffer */
+    if (opts & AHCI_CMD_ATAPI) {
+        memcpy(ide_state->io_buffer, &cmd_fis[AHCI_COMMAND_TABLE_ACMD], 0x10);
+        debug_print_fis(ide_state->io_buffer, 0x10);
+        s->dev[port].done_atapi_packet = false;
+        /* XXX send PIO setup FIS */
     }
+
+    ide_state->error = 0;
+
+    /* Reset transferred byte counter */
+    cmd->status = 0;
+
+    /* We're ready to process the command in FIS byte 2. */
+    ide_exec_cmd(&s->dev[port].port, cmd_fis[2]);
 }
 
 static int handle_cmd(AHCIState *s, int port, int slot)
 {
     IDEState *ide_state;
-    uint32_t opts;
     uint64_t tbl_addr;
     AHCICmdHdr *cmd;
     uint8_t *cmd_fis;
@@ -835,147 +1045,48 @@ static int handle_cmd(AHCIState *s, int port, int slot)
         return -1;
     }
 
-    cmd = &((AHCICmdHdr *)s->dev[port].lst)[slot];
-
     if (!s->dev[port].lst) {
         DPRINTF(port, "error: lst not given but cmd handled");
         return -1;
     }
-
+    cmd = &((AHCICmdHdr *)s->dev[port].lst)[slot];
     /* remember current slot handle for later */
     s->dev[port].cur_cmd = cmd;
 
-    opts = le32_to_cpu(cmd->opts);
-    tbl_addr = le64_to_cpu(cmd->tbl_addr);
+    /* The device we are working for */
+    ide_state = &s->dev[port].port.ifs[0];
+    if (!ide_state->blk) {
+        DPRINTF(port, "error: guest accessed unused port");
+        return -1;
+    }
 
+    tbl_addr = le64_to_cpu(cmd->tbl_addr);
     cmd_len = 0x80;
     cmd_fis = dma_memory_map(s->as, tbl_addr, &cmd_len,
                              DMA_DIRECTION_FROM_DEVICE);
-
     if (!cmd_fis) {
         DPRINTF(port, "error: guest passed us an invalid cmd fis\n");
         return -1;
-    }
-
-    /* The device we are working for */
-    ide_state = &s->dev[port].port.ifs[0];
-
-    if (!ide_state->bs) {
-        DPRINTF(port, "error: guest accessed unused port");
+    } else if (cmd_len != 0x80) {
+        ahci_trigger_irq(s, &s->dev[port], PORT_IRQ_HBUS_ERR);
+        DPRINTF(port, "error: dma_memory_map failed: "
+                "(len(%02"PRIx64") != 0x80)\n",
+                cmd_len);
         goto out;
     }
-
-    debug_print_fis(cmd_fis, 0x90);
-    //debug_print_fis(cmd_fis, (opts & AHCI_CMD_HDR_CMD_FIS_LEN) * 4);
+    debug_print_fis(cmd_fis, 0x80);
 
     switch (cmd_fis[0]) {
         case SATA_FIS_TYPE_REGISTER_H2D:
+            handle_reg_h2d_fis(s, port, slot, cmd_fis);
             break;
         default:
             DPRINTF(port, "unknown command cmd_fis[0]=%02x cmd_fis[1]=%02x "
                           "cmd_fis[2]=%02x\n", cmd_fis[0], cmd_fis[1],
                           cmd_fis[2]);
-            goto out;
-            break;
-    }
-
-    switch (cmd_fis[1]) {
-        case SATA_FIS_REG_H2D_UPDATE_COMMAND_REGISTER:
-            break;
-        case 0:
-            break;
-        default:
-            DPRINTF(port, "unknown command cmd_fis[0]=%02x cmd_fis[1]=%02x "
-                          "cmd_fis[2]=%02x\n", cmd_fis[0], cmd_fis[1],
-                          cmd_fis[2]);
-            goto out;
-            break;
-    }
-
-    switch (s->dev[port].port_state) {
-        case STATE_RUN:
-            if (cmd_fis[15] & ATA_SRST) {
-                s->dev[port].port_state = STATE_RESET;
-            }
-            break;
-        case STATE_RESET:
-            if (!(cmd_fis[15] & ATA_SRST)) {
-                ahci_reset_port(s, port);
-            }
             break;
     }
 
-    if (cmd_fis[1] == SATA_FIS_REG_H2D_UPDATE_COMMAND_REGISTER) {
-
-        /* Check for NCQ command */
-        if ((cmd_fis[2] == READ_FPDMA_QUEUED) ||
-            (cmd_fis[2] == WRITE_FPDMA_QUEUED)) {
-            process_ncq_command(s, port, cmd_fis, slot);
-            goto out;
-        }
-
-        /* Decompose the FIS  */
-        ide_state->nsector = (int64_t)((cmd_fis[13] << 8) | cmd_fis[12]);
-        ide_state->feature = cmd_fis[3];
-        if (!ide_state->nsector) {
-            ide_state->nsector = 256;
-        }
-
-        if (ide_state->drive_kind != IDE_CD) {
-            /*
-             * We set the sector depending on the sector defined in the FIS.
-             * Unfortunately, the spec isn't exactly obvious on this one.
-             *
-             * Apparently LBA48 commands set fis bytes 10,9,8,6,5,4 to the
-             * 48 bit sector number. ATA_CMD_READ_DMA_EXT is an example for
-             * such a command.
-             *
-             * Non-LBA48 commands however use 7[lower 4 bits],6,5,4 to define a
-             * 28-bit sector number. ATA_CMD_READ_DMA is an example for such
-             * a command.
-             *
-             * Since the spec doesn't explicitly state what each field should
-             * do, I simply assume non-used fields as reserved and OR everything
-             * together, independent of the command.
-             */
-            ide_set_sector(ide_state, ((uint64_t)cmd_fis[10] << 40)
-                                    | ((uint64_t)cmd_fis[9] << 32)
-                                    /* This is used for LBA48 commands */
-                                    | ((uint64_t)cmd_fis[8] << 24)
-                                    /* This is used for non-LBA48 commands */
-                                    | ((uint64_t)(cmd_fis[7] & 0xf) << 24)
-                                    | ((uint64_t)cmd_fis[6] << 16)
-                                    | ((uint64_t)cmd_fis[5] << 8)
-                                    | cmd_fis[4]);
-        }
-
-        /* Copy the ACMD field (ATAPI packet, if any) from the AHCI command
-         * table to ide_state->io_buffer
-         */
-        if (opts & AHCI_CMD_ATAPI) {
-            memcpy(ide_state->io_buffer, &cmd_fis[AHCI_COMMAND_TABLE_ACMD], 0x10);
-            ide_state->lcyl = 0x14;
-            ide_state->hcyl = 0xeb;
-            debug_print_fis(ide_state->io_buffer, 0x10);
-            ide_state->feature = IDE_FEATURE_DMA;
-            s->dev[port].done_atapi_packet = false;
-            /* XXX send PIO setup FIS */
-        }
-
-        ide_state->error = 0;
-
-        /* Reset transferred byte counter */
-        cmd->status = 0;
-
-        /* We're ready to process the command in FIS byte 2. */
-        ide_exec_cmd(&s->dev[port].port, cmd_fis[2]);
-
-        if ((s->dev[port].port.ifs[0].status & (READY_STAT|DRQ_STAT|BUSY_STAT)) ==
-            READY_STAT) {
-            ahci_write_fis_d2h(&s->dev[port], cmd_fis);
-        }
-    }
-
 out:
     dma_memory_unmap(s->as, cmd_fis, cmd_len, DMA_DIRECTION_FROM_DEVICE,
                      cmd_len);
@@ -991,7 +1102,7 @@ out:
 }
 
 /* DMA dev <-> ram */
-static int ahci_start_transfer(IDEDMA *dma)
+static void ahci_start_transfer(IDEDMA *dma)
 {
     AHCIDevice *ad = DO_UPCAST(AHCIDevice, dma, dma);
     IDEState *s = &ad->port.ifs[0];
@@ -1005,10 +1116,11 @@ static int ahci_start_transfer(IDEDMA *dma)
     if (is_atapi && !ad->done_atapi_packet) {
         /* already prepopulated iobuffer */
         ad->done_atapi_packet = true;
+        size = 0;
         goto out;
     }
 
-    if (!ahci_populate_sglist(ad, &s->sg, 0)) {
+    if (ahci_dma_prepare_buf(dma, is_write)) {
         has_sglist = 1;
     }
 
@@ -1024,29 +1136,23 @@ static int ahci_start_transfer(IDEDMA *dma)
         }
     }
 
-    /* update number of transferred bytes */
-    ad->cur_cmd->status = cpu_to_le32(le32_to_cpu(ad->cur_cmd->status) + size);
-
 out:
     /* declare that we processed everything */
     s->data_ptr = s->data_end;
 
-    if (has_sglist) {
-        qemu_sglist_destroy(&s->sg);
-    }
+    /* Update number of transferred bytes, destroy sglist */
+    ahci_commit_buf(dma, size);
 
     s->end_transfer_func(s);
 
     if (!(s->status & DRQ_STAT)) {
-        /* done with DMA */
-        ahci_trigger_irq(ad->hba, ad, PORT_IRQ_STAT_DSS);
+        /* done with PIO send/receive */
+        ahci_write_fis_pio(ad, le32_to_cpu(ad->cur_cmd->status));
     }
-
-    return 0;
 }
 
 static void ahci_start_dma(IDEDMA *dma, IDEState *s,
-                           BlockDriverCompletionFunc *dma_cb)
+                           BlockCompletionFunc *dma_cb)
 {
 #ifdef DEBUG_AHCI
     AHCIDevice *ad = DO_UPCAST(AHCIDevice, dma, dma);
@@ -1056,16 +1162,42 @@ static void ahci_start_dma(IDEDMA *dma, IDEState *s,
     dma_cb(s, 0);
 }
 
-static int ahci_dma_prepare_buf(IDEDMA *dma, int is_write)
+/**
+ * Called in DMA R/W chains to read the PRDT, utilizing ahci_populate_sglist.
+ * Not currently invoked by PIO R/W chains,
+ * which invoke ahci_populate_sglist via ahci_start_transfer.
+ */
+static int32_t ahci_dma_prepare_buf(IDEDMA *dma, int is_write)
 {
     AHCIDevice *ad = DO_UPCAST(AHCIDevice, dma, dma);
     IDEState *s = &ad->port.ifs[0];
 
-    ahci_populate_sglist(ad, &s->sg, 0);
+    if (ahci_populate_sglist(ad, &s->sg, s->io_buffer_offset) == -1) {
+        DPRINTF(ad->port_no, "ahci_dma_prepare_buf failed.\n");
+        return -1;
+    }
     s->io_buffer_size = s->sg.size;
 
     DPRINTF(ad->port_no, "len=%#x\n", s->io_buffer_size);
-    return s->io_buffer_size != 0;
+    return s->io_buffer_size;
+}
+
+/**
+ * Destroys the scatter-gather list,
+ * and updates the command header with a bytes-read value.
+ * called explicitly via ahci_dma_rw_buf (ATAPI DMA),
+ * and ahci_start_transfer (PIO R/W),
+ * and called via callback from ide_dma_cb for DMA R/W paths.
+ */
+static void ahci_commit_buf(IDEDMA *dma, uint32_t tx_bytes)
+{
+    AHCIDevice *ad = DO_UPCAST(AHCIDevice, dma, dma);
+    IDEState *s = &ad->port.ifs[0];
+
+    tx_bytes += le32_to_cpu(ad->cur_cmd->status);
+    ad->cur_cmd->status = cpu_to_le32(tx_bytes);
+
+    qemu_sglist_destroy(&s->sg);
 }
 
 static int ahci_dma_rw_buf(IDEDMA *dma, int is_write)
@@ -1085,11 +1217,9 @@ static int ahci_dma_rw_buf(IDEDMA *dma, int is_write)
         dma_buf_write(p, l, &s->sg);
     }
 
-    /* free sglist that was created in ahci_populate_sglist() */
-    qemu_sglist_destroy(&s->sg);
+    /* free sglist, update byte count */
+    ahci_commit_buf(dma, l);
 
-    /* update number of transferred bytes */
-    ad->cur_cmd->status = cpu_to_le32(le32_to_cpu(ad->cur_cmd->status) + l);
     s->io_buffer_index += l;
     s->io_buffer_offset += l;
 
@@ -1104,28 +1234,11 @@ static int ahci_dma_set_unit(IDEDMA *dma, int unit)
     return 0;
 }
 
-static int ahci_dma_add_status(IDEDMA *dma, int status)
-{
-    AHCIDevice *ad = DO_UPCAST(AHCIDevice, dma, dma);
-    DPRINTF(ad->port_no, "set status: %x\n", status);
-
-    if (status & BM_STATUS_INT) {
-        ahci_trigger_irq(ad->hba, ad, PORT_IRQ_STAT_DSS);
-    }
-
-    return 0;
-}
-
-static int ahci_dma_set_inactive(IDEDMA *dma)
-{
-    return 0;
-}
-
-static int ahci_async_cmd_done(IDEDMA *dma)
+static void ahci_cmd_done(IDEDMA *dma)
 {
     AHCIDevice *ad = DO_UPCAST(AHCIDevice, dma, dma);
 
-    DPRINTF(ad->port_no, "async cmd done\n");
+    DPRINTF(ad->port_no, "cmd done\n");
 
     /* update d2h status */
     ahci_write_fis_d2h(ad, NULL);
@@ -1135,8 +1248,6 @@ static int ahci_async_cmd_done(IDEDMA *dma)
         ad->check_bh = qemu_bh_new(ahci_check_cmd_bh, ad);
         qemu_bh_schedule(ad->check_bh);
     }
-
-    return 0;
 }
 
 static void ahci_irq_set(void *opaque, int n, int level)
@@ -1147,22 +1258,15 @@ static void ahci_dma_restart_cb(void *opaque, int running, RunState state)
 {
 }
 
-static int ahci_dma_reset(IDEDMA *dma)
-{
-    return 0;
-}
-
 static const IDEDMAOps ahci_dma_ops = {
     .start_dma = ahci_start_dma,
     .start_transfer = ahci_start_transfer,
     .prepare_buf = ahci_dma_prepare_buf,
+    .commit_buf = ahci_commit_buf,
     .rw_buf = ahci_dma_rw_buf,
     .set_unit = ahci_dma_set_unit,
-    .add_status = ahci_dma_add_status,
-    .set_inactive = ahci_dma_set_inactive,
-    .async_cmd_done = ahci_async_cmd_done,
+    .cmd_done = ahci_cmd_done,
     .restart_cb = ahci_dma_restart_cb,
-    .reset = ahci_dma_reset,
 };
 
 void ahci_init(AHCIState *s, DeviceState *qdev, AddressSpace *as, int ports)
@@ -1172,7 +1276,7 @@ void ahci_init(AHCIState *s, DeviceState *qdev, AddressSpace *as, int ports)
 
     s->as = as;
     s->ports = ports;
-    s->dev = g_malloc0(sizeof(AHCIDevice) * ports);
+    s->dev = g_new0(AHCIDevice, ports);
     ahci_reg_init(s);
     /* XXX BAR size should be 1k, but that breaks, so bump it to 4k for now */
     memory_region_init_io(&s->mem, OBJECT(qdev), &ahci_mem_ops, s,
@@ -1197,8 +1301,6 @@ void ahci_init(AHCIState *s, DeviceState *qdev, AddressSpace *as, int ports)
 
 void ahci_uninit(AHCIState *s)
 {
-    memory_region_destroy(&s->mem);
-    memory_region_destroy(&s->idp);
     g_free(s->dev);
 }
 
@@ -1372,3 +1474,18 @@ static void sysbus_ahci_register_types(void)
 }
 
 type_init(sysbus_ahci_register_types)
+
+void ahci_ide_create_devs(PCIDevice *dev, DriveInfo **hd)
+{
+    AHCIPCIState *d = ICH_AHCI(dev);
+    AHCIState *ahci = &d->ahci;
+    int i;
+
+    for (i = 0; i < ahci->ports; i++) {
+        if (hd[i] == NULL) {
+            continue;
+        }
+        ide_create_drive(&ahci->dev[i].port, 0, hd[i]);
+    }
+
+}
diff --git a/hw/ide/ahci.h b/hw/ide/ahci.h
index f418b30ce..e0d2eb8f1 100644
--- a/hw/ide/ahci.h
+++ b/hw/ide/ahci.h
@@ -132,27 +132,6 @@
 #define PORT_CMD_ICC_PARTIAL      (0x2 << 28) /* Put i/f in partial state */
 #define PORT_CMD_ICC_SLUMBER      (0x6 << 28) /* Put i/f in slumber state */
 
-#define PORT_IRQ_STAT_DHRS        (1 << 0) /* Device to Host Register FIS */
-#define PORT_IRQ_STAT_PSS         (1 << 1) /* PIO Setup FIS */
-#define PORT_IRQ_STAT_DSS         (1 << 2) /* DMA Setup FIS */
-#define PORT_IRQ_STAT_SDBS        (1 << 3) /* Set Device Bits */
-#define PORT_IRQ_STAT_UFS         (1 << 4) /* Unknown FIS */
-#define PORT_IRQ_STAT_DPS         (1 << 5) /* Descriptor Processed */
-#define PORT_IRQ_STAT_PCS         (1 << 6) /* Port Connect Change Status */
-#define PORT_IRQ_STAT_DMPS        (1 << 7) /* Device Mechanical Presence
-                                              Status */
-#define PORT_IRQ_STAT_PRCS        (1 << 22) /* File Ready Status */
-#define PORT_IRQ_STAT_IPMS        (1 << 23) /* Incorrect Port Multiplier
-                                               Status */
-#define PORT_IRQ_STAT_OFS         (1 << 24) /* Overflow Status */
-#define PORT_IRQ_STAT_INFS        (1 << 26) /* Interface Non-Fatal Error
-                                               Status */
-#define PORT_IRQ_STAT_IFS         (1 << 27) /* Interface Fatal Error */
-#define PORT_IRQ_STAT_HBDS        (1 << 28) /* Host Bus Data Error Status */
-#define PORT_IRQ_STAT_HBFS        (1 << 29) /* Host Bus Fatal Error Status */
-#define PORT_IRQ_STAT_TFES        (1 << 30) /* Task File Error Status */
-#define PORT_IRQ_STAT_CPDS        (1U << 31) /* Code Port Detect Status */
-
 /* ap->flags bits */
 #define AHCI_FLAG_NO_NCQ                  (1 << 24)
 #define AHCI_FLAG_IGN_IRQ_IF_ERR          (1 << 25) /* ignore IRQ_IF_ERR */
@@ -207,6 +186,9 @@
 
 #define READ_FPDMA_QUEUED                  0x60
 #define WRITE_FPDMA_QUEUED                 0x61
+#define NCQ_NON_DATA                       0x63
+#define RECEIVE_FPDMA_QUEUED               0x65
+#define SEND_FPDMA_QUEUED                  0x64
 
 #define RES_FIS_DSFIS                      0x00
 #define RES_FIS_PSFIS                      0x20
@@ -262,7 +244,7 @@ typedef struct AHCIDevice AHCIDevice;
 
 typedef struct NCQTransferState {
     AHCIDevice *drive;
-    BlockDriverAIOCB *aiocb;
+    BlockAIOCB *aiocb;
     QEMUSGList sglist;
     BlockAcctCookie acct;
     uint16_t sector_count;
@@ -348,9 +330,19 @@ typedef struct NCQFrame {
     uint8_t reserved10;
 } QEMU_PACKED NCQFrame;
 
+typedef struct SDBFIS {
+    uint8_t type;
+    uint8_t flags;
+    uint8_t status;
+    uint8_t error;
+    uint32_t payload;
+} QEMU_PACKED SDBFIS;
+
 void ahci_init(AHCIState *s, DeviceState *qdev, AddressSpace *as, int ports);
 void ahci_uninit(AHCIState *s);
 
 void ahci_reset(AHCIState *s);
 
+void ahci_ide_create_devs(PCIDevice *dev, DriveInfo **hd);
+
 #endif /* HW_IDE_AHCI_H */
diff --git a/hw/ide/atapi.c b/hw/ide/atapi.c
index f7d2009c0..c63b7e556 100644
--- a/hw/ide/atapi.c
+++ b/hw/ide/atapi.c
@@ -25,6 +25,7 @@
 
 #include "hw/ide/internal.h"
 #include "hw/scsi/scsi.h"
+#include "sysemu/block-backend.h"
 
 static void ide_atapi_cmd_read_dma_cb(void *opaque, int ret);
 
@@ -110,14 +111,16 @@ static int cd_read_sector(IDEState *s, int lba, uint8_t *buf, int sector_size)
 
     switch(sector_size) {
     case 2048:
-        bdrv_acct_start(s->bs, &s->acct, 4 * BDRV_SECTOR_SIZE, BDRV_ACCT_READ);
-        ret = bdrv_read(s->bs, (int64_t)lba << 2, buf, 4);
-        bdrv_acct_done(s->bs, &s->acct);
+        block_acct_start(blk_get_stats(s->blk), &s->acct,
+                         4 * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
+        ret = blk_read(s->blk, (int64_t)lba << 2, buf, 4);
+        block_acct_done(blk_get_stats(s->blk), &s->acct);
         break;
     case 2352:
-        bdrv_acct_start(s->bs, &s->acct, 4 * BDRV_SECTOR_SIZE, BDRV_ACCT_READ);
-        ret = bdrv_read(s->bs, (int64_t)lba << 2, buf + 16, 4);
-        bdrv_acct_done(s->bs, &s->acct);
+        block_acct_start(blk_get_stats(s->blk), &s->acct,
+                         4 * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
+        ret = blk_read(s->blk, (int64_t)lba << 2, buf + 16, 4);
+        block_acct_done(blk_get_stats(s->blk), &s->acct);
         if (ret < 0)
             return ret;
         cd_data_to_raw(buf, lba);
@@ -134,6 +137,7 @@ void ide_atapi_cmd_ok(IDEState *s)
     s->error = 0;
     s->status = READY_STAT | SEEK_STAT;
     s->nsector = (s->nsector & ~7) | ATAPI_INT_REASON_IO | ATAPI_INT_REASON_CD;
+    ide_transfer_stop(s);
     ide_set_irq(s->bus);
 }
 
@@ -147,6 +151,7 @@ void ide_atapi_cmd_error(IDEState *s, int sense_key, int asc)
     s->nsector = (s->nsector & ~7) | ATAPI_INT_REASON_IO | ATAPI_INT_REASON_CD;
     s->sense_key = sense_key;
     s->asc = asc;
+    ide_transfer_stop(s);
     ide_set_irq(s->bus);
 }
 
@@ -174,9 +179,7 @@ void ide_atapi_cmd_reply_end(IDEState *s)
 #endif
     if (s->packet_transfer_size <= 0) {
         /* end of transfer */
-        ide_transfer_stop(s);
-        s->status = READY_STAT | SEEK_STAT;
-        s->nsector = (s->nsector & ~7) | ATAPI_INT_REASON_IO | ATAPI_INT_REASON_CD;
+        ide_atapi_cmd_ok(s);
         ide_set_irq(s->bus);
 #ifdef DEBUG_IDE_ATAPI
         printf("status=0x%x\n", s->status);
@@ -186,7 +189,6 @@ void ide_atapi_cmd_reply_end(IDEState *s)
         if (s->lba != -1 && s->io_buffer_index >= s->cd_sector_size) {
             ret = cd_read_sector(s, s->lba, s->io_buffer, s->cd_sector_size);
             if (ret < 0) {
-                ide_transfer_stop(s);
                 ide_atapi_io_error(s, ret);
                 return;
             }
@@ -253,10 +255,10 @@ static void ide_atapi_cmd_reply(IDEState *s, int size, int max_size)
     s->io_buffer_index = 0;
 
     if (s->atapi_dma) {
-        bdrv_acct_start(s->bs, &s->acct, size, BDRV_ACCT_READ);
+        block_acct_start(blk_get_stats(s->blk), &s->acct, size,
+                         BLOCK_ACCT_READ);
         s->status = READY_STAT | SEEK_STAT | DRQ_STAT;
-        s->bus->dma->ops->start_dma(s->bus->dma, s,
-                                   ide_atapi_cmd_read_dma_cb);
+        ide_start_dma(s, ide_atapi_cmd_read_dma_cb);
     } else {
         s->status = READY_STAT | SEEK_STAT;
         ide_atapi_cmd_reply_end(s);
@@ -349,15 +351,14 @@ static void ide_atapi_cmd_read_dma_cb(void *opaque, int ret)
     s->bus->dma->iov.iov_len = n * 4 * 512;
     qemu_iovec_init_external(&s->bus->dma->qiov, &s->bus->dma->iov, 1);
 
-    s->bus->dma->aiocb = bdrv_aio_readv(s->bs, (int64_t)s->lba << 2,
+    s->bus->dma->aiocb = blk_aio_readv(s->blk, (int64_t)s->lba << 2,
                                        &s->bus->dma->qiov, n * 4,
                                        ide_atapi_cmd_read_dma_cb, s);
     return;
 
 eot:
-    bdrv_acct_done(s->bs, &s->acct);
-    s->bus->dma->ops->add_status(s->bus->dma, BM_STATUS_INT);
-    ide_set_inactive(s);
+    block_acct_done(blk_get_stats(s->blk), &s->acct);
+    ide_set_inactive(s, false);
 }
 
 /* start a CD-CDROM read command with DMA */
@@ -371,12 +372,12 @@ static void ide_atapi_cmd_read_dma(IDEState *s, int lba, int nb_sectors,
     s->io_buffer_size = 0;
     s->cd_sector_size = sector_size;
 
-    bdrv_acct_start(s->bs, &s->acct, s->packet_transfer_size, BDRV_ACCT_READ);
+    block_acct_start(blk_get_stats(s->blk), &s->acct, s->packet_transfer_size,
+                     BLOCK_ACCT_READ);
 
     /* XXX: check if BUSY_STAT should be set */
     s->status = READY_STAT | SEEK_STAT | DRQ_STAT | BUSY_STAT;
-    s->bus->dma->ops->start_dma(s->bus->dma, s,
-                               ide_atapi_cmd_read_dma_cb);
+    ide_start_dma(s, ide_atapi_cmd_read_dma_cb);
 }
 
 static void ide_atapi_cmd_read(IDEState *s, int lba, int nb_sectors,
@@ -504,7 +505,7 @@ static unsigned int event_status_media(IDEState *s,
     media_status = 0;
     if (s->tray_open) {
         media_status = MS_TRAY_OPEN;
-    } else if (bdrv_is_inserted(s->bs)) {
+    } else if (blk_is_inserted(s->blk)) {
         media_status = MS_MEDIA_PRESENT;
     }
 
@@ -800,7 +801,7 @@ static void cmd_test_unit_ready(IDEState *s, uint8_t *buf)
 static void cmd_prevent_allow_medium_removal(IDEState *s, uint8_t* buf)
 {
     s->tray_locked = buf[4] & 1;
-    bdrv_lock_medium(s->bs, buf[4] & 1);
+    blk_lock_medium(s->blk, buf[4] & 1);
     ide_atapi_cmd_ok(s);
 }
 
@@ -884,14 +885,14 @@ static void cmd_start_stop_unit(IDEState *s, uint8_t* buf)
 
     if (loej) {
         if (!start && !s->tray_open && s->tray_locked) {
-            sense = bdrv_is_inserted(s->bs)
+            sense = blk_is_inserted(s->blk)
                 ? NOT_READY : ILLEGAL_REQUEST;
             ide_atapi_cmd_error(s, sense, ASC_MEDIA_REMOVAL_PREVENTED);
             return;
         }
 
         if (s->tray_open != !start) {
-            bdrv_eject(s->bs, !start);
+            blk_eject(s->blk, !start);
             s->tray_open = !start;
         }
     }
@@ -1125,7 +1126,7 @@ void ide_atapi_cmd(IDEState *s)
      * states rely on this behavior.
      */
     if (!(atapi_cmd_table[s->io_buffer[0]].flags & ALLOW_UA) &&
-        !s->tray_open && bdrv_is_inserted(s->bs) && s->cdrom_changed) {
+        !s->tray_open && blk_is_inserted(s->blk) && s->cdrom_changed) {
 
         if (s->cdrom_changed == 1) {
             ide_atapi_cmd_error(s, NOT_READY, ASC_MEDIUM_NOT_PRESENT);
@@ -1140,7 +1141,7 @@ void ide_atapi_cmd(IDEState *s)
 
     /* Report a Not Ready condition if appropriate for the command */
     if ((atapi_cmd_table[s->io_buffer[0]].flags & CHECK_READY) &&
-        (!media_present(s) || !bdrv_is_inserted(s->bs)))
+        (!media_present(s) || !blk_is_inserted(s->blk)))
     {
         ide_atapi_cmd_error(s, NOT_READY, ASC_MEDIUM_NOT_PRESENT);
         return;
diff --git a/hw/ide/cmd646.c b/hw/ide/cmd646.c
index a8e35fe38..c8b032215 100644
--- a/hw/ide/cmd646.c
+++ b/hw/ide/cmd646.c
@@ -26,13 +26,20 @@
 #include <hw/i386/pc.h>
 #include <hw/pci/pci.h>
 #include <hw/isa/isa.h>
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/dma.h"
 
 #include <hw/ide/pci.h>
 
 /* CMD646 specific */
+#define CFR		0x50
+#define   CFR_INTR_CH0	0x04
+#define CNTRL		0x51
+#define   CNTRL_EN_CH0	0x04
+#define   CNTRL_EN_CH1	0x08
+#define ARTTIM23	0x57
+#define    ARTTIM23_INTR_CH1	0x10
 #define MRDMODE		0x71
 #define   MRDMODE_INTR_CH0	0x04
 #define   MRDMODE_INTR_CH1	0x08
@@ -41,7 +48,7 @@
 #define UDIDETCR0	0x73
 #define UDIDETCR1	0x7B
 
-static void cmd646_update_irq(PCIIDEState *d);
+static void cmd646_update_irq(PCIDevice *pd);
 
 static uint64_t cmd646_cmd_read(void *opaque, hwaddr addr,
                                 unsigned size)
@@ -123,6 +130,38 @@ static void setup_cmd646_bar(PCIIDEState *d, int bus_num)
                           "cmd646-data", 8);
 }
 
+static void cmd646_update_dma_interrupts(PCIDevice *pd)
+{
+    /* Sync DMA interrupt status from UDMA interrupt status */
+    if (pd->config[MRDMODE] & MRDMODE_INTR_CH0) {
+        pd->config[CFR] |= CFR_INTR_CH0;
+    } else {
+        pd->config[CFR] &= ~CFR_INTR_CH0;
+    }
+
+    if (pd->config[MRDMODE] & MRDMODE_INTR_CH1) {
+        pd->config[ARTTIM23] |= ARTTIM23_INTR_CH1;
+    } else {
+        pd->config[ARTTIM23] &= ~ARTTIM23_INTR_CH1;
+    }
+}
+
+static void cmd646_update_udma_interrupts(PCIDevice *pd)
+{
+    /* Sync UDMA interrupt status from DMA interrupt status */
+    if (pd->config[CFR] & CFR_INTR_CH0) {
+        pd->config[MRDMODE] |= MRDMODE_INTR_CH0;
+    } else {
+        pd->config[MRDMODE] &= ~MRDMODE_INTR_CH0;
+    }
+
+    if (pd->config[ARTTIM23] & ARTTIM23_INTR_CH1) {
+        pd->config[MRDMODE] |= MRDMODE_INTR_CH1;
+    } else {
+        pd->config[MRDMODE] &= ~MRDMODE_INTR_CH1;
+    }
+}
+
 static uint64_t bmdma_read(void *opaque, hwaddr addr,
                            unsigned size)
 {
@@ -181,7 +220,8 @@ static void bmdma_write(void *opaque, hwaddr addr,
     case 1:
         pci_dev->config[MRDMODE] =
             (pci_dev->config[MRDMODE] & ~0x30) | (val & 0x30);
-        cmd646_update_irq(bm->pci_dev);
+        cmd646_update_dma_interrupts(pci_dev);
+        cmd646_update_irq(pci_dev);
         break;
     case 2:
         bm->status = (val & 0x60) | (bm->status & 1) | (bm->status & ~val & 0x06);
@@ -219,11 +259,8 @@ static void bmdma_setup_bar(PCIIDEState *d)
     }
 }
 
-/* XXX: call it also when the MRDMODE is changed from the PCI config
-   registers */
-static void cmd646_update_irq(PCIIDEState *d)
+static void cmd646_update_irq(PCIDevice *pd)
 {
-    PCIDevice *pd = PCI_DEVICE(d);
     int pci_level;
 
     pci_level = ((pd->config[MRDMODE] & MRDMODE_INTR_CH0) &&
@@ -246,7 +283,8 @@ static void cmd646_set_irq(void *opaque, int channel, int level)
     } else {
         pd->config[MRDMODE] &= ~irq_mask;
     }
-    cmd646_update_irq(d);
+    cmd646_update_dma_interrupts(pd);
+    cmd646_update_irq(pd);
 }
 
 static void cmd646_reset(void *opaque)
@@ -259,6 +297,34 @@ static void cmd646_reset(void *opaque)
     }
 }
 
+static uint32_t cmd646_pci_config_read(PCIDevice *d,
+                                       uint32_t address, int len)
+{
+    return pci_default_read_config(d, address, len);
+}
+
+static void cmd646_pci_config_write(PCIDevice *d, uint32_t addr, uint32_t val,
+                                    int l)
+{
+    uint32_t i;
+
+    pci_default_write_config(d, addr, val, l);
+
+    for (i = addr; i < addr + l; i++) {
+        switch (i) {
+        case CFR:
+        case ARTTIM23:
+            cmd646_update_udma_interrupts(d);
+            break;
+        case MRDMODE:
+            cmd646_update_dma_interrupts(d);
+            break;
+        }
+    }
+
+    cmd646_update_irq(d);
+}
+
 /* CMD646 PCI IDE controller */
 static int pci_cmd646_ide_initfn(PCIDevice *dev)
 {
@@ -269,12 +335,20 @@ static int pci_cmd646_ide_initfn(PCIDevice *dev)
 
     pci_conf[PCI_CLASS_PROG] = 0x8f;
 
-    pci_conf[0x51] = 0x04; // enable IDE0
+    pci_conf[CNTRL] = CNTRL_EN_CH0; // enable IDE0
     if (d->secondary) {
         /* XXX: if not enabled, really disable the seconday IDE controller */
-        pci_conf[0x51] |= 0x08; /* enable IDE1 */
+        pci_conf[CNTRL] |= CNTRL_EN_CH1; /* enable IDE1 */
     }
 
+    /* Set write-to-clear interrupt bits */
+    dev->wmask[CFR] = 0x0;
+    dev->w1cmask[CFR] = CFR_INTR_CH0;
+    dev->wmask[ARTTIM23] = 0x0;
+    dev->w1cmask[ARTTIM23] = ARTTIM23_INTR_CH1;
+    dev->wmask[MRDMODE] = 0x0;
+    dev->w1cmask[MRDMODE] = MRDMODE_INTR_CH0 | MRDMODE_INTR_CH1;
+
     setup_cmd646_bar(d, 0);
     setup_cmd646_bar(d, 1);
     pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_IO, &d->cmd646_bar[0].data);
@@ -310,13 +384,8 @@ static void pci_cmd646_ide_exitfn(PCIDevice *dev)
 
     for (i = 0; i < 2; ++i) {
         memory_region_del_subregion(&d->bmdma_bar, &d->bmdma[i].extra_io);
-        memory_region_destroy(&d->bmdma[i].extra_io);
         memory_region_del_subregion(&d->bmdma_bar, &d->bmdma[i].addr_ioport);
-        memory_region_destroy(&d->bmdma[i].addr_ioport);
-        memory_region_destroy(&d->cmd646_bar[i].cmd);
-        memory_region_destroy(&d->cmd646_bar[i].data);
     }
-    memory_region_destroy(&d->bmdma_bar);
 }
 
 void pci_cmd646_ide_init(PCIBus *bus, DriveInfo **hd_table,
@@ -347,6 +416,8 @@ static void cmd646_ide_class_init(ObjectClass *klass, void *data)
     k->device_id = PCI_DEVICE_ID_CMD_646;
     k->revision = 0x07;
     k->class_id = PCI_CLASS_STORAGE_IDE;
+    k->config_read = cmd646_pci_config_read;
+    k->config_write = cmd646_pci_config_write;
     dc->props = cmd646_ide_properties;
 }
 
diff --git a/hw/ide/core.c b/hw/ide/core.c
index db191a6c3..d4af5e2eb 100644
--- a/hw/ide/core.c
+++ b/hw/ide/core.c
@@ -31,7 +31,7 @@
 #include "sysemu/sysemu.h"
 #include "sysemu/dma.h"
 #include "hw/block/block.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 
 #include <hw/ide/internal.h>
 
@@ -75,19 +75,29 @@ static void put_le16(uint16_t *p, unsigned int v)
     *p = cpu_to_le16(v);
 }
 
+static void ide_identify_size(IDEState *s)
+{
+    uint16_t *p = (uint16_t *)s->identify_data;
+    put_le16(p + 60, s->nb_sectors);
+    put_le16(p + 61, s->nb_sectors >> 16);
+    put_le16(p + 100, s->nb_sectors);
+    put_le16(p + 101, s->nb_sectors >> 16);
+    put_le16(p + 102, s->nb_sectors >> 32);
+    put_le16(p + 103, s->nb_sectors >> 48);
+}
+
 static void ide_identify(IDEState *s)
 {
     uint16_t *p;
     unsigned int oldsize;
     IDEDevice *dev = s->unit ? s->bus->slave : s->bus->master;
 
+    p = (uint16_t *)s->identify_data;
     if (s->identify_set) {
-	memcpy(s->io_buffer, s->identify_data, sizeof(s->identify_data));
-	return;
+        goto fill_buffer;
     }
+    memset(p, 0, sizeof(s->identify_data));
 
-    memset(s->io_buffer, 0, 512);
-    p = (uint16_t *)s->io_buffer;
     put_le16(p + 0, 0x0040);
     put_le16(p + 1, s->cylinders);
     put_le16(p + 3, s->heads);
@@ -116,8 +126,8 @@ static void ide_identify(IDEState *s)
     put_le16(p + 58, oldsize >> 16);
     if (s->mult_sectors)
         put_le16(p + 59, 0x100 | s->mult_sectors);
-    put_le16(p + 60, s->nb_sectors);
-    put_le16(p + 61, s->nb_sectors >> 16);
+    /* *(p + 60) := nb_sectors       -- see ide_identify_size */
+    /* *(p + 61) := nb_sectors >> 16 -- see ide_identify_size */
     put_le16(p + 62, 0x07); /* single word dma0-2 supported */
     put_le16(p + 63, 0x07); /* mdma0-2 supported */
     put_le16(p + 64, 0x03); /* pio3-4 supported */
@@ -148,10 +158,11 @@ static void ide_identify(IDEState *s)
         put_le16(p + 84, (1 << 14) | 0);
     }
     /* 14 = NOP supported, 5=WCACHE enabled, 0=SMART feature set enabled */
-    if (bdrv_enable_write_cache(s->bs))
-         put_le16(p + 85, (1 << 14) | (1 << 5) | 1);
-    else
-         put_le16(p + 85, (1 << 14) | 1);
+    if (blk_enable_write_cache(s->blk)) {
+        put_le16(p + 85, (1 << 14) | (1 << 5) | 1);
+    } else {
+        put_le16(p + 85, (1 << 14) | 1);
+    }
     /* 13=flush_cache_ext,12=flush_cache,10=lba48 */
     put_le16(p + 86, (1 << 13) | (1 <<12) | (1 << 10));
     /* 14=set to 1, 8=has WWN, 1=SMART self test, 0=SMART error logging */
@@ -162,10 +173,10 @@ static void ide_identify(IDEState *s)
     }
     put_le16(p + 88, 0x3f | (1 << 13)); /* udma5 set and supported */
     put_le16(p + 93, 1 | (1 << 14) | 0x2000);
-    put_le16(p + 100, s->nb_sectors);
-    put_le16(p + 101, s->nb_sectors >> 16);
-    put_le16(p + 102, s->nb_sectors >> 32);
-    put_le16(p + 103, s->nb_sectors >> 48);
+    /* *(p + 100) := nb_sectors       -- see ide_identify_size */
+    /* *(p + 101) := nb_sectors >> 16 -- see ide_identify_size */
+    /* *(p + 102) := nb_sectors >> 32 -- see ide_identify_size */
+    /* *(p + 103) := nb_sectors >> 48 -- see ide_identify_size */
 
     if (dev && dev->conf.physical_block_size)
         put_le16(p + 106, 0x6000 | get_physical_block_exp(&dev->conf));
@@ -180,21 +191,23 @@ static void ide_identify(IDEState *s)
         put_le16(p + 169, 1); /* TRIM support */
     }
 
-    memcpy(s->identify_data, p, sizeof(s->identify_data));
+    ide_identify_size(s);
     s->identify_set = 1;
+
+fill_buffer:
+    memcpy(s->io_buffer, p, sizeof(s->identify_data));
 }
 
 static void ide_atapi_identify(IDEState *s)
 {
     uint16_t *p;
 
+    p = (uint16_t *)s->identify_data;
     if (s->identify_set) {
-	memcpy(s->io_buffer, s->identify_data, sizeof(s->identify_data));
-	return;
+        goto fill_buffer;
     }
+    memset(p, 0, sizeof(s->identify_data));
 
-    memset(s->io_buffer, 0, 512);
-    p = (uint16_t *)s->io_buffer;
     /* Removable CDROM, 50us response, 12 byte packets */
     put_le16(p + 0, (2 << 14) | (5 << 8) | (1 << 7) | (2 << 5) | (0 << 0));
     padstr((char *)(p + 10), s->drive_serial_str, 20); /* serial number */
@@ -230,11 +243,36 @@ static void ide_atapi_identify(IDEState *s)
     }
 
     put_le16(p + 80, 0x1e); /* support up to ATA/ATAPI-4 */
+    if (s->wwn) {
+        put_le16(p + 84, (1 << 8)); /* supports WWN for words 108-111 */
+        put_le16(p + 87, (1 << 8)); /* WWN enabled */
+    }
+
 #ifdef USE_DMA_CDROM
     put_le16(p + 88, 0x3f | (1 << 13)); /* udma5 set and supported */
 #endif
-    memcpy(s->identify_data, p, sizeof(s->identify_data));
+
+    if (s->wwn) {
+        /* LE 16-bit words 111-108 contain 64-bit World Wide Name */
+        put_le16(p + 108, s->wwn >> 48);
+        put_le16(p + 109, s->wwn >> 32);
+        put_le16(p + 110, s->wwn >> 16);
+        put_le16(p + 111, s->wwn);
+    }
+
     s->identify_set = 1;
+
+fill_buffer:
+    memcpy(s->io_buffer, p, sizeof(s->identify_data));
+}
+
+static void ide_cfata_identify_size(IDEState *s)
+{
+    uint16_t *p = (uint16_t *)s->identify_data;
+    put_le16(p + 7, s->nb_sectors >> 16);  /* Sectors per card */
+    put_le16(p + 8, s->nb_sectors);        /* Sectors per card */
+    put_le16(p + 60, s->nb_sectors);       /* Total LBA sectors */
+    put_le16(p + 61, s->nb_sectors >> 16); /* Total LBA sectors */
 }
 
 static void ide_cfata_identify(IDEState *s)
@@ -242,10 +280,10 @@ static void ide_cfata_identify(IDEState *s)
     uint16_t *p;
     uint32_t cur_sec;
 
-    p = (uint16_t *) s->identify_data;
-    if (s->identify_set)
+    p = (uint16_t *)s->identify_data;
+    if (s->identify_set) {
         goto fill_buffer;
-
+    }
     memset(p, 0, sizeof(s->identify_data));
 
     cur_sec = s->cylinders * s->heads * s->sectors;
@@ -254,8 +292,8 @@ static void ide_cfata_identify(IDEState *s)
     put_le16(p + 1, s->cylinders);		/* Default cylinders */
     put_le16(p + 3, s->heads);			/* Default heads */
     put_le16(p + 6, s->sectors);		/* Default sectors per track */
-    put_le16(p + 7, s->nb_sectors >> 16);	/* Sectors per card */
-    put_le16(p + 8, s->nb_sectors);		/* Sectors per card */
+    /* *(p + 7) := nb_sectors >> 16 -- see ide_cfata_identify_size */
+    /* *(p + 8) := nb_sectors       -- see ide_cfata_identify_size */
     padstr((char *)(p + 10), s->drive_serial_str, 20); /* serial number */
     put_le16(p + 22, 0x0004);			/* ECC bytes */
     padstr((char *) (p + 23), s->version, 8);	/* Firmware Revision */
@@ -276,8 +314,8 @@ static void ide_cfata_identify(IDEState *s)
     put_le16(p + 58, cur_sec >> 16);		/* Current capacity */
     if (s->mult_sectors)			/* Multiple sector setting */
         put_le16(p + 59, 0x100 | s->mult_sectors);
-    put_le16(p + 60, s->nb_sectors);		/* Total LBA sectors */
-    put_le16(p + 61, s->nb_sectors >> 16);	/* Total LBA sectors */
+    /* *(p + 60) := nb_sectors       -- see ide_cfata_identify_size */
+    /* *(p + 61) := nb_sectors >> 16 -- see ide_cfata_identify_size */
     put_le16(p + 63, 0x0203);			/* Multiword DMA capability */
     put_le16(p + 64, 0x0001);			/* Flow Control PIO support */
     put_le16(p + 65, 0x0096);			/* Min. Multiword DMA cycle */
@@ -297,6 +335,7 @@ static void ide_cfata_identify(IDEState *s)
     put_le16(p + 160, 0x8100);			/* Power requirement */
     put_le16(p + 161, 0x8001);			/* CF command set */
 
+    ide_cfata_identify_size(s);
     s->identify_set = 1;
 
 fill_buffer:
@@ -312,7 +351,7 @@ static void ide_set_signature(IDEState *s)
     if (s->drive_kind == IDE_CD) {
         s->lcyl = 0x14;
         s->hcyl = 0xeb;
-    } else if (s->bs) {
+    } else if (s->blk) {
         s->lcyl = 0;
         s->hcyl = 0;
     } else {
@@ -322,35 +361,34 @@ static void ide_set_signature(IDEState *s)
 }
 
 typedef struct TrimAIOCB {
-    BlockDriverAIOCB common;
+    BlockAIOCB common;
+    BlockBackend *blk;
     QEMUBH *bh;
     int ret;
     QEMUIOVector *qiov;
-    BlockDriverAIOCB *aiocb;
+    BlockAIOCB *aiocb;
     int i, j;
 } TrimAIOCB;
 
-static void trim_aio_cancel(BlockDriverAIOCB *acb)
+static void trim_aio_cancel(BlockAIOCB *acb)
 {
     TrimAIOCB *iocb = container_of(acb, TrimAIOCB, common);
 
-    /* Exit the loop in case bdrv_aio_cancel calls ide_issue_trim_cb again.  */
+    /* Exit the loop so ide_issue_trim_cb will not continue  */
     iocb->j = iocb->qiov->niov - 1;
     iocb->i = (iocb->qiov->iov[iocb->j].iov_len / 8) - 1;
 
-    /* Tell ide_issue_trim_cb not to trigger the completion, too.  */
-    qemu_bh_delete(iocb->bh);
-    iocb->bh = NULL;
+    iocb->ret = -ECANCELED;
 
     if (iocb->aiocb) {
-        bdrv_aio_cancel(iocb->aiocb);
+        blk_aio_cancel_async(iocb->aiocb);
+        iocb->aiocb = NULL;
     }
-    qemu_aio_release(iocb);
 }
 
 static const AIOCBInfo trim_aiocb_info = {
     .aiocb_size         = sizeof(TrimAIOCB),
-    .cancel             = trim_aio_cancel,
+    .cancel_async       = trim_aio_cancel,
 };
 
 static void ide_trim_bh_cb(void *opaque)
@@ -361,7 +399,7 @@ static void ide_trim_bh_cb(void *opaque)
 
     qemu_bh_delete(iocb->bh);
     iocb->bh = NULL;
-    qemu_aio_release(iocb);
+    qemu_aio_unref(iocb);
 }
 
 static void ide_issue_trim_cb(void *opaque, int ret)
@@ -384,8 +422,8 @@ static void ide_issue_trim_cb(void *opaque, int ret)
                 }
 
                 /* Got an entry! Submit and exit.  */
-                iocb->aiocb = bdrv_aio_discard(iocb->common.bs, sector, count,
-                                               ide_issue_trim_cb, opaque);
+                iocb->aiocb = blk_aio_discard(iocb->blk, sector, count,
+                                              ide_issue_trim_cb, opaque);
                 return;
             }
 
@@ -402,13 +440,14 @@ static void ide_issue_trim_cb(void *opaque, int ret)
     }
 }
 
-BlockDriverAIOCB *ide_issue_trim(BlockDriverState *bs,
+BlockAIOCB *ide_issue_trim(BlockBackend *blk,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque)
+        BlockCompletionFunc *cb, void *opaque)
 {
     TrimAIOCB *iocb;
 
-    iocb = qemu_aio_get(&trim_aiocb_info, bs, cb, opaque);
+    iocb = blk_aio_get(&trim_aiocb_info, blk, cb, opaque);
+    iocb->blk = blk;
     iocb->bh = qemu_bh_new(ide_trim_bh_cb, iocb);
     iocb->ret = 0;
     iocb->qiov = qiov;
@@ -420,6 +459,7 @@ BlockDriverAIOCB *ide_issue_trim(BlockDriverState *bs,
 
 static inline void ide_abort_command(IDEState *s)
 {
+    ide_transfer_stop(s);
     s->status = READY_STAT | ERR_STAT;
     s->error = ABRT_ERR;
 }
@@ -434,7 +474,16 @@ void ide_transfer_start(IDEState *s, uint8_t *buf, int size,
     if (!(s->status & ERR_STAT)) {
         s->status |= DRQ_STAT;
     }
-    s->bus->dma->ops->start_transfer(s->bus->dma);
+    if (s->bus->dma->ops->start_transfer) {
+        s->bus->dma->ops->start_transfer(s->bus->dma);
+    }
+}
+
+static void ide_cmd_done(IDEState *s)
+{
+    if (s->bus->dma->ops->cmd_done) {
+        s->bus->dma->ops->cmd_done(s->bus->dma);
+    }
 }
 
 void ide_transfer_stop(IDEState *s)
@@ -443,6 +492,7 @@ void ide_transfer_stop(IDEState *s)
     s->data_ptr = s->io_buffer;
     s->data_end = s->io_buffer;
     s->status &= ~DRQ_STAT;
+    ide_cmd_done(s);
 }
 
 int64_t ide_get_sector(IDEState *s)
@@ -504,7 +554,7 @@ static bool ide_sect_range_ok(IDEState *s,
 {
     uint64_t total_sectors;
 
-    bdrv_get_geometry(s->bs, &total_sectors);
+    blk_get_geometry(s->blk, &total_sectors);
     if (sector > total_sectors || nb_sectors > total_sectors - sector) {
         return false;
     }
@@ -519,10 +569,13 @@ static void ide_sector_read_cb(void *opaque, int ret)
     s->pio_aiocb = NULL;
     s->status &= ~BUSY_STAT;
 
-    bdrv_acct_done(s->bs, &s->acct);
+    if (ret == -ECANCELED) {
+        return;
+    }
+    block_acct_done(blk_get_stats(s->blk), &s->acct);
     if (ret != 0) {
-        if (ide_handle_rw_error(s, -ret, BM_STATUS_PIO_RETRY |
-                                BM_STATUS_RETRY_READ)) {
+        if (ide_handle_rw_error(s, -ret, IDE_RETRY_PIO |
+                                IDE_RETRY_READ)) {
             return;
         }
     }
@@ -539,6 +592,7 @@ static void ide_sector_read_cb(void *opaque, int ret)
 
     ide_set_sector(s, ide_get_sector(s) + n);
     s->nsector -= n;
+    s->io_buffer_offset += 512 * n;
 }
 
 void ide_sector_read(IDEState *s)
@@ -575,56 +629,53 @@ void ide_sector_read(IDEState *s)
     s->iov.iov_len  = n * BDRV_SECTOR_SIZE;
     qemu_iovec_init_external(&s->qiov, &s->iov, 1);
 
-    bdrv_acct_start(s->bs, &s->acct, n * BDRV_SECTOR_SIZE, BDRV_ACCT_READ);
-    s->pio_aiocb = bdrv_aio_readv(s->bs, sector_num, &s->qiov, n,
-                                  ide_sector_read_cb, s);
-}
-
-static void dma_buf_commit(IDEState *s)
-{
-    qemu_sglist_destroy(&s->sg);
+    block_acct_start(blk_get_stats(s->blk), &s->acct,
+                     n * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
+    s->pio_aiocb = blk_aio_readv(s->blk, sector_num, &s->qiov, n,
+                                 ide_sector_read_cb, s);
 }
 
-static void ide_async_cmd_done(IDEState *s)
+static void dma_buf_commit(IDEState *s, uint32_t tx_bytes)
 {
-    if (s->bus->dma->ops->async_cmd_done) {
-        s->bus->dma->ops->async_cmd_done(s->bus->dma);
+    if (s->bus->dma->ops->commit_buf) {
+        s->bus->dma->ops->commit_buf(s->bus->dma, tx_bytes);
     }
+    qemu_sglist_destroy(&s->sg);
 }
 
-void ide_set_inactive(IDEState *s)
+void ide_set_inactive(IDEState *s, bool more)
 {
     s->bus->dma->aiocb = NULL;
-    s->bus->dma->ops->set_inactive(s->bus->dma);
-    ide_async_cmd_done(s);
+    if (s->bus->dma->ops->set_inactive) {
+        s->bus->dma->ops->set_inactive(s->bus->dma, more);
+    }
+    ide_cmd_done(s);
 }
 
 void ide_dma_error(IDEState *s)
 {
-    ide_transfer_stop(s);
-    s->error = ABRT_ERR;
-    s->status = READY_STAT | ERR_STAT;
-    ide_set_inactive(s);
+    dma_buf_commit(s, 0);
+    ide_abort_command(s);
+    ide_set_inactive(s, false);
     ide_set_irq(s->bus);
 }
 
 static int ide_handle_rw_error(IDEState *s, int error, int op)
 {
-    bool is_read = (op & BM_STATUS_RETRY_READ) != 0;
-    BlockErrorAction action = bdrv_get_error_action(s->bs, is_read, error);
+    bool is_read = (op & IDE_RETRY_READ) != 0;
+    BlockErrorAction action = blk_get_error_action(s->blk, is_read, error);
 
     if (action == BLOCK_ERROR_ACTION_STOP) {
         s->bus->dma->ops->set_unit(s->bus->dma, s->unit);
         s->bus->error_status = op;
     } else if (action == BLOCK_ERROR_ACTION_REPORT) {
-        if (op & BM_STATUS_DMA_RETRY) {
-            dma_buf_commit(s);
+        if (op & IDE_RETRY_DMA) {
             ide_dma_error(s);
         } else {
             ide_rw_error(s);
         }
     }
-    bdrv_error_action(s->bs, action, is_read, error);
+    blk_error_action(s->blk, action, is_read, error);
     return action != BLOCK_ERROR_ACTION_IGNORE;
 }
 
@@ -635,13 +686,16 @@ void ide_dma_cb(void *opaque, int ret)
     int64_t sector_num;
     bool stay_active = false;
 
+    if (ret == -ECANCELED) {
+        return;
+    }
     if (ret < 0) {
-        int op = BM_STATUS_DMA_RETRY;
+        int op = IDE_RETRY_DMA;
 
         if (s->dma_cmd == IDE_DMA_READ)
-            op |= BM_STATUS_RETRY_READ;
+            op |= IDE_RETRY_READ;
         else if (s->dma_cmd == IDE_DMA_TRIM)
-            op |= BM_STATUS_RETRY_TRIM;
+            op |= IDE_RETRY_TRIM;
 
         if (ide_handle_rw_error(s, -ret, op)) {
             return;
@@ -659,7 +713,8 @@ void ide_dma_cb(void *opaque, int ret)
 
     sector_num = ide_get_sector(s);
     if (n > 0) {
-        dma_buf_commit(s);
+        assert(s->io_buffer_size == s->sg.size);
+        dma_buf_commit(s, s->io_buffer_size);
         sector_num += n;
         ide_set_sector(s, sector_num);
         s->nsector -= n;
@@ -676,10 +731,11 @@ void ide_dma_cb(void *opaque, int ret)
     n = s->nsector;
     s->io_buffer_index = 0;
     s->io_buffer_size = n * 512;
-    if (s->bus->dma->ops->prepare_buf(s->bus->dma, ide_cmd_is_read(s)) == 0) {
+    if (s->bus->dma->ops->prepare_buf(s->bus->dma, ide_cmd_is_read(s)) < 512) {
         /* The PRDs were too short. Reset the Active bit, but don't raise an
          * interrupt. */
         s->status = READY_STAT | SEEK_STAT;
+        dma_buf_commit(s, 0);
         goto eot;
     }
 
@@ -688,37 +744,34 @@ void ide_dma_cb(void *opaque, int ret)
            sector_num, n, s->dma_cmd);
 #endif
 
-    if (!ide_sect_range_ok(s, sector_num, n)) {
-        dma_buf_commit(s);
+    if ((s->dma_cmd == IDE_DMA_READ || s->dma_cmd == IDE_DMA_WRITE) &&
+        !ide_sect_range_ok(s, sector_num, n)) {
         ide_dma_error(s);
         return;
     }
 
     switch (s->dma_cmd) {
     case IDE_DMA_READ:
-        s->bus->dma->aiocb = dma_bdrv_read(s->bs, &s->sg, sector_num,
-                                           ide_dma_cb, s);
+        s->bus->dma->aiocb = dma_blk_read(s->blk, &s->sg, sector_num,
+                                          ide_dma_cb, s);
         break;
     case IDE_DMA_WRITE:
-        s->bus->dma->aiocb = dma_bdrv_write(s->bs, &s->sg, sector_num,
-                                            ide_dma_cb, s);
+        s->bus->dma->aiocb = dma_blk_write(s->blk, &s->sg, sector_num,
+                                           ide_dma_cb, s);
         break;
     case IDE_DMA_TRIM:
-        s->bus->dma->aiocb = dma_bdrv_io(s->bs, &s->sg, sector_num,
-                                         ide_issue_trim, ide_dma_cb, s,
-                                         DMA_DIRECTION_TO_DEVICE);
+        s->bus->dma->aiocb = dma_blk_io(s->blk, &s->sg, sector_num,
+                                        ide_issue_trim, ide_dma_cb, s,
+                                        DMA_DIRECTION_TO_DEVICE);
         break;
     }
     return;
 
 eot:
     if (s->dma_cmd == IDE_DMA_READ || s->dma_cmd == IDE_DMA_WRITE) {
-        bdrv_acct_done(s->bs, &s->acct);
-    }
-    ide_set_inactive(s);
-    if (stay_active) {
-        s->bus->dma->ops->add_status(s->bus->dma, BM_STATUS_DMAING);
+        block_acct_done(blk_get_stats(s->blk), &s->acct);
     }
+    ide_set_inactive(s, stay_active);
 }
 
 static void ide_sector_start_dma(IDEState *s, enum ide_dma_cmd dma_cmd)
@@ -730,18 +783,25 @@ static void ide_sector_start_dma(IDEState *s, enum ide_dma_cmd dma_cmd)
 
     switch (dma_cmd) {
     case IDE_DMA_READ:
-        bdrv_acct_start(s->bs, &s->acct, s->nsector * BDRV_SECTOR_SIZE,
-                        BDRV_ACCT_READ);
+        block_acct_start(blk_get_stats(s->blk), &s->acct,
+                         s->nsector * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
         break;
     case IDE_DMA_WRITE:
-        bdrv_acct_start(s->bs, &s->acct, s->nsector * BDRV_SECTOR_SIZE,
-                        BDRV_ACCT_WRITE);
+        block_acct_start(blk_get_stats(s->blk), &s->acct,
+                         s->nsector * BDRV_SECTOR_SIZE, BLOCK_ACCT_WRITE);
         break;
     default:
         break;
     }
 
-    s->bus->dma->ops->start_dma(s->bus->dma, s, ide_dma_cb);
+    ide_start_dma(s, ide_dma_cb);
+}
+
+void ide_start_dma(IDEState *s, BlockCompletionFunc *cb)
+{
+    if (s->bus->dma->ops->start_dma) {
+        s->bus->dma->ops->start_dma(s->bus->dma, s, cb);
+    }
 }
 
 static void ide_sector_write_timer_cb(void *opaque)
@@ -755,13 +815,16 @@ static void ide_sector_write_cb(void *opaque, int ret)
     IDEState *s = opaque;
     int n;
 
-    bdrv_acct_done(s->bs, &s->acct);
+    if (ret == -ECANCELED) {
+        return;
+    }
+    block_acct_done(blk_get_stats(s->blk), &s->acct);
 
     s->pio_aiocb = NULL;
     s->status &= ~BUSY_STAT;
 
     if (ret != 0) {
-        if (ide_handle_rw_error(s, -ret, BM_STATUS_PIO_RETRY)) {
+        if (ide_handle_rw_error(s, -ret, IDE_RETRY_PIO)) {
             return;
         }
     }
@@ -771,6 +834,8 @@ static void ide_sector_write_cb(void *opaque, int ret)
         n = s->req_nb_sectors;
     }
     s->nsector -= n;
+    s->io_buffer_offset += 512 * n;
+
     if (s->nsector == 0) {
         /* no more sectors to write */
         ide_transfer_stop(s);
@@ -822,38 +887,46 @@ void ide_sector_write(IDEState *s)
     s->iov.iov_len  = n * BDRV_SECTOR_SIZE;
     qemu_iovec_init_external(&s->qiov, &s->iov, 1);
 
-    bdrv_acct_start(s->bs, &s->acct, n * BDRV_SECTOR_SIZE, BDRV_ACCT_READ);
-    s->pio_aiocb = bdrv_aio_writev(s->bs, sector_num, &s->qiov, n,
-                                   ide_sector_write_cb, s);
+    block_acct_start(blk_get_stats(s->blk), &s->acct,
+                     n * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
+    s->pio_aiocb = blk_aio_writev(s->blk, sector_num, &s->qiov, n,
+                                  ide_sector_write_cb, s);
 }
 
 static void ide_flush_cb(void *opaque, int ret)
 {
     IDEState *s = opaque;
 
+    s->pio_aiocb = NULL;
+
+    if (ret == -ECANCELED) {
+        return;
+    }
     if (ret < 0) {
         /* XXX: What sector number to set here? */
-        if (ide_handle_rw_error(s, -ret, BM_STATUS_RETRY_FLUSH)) {
+        if (ide_handle_rw_error(s, -ret, IDE_RETRY_FLUSH)) {
             return;
         }
     }
 
-    bdrv_acct_done(s->bs, &s->acct);
+    if (s->blk) {
+        block_acct_done(blk_get_stats(s->blk), &s->acct);
+    }
     s->status = READY_STAT | SEEK_STAT;
-    ide_async_cmd_done(s);
+    ide_cmd_done(s);
     ide_set_irq(s->bus);
 }
 
 void ide_flush_cache(IDEState *s)
 {
-    if (s->bs == NULL) {
+    if (s->blk == NULL) {
         ide_flush_cb(s, 0);
         return;
     }
 
     s->status |= BUSY_STAT;
-    bdrv_acct_start(s->bs, &s->acct, 0, BDRV_ACCT_FLUSH);
-    bdrv_aio_flush(s->bs, ide_flush_cb, s);
+    block_acct_start(blk_get_stats(s->blk), &s->acct, 0, BLOCK_ACCT_FLUSH);
+    s->pio_aiocb = blk_aio_flush(s->blk, ide_flush_cb, s);
 }
 
 static void ide_cfata_metadata_inquiry(IDEState *s)
@@ -916,7 +989,7 @@ static void ide_cd_change_cb(void *opaque, bool load)
     uint64_t nb_sectors;
 
     s->tray_open = !load;
-    bdrv_get_geometry(s->bs, &nb_sectors);
+    blk_get_geometry(s->blk, &nb_sectors);
     s->nb_sectors = nb_sectors;
 
     /*
@@ -1050,7 +1123,7 @@ static bool cmd_data_set_management(IDEState *s, uint8_t cmd)
 {
     switch (s->feature) {
     case DSM_TRIM:
-        if (s->bs) {
+        if (s->blk) {
             ide_sector_start_dma(s, IDE_DMA_TRIM);
             return false;
         }
@@ -1063,7 +1136,7 @@ static bool cmd_data_set_management(IDEState *s, uint8_t cmd)
 
 static bool cmd_identify(IDEState *s, uint8_t cmd)
 {
-    if (s->bs && s->drive_kind != IDE_CD) {
+    if (s->blk && s->drive_kind != IDE_CD) {
         if (s->drive_kind != IDE_CFATA) {
             ide_identify(s);
         } else {
@@ -1113,7 +1186,7 @@ static bool cmd_read_multiple(IDEState *s, uint8_t cmd)
 {
     bool lba48 = (cmd == WIN_MULTREAD_EXT);
 
-    if (!s->bs || !s->mult_sectors) {
+    if (!s->blk || !s->mult_sectors) {
         ide_abort_command(s);
         return true;
     }
@@ -1129,7 +1202,7 @@ static bool cmd_write_multiple(IDEState *s, uint8_t cmd)
     bool lba48 = (cmd == WIN_MULTWRITE_EXT);
     int n;
 
-    if (!s->bs || !s->mult_sectors) {
+    if (!s->blk || !s->mult_sectors) {
         ide_abort_command(s);
         return true;
     }
@@ -1157,7 +1230,7 @@ static bool cmd_read_pio(IDEState *s, uint8_t cmd)
         return true;
     }
 
-    if (!s->bs) {
+    if (!s->blk) {
         ide_abort_command(s);
         return true;
     }
@@ -1173,7 +1246,7 @@ static bool cmd_write_pio(IDEState *s, uint8_t cmd)
 {
     bool lba48 = (cmd == WIN_WRITE_EXT);
 
-    if (!s->bs) {
+    if (!s->blk) {
         ide_abort_command(s);
         return true;
     }
@@ -1193,7 +1266,7 @@ static bool cmd_read_dma(IDEState *s, uint8_t cmd)
 {
     bool lba48 = (cmd == WIN_READDMA_EXT);
 
-    if (!s->bs) {
+    if (!s->blk) {
         ide_abort_command(s);
         return true;
     }
@@ -1208,7 +1281,7 @@ static bool cmd_write_dma(IDEState *s, uint8_t cmd)
 {
     bool lba48 = (cmd == WIN_WRITEDMA_EXT);
 
-    if (!s->bs) {
+    if (!s->blk) {
         ide_abort_command(s);
         return true;
     }
@@ -1259,7 +1332,7 @@ static bool cmd_set_features(IDEState *s, uint8_t cmd)
 {
     uint16_t *identify_data;
 
-    if (!s->bs) {
+    if (!s->blk) {
         ide_abort_command(s);
         return true;
     }
@@ -1267,12 +1340,12 @@ static bool cmd_set_features(IDEState *s, uint8_t cmd)
     /* XXX: valid for CDROM ? */
     switch (s->feature) {
     case 0x02: /* write cache enable */
-        bdrv_set_enable_write_cache(s->bs, true);
+        blk_set_enable_write_cache(s->blk, true);
         identify_data = (uint16_t *)s->identify_data;
         put_le16(identify_data + 85, (1 << 14) | (1 << 5) | 1);
         return true;
     case 0x82: /* write cache disable */
-        bdrv_set_enable_write_cache(s->bs, false);
+        blk_set_enable_write_cache(s->blk, false);
         identify_data = (uint16_t *)s->identify_data;
         put_le16(identify_data + 85, (1 << 14) | 1);
         ide_flush_cache(s);
@@ -1739,8 +1812,9 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
 #endif
     s = idebus_active_if(bus);
     /* ignore commands to non existent slave */
-    if (s != bus->ifs && !s->bs)
+    if (s != bus->ifs && !s->blk) {
         return;
+    }
 
     /* Only DEVICE RESET is allowed while BSY or/and DRQ are set */
     if ((s->status & (BUSY_STAT|DRQ_STAT)) && val != WIN_DEVICE_RESET)
@@ -1754,6 +1828,7 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
 
     s->status = READY_STAT | BUSY_STAT;
     s->error = 0;
+    s->io_buffer_offset = 0;
 
     complete = ide_cmd_table[val].handler(s, val);
     if (complete) {
@@ -1764,6 +1839,7 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
             s->status |= SEEK_STAT;
         }
 
+        ide_cmd_done(s);
         ide_set_irq(s->bus);
     }
 }
@@ -1784,59 +1860,66 @@ uint32_t ide_ioport_read(void *opaque, uint32_t addr1)
         ret = 0xff;
         break;
     case 1:
-        if ((!bus->ifs[0].bs && !bus->ifs[1].bs) ||
-            (s != bus->ifs && !s->bs))
+        if ((!bus->ifs[0].blk && !bus->ifs[1].blk) ||
+            (s != bus->ifs && !s->blk)) {
             ret = 0;
-        else if (!hob)
+        } else if (!hob) {
             ret = s->error;
-	else
+        } else {
 	    ret = s->hob_feature;
+        }
         break;
     case 2:
-        if (!bus->ifs[0].bs && !bus->ifs[1].bs)
+        if (!bus->ifs[0].blk && !bus->ifs[1].blk) {
             ret = 0;
-        else if (!hob)
+        } else if (!hob) {
             ret = s->nsector & 0xff;
-	else
+        } else {
 	    ret = s->hob_nsector;
+        }
         break;
     case 3:
-        if (!bus->ifs[0].bs && !bus->ifs[1].bs)
+        if (!bus->ifs[0].blk && !bus->ifs[1].blk) {
             ret = 0;
-        else if (!hob)
+        } else if (!hob) {
             ret = s->sector;
-	else
+        } else {
 	    ret = s->hob_sector;
+        }
         break;
     case 4:
-        if (!bus->ifs[0].bs && !bus->ifs[1].bs)
+        if (!bus->ifs[0].blk && !bus->ifs[1].blk) {
             ret = 0;
-        else if (!hob)
+        } else if (!hob) {
             ret = s->lcyl;
-	else
+        } else {
 	    ret = s->hob_lcyl;
+        }
         break;
     case 5:
-        if (!bus->ifs[0].bs && !bus->ifs[1].bs)
+        if (!bus->ifs[0].blk && !bus->ifs[1].blk) {
             ret = 0;
-        else if (!hob)
+        } else if (!hob) {
             ret = s->hcyl;
-	else
+        } else {
 	    ret = s->hob_hcyl;
+        }
         break;
     case 6:
-        if (!bus->ifs[0].bs && !bus->ifs[1].bs)
+        if (!bus->ifs[0].blk && !bus->ifs[1].blk) {
             ret = 0;
-        else
+        } else {
             ret = s->select;
+        }
         break;
     default:
     case 7:
-        if ((!bus->ifs[0].bs && !bus->ifs[1].bs) ||
-            (s != bus->ifs && !s->bs))
+        if ((!bus->ifs[0].blk && !bus->ifs[1].blk) ||
+            (s != bus->ifs && !s->blk)) {
             ret = 0;
-        else
+        } else {
             ret = s->status;
+        }
         qemu_irq_lower(bus->irq);
         break;
     }
@@ -1852,11 +1935,12 @@ uint32_t ide_status_read(void *opaque, uint32_t addr)
     IDEState *s = idebus_active_if(bus);
     int ret;
 
-    if ((!bus->ifs[0].bs && !bus->ifs[1].bs) ||
-        (s != bus->ifs && !s->bs))
+    if ((!bus->ifs[0].blk && !bus->ifs[1].blk) ||
+        (s != bus->ifs && !s->blk)) {
         ret = 0;
-    else
+    } else {
         ret = s->status;
+    }
 #ifdef DEBUG_IDE
     printf("ide: read status addr=0x%x val=%02x\n", addr, ret);
 #endif
@@ -2017,7 +2101,7 @@ static void ide_reset(IDEState *s)
 #endif
 
     if (s->pio_aiocb) {
-        bdrv_aio_cancel(s->pio_aiocb);
+        blk_aio_cancel(s->pio_aiocb);
         s->pio_aiocb = NULL;
     }
 
@@ -2081,12 +2165,14 @@ void ide_bus_reset(IDEBus *bus)
 #ifdef DEBUG_AIO
         printf("aio_cancel\n");
 #endif
-        bdrv_aio_cancel(bus->dma->aiocb);
+        blk_aio_cancel(bus->dma->aiocb);
         bus->dma->aiocb = NULL;
     }
 
     /* reset dma provider too */
-    bus->dma->ops->reset(bus->dma);
+    if (bus->dma->ops->reset) {
+        bus->dma->ops->reset(bus->dma);
+    }
 }
 
 static bool ide_cd_is_tray_open(void *opaque)
@@ -2099,6 +2185,28 @@ static bool ide_cd_is_medium_locked(void *opaque)
     return ((IDEState *)opaque)->tray_locked;
 }
 
+static void ide_resize_cb(void *opaque)
+{
+    IDEState *s = opaque;
+    uint64_t nb_sectors;
+
+    if (!s->identify_set) {
+        return;
+    }
+
+    blk_get_geometry(s->blk, &nb_sectors);
+    s->nb_sectors = nb_sectors;
+
+    /* Update the identify data buffer. */
+    if (s->drive_kind == IDE_CFATA) {
+        ide_cfata_identify_size(s);
+    } else {
+        /* IDE_CD uses a different set of callbacks entirely. */
+        assert(s->drive_kind != IDE_CD);
+        ide_identify_size(s);
+    }
+}
+
 static const BlockDevOps ide_cd_block_ops = {
     .change_media_cb = ide_cd_change_cb,
     .eject_request_cb = ide_cd_eject_request_cb,
@@ -2106,7 +2214,11 @@ static const BlockDevOps ide_cd_block_ops = {
     .is_medium_locked = ide_cd_is_medium_locked,
 };
 
-int ide_init_drive(IDEState *s, BlockDriverState *bs, IDEDriveKind kind,
+static const BlockDevOps ide_hd_block_ops = {
+    .resize_cb = ide_resize_cb,
+};
+
+int ide_init_drive(IDEState *s, BlockBackend *blk, IDEDriveKind kind,
                    const char *version, const char *serial, const char *model,
                    uint64_t wwn,
                    uint32_t cylinders, uint32_t heads, uint32_t secs,
@@ -2114,10 +2226,10 @@ int ide_init_drive(IDEState *s, BlockDriverState *bs, IDEDriveKind kind,
 {
     uint64_t nb_sectors;
 
-    s->bs = bs;
+    s->blk = blk;
     s->drive_kind = kind;
 
-    bdrv_get_geometry(bs, &nb_sectors);
+    blk_get_geometry(blk, &nb_sectors);
     s->cylinders = cylinders;
     s->heads = heads;
     s->sectors = secs;
@@ -2131,17 +2243,18 @@ int ide_init_drive(IDEState *s, BlockDriverState *bs, IDEDriveKind kind,
     s->smart_errors = 0;
     s->smart_selftest_count = 0;
     if (kind == IDE_CD) {
-        bdrv_set_dev_ops(bs, &ide_cd_block_ops, s);
-        bdrv_set_guest_block_size(bs, 2048);
+        blk_set_dev_ops(blk, &ide_cd_block_ops, s);
+        blk_set_guest_block_size(blk, 2048);
     } else {
-        if (!bdrv_is_inserted(s->bs)) {
+        if (!blk_is_inserted(s->blk)) {
             error_report("Device needs media, but drive is empty");
             return -1;
         }
-        if (bdrv_is_read_only(bs)) {
+        if (blk_is_read_only(blk)) {
             error_report("Can't use a read-only drive");
             return -1;
         }
+        blk_set_dev_ops(blk, &ide_hd_block_ops, s);
     }
     if (serial) {
         pstrcpy(s->drive_serial_str, sizeof(s->drive_serial_str), serial);
@@ -2172,7 +2285,7 @@ int ide_init_drive(IDEState *s, BlockDriverState *bs, IDEDriveKind kind,
     }
 
     ide_reset(s);
-    bdrv_iostatus_enable(bs);
+    blk_iostatus_enable(blk);
     return 0;
 }
 
@@ -2189,24 +2302,19 @@ static void ide_init1(IDEBus *bus, int unit)
     s->io_buffer = qemu_memalign(2048, s->io_buffer_total_len);
     memset(s->io_buffer, 0, s->io_buffer_total_len);
 
-    s->smart_selftest_data = qemu_blockalign(s->bs, 512);
+    s->smart_selftest_data = blk_blockalign(s->blk, 512);
     memset(s->smart_selftest_data, 0, 512);
 
     s->sector_write_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
                                            ide_sector_write_timer_cb, s);
 }
 
-static void ide_nop_start(IDEDMA *dma, IDEState *s,
-                          BlockDriverCompletionFunc *cb)
-{
-}
-
-static int ide_nop(IDEDMA *dma)
+static int ide_nop_int(IDEDMA *dma, int x)
 {
     return 0;
 }
 
-static int ide_nop_int(IDEDMA *dma, int x)
+static int32_t ide_nop_int32(IDEDMA *dma, int x)
 {
     return 0;
 }
@@ -2216,15 +2324,10 @@ static void ide_nop_restart(void *opaque, int x, RunState y)
 }
 
 static const IDEDMAOps ide_dma_nop_ops = {
-    .start_dma      = ide_nop_start,
-    .start_transfer = ide_nop,
-    .prepare_buf    = ide_nop_int,
+    .prepare_buf    = ide_nop_int32,
     .rw_buf         = ide_nop_int,
     .set_unit       = ide_nop_int,
-    .add_status     = ide_nop_int,
-    .set_inactive   = ide_nop,
     .restart_cb     = ide_nop_restart,
-    .reset          = ide_nop,
 };
 
 static IDEDMA ide_dma_nop = {
@@ -2298,8 +2401,8 @@ static int ide_drive_post_load(void *opaque, int version_id)
 {
     IDEState *s = opaque;
 
-    if (s->identify_set) {
-        bdrv_set_enable_write_cache(s->bs, !!(s->identify_data[85] & (1 << 5)));
+    if (s->blk && s->identify_set) {
+        blk_set_enable_write_cache(s->blk, !!(s->identify_data[85] & (1 << 5)));
     }
     return 0;
 }
@@ -2341,7 +2444,7 @@ static bool ide_drive_pio_state_needed(void *opaque)
     IDEState *s = opaque;
 
     return ((s->status & DRQ_STAT) != 0)
-        || (s->bus->error_status & BM_STATUS_PIO_RETRY);
+        || (s->bus->error_status & IDE_RETRY_PIO);
 }
 
 static bool ide_tray_state_needed(void *opaque)
@@ -2480,16 +2583,28 @@ const VMStateDescription vmstate_ide_bus = {
     }
 };
 
-void ide_drive_get(DriveInfo **hd, int max_bus)
+void ide_drive_get(DriveInfo **hd, int n)
 {
     int i;
+    int highest_bus = drive_get_max_bus(IF_IDE) + 1;
+    int max_devs = drive_get_max_devs(IF_IDE);
+    int n_buses = max_devs ? (n / max_devs) : n;
+
+    /*
+     * Note: The number of actual buses available is not known.
+     * We compute this based on the size of the DriveInfo* array, n.
+     * If it is less than max_devs * <num_real_buses>,
+     * We will stop looking for drives prematurely instead of overfilling
+     * the array.
+     */
 
-    if (drive_get_max_bus(IF_IDE) >= max_bus) {
-        fprintf(stderr, "qemu: too many IDE bus: %d\n", max_bus);
+    if (highest_bus > n_buses) {
+        error_report("Too many IDE buses defined (%d > %d)",
+                     highest_bus, n_buses);
         exit(1);
     }
 
-    for(i = 0; i < max_bus * MAX_IDE_DEVS; i++) {
-        hd[i] = drive_get(IF_IDE, i / MAX_IDE_DEVS, i % MAX_IDE_DEVS);
+    for (i = 0; i < n; i++) {
+        hd[i] = drive_get_by_index(IF_IDE, i);
     }
 }
diff --git a/hw/ide/ich.c b/hw/ide/ich.c
index a2f163931..fb1d09550 100644
--- a/hw/ide/ich.c
+++ b/hw/ide/ich.c
@@ -65,12 +65,13 @@
 #include <hw/i386/pc.h>
 #include <hw/pci/pci.h>
 #include <hw/isa/isa.h>
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/dma.h"
 
 #include <hw/ide/pci.h>
 #include <hw/ide/ahci.h>
 
+#define ICH9_MSI_CAP_OFFSET     0x80
 #define ICH9_SATA_CAP_OFFSET    0xA8
 
 #define ICH9_IDP_BAR            4
@@ -115,7 +116,6 @@ static int pci_ich9_ahci_init(PCIDevice *dev)
     /* XXX Software should program this register */
     dev->config[0x90]   = 1 << 6; /* Address Map Register - AHCI mode */
 
-    msi_init(dev, 0x50, 1, true, false);
     d->ahci.irq = pci_allocate_irq(dev);
 
     pci_register_bar(dev, ICH9_IDP_BAR, PCI_BASE_ADDRESS_SPACE_IO,
@@ -135,6 +135,11 @@ static int pci_ich9_ahci_init(PCIDevice *dev)
                  (ICH9_IDP_BAR + 0x4) | (ICH9_IDP_INDEX_LOG2 << 4));
     d->ahci.idp_offset = ICH9_IDP_INDEX;
 
+    /* Although the AHCI 1.3 specification states that the first capability
+     * should be PMCAP, the Intel ICH9 data sheet specifies that the ICH9
+     * AHCI device puts the MSI capability first, pointing to 0x80. */
+    msi_init(dev, ICH9_MSI_CAP_OFFSET, 1, true, false);
+
     return 0;
 }
 
diff --git a/hw/ide/internal.h b/hw/ide/internal.h
index 0567a522f..8a3eca40d 100644
--- a/hw/ide/internal.h
+++ b/hw/ide/internal.h
@@ -319,9 +319,12 @@ typedef enum { IDE_HD, IDE_CD, IDE_CFATA } IDEDriveKind;
 
 typedef void EndTransferFunc(IDEState *);
 
-typedef void DMAStartFunc(IDEDMA *, IDEState *, BlockDriverCompletionFunc *);
-typedef int DMAFunc(IDEDMA *);
+typedef void DMAStartFunc(IDEDMA *, IDEState *, BlockCompletionFunc *);
+typedef void DMAVoidFunc(IDEDMA *);
 typedef int DMAIntFunc(IDEDMA *, int);
+typedef int32_t DMAInt32Func(IDEDMA *, int);
+typedef void DMAu32Func(IDEDMA *, uint32_t);
+typedef void DMAStopFunc(IDEDMA *, bool);
 typedef void DMARestartFunc(void *, int, RunState);
 
 struct unreported_events {
@@ -372,7 +375,7 @@ struct IDEState {
 
     /* set for lba48 access */
     uint8_t lba48;
-    BlockDriverState *bs;
+    BlockBackend *blk;
     char version[9];
     /* ATAPI specific */
     struct unreported_events events;
@@ -383,17 +386,17 @@ struct IDEState {
     uint8_t cdrom_changed;
     int packet_transfer_size;
     int elementary_transfer_size;
-    int io_buffer_index;
+    int32_t io_buffer_index;
     int lba;
     int cd_sector_size;
     int atapi_dma; /* true if dma is requested for the packet cmd */
     BlockAcctCookie acct;
-    BlockDriverAIOCB *pio_aiocb;
+    BlockAIOCB *pio_aiocb;
     struct iovec iov;
     QEMUIOVector qiov;
     /* ATA DMA state */
-    int io_buffer_offset;
-    int io_buffer_size;
+    int32_t io_buffer_offset;
+    int32_t io_buffer_size;
     QEMUSGList sg;
     /* PIO transfer handling */
     int req_nb_sectors; /* number of sectors per interrupt */
@@ -403,8 +406,8 @@ struct IDEState {
     uint8_t *io_buffer;
     /* PIO save/restore */
     int32_t io_buffer_total_len;
-    int cur_io_buffer_offset;
-    int cur_io_buffer_len;
+    int32_t cur_io_buffer_offset;
+    int32_t cur_io_buffer_len;
     uint8_t end_transfer_fn_idx;
     QEMUTimer *sector_write_timer; /* only used for win2k install hack */
     uint32_t irq_count; /* counts IRQs when using win2k install hack */
@@ -427,22 +430,22 @@ struct IDEState {
 
 struct IDEDMAOps {
     DMAStartFunc *start_dma;
-    DMAFunc *start_transfer;
-    DMAIntFunc *prepare_buf;
+    DMAVoidFunc *start_transfer;
+    DMAInt32Func *prepare_buf;
+    DMAu32Func *commit_buf;
     DMAIntFunc *rw_buf;
     DMAIntFunc *set_unit;
-    DMAIntFunc *add_status;
-    DMAFunc *set_inactive;
-    DMAFunc *async_cmd_done;
+    DMAStopFunc *set_inactive;
+    DMAVoidFunc *cmd_done;
     DMARestartFunc *restart_cb;
-    DMAFunc *reset;
+    DMAVoidFunc *reset;
 };
 
 struct IDEDMA {
     const struct IDEDMAOps *ops;
     struct iovec iov;
     QEMUIOVector qiov;
-    BlockDriverAIOCB *aiocb;
+    BlockAIOCB *aiocb;
 };
 
 struct IDEBus {
@@ -484,23 +487,12 @@ struct IDEDevice {
     uint64_t wwn;
 };
 
-#define BM_STATUS_DMAING 0x01
-#define BM_STATUS_ERROR  0x02
-#define BM_STATUS_INT    0x04
-
-/* FIXME These are not status register bits */
-#define BM_STATUS_DMA_RETRY  0x08
-#define BM_STATUS_PIO_RETRY  0x10
-#define BM_STATUS_RETRY_READ  0x20
-#define BM_STATUS_RETRY_FLUSH 0x40
-#define BM_STATUS_RETRY_TRIM 0x80
-
-#define BM_MIGRATION_COMPAT_STATUS_BITS \
-        (BM_STATUS_DMA_RETRY | BM_STATUS_PIO_RETRY | \
-        BM_STATUS_RETRY_READ | BM_STATUS_RETRY_FLUSH)
-
-#define BM_CMD_START     0x01
-#define BM_CMD_READ      0x08
+/* These are used for the error_status field of IDEBus */
+#define IDE_RETRY_DMA  0x08
+#define IDE_RETRY_PIO  0x10
+#define IDE_RETRY_READ  0x20
+#define IDE_RETRY_FLUSH 0x40
+#define IDE_RETRY_TRIM 0x80
 
 static inline IDEState *idebus_active_if(IDEBus *bus)
 {
@@ -532,6 +524,7 @@ void ide_bus_reset(IDEBus *bus);
 int64_t ide_get_sector(IDEState *s);
 void ide_set_sector(IDEState *s, int64_t sector_num);
 
+void ide_start_dma(IDEState *s, BlockCompletionFunc *cb);
 void ide_dma_error(IDEState *s);
 
 void ide_atapi_cmd_ok(IDEState *s);
@@ -547,7 +540,7 @@ uint32_t ide_data_readw(void *opaque, uint32_t addr);
 void ide_data_writel(void *opaque, uint32_t addr, uint32_t val);
 uint32_t ide_data_readl(void *opaque, uint32_t addr);
 
-int ide_init_drive(IDEState *s, BlockDriverState *bs, IDEDriveKind kind,
+int ide_init_drive(IDEState *s, BlockBackend *blk, IDEDriveKind kind,
                    const char *version, const char *serial, const char *model,
                    uint64_t wwn,
                    uint32_t cylinders, uint32_t heads, uint32_t secs,
@@ -564,10 +557,10 @@ void ide_flush_cache(IDEState *s);
 void ide_transfer_start(IDEState *s, uint8_t *buf, int size,
                         EndTransferFunc *end_transfer_func);
 void ide_transfer_stop(IDEState *s);
-void ide_set_inactive(IDEState *s);
-BlockDriverAIOCB *ide_issue_trim(BlockDriverState *bs,
+void ide_set_inactive(IDEState *s, bool more);
+BlockAIOCB *ide_issue_trim(BlockBackend *blk,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque);
+        BlockCompletionFunc *cb, void *opaque);
 
 /* hw/ide/atapi.c */
 void ide_atapi_cmd(IDEState *s);
diff --git a/hw/ide/isa.c b/hw/ide/isa.c
index 4cbcb1cdf..b084162dd 100644
--- a/hw/ide/isa.c
+++ b/hw/ide/isa.c
@@ -25,7 +25,7 @@
 #include <hw/hw.h>
 #include <hw/i386/pc.h>
 #include <hw/isa/isa.h>
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/dma.h"
 
 #include <hw/ide/internal.h>
diff --git a/hw/ide/macio.c b/hw/ide/macio.c
index c14a1dddd..f6074f202 100644
--- a/hw/ide/macio.c
+++ b/hw/ide/macio.c
@@ -25,7 +25,7 @@
 #include "hw/hw.h"
 #include "hw/ppc/mac.h"
 #include "hw/ppc/mac_dbdma.h"
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/dma.h"
 
 #include <hw/ide/internal.h>
@@ -134,7 +134,7 @@ static void pmac_ide_atapi_transfer_cb(void *opaque, int ret)
         MACIO_DPRINTF("precopying unaligned %d bytes to %#" HWADDR_PRIx "\n",
                       unaligned, io->addr + io->len - unaligned);
 
-        bdrv_read(s->bs, sector_num + nsector, io->remainder, 1);
+        blk_read(s->blk, sector_num + nsector, io->remainder, 1);
         cpu_physical_memory_write(io->addr + io->len - unaligned,
                                   io->remainder, unaligned);
 
@@ -164,14 +164,14 @@ static void pmac_ide_atapi_transfer_cb(void *opaque, int ret)
                   (s->lba << 2) + (s->io_buffer_index >> 9),
                   s->packet_transfer_size, s->dma_cmd);
 
-    m->aiocb = dma_bdrv_read(s->bs, &s->sg,
-                             (int64_t)(s->lba << 2) + (s->io_buffer_index >> 9),
-                             pmac_ide_atapi_transfer_cb, io);
+    m->aiocb = dma_blk_read(s->blk, &s->sg,
+                            (int64_t)(s->lba << 2) + (s->io_buffer_index >> 9),
+                            pmac_ide_atapi_transfer_cb, io);
     return;
 
 done:
     MACIO_DPRINTF("done DMA\n");
-    bdrv_acct_done(s->bs, &s->acct);
+    block_acct_done(blk_get_stats(s->blk), &s->acct);
     io->dma_end(opaque);
 }
 
@@ -254,8 +254,8 @@ static void pmac_ide_transfer_cb(void *opaque, int ret)
             qemu_iovec_reset(&io->iov);
             qemu_iovec_add(&io->iov, io->remainder, 0x200);
 
-            m->aiocb = bdrv_aio_writev(s->bs, sector_num - 1, &io->iov, 1,
-                                       pmac_ide_transfer_cb, io);
+            m->aiocb = blk_aio_writev(s->blk, sector_num - 1, &io->iov, 1,
+                                      pmac_ide_transfer_cb, io);
         }
     }
 
@@ -294,8 +294,8 @@ static void pmac_ide_transfer_cb(void *opaque, int ret)
             qemu_iovec_reset(&io->iov);
             qemu_iovec_add(&io->iov, io->remainder, 0x200);
 
-            m->aiocb = bdrv_aio_readv(s->bs, sector_num + nsector, &io->iov, 1,
-                                      pmac_ide_transfer_cb, io);
+            m->aiocb = blk_aio_readv(s->blk, sector_num + nsector, &io->iov, 1,
+                                     pmac_ide_transfer_cb, io);
             break;
         case IDE_DMA_WRITE:
             /* cache the contents in our io struct */
@@ -333,17 +333,17 @@ static void pmac_ide_transfer_cb(void *opaque, int ret)
 
     switch (s->dma_cmd) {
     case IDE_DMA_READ:
-        m->aiocb = dma_bdrv_read(s->bs, &s->sg, sector_num,
-                                 pmac_ide_transfer_cb, io);
+        m->aiocb = dma_blk_read(s->blk, &s->sg, sector_num,
+                                pmac_ide_transfer_cb, io);
         break;
     case IDE_DMA_WRITE:
-        m->aiocb = dma_bdrv_write(s->bs, &s->sg, sector_num,
-                                  pmac_ide_transfer_cb, io);
+        m->aiocb = dma_blk_write(s->blk, &s->sg, sector_num,
+                                 pmac_ide_transfer_cb, io);
         break;
     case IDE_DMA_TRIM:
-        m->aiocb = dma_bdrv_io(s->bs, &s->sg, sector_num,
-                               ide_issue_trim, pmac_ide_transfer_cb, io,
-                               DMA_DIRECTION_TO_DEVICE);
+        m->aiocb = dma_blk_io(s->blk, &s->sg, sector_num,
+                              ide_issue_trim, pmac_ide_transfer_cb, io,
+                              DMA_DIRECTION_TO_DEVICE);
         break;
     }
 
@@ -352,7 +352,7 @@ static void pmac_ide_transfer_cb(void *opaque, int ret)
 
 done:
     if (s->dma_cmd == IDE_DMA_READ || s->dma_cmd == IDE_DMA_WRITE) {
-        bdrv_acct_done(s->bs, &s->acct);
+        block_acct_done(blk_get_stats(s->blk), &s->acct);
     }
     io->dma_end(io);
 }
@@ -370,8 +370,8 @@ static void pmac_ide_transfer(DBDMA_io *io)
         /* Handle non-block ATAPI DMA transfers */
         if (s->lba == -1) {
             s->io_buffer_size = MIN(io->len, s->packet_transfer_size);
-            bdrv_acct_start(s->bs, &s->acct, s->io_buffer_size,
-                            BDRV_ACCT_READ);
+            block_acct_start(blk_get_stats(s->blk), &s->acct, s->io_buffer_size,
+                             BLOCK_ACCT_READ);
             MACIO_DPRINTF("non-block ATAPI DMA transfer size: %d\n",
                           s->io_buffer_size);
 
@@ -382,22 +382,25 @@ static void pmac_ide_transfer(DBDMA_io *io)
             m->dma_active = false;
 
             MACIO_DPRINTF("end of non-block ATAPI DMA transfer\n");
-            bdrv_acct_done(s->bs, &s->acct);
+            block_acct_done(blk_get_stats(s->blk), &s->acct);
             io->dma_end(io);
             return;
         }
 
-        bdrv_acct_start(s->bs, &s->acct, io->len, BDRV_ACCT_READ);
+        block_acct_start(blk_get_stats(s->blk), &s->acct, io->len,
+                         BLOCK_ACCT_READ);
         pmac_ide_atapi_transfer_cb(io, 0);
         return;
     }
 
     switch (s->dma_cmd) {
     case IDE_DMA_READ:
-        bdrv_acct_start(s->bs, &s->acct, io->len, BDRV_ACCT_READ);
+        block_acct_start(blk_get_stats(s->blk), &s->acct, io->len,
+                         BLOCK_ACCT_READ);
         break;
     case IDE_DMA_WRITE:
-        bdrv_acct_start(s->bs, &s->acct, io->len, BDRV_ACCT_WRITE);
+        block_acct_start(blk_get_stats(s->blk), &s->acct, io->len,
+                         BLOCK_ACCT_WRITE);
         break;
     default:
         break;
@@ -412,7 +415,7 @@ static void pmac_ide_flush(DBDMA_io *io)
     MACIOIDEState *m = io->opaque;
 
     if (m->aiocb) {
-        bdrv_drain_all();
+        blk_drain_all();
     }
 }
 
@@ -545,12 +548,12 @@ static void macio_ide_reset(DeviceState *dev)
     ide_bus_reset(&d->bus);
 }
 
-static int ide_nop(IDEDMA *dma)
+static int ide_nop_int(IDEDMA *dma, int x)
 {
     return 0;
 }
 
-static int ide_nop_int(IDEDMA *dma, int x)
+static int32_t ide_nop_int32(IDEDMA *dma, int x)
 {
     return 0;
 }
@@ -560,7 +563,7 @@ static void ide_nop_restart(void *opaque, int x, RunState y)
 }
 
 static void ide_dbdma_start(IDEDMA *dma, IDEState *s,
-                            BlockDriverCompletionFunc *cb)
+                            BlockCompletionFunc *cb)
 {
     MACIOIDEState *m = container_of(dma, MACIOIDEState, dma);
 
@@ -571,14 +574,10 @@ static void ide_dbdma_start(IDEDMA *dma, IDEState *s,
 
 static const IDEDMAOps dbdma_ops = {
     .start_dma      = ide_dbdma_start,
-    .start_transfer = ide_nop,
-    .prepare_buf    = ide_nop_int,
+    .prepare_buf    = ide_nop_int32,
     .rw_buf         = ide_nop_int,
     .set_unit       = ide_nop_int,
-    .add_status     = ide_nop_int,
-    .set_inactive   = ide_nop,
     .restart_cb     = ide_nop_restart,
-    .reset          = ide_nop,
 };
 
 static void macio_ide_realizefn(DeviceState *dev, Error **errp)
diff --git a/hw/ide/microdrive.c b/hw/ide/microdrive.c
index 2d70ddb75..6639dd488 100644
--- a/hw/ide/microdrive.c
+++ b/hw/ide/microdrive.c
@@ -25,7 +25,7 @@
 #include <hw/hw.h>
 #include <hw/i386/pc.h>
 #include <hw/pcmcia.h>
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/dma.h"
 
 #include <hw/ide/internal.h>
@@ -247,7 +247,7 @@ static uint16_t md_common_read(PCMCIACardState *card, uint32_t at)
         return ide_ioport_read(&s->bus, 0x1);
     case 0xe:	/* Alternate Status */
         ifs = idebus_active_if(&s->bus);
-        if (ifs->bs) {
+        if (ifs->blk) {
             return ifs->status;
         } else {
             return 0;
@@ -543,7 +543,6 @@ static int dscm1xxxx_attach(PCMCIACardState *card)
     device_reset(DEVICE(md));
     md_interrupt_update(md);
 
-    card->slot->card_string = "DSCM-1xxxx Hitachi Microdrive";
     return 0;
 }
 
@@ -567,7 +566,7 @@ PCMCIACardState *dscm1xxxx_init(DriveInfo *dinfo)
     }
     md->bus.ifs[0].drive_kind = IDE_CFATA;
     md->bus.ifs[0].mdata_size = METADATA_SIZE;
-    md->bus.ifs[0].mdata_storage = (uint8_t *) g_malloc0(METADATA_SIZE);
+    md->bus.ifs[0].mdata_storage = g_malloc0(METADATA_SIZE);
 
     return PCMCIA_CARD(md);
 }
diff --git a/hw/ide/mmio.c b/hw/ide/mmio.c
index 01c1d0e6c..b6ce62ac5 100644
--- a/hw/ide/mmio.c
+++ b/hw/ide/mmio.c
@@ -24,7 +24,7 @@
  */
 #include "hw/hw.h"
 #include "hw/sysbus.h"
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/dma.h"
 
 #include <hw/ide/internal.h>
@@ -82,7 +82,7 @@ static void mmio_ide_write(void *opaque, hwaddr addr,
 static const MemoryRegionOps mmio_ide_ops = {
     .read = mmio_ide_read,
     .write = mmio_ide_write,
-    .endianness = DEVICE_NATIVE_ENDIAN,
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static uint64_t mmio_ide_status_read(void *opaque, hwaddr addr,
@@ -102,7 +102,7 @@ static void mmio_ide_cmd_write(void *opaque, hwaddr addr,
 static const MemoryRegionOps mmio_ide_cs_ops = {
     .read = mmio_ide_status_read,
     .write = mmio_ide_cmd_write,
-    .endianness = DEVICE_NATIVE_ENDIAN,
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static const VMStateDescription vmstate_ide_mmio = {
diff --git a/hw/ide/pci.c b/hw/ide/pci.c
index 6257a21ed..bee5ad39f 100644
--- a/hw/ide/pci.c
+++ b/hw/ide/pci.c
@@ -26,15 +26,19 @@
 #include <hw/i386/pc.h>
 #include <hw/pci/pci.h>
 #include <hw/isa/isa.h>
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/dma.h"
-
+#include "qemu/error-report.h"
 #include <hw/ide/pci.h>
 
 #define BMDMA_PAGE_SIZE 4096
 
+#define BM_MIGRATION_COMPAT_STATUS_BITS \
+        (IDE_RETRY_DMA | IDE_RETRY_PIO | \
+        IDE_RETRY_READ | IDE_RETRY_FLUSH)
+
 static void bmdma_start_dma(IDEDMA *dma, IDEState *s,
-                            BlockDriverCompletionFunc *dma_cb)
+                            BlockCompletionFunc *dma_cb)
 {
     BMDMAState *bm = DO_UPCAST(BMDMAState, dma, dma);
 
@@ -51,8 +55,11 @@ static void bmdma_start_dma(IDEDMA *dma, IDEState *s,
     }
 }
 
-/* return 0 if buffer completed */
-static int bmdma_prepare_buf(IDEDMA *dma, int is_write)
+/**
+ * Return the number of bytes successfully prepared.
+ * -1 on error.
+ */
+static int32_t bmdma_prepare_buf(IDEDMA *dma, int is_write)
 {
     BMDMAState *bm = DO_UPCAST(BMDMAState, dma, dma);
     IDEState *s = bmdma_active_if(bm);
@@ -70,8 +77,9 @@ static int bmdma_prepare_buf(IDEDMA *dma, int is_write)
         if (bm->cur_prd_len == 0) {
             /* end of table (with a fail safe of one page) */
             if (bm->cur_prd_last ||
-                (bm->cur_addr - bm->addr) >= BMDMA_PAGE_SIZE)
-                return s->io_buffer_size != 0;
+                (bm->cur_addr - bm->addr) >= BMDMA_PAGE_SIZE) {
+                return s->io_buffer_size;
+            }
             pci_dma_read(pci_dev, bm->cur_addr, &prd, 8);
             bm->cur_addr += 8;
             prd.addr = le32_to_cpu(prd.addr);
@@ -86,12 +94,23 @@ static int bmdma_prepare_buf(IDEDMA *dma, int is_write)
         l = bm->cur_prd_len;
         if (l > 0) {
             qemu_sglist_add(&s->sg, bm->cur_prd_addr, l);
+
+            /* Note: We limit the max transfer to be 2GiB.
+             * This should accommodate the largest ATA transaction
+             * for LBA48 (65,536 sectors) and 32K sector sizes. */
+            if (s->sg.size > INT32_MAX) {
+                error_report("IDE: sglist describes more than 2GiB.\n");
+                break;
+            }
             bm->cur_prd_addr += l;
             bm->cur_prd_len -= l;
             s->io_buffer_size += l;
         }
     }
-    return 1;
+
+    qemu_sglist_destroy(&s->sg);
+    s->io_buffer_size = 0;
+    return -1;
 }
 
 /* return 0 if buffer completed */
@@ -152,23 +171,17 @@ static int bmdma_set_unit(IDEDMA *dma, int unit)
     return 0;
 }
 
-static int bmdma_add_status(IDEDMA *dma, int status)
-{
-    BMDMAState *bm = DO_UPCAST(BMDMAState, dma, dma);
-    bm->status |= status;
-
-    return 0;
-}
-
-static int bmdma_set_inactive(IDEDMA *dma)
+static void bmdma_set_inactive(IDEDMA *dma, bool more)
 {
     BMDMAState *bm = DO_UPCAST(BMDMAState, dma, dma);
 
-    bm->status &= ~BM_STATUS_DMAING;
     bm->dma_cb = NULL;
     bm->unit = -1;
-
-    return 0;
+    if (more) {
+        bm->status |= BM_STATUS_DMAING;
+    } else {
+        bm->status &= ~BM_STATUS_DMAING;
+    }
 }
 
 static void bmdma_restart_dma(BMDMAState *bm, enum ide_dma_cmd dma_cmd)
@@ -200,7 +213,7 @@ static void bmdma_restart_bh(void *opaque)
         return;
     }
 
-    is_read = (bus->error_status & BM_STATUS_RETRY_READ) != 0;
+    is_read = (bus->error_status & IDE_RETRY_READ) != 0;
 
     /* The error status must be cleared before resubmitting the request: The
      * request may fail again, and this case can only be distinguished if the
@@ -208,19 +221,19 @@ static void bmdma_restart_bh(void *opaque)
     error_status = bus->error_status;
     bus->error_status = 0;
 
-    if (error_status & BM_STATUS_DMA_RETRY) {
-        if (error_status & BM_STATUS_RETRY_TRIM) {
+    if (error_status & IDE_RETRY_DMA) {
+        if (error_status & IDE_RETRY_TRIM) {
             bmdma_restart_dma(bm, IDE_DMA_TRIM);
         } else {
             bmdma_restart_dma(bm, is_read ? IDE_DMA_READ : IDE_DMA_WRITE);
         }
-    } else if (error_status & BM_STATUS_PIO_RETRY) {
+    } else if (error_status & IDE_RETRY_PIO) {
         if (is_read) {
             ide_sector_read(bmdma_active_if(bm));
         } else {
             ide_sector_write(bmdma_active_if(bm));
         }
-    } else if (error_status & BM_STATUS_RETRY_FLUSH) {
+    } else if (error_status & IDE_RETRY_FLUSH) {
         ide_flush_cache(bmdma_active_if(bm));
     }
 }
@@ -243,11 +256,11 @@ static void bmdma_cancel(BMDMAState *bm)
 {
     if (bm->status & BM_STATUS_DMAING) {
         /* cancel DMA request */
-        bmdma_set_inactive(&bm->dma);
+        bmdma_set_inactive(&bm->dma, false);
     }
 }
 
-static int bmdma_reset(IDEDMA *dma)
+static void bmdma_reset(IDEDMA *dma)
 {
     BMDMAState *bm = DO_UPCAST(BMDMAState, dma, dma);
 
@@ -264,13 +277,6 @@ static int bmdma_reset(IDEDMA *dma)
     bm->cur_prd_len = 0;
     bm->sector_num = 0;
     bm->nsector = 0;
-
-    return 0;
-}
-
-static int bmdma_start_transfer(IDEDMA *dma)
-{
-    return 0;
 }
 
 static void bmdma_irq(void *opaque, int n, int level)
@@ -311,7 +317,7 @@ void bmdma_cmd_writeb(BMDMAState *bm, uint32_t val)
              * aio operation with preadv/pwritev.
              */
             if (bm->bus->dma->aiocb) {
-                bdrv_drain_all();
+                blk_drain_all();
                 assert(bm->bus->dma->aiocb == NULL);
             }
             bm->status &= ~BM_STATUS_DMAING;
@@ -504,11 +510,9 @@ void pci_ide_create_devs(PCIDevice *dev, DriveInfo **hd_table)
 
 static const struct IDEDMAOps bmdma_ops = {
     .start_dma = bmdma_start_dma,
-    .start_transfer = bmdma_start_transfer,
     .prepare_buf = bmdma_prepare_buf,
     .rw_buf = bmdma_rw_buf,
     .set_unit = bmdma_set_unit,
-    .add_status = bmdma_add_status,
     .set_inactive = bmdma_set_inactive,
     .restart_cb = bmdma_restart_cb,
     .reset = bmdma_reset,
diff --git a/hw/ide/pci.h b/hw/ide/pci.h
index 2428275c8..2e9314ad8 100644
--- a/hw/ide/pci.h
+++ b/hw/ide/pci.h
@@ -3,6 +3,13 @@
 
 #include <hw/ide/internal.h>
 
+#define BM_STATUS_DMAING 0x01
+#define BM_STATUS_ERROR  0x02
+#define BM_STATUS_INT    0x04
+
+#define BM_CMD_START     0x01
+#define BM_CMD_READ      0x08
+
 typedef struct BMDMAState {
     IDEDMA dma;
     uint8_t cmd;
@@ -16,7 +23,7 @@ typedef struct BMDMAState {
     uint32_t cur_prd_addr;
     uint32_t cur_prd_len;
     uint8_t unit;
-    BlockDriverCompletionFunc *dma_cb;
+    BlockCompletionFunc *dma_cb;
     int64_t sector_num;
     uint32_t nsector;
     MemoryRegion addr_ioport;
diff --git a/hw/ide/piix.c b/hw/ide/piix.c
index 8651726f5..b0172fbf5 100644
--- a/hw/ide/piix.c
+++ b/hw/ide/piix.c
@@ -27,7 +27,7 @@
 #include <hw/i386/pc.h>
 #include <hw/pci/pci.h>
 #include <hw/isa/isa.h>
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/dma.h"
 
@@ -171,20 +171,20 @@ int pci_piix3_xen_ide_unplug(DeviceState *dev)
 {
     PCIIDEState *pci_ide;
     DriveInfo *di;
-    int i = 0;
+    int i;
 
     pci_ide = PCI_IDE(dev);
 
-    for (; i < 3; i++) {
+    for (i = 0; i < 4; i++) {
         di = drive_get_by_index(IF_IDE, i);
         if (di != NULL && !di->media_cd) {
-            DeviceState *ds = bdrv_get_attached_dev(di->bdrv);
+            BlockBackend *blk = blk_by_legacy_dinfo(di);
+            DeviceState *ds = blk_get_attached_dev(blk);
             if (ds) {
-                bdrv_detach_dev(di->bdrv, ds);
+                blk_detach_dev(blk, ds);
             }
-            bdrv_close(di->bdrv);
-            pci_ide->bus[di->bus].ifs[di->unit].bs = NULL;
-            drive_del(di);
+            pci_ide->bus[di->bus].ifs[di->unit].blk = NULL;
+            blk_unref(blk);
         }
     }
     qdev_reset_all(DEVICE(dev));
@@ -207,11 +207,8 @@ static void pci_piix_ide_exitfn(PCIDevice *dev)
 
     for (i = 0; i < 2; ++i) {
         memory_region_del_subregion(&d->bmdma_bar, &d->bmdma[i].extra_io);
-        memory_region_destroy(&d->bmdma[i].extra_io);
         memory_region_del_subregion(&d->bmdma_bar, &d->bmdma[i].addr_ioport);
-        memory_region_destroy(&d->bmdma[i].addr_ioport);
     }
-    memory_region_destroy(&d->bmdma_bar);
 }
 
 /* hd_table must contain 4 block drivers */
diff --git a/hw/ide/qdev.c b/hw/ide/qdev.c
index 6e475e697..b4f096e12 100644
--- a/hw/ide/qdev.c
+++ b/hw/ide/qdev.c
@@ -20,9 +20,11 @@
 #include "sysemu/dma.h"
 #include "qemu/error-report.h"
 #include <hw/ide/internal.h>
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "hw/block/block.h"
 #include "sysemu/sysemu.h"
+#include "qapi/visitor.h"
 
 /* --------------------------------- */
 
@@ -59,7 +61,7 @@ static char *idebus_get_fw_dev_path(DeviceState *dev)
 {
     char path[30];
 
-    snprintf(path, sizeof(path), "%s@%d", qdev_fw_name(dev),
+    snprintf(path, sizeof(path), "%s@%x", qdev_fw_name(dev),
              ((IDEBus*)dev->parent_bus)->bus_id);
 
     return g_strdup(path);
@@ -71,7 +73,7 @@ static int ide_qdev_init(DeviceState *qdev)
     IDEDeviceClass *dc = IDE_DEVICE_GET_CLASS(dev);
     IDEBus *bus = DO_UPCAST(IDEBus, qbus, qdev->parent_bus);
 
-    if (!dev->conf.bs) {
+    if (!dev->conf.blk) {
         error_report("No drive specified");
         goto err;
     }
@@ -116,7 +118,7 @@ IDEDevice *ide_create_drive(IDEBus *bus, int unit, DriveInfo *drive)
 
     dev = qdev_create(&bus->qbus, drive->media_cd ? "ide-cd" : "ide-hd");
     qdev_prop_set_uint32(dev, "unit", unit);
-    qdev_prop_set_drive_nofail(dev, "drive", drive->bdrv);
+    qdev_prop_set_drive_nofail(dev, "drive", blk_by_legacy_dinfo(drive));
     qdev_init_nofail(dev);
     return DO_UPCAST(IDEDevice, qdev, dev);
 }
@@ -126,7 +128,7 @@ int ide_get_geometry(BusState *bus, int unit,
 {
     IDEState *s = &DO_UPCAST(IDEBus, qbus, bus)->ifs[unit];
 
-    if (s->drive_kind != IDE_HD || !s->bs) {
+    if (s->drive_kind != IDE_HD || !s->blk) {
         return -1;
     }
 
@@ -151,6 +153,7 @@ static int ide_dev_initfn(IDEDevice *dev, IDEDriveKind kind)
 {
     IDEBus *bus = DO_UPCAST(IDEBus, qbus, dev->qdev.parent_bus);
     IDEState *s = bus->ifs + dev->unit;
+    Error *err = NULL;
 
     if (dev->conf.discard_granularity == -1) {
         dev->conf.discard_granularity = 512;
@@ -161,12 +164,16 @@ static int ide_dev_initfn(IDEDevice *dev, IDEDriveKind kind)
     }
 
     blkconf_serial(&dev->conf, &dev->serial);
-    if (kind != IDE_CD
-        && blkconf_geometry(&dev->conf, &dev->chs_trans, 65536, 16, 255) < 0) {
-        return -1;
+    if (kind != IDE_CD) {
+        blkconf_geometry(&dev->conf, &dev->chs_trans, 65536, 16, 255, &err);
+        if (err) {
+            error_report("%s", error_get_pretty(err));
+            error_free(err);
+            return -1;
+        }
     }
 
-    if (ide_init_drive(s, dev->conf.bs, kind,
+    if (ide_init_drive(s, dev->conf.blk, kind,
                        dev->version, dev->serial, dev->model, dev->wwn,
                        dev->conf.cyls, dev->conf.heads, dev->conf.secs,
                        dev->chs_trans) < 0) {
@@ -186,6 +193,51 @@ static int ide_dev_initfn(IDEDevice *dev, IDEDriveKind kind)
     return 0;
 }
 
+static void ide_dev_get_bootindex(Object *obj, Visitor *v, void *opaque,
+                                  const char *name, Error **errp)
+{
+    IDEDevice *d = IDE_DEVICE(obj);
+
+    visit_type_int32(v, &d->conf.bootindex, name, errp);
+}
+
+static void ide_dev_set_bootindex(Object *obj, Visitor *v, void *opaque,
+                                  const char *name, Error **errp)
+{
+    IDEDevice *d = IDE_DEVICE(obj);
+    int32_t boot_index;
+    Error *local_err = NULL;
+
+    visit_type_int32(v, &boot_index, name, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    /* check whether bootindex is present in fw_boot_order list  */
+    check_boot_index(boot_index, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    /* change bootindex to a new one */
+    d->conf.bootindex = boot_index;
+
+    if (d->unit != -1) {
+        add_boot_device_path(d->conf.bootindex, &d->qdev,
+                             d->unit ? "/disk@1" : "/disk@0");
+    }
+out:
+    if (local_err) {
+        error_propagate(errp, local_err);
+    }
+}
+
+static void ide_dev_instance_init(Object *obj)
+{
+    object_property_add(obj, "bootindex", "int32",
+                        ide_dev_get_bootindex,
+                        ide_dev_set_bootindex, NULL, NULL, NULL);
+    object_property_set_int(obj, -1, "bootindex", NULL);
+}
+
 static int ide_hd_initfn(IDEDevice *dev)
 {
     return ide_dev_initfn(dev, IDE_HD);
@@ -198,9 +250,9 @@ static int ide_cd_initfn(IDEDevice *dev)
 
 static int ide_drive_initfn(IDEDevice *dev)
 {
-    DriveInfo *dinfo = drive_get_by_blockdev(dev->conf.bs);
+    DriveInfo *dinfo = blk_legacy_dinfo(dev->conf.blk);
 
-    return ide_dev_initfn(dev, dinfo->media_cd ? IDE_CD : IDE_HD);
+    return ide_dev_initfn(dev, dinfo && dinfo->media_cd ? IDE_CD : IDE_HD);
 }
 
 #define DEFINE_IDE_DEV_PROPERTIES()                     \
@@ -295,6 +347,7 @@ static const TypeInfo ide_device_type_info = {
     .abstract = true,
     .class_size = sizeof(IDEDeviceClass),
     .class_init = ide_device_class_init,
+    .instance_init = ide_dev_instance_init,
 };
 
 static void ide_register_types(void)
diff --git a/hw/ide/via.c b/hw/ide/via.c
index 198123b02..4d8089de7 100644
--- a/hw/ide/via.c
+++ b/hw/ide/via.c
@@ -27,7 +27,7 @@
 #include <hw/i386/pc.h>
 #include <hw/pci/pci.h>
 #include <hw/isa/isa.h>
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/dma.h"
 
@@ -198,11 +198,8 @@ static void vt82c686b_ide_exitfn(PCIDevice *dev)
 
     for (i = 0; i < 2; ++i) {
         memory_region_del_subregion(&d->bmdma_bar, &d->bmdma[i].extra_io);
-        memory_region_destroy(&d->bmdma[i].extra_io);
         memory_region_del_subregion(&d->bmdma_bar, &d->bmdma[i].addr_ioport);
-        memory_region_destroy(&d->bmdma[i].addr_ioport);
     }
-    memory_region_destroy(&d->bmdma_bar);
 }
 
 void vt82c686b_ide_init(PCIBus *bus, DriveInfo **hd_table, int devfn)
diff --git a/hw/input/milkymist-softusb.c b/hw/input/milkymist-softusb.c
index 1b4b8d441..5a427f0b3 100644
--- a/hw/input/milkymist-softusb.c
+++ b/hw/input/milkymist-softusb.c
@@ -250,12 +250,12 @@ static int milkymist_softusb_init(SysBusDevice *dev)
 
     /* register pmem and dmem */
     memory_region_init_ram(&s->pmem, OBJECT(s), "milkymist-softusb.pmem",
-                           s->pmem_size);
+                           s->pmem_size, &error_abort);
     vmstate_register_ram_global(&s->pmem);
     s->pmem_ptr = memory_region_get_ram_ptr(&s->pmem);
     sysbus_init_mmio(dev, &s->pmem);
     memory_region_init_ram(&s->dmem, OBJECT(s), "milkymist-softusb.dmem",
-                           s->dmem_size);
+                           s->dmem_size, &error_abort);
     vmstate_register_ram_global(&s->dmem);
     s->dmem_ptr = memory_region_get_ram_ptr(&s->dmem);
     sysbus_init_mmio(dev, &s->dmem);
diff --git a/hw/input/pckbd.c b/hw/input/pckbd.c
index ca1cffcc0..2b0cd3dbb 100644
--- a/hw/input/pckbd.c
+++ b/hw/input/pckbd.c
@@ -131,6 +131,7 @@ typedef struct KBDState {
     uint8_t status;
     uint8_t mode;
     uint8_t outport;
+    bool outport_present;
     /* Bitmask of devices with data available.  */
     uint8_t pending;
     void *kbd;
@@ -229,7 +230,7 @@ static void kbd_write_command(void *opaque, hwaddr addr,
 {
     KBDState *s = opaque;
 
-    DPRINTF("kbd: write cmd=0x%02x\n", val);
+    DPRINTF("kbd: write cmd=0x%02" PRIx64 "\n", val);
 
     /* Bits 3-0 of the output port P2 of the keyboard controller may be pulsed
      * low for approximately 6 micro seconds. Bits 3-0 of the KBD_CCMD_PULSE
@@ -330,7 +331,7 @@ static void kbd_write_data(void *opaque, hwaddr addr,
 {
     KBDState *s = opaque;
 
-    DPRINTF("kbd: write data=0x%02x\n", val);
+    DPRINTF("kbd: write data=0x%02" PRIx64 "\n", val);
 
     switch(s->write_cmd) {
     case 0:
@@ -367,18 +368,68 @@ static void kbd_reset(void *opaque)
     s->mode = KBD_MODE_KBD_INT | KBD_MODE_MOUSE_INT;
     s->status = KBD_STAT_CMD | KBD_STAT_UNLOCKED;
     s->outport = KBD_OUT_RESET | KBD_OUT_A20;
+    s->outport_present = false;
+}
+
+static uint8_t kbd_outport_default(KBDState *s)
+{
+    return KBD_OUT_RESET | KBD_OUT_A20
+           | (s->status & KBD_STAT_OBF ? KBD_OUT_OBF : 0)
+           | (s->status & KBD_STAT_MOUSE_OBF ? KBD_OUT_MOUSE_OBF : 0);
+}
+
+static int kbd_outport_post_load(void *opaque, int version_id)
+{
+    KBDState *s = opaque;
+    s->outport_present = true;
+    return 0;
+}
+
+static const VMStateDescription vmstate_kbd_outport = {
+    .name = "pckbd_outport",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .post_load = kbd_outport_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(outport, KBDState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool kbd_outport_needed(void *opaque)
+{
+    KBDState *s = opaque;
+    return s->outport != kbd_outport_default(s);
+}
+
+static int kbd_post_load(void *opaque, int version_id)
+{
+    KBDState *s = opaque;
+    if (!s->outport_present) {
+        s->outport = kbd_outport_default(s);
+    }
+    s->outport_present = false;
+    return 0;
 }
 
 static const VMStateDescription vmstate_kbd = {
     .name = "pckbd",
     .version_id = 3,
     .minimum_version_id = 3,
+    .post_load = kbd_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_UINT8(write_cmd, KBDState),
         VMSTATE_UINT8(status, KBDState),
         VMSTATE_UINT8(mode, KBDState),
         VMSTATE_UINT8(pending, KBDState),
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (VMStateSubsection[]) {
+        {
+            .vmsd = &vmstate_kbd_outport,
+            .needed = kbd_outport_needed,
+        },
+        VMSTATE_END_OF_LIST()
     }
 };
 
diff --git a/hw/input/tsc210x.c b/hw/input/tsc210x.c
index aa5b6886e..fae338563 100644
--- a/hw/input/tsc210x.c
+++ b/hw/input/tsc210x.c
@@ -215,36 +215,6 @@ typedef struct {
     int fsref;
 } TSC210xRateInfo;
 
-/*  { rate,  dsor,  fsref } */
-static const TSC210xRateInfo tsc2101_rates[] = {
-    /* Fsref / 6.0 */
-    { 7350,	7,	1 },
-    { 8000,	7,	0 },
-    /* Fsref / 5.5 */
-    { 8018,	6,	1 },
-    { 8727,	6,	0 },
-    /* Fsref / 5.0 */
-    { 8820,	5,	1 },
-    { 9600,	5,	0 },
-    /* Fsref / 4.0 */
-    { 11025,	4,	1 },
-    { 12000,	4,	0 },
-    /* Fsref / 3.0 */
-    { 14700,	3,	1 },
-    { 16000,	3,	0 },
-    /* Fsref / 2.0 */
-    { 22050,	2,	1 },
-    { 24000,	2,	0 },
-    /* Fsref / 1.5 */
-    { 29400,	1,	1 },
-    { 32000,	1,	0 },
-    /* Fsref */
-    { 44100,	0,	1 },
-    { 48000,	0,	0 },
-
-    { 0,	0, 	0 },
-};
-
 /*  { rate,   dsor, fsref }	*/
 static const TSC210xRateInfo tsc2102_rates[] = {
     /* Fsref / 6.0 */
diff --git a/hw/intc/apic.c b/hw/intc/apic.c
index ef19e5515..0f97b4792 100644
--- a/hw/intc/apic.c
+++ b/hw/intc/apic.c
@@ -188,7 +188,7 @@ void apic_deliver_pic_intr(DeviceState *dev, int level)
             apic_reset_bit(s->irr, lvt & 0xff);
             /* fall through */
         case APIC_DM_EXTINT:
-            cpu_reset_interrupt(CPU(s->cpu), CPU_INTERRUPT_HARD);
+            apic_update_irq(s);
             break;
         }
     }
@@ -349,6 +349,11 @@ static int apic_get_arb_pri(APICCommonState *s)
 static int apic_irq_pending(APICCommonState *s)
 {
     int irrv, ppr;
+
+    if (!(s->spurious_vec & APIC_SV_ENABLE)) {
+        return 0;
+    }
+
     irrv = get_highest_priority_int(s->irr);
     if (irrv < 0) {
         return 0;
@@ -366,14 +371,13 @@ static void apic_update_irq(APICCommonState *s)
 {
     CPUState *cpu;
 
-    if (!(s->spurious_vec & APIC_SV_ENABLE)) {
-        return;
-    }
     cpu = CPU(s->cpu);
     if (!qemu_cpu_is_self(cpu)) {
         cpu_interrupt(cpu, CPU_INTERRUPT_POLL);
     } else if (apic_irq_pending(s) > 0) {
         cpu_interrupt(cpu, CPU_INTERRUPT_HARD);
+    } else if (!apic_accept_pic_intr(&s->busdev.qdev) || !pic_get_output(isa_pic)) {
+        cpu_reset_interrupt(cpu, CPU_INTERRUPT_HARD);
     }
 }
 
@@ -567,7 +571,10 @@ int apic_get_interrupt(DeviceState *dev)
     apic_sync_vapic(s, SYNC_FROM_VAPIC);
     intno = apic_irq_pending(s);
 
-    if (intno == 0) {
+    /* if there is an interrupt from the 8259, let the caller handle
+     * that first since ExtINT interrupts ignore the priority.
+     */
+    if (intno == 0 || apic_check_pic(s)) {
         apic_sync_vapic(s, SYNC_TO_VAPIC);
         return -1;
     } else if (intno < 0) {
@@ -578,9 +585,6 @@ int apic_get_interrupt(DeviceState *dev)
     apic_set_bit(s->isr, intno);
     apic_sync_vapic(s, SYNC_TO_VAPIC);
 
-    /* re-inject if there is still a pending PIC interrupt */
-    apic_check_pic(s);
-
     apic_update_irq(s);
 
     return intno;
@@ -698,7 +702,7 @@ static uint32_t apic_mem_readl(void *opaque, hwaddr addr)
         val = s->log_dest << 24;
         break;
     case 0x0e:
-        val = s->dest_mode << 28;
+        val = (s->dest_mode << 28) | 0xfffffff;
         break;
     case 0x0f:
         val = s->spurious_vec;
diff --git a/hw/intc/apic_common.c b/hw/intc/apic_common.c
index ce3d903b1..4e62f25ed 100644
--- a/hw/intc/apic_common.c
+++ b/hw/intc/apic_common.c
@@ -324,6 +324,19 @@ static void apic_common_realize(DeviceState *dev, Error **errp)
 
 }
 
+static int apic_pre_load(void *opaque)
+{
+    APICCommonState *s = APIC_COMMON(opaque);
+
+    /* The default is !cpu_is_bsp(s->cpu), but the common value is 0
+     * so that's what apic_common_sipi_needed checks for.  Reset to
+     * the value that is assumed when the apic_sipi subsection is
+     * absent.
+     */
+    s->wait_for_sipi = 0;
+    return 0;
+}
+
 static void apic_dispatch_pre_save(void *opaque)
 {
     APICCommonState *s = APIC_COMMON(opaque);
@@ -345,12 +358,30 @@ static int apic_dispatch_post_load(void *opaque, int version_id)
     return 0;
 }
 
+static bool apic_common_sipi_needed(void *opaque)
+{
+    APICCommonState *s = APIC_COMMON(opaque);
+    return s->wait_for_sipi != 0;
+}
+
+static const VMStateDescription vmstate_apic_common_sipi = {
+    .name = "apic_sipi",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT32(sipi_vector, APICCommonState),
+        VMSTATE_INT32(wait_for_sipi, APICCommonState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_apic_common = {
     .name = "apic",
     .version_id = 3,
     .minimum_version_id = 3,
     .minimum_version_id_old = 1,
     .load_state_old = apic_load_old,
+    .pre_load = apic_pre_load,
     .pre_save = apic_dispatch_pre_save,
     .post_load = apic_dispatch_post_load,
     .fields = (VMStateField[]) {
@@ -375,6 +406,13 @@ static const VMStateDescription vmstate_apic_common = {
         VMSTATE_INT64(timer_expiry,
                       APICCommonState), /* open-coded timer state */
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (VMStateSubsection[]) {
+        {
+            .vmsd = &vmstate_apic_common_sipi,
+            .needed = apic_common_sipi_needed,
+        },
+        VMSTATE_END_OF_LIST()
     }
 };
 
diff --git a/hw/intc/arm_gic.c b/hw/intc/arm_gic.c
index 1532ef948..270ce05b5 100644
--- a/hw/intc/arm_gic.c
+++ b/hw/intc/arm_gic.c
@@ -66,7 +66,8 @@ void gic_update(GICState *s)
         best_prio = 0x100;
         best_irq = 1023;
         for (irq = 0; irq < s->num_irq; irq++) {
-            if (GIC_TEST_ENABLED(irq, cm) && gic_test_pending(s, irq, cm)) {
+            if (GIC_TEST_ENABLED(irq, cm) && gic_test_pending(s, irq, cm) &&
+                (irq < GIC_INTERNAL || GIC_TARGET(irq) & cm)) {
                 if (GIC_GET_PRIORITY(irq, cpu) < best_prio) {
                     best_prio = GIC_GET_PRIORITY(irq, cpu);
                     best_irq = irq;
@@ -372,7 +373,7 @@ static uint32_t gic_dist_readb(void *opaque, hwaddr offset)
         }
     } else if (offset < 0xf00) {
         /* Interrupt Configuration.  */
-        irq = (offset - 0xc00) * 2 + GIC_BASE_IRQ;
+        irq = (offset - 0xc00) * 4 + GIC_BASE_IRQ;
         if (irq >= s->num_irq)
             goto bad_reg;
         res = 0;
@@ -558,13 +559,15 @@ static void gic_dist_writeb(void *opaque, hwaddr offset,
         irq = (offset - 0xc00) * 4 + GIC_BASE_IRQ;
         if (irq >= s->num_irq)
             goto bad_reg;
-        if (irq < GIC_INTERNAL)
+        if (irq < GIC_NR_SGIS)
             value |= 0xaa;
         for (i = 0; i < 4; i++) {
-            if (value & (1 << (i * 2))) {
-                GIC_SET_MODEL(irq + i);
-            } else {
-                GIC_CLEAR_MODEL(irq + i);
+            if (s->revision == REV_11MPCORE || s->revision == REV_NVIC) {
+                if (value & (1 << (i * 2))) {
+                    GIC_SET_MODEL(irq + i);
+                } else {
+                    GIC_CLEAR_MODEL(irq + i);
+                }
             }
             if (value & (2 << (i * 2))) {
                 GIC_SET_EDGE_TRIGGER(irq + i);
@@ -766,7 +769,7 @@ static const MemoryRegionOps gic_cpu_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-void gic_init_irqs_and_distributor(GICState *s, int num_irq)
+void gic_init_irqs_and_distributor(GICState *s)
 {
     SysBusDevice *sbd = SYS_BUS_DEVICE(s);
     int i;
@@ -805,7 +808,7 @@ static void arm_gic_realize(DeviceState *dev, Error **errp)
         return;
     }
 
-    gic_init_irqs_and_distributor(s, s->num_irq);
+    gic_init_irqs_and_distributor(s);
 
     /* Memory regions for the CPU interfaces (NVIC doesn't have these):
      * a region for "CPU interface for this core", then a region for
diff --git a/hw/intc/arm_gic_common.c b/hw/intc/arm_gic_common.c
index 6d884eca3..18b01ba0c 100644
--- a/hw/intc/arm_gic_common.c
+++ b/hw/intc/arm_gic_common.c
@@ -128,7 +128,7 @@ static void arm_gic_common_reset(DeviceState *dev)
         s->running_priority[i] = 0x100;
         s->cpu_enabled[i] = false;
     }
-    for (i = 0; i < 16; i++) {
+    for (i = 0; i < GIC_NR_SGIS; i++) {
         GIC_SET_ENABLED(i, ALL_CPU_MASK);
         GIC_SET_EDGE_TRIGGER(i);
     }
diff --git a/hw/intc/armv7m_nvic.c b/hw/intc/armv7m_nvic.c
index 1a7af450a..d0543d4b9 100644
--- a/hw/intc/armv7m_nvic.c
+++ b/hw/intc/armv7m_nvic.c
@@ -488,7 +488,7 @@ static void armv7m_nvic_realize(DeviceState *dev, Error **errp)
         error_propagate(errp, local_err);
         return;
     }
-    gic_init_irqs_and_distributor(&s->gic, s->num_irq);
+    gic_init_irqs_and_distributor(&s->gic);
     /* The NVIC and system controller register area looks like this:
      *  0..0xff : system control registers, including systick
      *  0x100..0xcff : GIC-like registers
diff --git a/hw/intc/gic_internal.h b/hw/intc/gic_internal.h
index 48a58d789..e87ef36d0 100644
--- a/hw/intc/gic_internal.h
+++ b/hw/intc/gic_internal.h
@@ -59,7 +59,7 @@ void gic_set_pending_private(GICState *s, int cpu, int irq);
 uint32_t gic_acknowledge_irq(GICState *s, int cpu);
 void gic_complete_irq(GICState *s, int cpu, int irq);
 void gic_update(GICState *s);
-void gic_init_irqs_and_distributor(GICState *s, int num_irq);
+void gic_init_irqs_and_distributor(GICState *s);
 void gic_set_priority(GICState *s, int cpu, int irq, uint8_t val);
 
 static inline bool gic_test_pending(GICState *s, int irq, int cm)
diff --git a/hw/intc/i8259.c b/hw/intc/i8259.c
index d0b0c52b9..c51901bfb 100644
--- a/hw/intc/i8259.c
+++ b/hw/intc/i8259.c
@@ -370,7 +370,7 @@ static uint64_t pic_ioport_read(void *opaque, hwaddr addr,
             ret = s->imr;
         }
     }
-    DPRINTF("read: addr=0x%02x val=0x%02x\n", addr, ret);
+    DPRINTF("read: addr=0x%02" HWADDR_PRIx " val=0x%02x\n", addr, ret);
     return ret;
 }
 
@@ -472,7 +472,7 @@ qemu_irq *i8259_init(ISABus *bus, qemu_irq parent_irq)
     ISADevice *isadev;
     int i;
 
-    irq_set = g_malloc(ISA_NUM_IRQS * sizeof(qemu_irq));
+    irq_set = g_new0(qemu_irq, ISA_NUM_IRQS);
 
     isadev = i8259_init_chip(TYPE_I8259, bus, true);
     dev = DEVICE(isadev);
diff --git a/hw/intc/imx_avic.c b/hw/intc/imx_avic.c
index ec5f9ad81..e48f66c8f 100644
--- a/hw/intc/imx_avic.c
+++ b/hw/intc/imx_avic.c
@@ -97,15 +97,6 @@ static inline int imx_avic_prio(IMXAVICState *s, int irq)
     return 0xf & (s->prio[word] >> part);
 }
 
-static inline void imx_avic_set_prio(IMXAVICState *s, int irq, int prio)
-{
-    uint32_t word = irq / PRIO_PER_WORD;
-    uint32_t part = 4 * (irq % PRIO_PER_WORD);
-    uint32_t mask = ~(0xf << part);
-    s->prio[word] &= mask;
-    s->prio[word] |= prio << part;
-}
-
 /* Update interrupts.  */
 static void imx_avic_update(IMXAVICState *s)
 {
diff --git a/hw/intc/openpic.c b/hw/intc/openpic.c
index 028529e13..7d1f3b949 100644
--- a/hw/intc/openpic.c
+++ b/hw/intc/openpic.c
@@ -1627,7 +1627,7 @@ static void openpic_realize(DeviceState *dev, Error **errp)
     }
 
     for (i = 0; i < opp->nb_cpus; i++) {
-        opp->dst[i].irqs = g_new(qemu_irq, OPENPIC_OUTPUT_NB);
+        opp->dst[i].irqs = g_new0(qemu_irq, OPENPIC_OUTPUT_NB);
         for (j = 0; j < OPENPIC_OUTPUT_NB; j++) {
             sysbus_init_irq(d, &opp->dst[i].irqs[j]);
         }
diff --git a/hw/intc/openpic_kvm.c b/hw/intc/openpic_kvm.c
index e3bce043a..3e2cd189f 100644
--- a/hw/intc/openpic_kvm.c
+++ b/hw/intc/openpic_kvm.c
@@ -45,6 +45,7 @@ typedef struct KVMOpenPICState {
     MemoryListener mem_listener;
     uint32_t fd;
     uint32_t model;
+    hwaddr mapped;
 } KVMOpenPICState;
 
 static void kvm_openpic_set_irq(void *opaque, int n_IRQ, int level)
@@ -128,7 +129,16 @@ static void kvm_openpic_region_add(MemoryListener *listener,
         return;
     }
 
+    if (opp->mapped) {
+        /*
+         * We can only map the MPIC once. Since we are already mapped,
+         * the best we can do is ignore new maps.
+         */
+        return;
+    }
+
     reg_base = section->offset_within_address_space;
+    opp->mapped = reg_base;
 
     attr.group = KVM_DEV_MPIC_GRP_MISC;
     attr.attr = KVM_DEV_MPIC_BASE_ADDR;
@@ -155,6 +165,15 @@ static void kvm_openpic_region_del(MemoryListener *listener,
         return;
     }
 
+    if (section->offset_within_address_space != opp->mapped) {
+        /*
+         * We can only map the MPIC once. This mapping was a secondary
+         * one that we couldn't fulfill. Ignore it.
+         */
+        return;
+    }
+    opp->mapped = 0;
+
     attr.group = KVM_DEV_MPIC_GRP_MISC;
     attr.attr = KVM_DEV_MPIC_BASE_ADDR;
     attr.addr = (uint64_t)(unsigned long)&reg_base;
diff --git a/hw/ipack/tpci200.c b/hw/ipack/tpci200.c
index 42ca923c7..b7031a0b1 100644
--- a/hw/ipack/tpci200.c
+++ b/hw/ipack/tpci200.c
@@ -613,18 +613,6 @@ static int tpci200_initfn(PCIDevice *pci_dev)
     return 0;
 }
 
-static void tpci200_exitfn(PCIDevice *pci_dev)
-{
-    TPCI200State *s = TPCI200(pci_dev);
-
-    memory_region_destroy(&s->mmio);
-    memory_region_destroy(&s->io);
-    memory_region_destroy(&s->las0);
-    memory_region_destroy(&s->las1);
-    memory_region_destroy(&s->las2);
-    memory_region_destroy(&s->las3);
-}
-
 static const VMStateDescription vmstate_tpci200 = {
     .name = "tpci200",
     .version_id = 1,
@@ -645,7 +633,6 @@ static void tpci200_class_init(ObjectClass *klass, void *data)
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
     k->init = tpci200_initfn;
-    k->exit = tpci200_exitfn;
     k->vendor_id = PCI_VENDOR_ID_TEWS;
     k->device_id = PCI_DEVICE_ID_TEWS_TPCI200;
     k->class_id = PCI_CLASS_BRIDGE_OTHER;
diff --git a/hw/isa/apm.c b/hw/isa/apm.c
index 054d529b8..26ab17021 100644
--- a/hw/isa/apm.c
+++ b/hw/isa/apm.c
@@ -41,7 +41,8 @@ static void apm_ioport_writeb(void *opaque, hwaddr addr, uint64_t val,
 {
     APMState *apm = opaque;
     addr &= 1;
-    APM_DPRINTF("apm_ioport_writeb addr=0x%x val=0x%02x\n", addr, val);
+    APM_DPRINTF("apm_ioport_writeb addr=0x%" HWADDR_PRIx
+                " val=0x%02" PRIx64 "\n", addr, val);
     if (addr == 0) {
         apm->apmc = val;
 
@@ -64,7 +65,7 @@ static uint64_t apm_ioport_readb(void *opaque, hwaddr addr, unsigned size)
     } else {
         val = apm->apms;
     }
-    APM_DPRINTF("apm_ioport_readb addr=0x%x val=0x%02x\n", addr, val);
+    APM_DPRINTF("apm_ioport_readb addr=0x%" HWADDR_PRIx " val=0x%02x\n", addr, val);
     return val;
 }
 
diff --git a/hw/isa/isa-bus.c b/hw/isa/isa-bus.c
index b28981bfd..cc85e538b 100644
--- a/hw/isa/isa-bus.c
+++ b/hw/isa/isa-bus.c
@@ -50,7 +50,7 @@ ISABus *isa_bus_new(DeviceState *dev, MemoryRegion *address_space_io)
         fprintf(stderr, "Can't create a second ISA bus\n");
         return NULL;
     }
-    if (NULL == dev) {
+    if (!dev) {
         dev = qdev_create(NULL, "isabus-bridge");
         qdev_init_nofail(dev);
     }
diff --git a/hw/isa/lpc_ich9.c b/hw/isa/lpc_ich9.c
index b846d8199..530b07455 100644
--- a/hw/isa/lpc_ich9.c
+++ b/hw/isa/lpc_ich9.c
@@ -7,7 +7,7 @@
  *               VA Linux Systems Japan K.K.
  * Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
  *
- * This is based on piix_pci.c, but heavily modified.
+ * This is based on piix.c, but heavily modified.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -607,8 +607,8 @@ static void ich9_device_plug_cb(HotplugHandler *hotplug_dev,
     ich9_pm_device_plug_cb(&lpc->pm, dev, errp);
 }
 
-static void ich9_device_unplug_cb(HotplugHandler *hotplug_dev,
-                                  DeviceState *dev, Error **errp)
+static void ich9_device_unplug_request_cb(HotplugHandler *hotplug_dev,
+                                          DeviceState *dev, Error **errp)
 {
     error_setg(errp, "acpi: device unplug request for not supported device"
                " type: %s", object_get_typename(OBJECT(dev)));
@@ -676,7 +676,7 @@ static void ich9_lpc_class_init(ObjectClass *klass, void *data)
      */
     dc->cannot_instantiate_with_device_add_yet = true;
     hc->plug = ich9_device_plug_cb;
-    hc->unplug = ich9_device_unplug_cb;
+    hc->unplug_request = ich9_device_unplug_request_cb;
     adevc->ospm_status = ich9_pm_ospm_status;
 }
 
diff --git a/hw/isa/pc87312.c b/hw/isa/pc87312.c
index 9327c5313..40a11060a 100644
--- a/hw/isa/pc87312.c
+++ b/hw/isa/pc87312.c
@@ -25,6 +25,7 @@
 
 #include "hw/isa/pc87312.h"
 #include "qemu/error-report.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/char.h"
@@ -320,11 +321,13 @@ static void pc87312_realize(DeviceState *dev, Error **errp)
         qdev_prop_set_uint32(d, "irq", 6);
         drive = drive_get(IF_FLOPPY, 0, 0);
         if (drive != NULL) {
-            qdev_prop_set_drive_nofail(d, "driveA", drive->bdrv);
+            qdev_prop_set_drive_nofail(d, "driveA",
+                                       blk_by_legacy_dinfo(drive));
         }
         drive = drive_get(IF_FLOPPY, 0, 1);
         if (drive != NULL) {
-            qdev_prop_set_drive_nofail(d, "driveB", drive->bdrv);
+            qdev_prop_set_drive_nofail(d, "driveB",
+                                       blk_by_legacy_dinfo(drive));
         }
         qdev_init_nofail(d);
         s->fdc.dev = isa;
diff --git a/hw/lm32/lm32_boards.c b/hw/lm32/lm32_boards.c
index 0e013408f..af0abdbed 100644
--- a/hw/lm32/lm32_boards.c
+++ b/hw/lm32/lm32_boards.c
@@ -23,7 +23,7 @@
 #include "hw/devices.h"
 #include "hw/boards.h"
 #include "hw/loader.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "elf.h"
 #include "lm32_hwsetup.h"
 #include "lm32.h"
@@ -111,16 +111,17 @@ static void lm32_evr_init(MachineState *machine)
 
     reset_info->flash_base = flash_base;
 
-    memory_region_init_ram(phys_ram, NULL, "lm32_evr.sdram", ram_size);
+    memory_region_init_ram(phys_ram, NULL, "lm32_evr.sdram", ram_size,
+                           &error_abort);
     vmstate_register_ram_global(phys_ram);
     memory_region_add_subregion(address_space_mem, ram_base, phys_ram);
 
     dinfo = drive_get(IF_PFLASH, 0, 0);
     /* Spansion S29NS128P */
     pflash_cfi02_register(flash_base, NULL, "lm32_evr.flash", flash_size,
-                          dinfo ? dinfo->bdrv : NULL, flash_sector_size,
-                          flash_size / flash_sector_size, 1, 2,
-                          0x01, 0x7e, 0x43, 0x00, 0x555, 0x2aa, 1);
+                          dinfo ? blk_by_legacy_dinfo(dinfo) : NULL,
+                          flash_sector_size, flash_size / flash_sector_size,
+                          1, 2, 0x01, 0x7e, 0x43, 0x00, 0x555, 0x2aa, 1);
 
     /* create irq lines */
     cpu_irq = qemu_allocate_irqs(cpu_irq_handler, cpu, 1);
@@ -213,16 +214,17 @@ static void lm32_uclinux_init(MachineState *machine)
 
     reset_info->flash_base = flash_base;
 
-    memory_region_init_ram(phys_ram, NULL, "lm32_uclinux.sdram", ram_size);
+    memory_region_init_ram(phys_ram, NULL, "lm32_uclinux.sdram", ram_size,
+                           &error_abort);
     vmstate_register_ram_global(phys_ram);
     memory_region_add_subregion(address_space_mem, ram_base, phys_ram);
 
     dinfo = drive_get(IF_PFLASH, 0, 0);
     /* Spansion S29NS128P */
     pflash_cfi02_register(flash_base, NULL, "lm32_uclinux.flash", flash_size,
-                          dinfo ? dinfo->bdrv : NULL, flash_sector_size,
-                          flash_size / flash_sector_size, 1, 2,
-                          0x01, 0x7e, 0x43, 0x00, 0x555, 0x2aa, 1);
+                          dinfo ? blk_by_legacy_dinfo(dinfo) : NULL,
+                          flash_sector_size, flash_size / flash_sector_size,
+                          1, 2, 0x01, 0x7e, 0x43, 0x00, 0x555, 0x2aa, 1);
 
     /* create irq lines */
     cpu_irq = qemu_allocate_irqs(cpu_irq_handler, env, 1);
diff --git a/hw/lm32/milkymist.c b/hw/lm32/milkymist.c
index 81c3933e6..256c1029c 100644
--- a/hw/lm32/milkymist.c
+++ b/hw/lm32/milkymist.c
@@ -26,7 +26,7 @@
 #include "hw/boards.h"
 #include "hw/loader.h"
 #include "elf.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "milkymist-hw.h"
 #include "lm32.h"
 #include "exec/address-spaces.h"
@@ -118,16 +118,17 @@ milkymist_init(MachineState *machine)
 
     cpu_lm32_set_phys_msb_ignore(env, 1);
 
-    memory_region_init_ram(phys_sdram, NULL, "milkymist.sdram", sdram_size);
+    memory_region_init_ram(phys_sdram, NULL, "milkymist.sdram", sdram_size,
+                           &error_abort);
     vmstate_register_ram_global(phys_sdram);
     memory_region_add_subregion(address_space_mem, sdram_base, phys_sdram);
 
     dinfo = drive_get(IF_PFLASH, 0, 0);
     /* Numonyx JS28F256J3F105 */
     pflash_cfi01_register(flash_base, NULL, "milkymist.flash", flash_size,
-                          dinfo ? dinfo->bdrv : NULL, flash_sector_size,
-                          flash_size / flash_sector_size, 2,
-                          0x00, 0x89, 0x00, 0x1d, 1);
+                          dinfo ? blk_by_legacy_dinfo(dinfo) : NULL,
+                          flash_sector_size, flash_size / flash_sector_size,
+                          2, 0x00, 0x89, 0x00, 0x1d, 1);
 
     /* create irq lines */
     cpu_irq = qemu_allocate_irqs(cpu_irq_handler, cpu, 1);
diff --git a/hw/m68k/an5206.c b/hw/m68k/an5206.c
index 684496a94..f1f13504c 100644
--- a/hw/m68k/an5206.c
+++ b/hw/m68k/an5206.c
@@ -50,12 +50,12 @@ static void an5206_init(MachineState *machine)
     env->rambar0 = AN5206_RAMBAR_ADDR | 1;
 
     /* DRAM at address zero */
-    memory_region_init_ram(ram, NULL, "an5206.ram", ram_size);
+    memory_region_init_ram(ram, NULL, "an5206.ram", ram_size, &error_abort);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(address_space_mem, 0, ram);
 
     /* Internal SRAM.  */
-    memory_region_init_ram(sram, NULL, "an5206.sram", 512);
+    memory_region_init_ram(sram, NULL, "an5206.sram", 512, &error_abort);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(address_space_mem, AN5206_RAMBAR_ADDR, sram);
 
@@ -74,7 +74,8 @@ static void an5206_init(MachineState *machine)
                            NULL, NULL, 1, ELF_MACHINE, 0);
     entry = elf_entry;
     if (kernel_size < 0) {
-        kernel_size = load_uimage(kernel_filename, &entry, NULL, NULL);
+        kernel_size = load_uimage(kernel_filename, &entry, NULL, NULL,
+                                  NULL, NULL);
     }
     if (kernel_size < 0) {
         kernel_size = load_image_targphys(kernel_filename, KERNEL_LOAD_ADDR,
diff --git a/hw/m68k/dummy_m68k.c b/hw/m68k/dummy_m68k.c
index 6db1b7164..facd561ef 100644
--- a/hw/m68k/dummy_m68k.c
+++ b/hw/m68k/dummy_m68k.c
@@ -40,7 +40,7 @@ static void dummy_m68k_init(MachineState *machine)
     env->vbr = 0;
 
     /* RAM at address zero */
-    memory_region_init_ram(ram, NULL, "dummy_m68k.ram", ram_size);
+    memory_region_init_ram(ram, NULL, "dummy_m68k.ram", ram_size, &error_abort);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(address_space_mem, 0, ram);
 
@@ -50,7 +50,8 @@ static void dummy_m68k_init(MachineState *machine)
                                NULL, NULL, 1, ELF_MACHINE, 0);
         entry = elf_entry;
         if (kernel_size < 0) {
-            kernel_size = load_uimage(kernel_filename, &entry, NULL, NULL);
+            kernel_size = load_uimage(kernel_filename, &entry, NULL, NULL,
+                                      NULL, NULL);
         }
         if (kernel_size < 0) {
             kernel_size = load_image_targphys(kernel_filename,
diff --git a/hw/m68k/mcf5208.c b/hw/m68k/mcf5208.c
index 2ef617f2b..a01a4458e 100644
--- a/hw/m68k/mcf5208.c
+++ b/hw/m68k/mcf5208.c
@@ -218,12 +218,12 @@ static void mcf5208evb_init(MachineState *machine)
     /* TODO: Configure BARs.  */
 
     /* DRAM at 0x40000000 */
-    memory_region_init_ram(ram, NULL, "mcf5208.ram", ram_size);
+    memory_region_init_ram(ram, NULL, "mcf5208.ram", ram_size, &error_abort);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(address_space_mem, 0x40000000, ram);
 
     /* Internal SRAM.  */
-    memory_region_init_ram(sram, NULL, "mcf5208.sram", 16384);
+    memory_region_init_ram(sram, NULL, "mcf5208.sram", 16384, &error_abort);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(address_space_mem, 0x80000000, sram);
 
@@ -279,7 +279,8 @@ static void mcf5208evb_init(MachineState *machine)
                            NULL, NULL, 1, ELF_MACHINE, 0);
     entry = elf_entry;
     if (kernel_size < 0) {
-        kernel_size = load_uimage(kernel_filename, &entry, NULL, NULL);
+        kernel_size = load_uimage(kernel_filename, &entry, NULL, NULL,
+                                  NULL, NULL);
     }
     if (kernel_size < 0) {
         kernel_size = load_image_targphys(kernel_filename, 0x40000000,
diff --git a/hw/mem/pc-dimm.c b/hw/mem/pc-dimm.c
index 08f49ed53..d43183403 100644
--- a/hw/mem/pc-dimm.c
+++ b/hw/mem/pc-dimm.c
@@ -139,19 +139,34 @@ static int pc_dimm_built_list(Object *obj, void *opaque)
 
 uint64_t pc_dimm_get_free_addr(uint64_t address_space_start,
                                uint64_t address_space_size,
-                               uint64_t *hint, uint64_t size,
+                               uint64_t *hint, uint64_t align, uint64_t size,
                                Error **errp)
 {
     GSList *list = NULL, *item;
     uint64_t new_addr, ret = 0;
     uint64_t address_space_end = address_space_start + address_space_size;
 
+    g_assert(QEMU_ALIGN_UP(address_space_start, align) == address_space_start);
+    g_assert(QEMU_ALIGN_UP(address_space_size, align) == address_space_size);
+
     if (!address_space_size) {
         error_setg(errp, "memory hotplug is not enabled, "
                          "please add maxmem option");
         goto out;
     }
 
+    if (hint && QEMU_ALIGN_UP(*hint, align) != *hint) {
+        error_setg(errp, "address must be aligned to 0x%" PRIx64 " bytes",
+                   align);
+        goto out;
+    }
+
+    if (QEMU_ALIGN_UP(size, align) != size) {
+        error_setg(errp, "backend memory size must be multiple of 0x%"
+                   PRIx64, align);
+        goto out;
+    }
+
     assert(address_space_end > address_space_start);
     object_child_foreach(qdev_get_machine(), pc_dimm_built_list, &list);
 
@@ -177,7 +192,7 @@ uint64_t pc_dimm_get_free_addr(uint64_t address_space_start,
                 error_setg(errp, "address range conflicts with '%s'", d->id);
                 goto out;
             }
-            new_addr = dimm->addr + dimm_size;
+            new_addr = QEMU_ALIGN_UP(dimm->addr + dimm_size, align);
         }
     }
     ret = new_addr;
@@ -252,6 +267,12 @@ static void pc_dimm_realize(DeviceState *dev, Error **errp)
         error_setg(errp, "'" PC_DIMM_MEMDEV_PROP "' property is not set");
         return;
     }
+    if ((nb_numa_nodes > 0) && (dimm->node >= nb_numa_nodes)) {
+        error_setg(errp, "'DIMM property " PC_DIMM_NODE_PROP " has value %"
+                   PRIu32 "' which exceeds the number of numa nodes: %d",
+                   dimm->node, nb_numa_nodes);
+        return;
+    }
 }
 
 static MemoryRegion *pc_dimm_get_memory_region(PCDIMMDevice *dimm)
diff --git a/hw/microblaze/boot.c b/hw/microblaze/boot.c
index 6bf36d046..a2843cdf9 100644
--- a/hw/microblaze/boot.c
+++ b/hw/microblaze/boot.c
@@ -154,7 +154,8 @@ void microblaze_load_kernel(MicroBlazeCPU *cpu, hwaddr ddr_base,
         if (kernel_size < 0) {
             hwaddr uentry, loadaddr;
 
-            kernel_size = load_uimage(kernel_filename, &uentry, &loadaddr, 0);
+            kernel_size = load_uimage(kernel_filename, &uentry, &loadaddr, 0,
+                                      NULL, NULL);
             boot_info.bootstrap_pc = uentry;
             high = (loadaddr + kernel_size + 3) & ~3;
         }
diff --git a/hw/microblaze/petalogix_ml605_mmu.c b/hw/microblaze/petalogix_ml605_mmu.c
index aea9c5b49..48c264b9d 100644
--- a/hw/microblaze/petalogix_ml605_mmu.c
+++ b/hw/microblaze/petalogix_ml605_mmu.c
@@ -32,7 +32,7 @@
 #include "sysemu/sysemu.h"
 #include "hw/devices.h"
 #include "hw/boards.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "hw/char/serial.h"
 #include "exec/address-spaces.h"
 #include "hw/ssi.h"
@@ -89,7 +89,6 @@ petalogix_ml605_init(MachineState *machine)
     SysBusDevice *busdev;
     DriveInfo *dinfo;
     int i;
-    hwaddr ddr_base = MEMORY_BASEADDR;
     MemoryRegion *phys_lmb_bram = g_new(MemoryRegion, 1);
     MemoryRegion *phys_ram = g_new(MemoryRegion, 1);
     qemu_irq irq[32];
@@ -100,21 +99,22 @@ petalogix_ml605_init(MachineState *machine)
 
     /* Attach emulated BRAM through the LMB.  */
     memory_region_init_ram(phys_lmb_bram, NULL, "petalogix_ml605.lmb_bram",
-                           LMB_BRAM_SIZE);
+                           LMB_BRAM_SIZE, &error_abort);
     vmstate_register_ram_global(phys_lmb_bram);
     memory_region_add_subregion(address_space_mem, 0x00000000, phys_lmb_bram);
 
-    memory_region_init_ram(phys_ram, NULL, "petalogix_ml605.ram", ram_size);
+    memory_region_init_ram(phys_ram, NULL, "petalogix_ml605.ram", ram_size,
+                           &error_abort);
     vmstate_register_ram_global(phys_ram);
-    memory_region_add_subregion(address_space_mem, ddr_base, phys_ram);
+    memory_region_add_subregion(address_space_mem, MEMORY_BASEADDR, phys_ram);
 
     dinfo = drive_get(IF_PFLASH, 0, 0);
     /* 5th parameter 2 means bank-width
      * 10th paremeter 0 means little-endian */
     pflash_cfi01_register(FLASH_BASEADDR,
                           NULL, "petalogix_ml605.flash", FLASH_SIZE,
-                          dinfo ? dinfo->bdrv : NULL, (64 * 1024),
-                          FLASH_SIZE >> 16,
+                          dinfo ? blk_by_legacy_dinfo(dinfo) : NULL,
+                          (64 * 1024), FLASH_SIZE >> 16,
                           2, 0x89, 0x18, 0x0000, 0x0, 0);
 
 
@@ -201,7 +201,7 @@ petalogix_ml605_init(MachineState *machine)
         }
     }
 
-    microblaze_load_kernel(cpu, ddr_base, ram_size,
+    microblaze_load_kernel(cpu, MEMORY_BASEADDR, ram_size,
                            machine->initrd_filename,
                            BINARY_DEVICE_TREE_FILE,
                            machine_cpu_reset);
diff --git a/hw/microblaze/petalogix_s3adsp1800_mmu.c b/hw/microblaze/petalogix_s3adsp1800_mmu.c
index 49dc6d194..84f6e7465 100644
--- a/hw/microblaze/petalogix_s3adsp1800_mmu.c
+++ b/hw/microblaze/petalogix_s3adsp1800_mmu.c
@@ -30,7 +30,7 @@
 #include "sysemu/sysemu.h"
 #include "hw/devices.h"
 #include "hw/boards.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 
 #include "boot.h"
@@ -81,19 +81,21 @@ petalogix_s3adsp1800_init(MachineState *machine)
 
     /* Attach emulated BRAM through the LMB.  */
     memory_region_init_ram(phys_lmb_bram, NULL,
-                           "petalogix_s3adsp1800.lmb_bram", LMB_BRAM_SIZE);
+                           "petalogix_s3adsp1800.lmb_bram", LMB_BRAM_SIZE,
+                           &error_abort);
     vmstate_register_ram_global(phys_lmb_bram);
     memory_region_add_subregion(sysmem, 0x00000000, phys_lmb_bram);
 
-    memory_region_init_ram(phys_ram, NULL, "petalogix_s3adsp1800.ram", ram_size);
+    memory_region_init_ram(phys_ram, NULL, "petalogix_s3adsp1800.ram",
+                           ram_size, &error_abort);
     vmstate_register_ram_global(phys_ram);
     memory_region_add_subregion(sysmem, ddr_base, phys_ram);
 
     dinfo = drive_get(IF_PFLASH, 0, 0);
     pflash_cfi01_register(FLASH_BASEADDR,
                           NULL, "petalogix_s3adsp1800.flash", FLASH_SIZE,
-                          dinfo ? dinfo->bdrv : NULL, (64 * 1024),
-                          FLASH_SIZE >> 16,
+                          dinfo ? blk_by_legacy_dinfo(dinfo) : NULL,
+                          (64 * 1024), FLASH_SIZE >> 16,
                           1, 0x89, 0x18, 0x0000, 0x0, 1);
 
     dev = qdev_create(NULL, "xlnx.xps-intc");
diff --git a/hw/mips/gt64xxx_pci.c b/hw/mips/gt64xxx_pci.c
index 22f63ce0c..1f2fe5fab 100644
--- a/hw/mips/gt64xxx_pci.c
+++ b/hw/mips/gt64xxx_pci.c
@@ -297,7 +297,7 @@ static void gt64120_pci_mapping(GT64120State *s)
       if (s->PCI0IO_length)
       {
           memory_region_del_subregion(get_system_memory(), &s->PCI0IO_mem);
-          memory_region_destroy(&s->PCI0IO_mem);
+          object_unparent(OBJECT(&s->PCI0IO_mem));
       }
       /* Map new IO address */
       s->PCI0IO_start = s->regs[GT_PCI0IOLD] << 21;
diff --git a/hw/mips/mips_fulong2e.c b/hw/mips/mips_fulong2e.c
index f7533ed20..6a9ebfa91 100644
--- a/hw/mips/mips_fulong2e.c
+++ b/hw/mips/mips_fulong2e.c
@@ -25,7 +25,7 @@
 #include "net/net.h"
 #include "hw/boards.h"
 #include "hw/i2c/smbus.h"
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "hw/block/flash.h"
 #include "hw/mips/mips.h"
 #include "hw/mips/cpudevs.h"
@@ -302,9 +302,10 @@ static void mips_fulong2e_init(MachineState *machine)
     bios_size = 1024 * 1024;
 
     /* allocate RAM */
-    memory_region_init_ram(ram, NULL, "fulong2e.ram", ram_size);
+    memory_region_init_ram(ram, NULL, "fulong2e.ram", ram_size, &error_abort);
     vmstate_register_ram_global(ram);
-    memory_region_init_ram(bios, NULL, "fulong2e.bios", bios_size);
+    memory_region_init_ram(bios, NULL, "fulong2e.bios", bios_size,
+                           &error_abort);
     vmstate_register_ram_global(bios);
     memory_region_set_readonly(bios, true);
 
@@ -349,7 +350,7 @@ static void mips_fulong2e_init(MachineState *machine)
     pci_bus = bonito_init((qemu_irq *)&(env->irq[2]));
 
     /* South bridge */
-    ide_drive_get(hd, MAX_IDE_BUS);
+    ide_drive_get(hd, ARRAY_SIZE(hd));
 
     isa_bus = vt82c686b_init(pci_bus, PCI_DEVFN(FULONG2E_VIA_SLOT, 0));
     if (!isa_bus) {
diff --git a/hw/mips/mips_jazz.c b/hw/mips/mips_jazz.c
index c113a8082..3f33093fd 100644
--- a/hw/mips/mips_jazz.c
+++ b/hw/mips/mips_jazz.c
@@ -39,7 +39,7 @@
 #include "hw/timer/mc146818rtc.h"
 #include "hw/timer/i8254.h"
 #include "hw/audio/pcspk.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "hw/sysbus.h"
 #include "exec/address-spaces.h"
 #include "sysemu/qtest.h"
@@ -179,11 +179,12 @@ static void mips_jazz_init(MemoryRegion *address_space,
     cc->do_unassigned_access = mips_jazz_do_unassigned_access;
 
     /* allocate RAM */
-    memory_region_init_ram(ram, NULL, "mips_jazz.ram", ram_size);
+    memory_region_init_ram(ram, NULL, "mips_jazz.ram", ram_size, &error_abort);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(address_space, 0, ram);
 
-    memory_region_init_ram(bios, NULL, "mips_jazz.bios", MAGNUM_BIOS_SIZE);
+    memory_region_init_ram(bios, NULL, "mips_jazz.bios", MAGNUM_BIOS_SIZE,
+                           &error_abort);
     vmstate_register_ram_global(bios);
     memory_region_set_readonly(bios, true);
     memory_region_init_alias(bios2, NULL, "mips_jazz.bios", bios,
@@ -244,7 +245,8 @@ static void mips_jazz_init(MemoryRegion *address_space,
         {
             /* Simple ROM, so user doesn't have to provide one */
             MemoryRegion *rom_mr = g_new(MemoryRegion, 1);
-            memory_region_init_ram(rom_mr, NULL, "g364fb.rom", 0x80000);
+            memory_region_init_ram(rom_mr, NULL, "g364fb.rom", 0x80000,
+                                   &error_abort);
             vmstate_register_ram_global(rom_mr);
             memory_region_set_readonly(rom_mr, true);
             uint8_t *rom = memory_region_get_ram_ptr(rom_mr);
diff --git a/hw/mips/mips_malta.c b/hw/mips/mips_malta.c
index cfb60aff9..5845158a7 100644
--- a/hw/mips/mips_malta.c
+++ b/hw/mips/mips_malta.c
@@ -29,7 +29,7 @@
 #include "net/net.h"
 #include "hw/boards.h"
 #include "hw/i2c/smbus.h"
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "hw/block/flash.h"
 #include "hw/mips/mips.h"
 #include "hw/mips/cpudevs.h"
@@ -44,6 +44,7 @@
 #include "elf.h"
 #include "hw/timer/mc146818rtc.h"
 #include "hw/timer/i8254.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "exec/address-spaces.h"
 #include "hw/sysbus.h"             /* SysBusDevice */
@@ -697,12 +698,12 @@ static void write_bootloader (CPUMIPSState *env, uint8_t *base,
     /* Jump to kernel code */
     stl_p(p++, 0x3c1f0000 | ((kernel_entry >> 16) & 0xffff));    /* lui ra, high(kernel_entry) */
     stl_p(p++, 0x37ff0000 | (kernel_entry & 0xffff));            /* ori ra, ra, low(kernel_entry) */
-    stl_p(p++, 0x03e00008);                                      /* jr ra */
+    stl_p(p++, 0x03e00009);                                      /* jalr ra */
     stl_p(p++, 0x00000000);                                      /* nop */
 
     /* YAMON subroutines */
     p = (uint32_t *) (base + 0x800);
-    stl_p(p++, 0x03e00008);                                     /* jr ra */
+    stl_p(p++, 0x03e00009);                                     /* jalr ra */
     stl_p(p++, 0x24020000);                                     /* li v0,0 */
     /* 808 YAMON print */
     stl_p(p++, 0x03e06821);                                     /* move t5,ra */
@@ -716,7 +717,7 @@ static void write_bootloader (CPUMIPSState *env, uint8_t *base,
     stl_p(p++, 0x00000000);                                     /* nop */
     stl_p(p++, 0x08000205);                                     /* j 814 */
     stl_p(p++, 0x00000000);                                     /* nop */
-    stl_p(p++, 0x01a00008);                                     /* jr t5 */
+    stl_p(p++, 0x01a00009);                                     /* jalr t5 */
     stl_p(p++, 0x01602021);                                     /* move a0,t3 */
     /* 0x83c YAMON print_count */
     stl_p(p++, 0x03e06821);                                     /* move t5,ra */
@@ -730,7 +731,7 @@ static void write_bootloader (CPUMIPSState *env, uint8_t *base,
     stl_p(p++, 0x258cffff);                                     /* addiu t4,t4,-1 */
     stl_p(p++, 0x1580fffa);                                     /* bnez t4,84c */
     stl_p(p++, 0x00000000);                                     /* nop */
-    stl_p(p++, 0x01a00008);                                     /* jr t5 */
+    stl_p(p++, 0x01a00009);                                     /* jalr t5 */
     stl_p(p++, 0x01602021);                                     /* move a0,t3 */
     /* 0x870 */
     stl_p(p++, 0x3c08b800);                                     /* lui t0,0xb400 */
@@ -740,7 +741,7 @@ static void write_bootloader (CPUMIPSState *env, uint8_t *base,
     stl_p(p++, 0x31290040);                                     /* andi t1,t1,0x40 */
     stl_p(p++, 0x1120fffc);                                     /* beqz t1,878 <outch+0x8> */
     stl_p(p++, 0x00000000);                                     /* nop */
-    stl_p(p++, 0x03e00008);                                     /* jr ra */
+    stl_p(p++, 0x03e00009);                                     /* jalr ra */
     stl_p(p++, 0xa1040000);                                     /* sb a0,0(t0) */
 
 }
@@ -992,7 +993,8 @@ void mips_malta_init(MachineState *machine)
     }
 
     /* register RAM at high address where it is undisturbed by IO */
-    memory_region_init_ram(ram_high, NULL, "mips_malta.ram", ram_size);
+    memory_region_init_ram(ram_high, NULL, "mips_malta.ram", ram_size,
+                           &error_abort);
     vmstate_register_ram_global(ram_high);
     memory_region_add_subregion(system_memory, 0x80000000, ram_high);
 
@@ -1031,11 +1033,12 @@ void mips_malta_init(MachineState *machine)
         printf("Register parallel flash %d size " TARGET_FMT_lx " at "
                "addr %08llx '%s' %x\n",
                fl_idx, bios_size, FLASH_ADDRESS,
-               bdrv_get_device_name(dinfo->bdrv), fl_sectors);
+               blk_name(dinfo->bdrv), fl_sectors);
     }
 #endif
     fl = pflash_cfi01_register(FLASH_ADDRESS, NULL, "mips_malta.bios",
-                               BIOS_SIZE, dinfo ? dinfo->bdrv : NULL,
+                               BIOS_SIZE,
+                               dinfo ? blk_by_legacy_dinfo(dinfo) : NULL,
                                65536, fl_sectors,
                                4, 0x0000, 0x0000, 0x0000, 0x0000, be);
     bios = pflash_cfi01_get_memory(fl);
@@ -1116,7 +1119,8 @@ void mips_malta_init(MachineState *machine)
      * handled by an overlapping region as the resulting ROM code subpage
      * regions are not executable.
      */
-    memory_region_init_ram(bios_copy, NULL, "bios.1fc", BIOS_SIZE);
+    memory_region_init_ram(bios_copy, NULL, "bios.1fc", BIOS_SIZE,
+                           &error_abort);
     if (!rom_copy(memory_region_get_ram_ptr(bios_copy),
                   FLASH_ADDRESS, BIOS_SIZE)) {
         memcpy(memory_region_get_ram_ptr(bios_copy),
@@ -1145,7 +1149,7 @@ void mips_malta_init(MachineState *machine)
     pci_bus = gt64120_register(isa_irq);
 
     /* Southbridge */
-    ide_drive_get(hd, MAX_IDE_BUS);
+    ide_drive_get(hd, ARRAY_SIZE(hd));
 
     piix4_devfn = piix4_init(pci_bus, &isa_bus, 80);
 
diff --git a/hw/mips/mips_mipssim.c b/hw/mips/mips_mipssim.c
index 413e64d16..5d44c3f73 100644
--- a/hw/mips/mips_mipssim.c
+++ b/hw/mips/mips_mipssim.c
@@ -171,9 +171,11 @@ mips_mipssim_init(MachineState *machine)
     qemu_register_reset(main_cpu_reset, reset_info);
 
     /* Allocate RAM. */
-    memory_region_init_ram(ram, NULL, "mips_mipssim.ram", ram_size);
+    memory_region_init_ram(ram, NULL, "mips_mipssim.ram", ram_size,
+                           &error_abort);
     vmstate_register_ram_global(ram);
-    memory_region_init_ram(bios, NULL, "mips_mipssim.bios", BIOS_SIZE);
+    memory_region_init_ram(bios, NULL, "mips_mipssim.bios", BIOS_SIZE,
+                           &error_abort);
     vmstate_register_ram_global(bios);
     memory_region_set_readonly(bios, true);
 
@@ -195,7 +197,7 @@ mips_mipssim_init(MachineState *machine)
         !kernel_filename && !qtest_enabled()) {
         /* Bail out if we have neither a kernel image nor boot vector code. */
         error_report("Could not load MIPS bios '%s', and no "
-                     "-kernel argument was specified", filename);
+                     "-kernel argument was specified", bios_name);
         exit(1);
     } else {
         /* We have a boot vector start address. */
diff --git a/hw/mips/mips_r4k.c b/hw/mips/mips_r4k.c
index 71202931b..a7fe0cead 100644
--- a/hw/mips/mips_r4k.c
+++ b/hw/mips/mips_r4k.c
@@ -24,7 +24,7 @@
 #include "elf.h"
 #include "hw/timer/mc146818rtc.h"
 #include "hw/timer/i8254.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 #include "sysemu/qtest.h"
 
@@ -204,7 +204,7 @@ void mips_r4k_init(MachineState *machine)
                 ((unsigned int)ram_size / (1 << 20)));
         exit(1);
     }
-    memory_region_init_ram(ram, NULL, "mips_r4k.ram", ram_size);
+    memory_region_init_ram(ram, NULL, "mips_r4k.ram", ram_size, &error_abort);
     vmstate_register_ram_global(ram);
 
     memory_region_add_subregion(address_space_mem, 0, ram);
@@ -231,7 +231,8 @@ void mips_r4k_init(MachineState *machine)
 #endif
     if ((bios_size > 0) && (bios_size <= BIOS_SIZE)) {
         bios = g_new(MemoryRegion, 1);
-        memory_region_init_ram(bios, NULL, "mips_r4k.bios", BIOS_SIZE);
+        memory_region_init_ram(bios, NULL, "mips_r4k.bios", BIOS_SIZE,
+                               &error_abort);
         vmstate_register_ram_global(bios);
         memory_region_set_readonly(bios, true);
         memory_region_add_subregion(get_system_memory(), 0x1fc00000, bios);
@@ -240,8 +241,8 @@ void mips_r4k_init(MachineState *machine)
     } else if ((dinfo = drive_get(IF_PFLASH, 0, 0)) != NULL) {
         uint32_t mips_rom = 0x00400000;
         if (!pflash_cfi01_register(0x1fc00000, NULL, "mips_r4k.bios", mips_rom,
-                                   dinfo->bdrv, sector_len,
-                                   mips_rom / sector_len,
+                                   blk_by_legacy_dinfo(dinfo),
+                                   sector_len, mips_rom / sector_len,
                                    4, 0, 0, 0, 0, be)) {
             fprintf(stderr, "qemu: Error registering flash memory.\n");
 	}
@@ -293,7 +294,7 @@ void mips_r4k_init(MachineState *machine)
     if (nd_table[0].used)
         isa_ne2000_init(isa_bus, 0x300, 9, &nd_table[0]);
 
-    ide_drive_get(hd, MAX_IDE_BUS);
+    ide_drive_get(hd, ARRAY_SIZE(hd));
     for(i = 0; i < MAX_IDE_BUS; i++)
         isa_ide_init(isa_bus, ide_iobase[i], ide_iobase2[i], ide_irq[i],
                      hd[MAX_IDE_DEVS * i],
diff --git a/hw/misc/ivshmem.c b/hw/misc/ivshmem.c
index 768e5288b..5d272c84e 100644
--- a/hw/misc/ivshmem.c
+++ b/hw/misc/ivshmem.c
@@ -24,10 +24,12 @@
 #include "migration/migration.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/event_notifier.h"
+#include "qemu/fifo8.h"
 #include "sysemu/char.h"
 
 #include <sys/mman.h>
 #include <sys/types.h>
+#include <limits.h>
 
 #define PCI_VENDOR_ID_IVSHMEM   PCI_VENDOR_ID_REDHAT_QUMRANET
 #define PCI_DEVICE_ID_IVSHMEM   0x1110
@@ -73,6 +75,7 @@ typedef struct IVShmemState {
 
     CharDriverState **eventfd_chr;
     CharDriverState *server_chr;
+    Fifo8 incoming_fifo;
     MemoryRegion ivshmem_mmio;
 
     /* We might need to register the BAR before we actually have the memory.
@@ -130,7 +133,7 @@ static void ivshmem_update_irq(IVShmemState *s, int val)
     /* don't print ISR resets */
     if (isr) {
         IVSHMEM_DPRINTF("Set IRQ to %d (%04x %04x)\n",
-           isr ? 1 : 0, s->intrstatus, s->intrmask);
+                        isr ? 1 : 0, s->intrstatus, s->intrmask);
     }
 
     pci_set_irq(d, (isr != 0));
@@ -297,8 +300,8 @@ static CharDriverState* create_eventfd_chr_device(void * opaque, EventNotifier *
     chr = qemu_chr_open_eventfd(eventfd);
 
     if (chr == NULL) {
-        fprintf(stderr, "creating eventfd for eventfd %d failed\n", eventfd);
-        exit(-1);
+        error_report("creating eventfd for eventfd %d failed", eventfd);
+        exit(1);
     }
     qemu_chr_fe_claim_no_fail(chr);
 
@@ -324,13 +327,16 @@ static int check_shm_size(IVShmemState *s, int fd) {
 
     struct stat buf;
 
-    fstat(fd, &buf);
+    if (fstat(fd, &buf) < 0) {
+        error_report("exiting: fstat on fd %d failed: %s",
+                     fd, strerror(errno));
+        return -1;
+    }
 
     if (s->ivshmem_size > buf.st_size) {
-        fprintf(stderr,
-                "IVSHMEM ERROR: Requested memory size greater"
-                " than shared object size (%" PRIu64 " > %" PRIu64")\n",
-                s->ivshmem_size, (uint64_t)buf.st_size);
+        error_report("Requested memory size greater"
+                     " than shared object size (%" PRIu64 " > %" PRIu64")",
+                     s->ivshmem_size, (uint64_t)buf.st_size);
         return -1;
     } else {
         return 0;
@@ -383,6 +389,9 @@ static void close_guest_eventfds(IVShmemState *s, int posn)
     if (!ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
         return;
     }
+    if (posn < 0 || posn >= s->nb_peers) {
+        return;
+    }
 
     guest_curr_max = s->peers[posn].nb_eventfds;
 
@@ -401,14 +410,24 @@ static void close_guest_eventfds(IVShmemState *s, int posn)
 
 /* this function increase the dynamic storage need to store data about other
  * guests */
-static void increase_dynamic_storage(IVShmemState *s, int new_min_size) {
+static int increase_dynamic_storage(IVShmemState *s, int new_min_size)
+{
 
     int j, old_nb_alloc;
 
+    /* check for integer overflow */
+    if (new_min_size >= INT_MAX / sizeof(Peer) - 1 || new_min_size <= 0) {
+        return -1;
+    }
+
     old_nb_alloc = s->nb_peers;
 
-    while (new_min_size >= s->nb_peers)
-        s->nb_peers = s->nb_peers * 2;
+    if (new_min_size >= s->nb_peers) {
+        /* +1 because #new_min_size is used as last array index */
+        s->nb_peers = new_min_size + 1;
+    } else {
+        return 0;
+    }
 
     IVSHMEM_DPRINTF("bumping storage to %d guests\n", s->nb_peers);
     s->peers = g_realloc(s->peers, s->nb_peers * sizeof(Peer));
@@ -418,23 +437,57 @@ static void increase_dynamic_storage(IVShmemState *s, int new_min_size) {
         s->peers[j].eventfds = NULL;
         s->peers[j].nb_eventfds = 0;
     }
+
+    return 0;
 }
 
-static void ivshmem_read(void *opaque, const uint8_t * buf, int flags)
+static void ivshmem_read(void *opaque, const uint8_t *buf, int size)
 {
     IVShmemState *s = opaque;
     int incoming_fd, tmp_fd;
     int guest_max_eventfd;
     long incoming_posn;
 
-    memcpy(&incoming_posn, buf, sizeof(long));
+    if (fifo8_is_empty(&s->incoming_fifo) && size == sizeof(incoming_posn)) {
+        memcpy(&incoming_posn, buf, size);
+    } else {
+        const uint8_t *p;
+        uint32_t num;
+
+        IVSHMEM_DPRINTF("short read of %d bytes\n", size);
+        num = MAX(size, sizeof(long) - fifo8_num_used(&s->incoming_fifo));
+        fifo8_push_all(&s->incoming_fifo, buf, num);
+        if (fifo8_num_used(&s->incoming_fifo) < sizeof(incoming_posn)) {
+            return;
+        }
+        size -= num;
+        buf += num;
+        p = fifo8_pop_buf(&s->incoming_fifo, sizeof(incoming_posn), &num);
+        g_assert(num == sizeof(incoming_posn));
+        memcpy(&incoming_posn, p, sizeof(incoming_posn));
+        if (size > 0) {
+            fifo8_push_all(&s->incoming_fifo, buf, size);
+        }
+    }
+
+    if (incoming_posn < -1) {
+        IVSHMEM_DPRINTF("invalid incoming_posn %ld\n", incoming_posn);
+        return;
+    }
+
     /* pick off s->server_chr->msgfd and store it, posn should accompany msg */
     tmp_fd = qemu_chr_fe_get_msgfd(s->server_chr);
     IVSHMEM_DPRINTF("posn is %ld, fd is %d\n", incoming_posn, tmp_fd);
 
     /* make sure we have enough space for this guest */
     if (incoming_posn >= s->nb_peers) {
-        increase_dynamic_storage(s, incoming_posn);
+        if (increase_dynamic_storage(s, incoming_posn) < 0) {
+            error_report("increase_dynamic_storage() failed");
+            if (tmp_fd != -1) {
+                close(tmp_fd);
+            }
+            return;
+        }
     }
 
     if (tmp_fd == -1) {
@@ -456,8 +509,8 @@ static void ivshmem_read(void *opaque, const uint8_t * buf, int flags)
     incoming_fd = dup(tmp_fd);
 
     if (incoming_fd == -1) {
-        fprintf(stderr, "could not allocate file descriptor %s\n",
-                                                            strerror(errno));
+        error_report("could not allocate file descriptor %s", strerror(errno));
+        close(tmp_fd);
         return;
     }
 
@@ -469,7 +522,7 @@ static void ivshmem_read(void *opaque, const uint8_t * buf, int flags)
         s->max_peer = 0;
 
         if (check_shm_size(s, incoming_fd) == -1) {
-            exit(-1);
+            exit(1);
         }
 
         /* mmap the region and map into the BAR2 */
@@ -479,8 +532,8 @@ static void ivshmem_read(void *opaque, const uint8_t * buf, int flags)
                                    "ivshmem.bar2", s->ivshmem_size, map_ptr);
         vmstate_register_ram(&s->ivshmem, DEVICE(s));
 
-        IVSHMEM_DPRINTF("guest h/w addr = %" PRIu64 ", size = %" PRIu64 "\n",
-                         s->ivshmem_offset, s->ivshmem_size);
+        IVSHMEM_DPRINTF("guest h/w addr = %p, size = %" PRIu64 "\n",
+                        map_ptr, s->ivshmem_size);
 
         memory_region_add_subregion(&s->bar, 0, &s->ivshmem);
 
@@ -501,7 +554,7 @@ static void ivshmem_read(void *opaque, const uint8_t * buf, int flags)
 
     /* this is an eventfd for a particular guest VM */
     IVSHMEM_DPRINTF("eventfds[%ld][%d] = %d\n", incoming_posn,
-                                            guest_max_eventfd, incoming_fd);
+                    guest_max_eventfd, incoming_fd);
     event_notifier_init_fd(&s->peers[incoming_posn].eventfds[guest_max_eventfd],
                            incoming_fd);
 
@@ -563,13 +616,13 @@ static uint64_t ivshmem_get_size(IVShmemState * s) {
             value <<= 30;
             break;
         default:
-            fprintf(stderr, "qemu: invalid ram size: %s\n", s->sizearg);
+            error_report("invalid ram size: %s", s->sizearg);
             exit(1);
     }
 
     /* BARs must be a power of 2 */
     if (!is_power_of_two(value)) {
-        fprintf(stderr, "ivshmem: size must be power of 2\n");
+        error_report("size must be power of 2");
         exit(1);
     }
 
@@ -621,7 +674,7 @@ static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)
     }
 
     if (proxy->role_val == IVSHMEM_PEER) {
-        fprintf(stderr, "ivshmem: 'peer' devices are not migratable\n");
+        error_report("'peer' devices are not migratable");
         return -EINVAL;
     }
 
@@ -659,13 +712,15 @@ static int pci_ivshmem_init(PCIDevice *dev)
         s->ivshmem_size = ivshmem_get_size(s);
     }
 
+    fifo8_create(&s->incoming_fifo, sizeof(long));
+
     register_savevm(DEVICE(dev), "ivshmem", 0, 0, ivshmem_save, ivshmem_load,
                                                                         dev);
 
     /* IRQFD requires MSI */
     if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD) &&
         !ivshmem_has_feature(s, IVSHMEM_MSI)) {
-        fprintf(stderr, "ivshmem: ioeventfd/irqfd requires MSI\n");
+        error_report("ioeventfd/irqfd requires MSI");
         exit(1);
     }
 
@@ -676,7 +731,7 @@ static int pci_ivshmem_init(PCIDevice *dev)
         } else if (strncmp(s->role, "master", 7) == 0) {
             s->role_val = IVSHMEM_MASTER;
         } else {
-            fprintf(stderr, "ivshmem: 'role' must be 'peer' or 'master'\n");
+            error_report("'role' must be 'peer' or 'master'");
             exit(1);
         }
     } else {
@@ -716,12 +771,12 @@ static int pci_ivshmem_init(PCIDevice *dev)
          * to the ivshmem server to receive the memory region */
 
         if (s->shmobj != NULL) {
-            fprintf(stderr, "WARNING: do not specify both 'chardev' "
-                                                "and 'shm' with ivshmem\n");
+            error_report("WARNING: do not specify both 'chardev' "
+                         "and 'shm' with ivshmem");
         }
 
         IVSHMEM_DPRINTF("using shared memory server (socket = %s)\n",
-                                                    s->server_chr->filename);
+                        s->server_chr->filename);
 
         if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
             ivshmem_setup_msi(s);
@@ -745,7 +800,7 @@ static int pci_ivshmem_init(PCIDevice *dev)
         int fd;
 
         if (s->shmobj == NULL) {
-            fprintf(stderr, "Must specify 'chardev' or 'shm' to ivshmem\n");
+            error_report("Must specify 'chardev' or 'shm' to ivshmem");
             exit(1);
         }
 
@@ -757,18 +812,18 @@ static int pci_ivshmem_init(PCIDevice *dev)
                         S_IRWXU|S_IRWXG|S_IRWXO)) > 0) {
            /* truncate file to length PCI device's memory */
             if (ftruncate(fd, s->ivshmem_size) != 0) {
-                fprintf(stderr, "ivshmem: could not truncate shared file\n");
+                error_report("could not truncate shared file");
             }
 
         } else if ((fd = shm_open(s->shmobj, O_CREAT|O_RDWR,
                         S_IRWXU|S_IRWXG|S_IRWXO)) < 0) {
-            fprintf(stderr, "ivshmem: could not open shared file\n");
-            exit(-1);
+            error_report("could not open shared file");
+            exit(1);
 
         }
 
         if (check_shm_size(s, fd) == -1) {
-            exit(-1);
+            exit(1);
         }
 
         create_shared_memory_BAR(s, fd);
@@ -789,12 +844,10 @@ static void pci_ivshmem_uninit(PCIDevice *dev)
         error_free(s->migration_blocker);
     }
 
-    memory_region_destroy(&s->ivshmem_mmio);
     memory_region_del_subregion(&s->bar, &s->ivshmem);
     vmstate_unregister_ram(&s->ivshmem, DEVICE(dev));
-    memory_region_destroy(&s->ivshmem);
-    memory_region_destroy(&s->bar);
     unregister_savevm(DEVICE(dev), "ivshmem", s);
+    fifo8_destroy(&s->incoming_fifo);
 }
 
 static Property ivshmem_properties[] = {
diff --git a/hw/misc/macio/cuda.c b/hw/misc/macio/cuda.c
index ff6051def..b4273aa17 100644
--- a/hw/misc/macio/cuda.c
+++ b/hw/misc/macio/cuda.c
@@ -123,13 +123,22 @@ static void cuda_update_irq(CUDAState *s)
     }
 }
 
+static uint64_t get_tb(uint64_t freq)
+{
+    return muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
+                    freq, get_ticks_per_sec());
+}
+
 static unsigned int get_counter(CUDATimer *s)
 {
     int64_t d;
     unsigned int counter;
+    uint64_t tb_diff;
+
+    /* Reverse of the tb calculation algorithm that Mac OS X uses on bootup. */
+    tb_diff = get_tb(s->frequency) - s->load_time;
+    d = (tb_diff * 0xBF401675E5DULL) / (s->frequency << 24);
 
-    d = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - s->load_time,
-                 CUDA_TIMER_FREQ, get_ticks_per_sec());
     if (s->index == 0) {
         /* the timer goes down from latch to -1 (period of latch + 2) */
         if (d <= (s->counter_value + 1)) {
@@ -147,7 +156,7 @@ static unsigned int get_counter(CUDATimer *s)
 static void set_counter(CUDAState *s, CUDATimer *ti, unsigned int val)
 {
     CUDA_DPRINTF("T%d.counter=%d\n", 1 + (ti->timer == NULL), val);
-    ti->load_time = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+    ti->load_time = get_tb(s->frequency);
     ti->counter_value = val;
     cuda_timer_update(s, ti, ti->load_time);
 }
@@ -688,6 +697,8 @@ static void cuda_realizefn(DeviceState *dev, Error **errp)
     struct tm tm;
 
     s->timers[0].timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, cuda_timer1, s);
+    s->timers[0].frequency = s->frequency;
+    s->timers[1].frequency = s->frequency;
 
     qemu_get_timedate(&tm, 0);
     s->tick_offset = (uint32_t)mktimegm(&tm) + RTC_OFFSET;
@@ -713,6 +724,11 @@ static void cuda_initfn(Object *obj)
                         DEVICE(obj), "adb.0");
 }
 
+static Property cuda_properties[] = {
+    DEFINE_PROP_UINT64("frequency", CUDAState, frequency, 0),
+    DEFINE_PROP_END_OF_LIST()
+};
+
 static void cuda_class_init(ObjectClass *oc, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(oc);
@@ -720,6 +736,7 @@ static void cuda_class_init(ObjectClass *oc, void *data)
     dc->realize = cuda_realizefn;
     dc->reset = cuda_reset;
     dc->vmsd = &vmstate_cuda;
+    dc->props = cuda_properties;
 }
 
 static const TypeInfo cuda_type_info = {
diff --git a/hw/misc/macio/macio.c b/hw/misc/macio/macio.c
index 47f45f5d2..e0f1e88f5 100644
--- a/hw/misc/macio/macio.c
+++ b/hw/misc/macio/macio.c
@@ -42,6 +42,7 @@ typedef struct MacIOState
     void *dbdma;
     MemoryRegion *pic_mem;
     MemoryRegion *escc_mem;
+    uint64_t frequency;
 } MacIOState;
 
 #define OLDWORLD_MACIO(obj) \
@@ -243,13 +244,18 @@ static void timer_write(void *opaque, hwaddr addr, uint64_t value,
 static uint64_t timer_read(void *opaque, hwaddr addr, unsigned size)
 {
     uint32_t value = 0;
+    uint64_t systime = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+    uint64_t kltime;
+
+    kltime = muldiv64(systime, 4194300, get_ticks_per_sec() * 4);
+    kltime = muldiv64(kltime, 18432000, 1048575);
 
     switch (addr) {
     case 0x38:
-        value = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+        value = kltime;
         break;
     case 0x3c:
-        value = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) >> 32;
+        value = kltime >> 32;
         break;
     }
 
@@ -346,12 +352,19 @@ static void macio_newworld_class_init(ObjectClass *oc, void *data)
     pdc->device_id = PCI_DEVICE_ID_APPLE_UNI_N_KEYL;
 }
 
+static Property macio_properties[] = {
+    DEFINE_PROP_UINT64("frequency", MacIOState, frequency, 0),
+    DEFINE_PROP_END_OF_LIST()
+};
+
 static void macio_class_init(ObjectClass *klass, void *data)
 {
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
     k->vendor_id = PCI_VENDOR_ID_APPLE;
     k->class_id = PCI_CLASS_OTHERS << 8;
+    dc->props = macio_properties;
 }
 
 static const TypeInfo macio_oldworld_type_info = {
@@ -398,6 +411,8 @@ void macio_init(PCIDevice *d,
     macio_state->escc_mem = escc_mem;
     /* Note: this code is strongly inspirated from the corresponding code
        in PearPC */
+    qdev_prop_set_uint64(DEVICE(&macio_state->cuda), "frequency",
+                         macio_state->frequency);
 
     qdev_init_nofail(DEVICE(d));
 }
diff --git a/hw/misc/omap_gpmc.c b/hw/misc/omap_gpmc.c
index cddea241d..a0de52f9b 100644
--- a/hw/misc/omap_gpmc.c
+++ b/hw/misc/omap_gpmc.c
@@ -436,7 +436,7 @@ static void omap_gpmc_cs_unmap(struct omap_gpmc_s *s, int cs)
     }
     memory_region_del_subregion(get_system_memory(), &f->container);
     memory_region_del_subregion(&f->container, omap_gpmc_cs_memregion(s, cs));
-    memory_region_destroy(&f->container);
+    object_unparent(OBJECT(&f->container));
 }
 
 void omap_gpmc_reset(struct omap_gpmc_s *s)
@@ -466,8 +466,6 @@ void omap_gpmc_reset(struct omap_gpmc_s *s)
         s->cs_file[i].config[3] = 0x10031003;
         s->cs_file[i].config[4] = 0x10f1111;
         s->cs_file[i].config[5] = 0;
-        s->cs_file[i].config[6] = 0xf00 | (i ? 0 : 1 << 6);
-
         s->cs_file[i].config[6] = 0xf00;
         /* In theory we could probe attached devices for some CFG1
          * bits here, but we just retain them across resets as they
diff --git a/hw/misc/pci-testdev.c b/hw/misc/pci-testdev.c
index ca53b3f50..c78a63e3a 100644
--- a/hw/misc/pci-testdev.c
+++ b/hw/misc/pci-testdev.c
@@ -293,8 +293,6 @@ pci_testdev_uninit(PCIDevice *dev)
         g_free(d->tests[i].hdr);
     }
     g_free(d->tests);
-    memory_region_destroy(&d->mmio);
-    memory_region_destroy(&d->portio);
 }
 
 static void qdev_pci_testdev_reset(DeviceState *dev)
diff --git a/hw/misc/vfio.c b/hw/misc/vfio.c
index 0b9eba0c8..fd318a122 100644
--- a/hw/misc/vfio.c
+++ b/hw/misc/vfio.c
@@ -120,11 +120,19 @@ typedef struct VFIOINTx {
 } VFIOINTx;
 
 typedef struct VFIOMSIVector {
-    EventNotifier interrupt; /* eventfd triggered on interrupt */
-    EventNotifier kvm_interrupt; /* eventfd triggered for KVM irqfd bypass */
+    /*
+     * Two interrupt paths are configured per vector.  The first, is only used
+     * for interrupts injected via QEMU.  This is typically the non-accel path,
+     * but may also be used when we want QEMU to handle masking and pending
+     * bits.  The KVM path bypasses QEMU and is therefore higher performance,
+     * but requires masking at the device.  virq is used to track the MSI route
+     * through KVM, thus kvm_interrupt is only available when virq is set to a
+     * valid (>= 0) value.
+     */
+    EventNotifier interrupt;
+    EventNotifier kvm_interrupt;
     struct VFIODevice *vdev; /* back pointer to device */
-    MSIMessage msg; /* cache the MSI message so we know when it changes */
-    int virq; /* KVM irqchip route for QEMU bypass */
+    int virq;
     bool use;
 } VFIOMSIVector;
 
@@ -681,13 +689,24 @@ static int vfio_enable_vectors(VFIODevice *vdev, bool msix)
     fds = (int32_t *)&irq_set->data;
 
     for (i = 0; i < vdev->nr_vectors; i++) {
-        if (!vdev->msi_vectors[i].use) {
-            fds[i] = -1;
-        } else if (vdev->msi_vectors[i].virq >= 0) {
-            fds[i] = event_notifier_get_fd(&vdev->msi_vectors[i].kvm_interrupt);
-        } else {
-            fds[i] = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt);
+        int fd = -1;
+
+        /*
+         * MSI vs MSI-X - The guest has direct access to MSI mask and pending
+         * bits, therefore we always use the KVM signaling path when setup.
+         * MSI-X mask and pending bits are emulated, so we want to use the
+         * KVM signaling path only when configured and unmasked.
+         */
+        if (vdev->msi_vectors[i].use) {
+            if (vdev->msi_vectors[i].virq < 0 ||
+                (msix && msix_is_masked(&vdev->pdev, i))) {
+                fd = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt);
+            } else {
+                fd = event_notifier_get_fd(&vdev->msi_vectors[i].kvm_interrupt);
+            }
         }
+
+        fds[i] = fd;
     }
 
     ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
@@ -724,7 +743,6 @@ static void vfio_add_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage *msg,
         return;
     }
 
-    vector->msg = *msg;
     vector->virq = virq;
 }
 
@@ -740,7 +758,6 @@ static void vfio_remove_kvm_msi_virq(VFIOMSIVector *vector)
 static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg)
 {
     kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg);
-    vector->msg = msg;
 }
 
 static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
@@ -919,6 +936,7 @@ retry:
 
     for (i = 0; i < vdev->nr_vectors; i++) {
         VFIOMSIVector *vector = &vdev->msi_vectors[i];
+        MSIMessage msg = msi_get_message(&vdev->pdev, i);
 
         vector->vdev = vdev;
         vector->virq = -1;
@@ -931,13 +949,11 @@ retry:
         qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
                             vfio_msi_interrupt, NULL, vector);
 
-        vector->msg = msi_get_message(&vdev->pdev, i);
-
         /*
          * Attempt to enable route through KVM irqchip,
          * default to userspace handling if unavailable.
          */
-        vfio_add_kvm_msi_virq(vector, &vector->msg, false);
+        vfio_add_kvm_msi_virq(vector, &msg, false);
     }
 
     /* Set interrupt type prior to possible interrupts */
@@ -1082,10 +1098,10 @@ static void vfio_bar_write(void *opaque, hwaddr addr,
         buf.byte = data;
         break;
     case 2:
-        buf.word = data;
+        buf.word = cpu_to_le16(data);
         break;
     case 4:
-        buf.dword = data;
+        buf.dword = cpu_to_le32(data);
         break;
     default:
         hw_error("vfio: unsupported write size, %d bytes", size);
@@ -1142,10 +1158,10 @@ static uint64_t vfio_bar_read(void *opaque,
         data = buf.byte;
         break;
     case 2:
-        data = buf.word;
+        data = le16_to_cpu(buf.word);
         break;
     case 4:
-        data = buf.dword;
+        data = le32_to_cpu(buf.dword);
         break;
     default:
         hw_error("vfio: unsupported read size, %d bytes", size);
@@ -1172,7 +1188,7 @@ static uint64_t vfio_bar_read(void *opaque,
 static const MemoryRegionOps vfio_bar_ops = {
     .read = vfio_bar_read,
     .write = vfio_bar_write,
-    .endianness = DEVICE_NATIVE_ENDIAN,
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static void vfio_pci_load_rom(VFIODevice *vdev)
@@ -1239,7 +1255,7 @@ static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size)
         uint16_t word;
         uint32_t dword;
         uint64_t qword;
-    } buf;
+    } val;
     uint64_t data = 0;
 
     /* Load the ROM lazily when the guest tries to read it */
@@ -1247,21 +1263,21 @@ static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size)
         vfio_pci_load_rom(vdev);
     }
 
-    memcpy(&buf, vdev->rom + addr,
+    memcpy(&val, vdev->rom + addr,
            (addr < vdev->rom_size) ? MIN(size, vdev->rom_size - addr) : 0);
 
     switch (size) {
     case 1:
-        data = buf.byte;
+        data = val.byte;
         break;
     case 2:
-        data = buf.word;
+        data = le16_to_cpu(val.word);
         break;
     case 4:
-        data = buf.dword;
+        data = le32_to_cpu(val.dword);
         break;
     default:
-        hw_error("vfio: unsupported read size, %d bytes", size);
+        hw_error("vfio: unsupported read size, %d bytes\n", size);
         break;
     }
 
@@ -1280,7 +1296,7 @@ static void vfio_rom_write(void *opaque, hwaddr addr,
 static const MemoryRegionOps vfio_rom_ops = {
     .read = vfio_rom_read,
     .write = vfio_rom_write,
-    .endianness = DEVICE_NATIVE_ENDIAN,
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static bool vfio_blacklist_opt_rom(VFIODevice *vdev)
@@ -2178,9 +2194,13 @@ static void vfio_probe_nvidia_bar0_88000_quirk(VFIODevice *vdev, int nr)
 {
     PCIDevice *pdev = &vdev->pdev;
     VFIOQuirk *quirk;
+    uint16_t vendor, class;
 
-    if (!vdev->has_vga || nr != 0 ||
-        pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA) {
+    vendor = pci_get_word(pdev->config + PCI_VENDOR_ID);
+    class = pci_get_word(pdev->config + PCI_CLASS_DEVICE);
+
+    if (nr != 0 || vendor != PCI_VENDOR_ID_NVIDIA ||
+        class != PCI_CLASS_DISPLAY_VGA) {
         return;
     }
 
@@ -2266,7 +2286,7 @@ static void vfio_vga_quirk_teardown(VFIODevice *vdev)
         while (!QLIST_EMPTY(&vdev->vga.region[i].quirks)) {
             VFIOQuirk *quirk = QLIST_FIRST(&vdev->vga.region[i].quirks);
             memory_region_del_subregion(&vdev->vga.region[i].mem, &quirk->mem);
-            memory_region_destroy(&quirk->mem);
+            object_unparent(OBJECT(&quirk->mem));
             QLIST_REMOVE(quirk, next);
             g_free(quirk);
         }
@@ -2290,7 +2310,7 @@ static void vfio_bar_quirk_teardown(VFIODevice *vdev, int nr)
     while (!QLIST_EMPTY(&bar->quirks)) {
         VFIOQuirk *quirk = QLIST_FIRST(&bar->quirks);
         memory_region_del_subregion(&bar->mem, &quirk->mem);
-        memory_region_destroy(&quirk->mem);
+        object_unparent(OBJECT(&quirk->mem));
         QLIST_REMOVE(quirk, next);
         g_free(quirk);
     }
@@ -2857,15 +2877,11 @@ static void vfio_unmap_bar(VFIODevice *vdev, int nr)
 
     memory_region_del_subregion(&bar->mem, &bar->mmap_mem);
     munmap(bar->mmap, memory_region_size(&bar->mmap_mem));
-    memory_region_destroy(&bar->mmap_mem);
 
     if (vdev->msix && vdev->msix->table_bar == nr) {
         memory_region_del_subregion(&bar->mem, &vdev->msix->mmap_mem);
         munmap(vdev->msix->mmap, memory_region_size(&vdev->msix->mmap_mem));
-        memory_region_destroy(&vdev->msix->mmap_mem);
     }
-
-    memory_region_destroy(&bar->mem);
 }
 
 static int vfio_mmap_bar(VFIODevice *vdev, VFIOBAR *bar,
@@ -2895,6 +2911,7 @@ static int vfio_mmap_bar(VFIODevice *vdev, VFIOBAR *bar,
         }
 
         memory_region_init_ram_ptr(submem, OBJECT(vdev), name, size, *map);
+        memory_region_set_skip_dump(submem);
     } else {
 empty_region:
         /* Create a zero sized sub-region to make cleanup easy. */
@@ -3018,9 +3035,6 @@ static void vfio_unmap_bars(VFIODevice *vdev)
     if (vdev->has_vga) {
         vfio_vga_quirk_teardown(vdev);
         pci_unregister_vga(&vdev->pdev);
-        memory_region_destroy(&vdev->vga.region[QEMU_PCI_VGA_MEM].mem);
-        memory_region_destroy(&vdev->vga.region[QEMU_PCI_VGA_IO_LO].mem);
-        memory_region_destroy(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem);
     }
 }
 
@@ -4283,7 +4297,6 @@ static int vfio_initfn(PCIDevice *pdev)
         }
     }
 
-    add_boot_device_path(vdev->bootindex, &pdev->qdev, NULL);
     vfio_register_err_notifier(vdev);
 
     return 0;
@@ -4352,13 +4365,22 @@ post_reset:
     vfio_pci_post_reset(vdev);
 }
 
+static void vfio_instance_init(Object *obj)
+{
+    PCIDevice *pci_dev = PCI_DEVICE(obj);
+    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, PCI_DEVICE(obj));
+
+    device_add_bootindex_property(obj, &vdev->bootindex,
+                                  "bootindex", NULL,
+                                  &pci_dev->qdev, NULL);
+}
+
 static Property vfio_pci_dev_properties[] = {
     DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIODevice, host),
     DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIODevice,
                        intx.mmap_timeout, 1100),
     DEFINE_PROP_BIT("x-vga", VFIODevice, features,
                     VFIO_FEATURE_ENABLE_VGA_BIT, false),
-    DEFINE_PROP_INT32("bootindex", VFIODevice, bootindex, -1),
     /*
      * TODO - support passed fds... is this necessary?
      * DEFINE_PROP_STRING("vfiofd", VFIODevice, vfiofd_name),
@@ -4394,6 +4416,7 @@ static const TypeInfo vfio_pci_dev_info = {
     .parent = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VFIODevice),
     .class_init = vfio_pci_dev_class_init,
+    .instance_init = vfio_instance_init,
 };
 
 static void register_vfio_pci_dev_type(void)
diff --git a/hw/moxie/moxiesim.c b/hw/moxie/moxiesim.c
index 430f8410d..80bcc5b4f 100644
--- a/hw/moxie/moxiesim.c
+++ b/hw/moxie/moxiesim.c
@@ -123,11 +123,11 @@ static void moxiesim_init(MachineState *machine)
     qemu_register_reset(main_cpu_reset, cpu);
 
     /* Allocate RAM. */
-    memory_region_init_ram(ram, NULL, "moxiesim.ram", ram_size);
+    memory_region_init_ram(ram, NULL, "moxiesim.ram", ram_size, &error_abort);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(address_space_mem, ram_base, ram);
 
-    memory_region_init_ram(rom, NULL, "moxie.rom", 128*0x1000);
+    memory_region_init_ram(rom, NULL, "moxie.rom", 128*0x1000, &error_abort);
     vmstate_register_ram_global(rom);
     memory_region_add_subregion(get_system_memory(), 0x1000, rom);
 
diff --git a/hw/net/dp8393x.c b/hw/net/dp8393x.c
index 789d38574..7eab7ad0c 100644
--- a/hw/net/dp8393x.c
+++ b/hw/net/dp8393x.c
@@ -863,9 +863,6 @@ static void nic_cleanup(NetClientState *nc)
 {
     dp8393xState *s = qemu_get_nic_opaque(nc);
 
-    memory_region_del_subregion(s->address_space, &s->mmio);
-    memory_region_destroy(&s->mmio);
-
     timer_del(s->watchdog);
     timer_free(s->watchdog);
 
diff --git a/hw/net/e1000.c b/hw/net/e1000.c
index 0fc29a0ae..e33a4da9f 100644
--- a/hw/net/e1000.c
+++ b/hw/net/e1000.c
@@ -186,21 +186,31 @@ e1000_link_up(E1000State *s)
     s->phy_reg[PHY_STATUS] |= MII_SR_LINK_STATUS;
 }
 
+static bool
+have_autoneg(E1000State *s)
+{
+    return (s->compat_flags & E1000_FLAG_AUTONEG) &&
+           (s->phy_reg[PHY_CTRL] & MII_CR_AUTO_NEG_EN);
+}
+
 static void
 set_phy_ctrl(E1000State *s, int index, uint16_t val)
 {
+    /* bits 0-5 reserved; MII_CR_[RESTART_AUTO_NEG,RESET] are self clearing */
+    s->phy_reg[PHY_CTRL] = val & ~(0x3f |
+                                   MII_CR_RESET |
+                                   MII_CR_RESTART_AUTO_NEG);
+
     /*
      * QEMU 1.3 does not support link auto-negotiation emulation, so if we
      * migrate during auto negotiation, after migration the link will be
      * down.
      */
-    if (!(s->compat_flags & E1000_FLAG_AUTONEG)) {
-        return;
-    }
-    if ((val & MII_CR_AUTO_NEG_EN) && (val & MII_CR_RESTART_AUTO_NEG)) {
+    if (have_autoneg(s) && (val & MII_CR_RESTART_AUTO_NEG)) {
         e1000_link_down(s);
         DBGOUT(PHY, "Start link auto negotiation\n");
-        timer_mod(s->autoneg_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 500);
+        timer_mod(s->autoneg_timer,
+                  qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 500);
     }
 }
 
@@ -223,13 +233,30 @@ static const char phy_regcap[0x20] = {
 
 /* PHY_ID2 documented in 8254x_GBe_SDM.pdf, pp. 250 */
 static const uint16_t phy_reg_init[] = {
-    [PHY_CTRL] = 0x1140,
-    [PHY_STATUS] = 0x794d, /* link initially up with not completed autoneg */
-    [PHY_ID1] = 0x141, /* [PHY_ID2] configured per DevId, from e1000_reset() */
-    [PHY_1000T_CTRL] = 0x0e00,			[M88E1000_PHY_SPEC_CTRL] = 0x360,
-    [M88E1000_EXT_PHY_SPEC_CTRL] = 0x0d60,	[PHY_AUTONEG_ADV] = 0xde1,
-    [PHY_LP_ABILITY] = 0x1e0,			[PHY_1000T_STATUS] = 0x3c00,
+    [PHY_CTRL] =   MII_CR_SPEED_SELECT_MSB |
+                   MII_CR_FULL_DUPLEX |
+                   MII_CR_AUTO_NEG_EN,
+
+    [PHY_STATUS] = MII_SR_EXTENDED_CAPS |
+                   MII_SR_LINK_STATUS |   /* link initially up */
+                   MII_SR_AUTONEG_CAPS |
+                   /* MII_SR_AUTONEG_COMPLETE: initially NOT completed */
+                   MII_SR_PREAMBLE_SUPPRESS |
+                   MII_SR_EXTENDED_STATUS |
+                   MII_SR_10T_HD_CAPS |
+                   MII_SR_10T_FD_CAPS |
+                   MII_SR_100X_HD_CAPS |
+                   MII_SR_100X_FD_CAPS,
+
+    [PHY_ID1] = 0x141,
+    /* [PHY_ID2] configured per DevId, from e1000_reset() */
+    [PHY_AUTONEG_ADV] = 0xde1,
+    [PHY_LP_ABILITY] = 0x1e0,
+    [PHY_1000T_CTRL] = 0x0e00,
+    [PHY_1000T_STATUS] = 0x3c00,
+    [M88E1000_PHY_SPEC_CTRL] = 0x360,
     [M88E1000_PHY_SPEC_STATUS] = 0xac00,
+    [M88E1000_EXT_PHY_SPEC_CTRL] = 0x0d60,
 };
 
 static const uint32_t mac_reg_init[] = {
@@ -446,8 +473,9 @@ set_mdic(E1000State *s, int index, uint32_t val)
         } else {
             if (addr < NPHYWRITEOPS && phyreg_writeops[addr]) {
                 phyreg_writeops[addr](s, index, data);
+            } else {
+                s->phy_reg[addr] = data;
             }
-            s->phy_reg[addr] = data;
         }
     }
     s->mac_reg[MDIC] = val | E1000_MDIC_READY;
@@ -848,14 +876,6 @@ receive_filter(E1000State *s, const uint8_t *buf, int size)
     return 0;
 }
 
-static bool
-have_autoneg(E1000State *s)
-{
-    return (s->compat_flags & E1000_FLAG_AUTONEG) &&
-           (s->phy_reg[PHY_CTRL] & MII_CR_AUTO_NEG_EN) &&
-           (s->phy_reg[PHY_CTRL] & MII_CR_RESTART_AUTO_NEG);
-}
-
 static void
 e1000_set_link_status(NetClientState *nc)
 {
@@ -1496,8 +1516,6 @@ pci_e1000_uninit(PCIDevice *dev)
     timer_free(d->autoneg_timer);
     timer_del(d->mit_timer);
     timer_free(d->mit_timer);
-    memory_region_destroy(&d->mmio);
-    memory_region_destroy(&d->io);
     qemu_del_nic(d->nic);
 }
 
@@ -1551,8 +1569,6 @@ static int pci_e1000_init(PCIDevice *pci_dev)
 
     qemu_format_nic_info_str(qemu_get_queue(d->nic), macaddr);
 
-    add_boot_device_path(d->conf.bootindex, dev, "/ethernet-phy@0");
-
     d->autoneg_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL, e1000_autoneg_timer, d);
     d->mit_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, e1000_mit_timer, d);
 
@@ -1603,10 +1619,19 @@ static void e1000_class_init(ObjectClass *klass, void *data)
     dc->props = e1000_properties;
 }
 
+static void e1000_instance_init(Object *obj)
+{
+    E1000State *n = E1000(obj);
+    device_add_bootindex_property(obj, &n->conf.bootindex,
+                                  "bootindex", "/ethernet-phy@0",
+                                  DEVICE(n), NULL);
+}
+
 static const TypeInfo e1000_base_info = {
     .name          = TYPE_E1000_BASE,
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(E1000State),
+    .instance_init = e1000_instance_init,
     .class_size    = sizeof(E1000BaseClass),
     .abstract      = true,
 };
@@ -1650,6 +1675,7 @@ static void e1000_register_types(void)
         type_info.parent = TYPE_E1000_BASE;
         type_info.class_data = (void *)info;
         type_info.class_init = e1000_class_init;
+        type_info.instance_init = e1000_instance_init;
 
         type_register(&type_info);
     }
diff --git a/hw/net/eepro100.c b/hw/net/eepro100.c
index 3263e3fe9..4877bfd4d 100644
--- a/hw/net/eepro100.c
+++ b/hw/net/eepro100.c
@@ -1843,9 +1843,6 @@ static void pci_nic_uninit(PCIDevice *pci_dev)
 {
     EEPRO100State *s = DO_UPCAST(EEPRO100State, dev, pci_dev);
 
-    memory_region_destroy(&s->mmio_bar);
-    memory_region_destroy(&s->io_bar);
-    memory_region_destroy(&s->flash_bar);
     vmstate_unregister(&pci_dev->qdev, s->vmstate, s);
     eeprom93xx_free(&pci_dev->qdev, s->eeprom);
     qemu_del_nic(s->nic);
@@ -1904,11 +1901,17 @@ static int e100_nic_init(PCIDevice *pci_dev)
     s->vmstate->name = qemu_get_queue(s->nic)->model;
     vmstate_register(&pci_dev->qdev, -1, s->vmstate, s);
 
-    add_boot_device_path(s->conf.bootindex, &pci_dev->qdev, "/ethernet-phy@0");
-
     return 0;
 }
 
+static void eepro100_instance_init(Object *obj)
+{
+    EEPRO100State *s = DO_UPCAST(EEPRO100State, dev, PCI_DEVICE(obj));
+    device_add_bootindex_property(obj, &s->conf.bootindex,
+                                  "bootindex", "/ethernet-phy@0",
+                                  DEVICE(s), NULL);
+}
+
 static E100PCIDeviceInfo e100_devices[] = {
     {
         .name = "i82550",
@@ -2107,7 +2110,8 @@ static void eepro100_register_types(void)
         type_info.parent = TYPE_PCI_DEVICE;
         type_info.class_init = eepro100_class_init;
         type_info.instance_size = sizeof(EEPRO100State);
-        
+        type_info.instance_init = eepro100_instance_init;
+
         type_register(&type_info);
     }
 }
diff --git a/hw/net/lance.c b/hw/net/lance.c
index 7811a9edc..a1c49f1b9 100644
--- a/hw/net/lance.c
+++ b/hw/net/lance.c
@@ -42,6 +42,7 @@
 #include "hw/sparc/sun4m.h"
 #include "pcnet.h"
 #include "trace.h"
+#include "sysemu/sysemu.h"
 
 #define TYPE_LANCE "lance"
 #define SYSBUS_PCNET(obj) \
@@ -143,6 +144,16 @@ static void lance_reset(DeviceState *dev)
     pcnet_h_reset(&d->state);
 }
 
+static void lance_instance_init(Object *obj)
+{
+    SysBusPCNetState *d = SYSBUS_PCNET(obj);
+    PCNetState *s = &d->state;
+
+    device_add_bootindex_property(obj, &s->conf.bootindex,
+                                  "bootindex", "/ethernet-phy@0",
+                                  DEVICE(obj), NULL);
+}
+
 static Property lance_properties[] = {
     DEFINE_PROP_PTR("dma", SysBusPCNetState, state.dma_opaque),
     DEFINE_NIC_PROPERTIES(SysBusPCNetState, state.conf),
@@ -169,6 +180,7 @@ static const TypeInfo lance_info = {
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysBusPCNetState),
     .class_init    = lance_class_init,
+    .instance_init = lance_instance_init,
 };
 
 static void lance_register_types(void)
diff --git a/hw/net/mcf_fec.c b/hw/net/mcf_fec.c
index 4bff3de34..22cd7cf87 100644
--- a/hw/net/mcf_fec.c
+++ b/hw/net/mcf_fec.c
@@ -443,9 +443,6 @@ static void mcf_fec_cleanup(NetClientState *nc)
 {
     mcf_fec_state *s = qemu_get_nic_opaque(nc);
 
-    memory_region_del_subregion(s->sysmem, &s->iomem);
-    memory_region_destroy(&s->iomem);
-
     g_free(s);
 }
 
diff --git a/hw/net/milkymist-minimac2.c b/hw/net/milkymist-minimac2.c
index c023351c0..c6326728e 100644
--- a/hw/net/milkymist-minimac2.c
+++ b/hw/net/milkymist-minimac2.c
@@ -472,7 +472,7 @@ static int milkymist_minimac2_init(SysBusDevice *sbd)
 
     /* register buffers memory */
     memory_region_init_ram(&s->buffers, OBJECT(dev), "milkymist-minimac2.buffers",
-                           buffers_size);
+                           buffers_size, &error_abort);
     vmstate_register_ram_global(&s->buffers);
     s->rx0_buf = memory_region_get_ram_ptr(&s->buffers);
     s->rx1_buf = s->rx0_buf + MINIMAC2_BUFFER_SIZE;
diff --git a/hw/net/ne2000-isa.c b/hw/net/ne2000-isa.c
index 6eb1dac8d..82e2ba17c 100644
--- a/hw/net/ne2000-isa.c
+++ b/hw/net/ne2000-isa.c
@@ -28,6 +28,7 @@
 #include "net/net.h"
 #include "ne2000.h"
 #include "exec/address-spaces.h"
+#include "qapi/visitor.h"
 
 #define TYPE_ISA_NE2000 "ne2k_isa"
 #define ISA_NE2000(obj) OBJECT_CHECK(ISANE2000State, (obj), TYPE_ISA_NE2000)
@@ -101,11 +102,54 @@ static void isa_ne2000_class_initfn(ObjectClass *klass, void *data)
     set_bit(DEVICE_CATEGORY_NETWORK, dc->categories);
 }
 
+static void isa_ne2000_get_bootindex(Object *obj, Visitor *v, void *opaque,
+                                     const char *name, Error **errp)
+{
+    ISANE2000State *isa = ISA_NE2000(obj);
+    NE2000State *s = &isa->ne2000;
+
+    visit_type_int32(v, &s->c.bootindex, name, errp);
+}
+
+static void isa_ne2000_set_bootindex(Object *obj, Visitor *v, void *opaque,
+                                     const char *name, Error **errp)
+{
+    ISANE2000State *isa = ISA_NE2000(obj);
+    NE2000State *s = &isa->ne2000;
+    int32_t boot_index;
+    Error *local_err = NULL;
+
+    visit_type_int32(v, &boot_index, name, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    /* check whether bootindex is present in fw_boot_order list  */
+    check_boot_index(boot_index, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    /* change bootindex to a new one */
+    s->c.bootindex = boot_index;
+
+out:
+    if (local_err) {
+        error_propagate(errp, local_err);
+    }
+}
+
+static void isa_ne2000_instance_init(Object *obj)
+{
+    object_property_add(obj, "bootindex", "int32",
+                        isa_ne2000_get_bootindex,
+                        isa_ne2000_set_bootindex, NULL, NULL, NULL);
+    object_property_set_int(obj, -1, "bootindex", NULL);
+}
 static const TypeInfo ne2000_isa_info = {
     .name          = TYPE_ISA_NE2000,
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(ISANE2000State),
     .class_init    = isa_ne2000_class_initfn,
+    .instance_init = isa_ne2000_instance_init,
 };
 
 static void ne2000_isa_register_types(void)
diff --git a/hw/net/ne2000.c b/hw/net/ne2000.c
index d558b8c67..3ab2d0369 100644
--- a/hw/net/ne2000.c
+++ b/hw/net/ne2000.c
@@ -738,8 +738,6 @@ static int pci_ne2000_init(PCIDevice *pci_dev)
                           object_get_typename(OBJECT(pci_dev)), pci_dev->qdev.id, s);
     qemu_format_nic_info_str(qemu_get_queue(s->nic), s->c.macaddr.a);
 
-    add_boot_device_path(s->c.bootindex, &pci_dev->qdev, "/ethernet-phy@0");
-
     return 0;
 }
 
@@ -748,11 +746,21 @@ static void pci_ne2000_exit(PCIDevice *pci_dev)
     PCINE2000State *d = DO_UPCAST(PCINE2000State, dev, pci_dev);
     NE2000State *s = &d->ne2000;
 
-    memory_region_destroy(&s->io);
     qemu_del_nic(s->nic);
     qemu_free_irq(s->irq);
 }
 
+static void ne2000_instance_init(Object *obj)
+{
+    PCIDevice *pci_dev = PCI_DEVICE(obj);
+    PCINE2000State *d = DO_UPCAST(PCINE2000State, dev, pci_dev);
+    NE2000State *s = &d->ne2000;
+
+    device_add_bootindex_property(obj, &s->c.bootindex,
+                                  "bootindex", "/ethernet-phy@0",
+                                  &pci_dev->qdev, NULL);
+}
+
 static Property ne2000_properties[] = {
     DEFINE_NIC_PROPERTIES(PCINE2000State, ne2000.c),
     DEFINE_PROP_END_OF_LIST(),
@@ -779,6 +787,7 @@ static const TypeInfo ne2000_info = {
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCINE2000State),
     .class_init    = ne2000_class_init,
+    .instance_init = ne2000_instance_init,
 };
 
 static void ne2000_register_types(void)
diff --git a/hw/net/pcnet-pci.c b/hw/net/pcnet-pci.c
index b25d789d2..fb5f5d623 100644
--- a/hw/net/pcnet-pci.c
+++ b/hw/net/pcnet-pci.c
@@ -32,6 +32,7 @@
 #include "hw/loader.h"
 #include "qemu/timer.h"
 #include "sysemu/dma.h"
+#include "sysemu/sysemu.h"
 
 #include "pcnet.h"
 
@@ -282,8 +283,6 @@ static void pci_pcnet_uninit(PCIDevice *dev)
     PCIPCNetState *d = PCI_PCNET(dev);
 
     qemu_free_irq(d->state.irq);
-    memory_region_destroy(&d->state.mmio);
-    memory_region_destroy(&d->io_bar);
     timer_del(d->state.poll_timer);
     timer_free(d->state.poll_timer);
     qemu_del_nic(d->state.nic);
@@ -346,6 +345,16 @@ static void pci_reset(DeviceState *dev)
     pcnet_h_reset(&d->state);
 }
 
+static void pcnet_instance_init(Object *obj)
+{
+    PCIPCNetState *d = PCI_PCNET(obj);
+    PCNetState *s = &d->state;
+
+    device_add_bootindex_property(obj, &s->conf.bootindex,
+                                  "bootindex", "/ethernet-phy@0",
+                                  DEVICE(obj), NULL);
+}
+
 static Property pcnet_properties[] = {
     DEFINE_NIC_PROPERTIES(PCIPCNetState, state.conf),
     DEFINE_PROP_END_OF_LIST(),
@@ -374,6 +383,7 @@ static const TypeInfo pcnet_info = {
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIPCNetState),
     .class_init    = pcnet_class_init,
+    .instance_init = pcnet_instance_init,
 };
 
 static void pci_pcnet_register_types(void)
diff --git a/hw/net/pcnet.c b/hw/net/pcnet.c
index 5299d52a8..f409b9293 100644
--- a/hw/net/pcnet.c
+++ b/hw/net/pcnet.c
@@ -1212,7 +1212,7 @@ static void pcnet_transmit(PCNetState *s)
     hwaddr xmit_cxda = 0;
     int count = CSR_XMTRL(s)-1;
     int add_crc = 0;
-
+    int bcnt;
     s->xmit_pos = -1;
 
     if (!CSR_TXON(s)) {
@@ -1247,35 +1247,40 @@ static void pcnet_transmit(PCNetState *s)
             s->xmit_pos = -1;
             goto txdone;
         }
+
+        if (s->xmit_pos < 0) {
+            goto txdone;
+        }
+
+        bcnt = 4096 - GET_FIELD(tmd.length, TMDL, BCNT);
+        s->phys_mem_read(s->dma_opaque, PHYSADDR(s, tmd.tbadr),
+                         s->buffer + s->xmit_pos, bcnt, CSR_BSWP(s));
+        s->xmit_pos += bcnt;
+        
         if (!GET_FIELD(tmd.status, TMDS, ENP)) {
-            int bcnt = 4096 - GET_FIELD(tmd.length, TMDL, BCNT);
-            s->phys_mem_read(s->dma_opaque, PHYSADDR(s, tmd.tbadr),
-                             s->buffer + s->xmit_pos, bcnt, CSR_BSWP(s));
-            s->xmit_pos += bcnt;
-        } else if (s->xmit_pos >= 0) {
-            int bcnt = 4096 - GET_FIELD(tmd.length, TMDL, BCNT);
-            s->phys_mem_read(s->dma_opaque, PHYSADDR(s, tmd.tbadr),
-                             s->buffer + s->xmit_pos, bcnt, CSR_BSWP(s));
-            s->xmit_pos += bcnt;
+            goto txdone;
+        }
+
 #ifdef PCNET_DEBUG
-            printf("pcnet_transmit size=%d\n", s->xmit_pos);
+        printf("pcnet_transmit size=%d\n", s->xmit_pos);
 #endif
-            if (CSR_LOOP(s)) {
-                if (BCR_SWSTYLE(s) == 1)
-                    add_crc = !GET_FIELD(tmd.status, TMDS, NOFCS);
-                s->looptest = add_crc ? PCNET_LOOPTEST_CRC : PCNET_LOOPTEST_NOCRC;
-                pcnet_receive(qemu_get_queue(s->nic), s->buffer, s->xmit_pos);
-                s->looptest = 0;
-            } else
-                if (s->nic)
-                    qemu_send_packet(qemu_get_queue(s->nic), s->buffer,
-                                     s->xmit_pos);
-
-            s->csr[0] &= ~0x0008;   /* clear TDMD */
-            s->csr[4] |= 0x0004;    /* set TXSTRT */
-            s->xmit_pos = -1;
+        if (CSR_LOOP(s)) {
+            if (BCR_SWSTYLE(s) == 1)
+                add_crc = !GET_FIELD(tmd.status, TMDS, NOFCS);
+            s->looptest = add_crc ? PCNET_LOOPTEST_CRC : PCNET_LOOPTEST_NOCRC;
+            pcnet_receive(qemu_get_queue(s->nic), s->buffer, s->xmit_pos);
+            s->looptest = 0;
+        } else {
+            if (s->nic) {
+                qemu_send_packet(qemu_get_queue(s->nic), s->buffer,
+                                 s->xmit_pos);
+            }
         }
 
+        s->csr[0] &= ~0x0008;   /* clear TDMD */
+        s->csr[4] |= 0x0004;    /* set TXSTRT */
+        s->xmit_pos = -1;
+
     txdone:
         SET_FIELD(&tmd.status, TMDS, OWN, 0);
         TMDSTORE(&tmd, PHYSADDR(s,CSR_CXDA(s)));
@@ -1735,8 +1740,6 @@ int pcnet_common_init(DeviceState *dev, PCNetState *s, NetClientInfo *info)
     s->nic = qemu_new_nic(info, &s->conf, object_get_typename(OBJECT(dev)), dev->id, s);
     qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
 
-    add_boot_device_path(s->conf.bootindex, dev, "/ethernet-phy@0");
-
     /* Initialize the PROM */
 
     /*
diff --git a/hw/net/pcnet.h b/hw/net/pcnet.h
index 9dee6f3e2..f8e8a6f6b 100644
--- a/hw/net/pcnet.h
+++ b/hw/net/pcnet.h
@@ -66,5 +66,4 @@ void pcnet_set_link_status(NetClientState *nc);
 void pcnet_common_cleanup(PCNetState *d);
 int pcnet_common_init(DeviceState *dev, PCNetState *s, NetClientInfo *info);
 extern const VMStateDescription vmstate_pcnet;
-
 #endif
diff --git a/hw/net/rtl8139.c b/hw/net/rtl8139.c
index 90bc5ecdd..5f0197c9d 100644
--- a/hw/net/rtl8139.c
+++ b/hw/net/rtl8139.c
@@ -1775,6 +1775,7 @@ static void rtl8139_transfer_frame(RTL8139State *s, uint8_t *buf, int size,
     int do_interrupt, const uint8_t *dot1q_buf)
 {
     struct iovec *iov = NULL;
+    struct iovec vlan_iov[3];
 
     if (!size)
     {
@@ -1789,6 +1790,9 @@ static void rtl8139_transfer_frame(RTL8139State *s, uint8_t *buf, int size,
             { .iov_base = buf + ETHER_ADDR_LEN * 2,
                 .iov_len = size - ETHER_ADDR_LEN * 2 },
         };
+
+        memcpy(vlan_iov, iov, sizeof(vlan_iov));
+        iov = vlan_iov;
     }
 
     if (TxLoopBack == (s->TxConfig & TxLoopBack))
@@ -3462,8 +3466,6 @@ static void pci_rtl8139_uninit(PCIDevice *dev)
 {
     RTL8139State *s = RTL8139(dev);
 
-    memory_region_destroy(&s->bar_io);
-    memory_region_destroy(&s->bar_mem);
     if (s->cplus_txbuffer) {
         g_free(s->cplus_txbuffer);
         s->cplus_txbuffer = NULL;
@@ -3540,11 +3542,18 @@ static int pci_rtl8139_init(PCIDevice *dev)
     s->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, rtl8139_timer, s);
     rtl8139_set_next_tctr_time(s, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
 
-    add_boot_device_path(s->conf.bootindex, d, "/ethernet-phy@0");
-
     return 0;
 }
 
+static void rtl8139_instance_init(Object *obj)
+{
+    RTL8139State *s = RTL8139(obj);
+
+    device_add_bootindex_property(obj, &s->conf.bootindex,
+                                  "bootindex", "/ethernet-phy@0",
+                                  DEVICE(obj), NULL);
+}
+
 static Property rtl8139_properties[] = {
     DEFINE_NIC_PROPERTIES(RTL8139State, conf),
     DEFINE_PROP_END_OF_LIST(),
@@ -3573,6 +3582,7 @@ static const TypeInfo rtl8139_info = {
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(RTL8139State),
     .class_init    = rtl8139_class_init,
+    .instance_init = rtl8139_instance_init,
 };
 
 static void rtl8139_register_types(void)
diff --git a/hw/net/spapr_llan.c b/hw/net/spapr_llan.c
index 2d47df693..2c8b03822 100644
--- a/hw/net/spapr_llan.c
+++ b/hw/net/spapr_llan.c
@@ -72,7 +72,14 @@ typedef uint64_t vlan_bd_t;
 #define VLAN_RXQ_BD_OFF      0
 #define VLAN_FILTER_BD_OFF   8
 #define VLAN_RX_BDS_OFF      16
-#define VLAN_MAX_BUFS        ((SPAPR_TCE_PAGE_SIZE - VLAN_RX_BDS_OFF) / 8)
+/*
+ * The final 8 bytes of the buffer list is a counter of frames dropped
+ * because there was not a buffer in the buffer list capable of holding
+ * the frame. We must avoid it, or the operating system will report garbage
+ * for this statistic.
+ */
+#define VLAN_RX_BDS_LEN      (SPAPR_TCE_PAGE_SIZE - VLAN_RX_BDS_OFF - 8)
+#define VLAN_MAX_BUFS        (VLAN_RX_BDS_LEN / 8)
 
 #define TYPE_VIO_SPAPR_VLAN_DEVICE "spapr-vlan"
 #define VIO_SPAPR_VLAN_DEVICE(obj) \
@@ -119,7 +126,7 @@ static ssize_t spapr_vlan_receive(NetClientState *nc, const uint8_t *buf,
 
     do {
         buf_ptr += 8;
-        if (buf_ptr >= SPAPR_TCE_PAGE_SIZE) {
+        if (buf_ptr >= (VLAN_RX_BDS_LEN + VLAN_RX_BDS_OFF)) {
             buf_ptr = VLAN_RX_BDS_OFF;
         }
 
@@ -214,11 +221,18 @@ static int spapr_vlan_init(VIOsPAPRDevice *sdev)
                             object_get_typename(OBJECT(sdev)), sdev->qdev.id, dev);
     qemu_format_nic_info_str(qemu_get_queue(dev->nic), dev->nicconf.macaddr.a);
 
-    add_boot_device_path(dev->nicconf.bootindex, DEVICE(dev), "");
-
     return 0;
 }
 
+static void spapr_vlan_instance_init(Object *obj)
+{
+    VIOsPAPRVLANDevice *dev = VIO_SPAPR_VLAN_DEVICE(obj);
+
+    device_add_bootindex_property(obj, &dev->nicconf.bootindex,
+                                  "bootindex", "",
+                                  DEVICE(dev), NULL);
+}
+
 void spapr_vlan_create(VIOsPAPRBus *bus, NICInfo *nd)
 {
     DeviceState *dev;
@@ -397,7 +411,7 @@ static target_ulong h_add_logical_lan_buffer(PowerPCCPU *cpu,
 
     do {
         dev->add_buf_ptr += 8;
-        if (dev->add_buf_ptr >= SPAPR_TCE_PAGE_SIZE) {
+        if (dev->add_buf_ptr >= (VLAN_RX_BDS_LEN + VLAN_RX_BDS_OFF)) {
             dev->add_buf_ptr = VLAN_RX_BDS_OFF;
         }
 
@@ -546,6 +560,7 @@ static const TypeInfo spapr_vlan_info = {
     .parent        = TYPE_VIO_SPAPR_DEVICE,
     .instance_size = sizeof(VIOsPAPRVLANDevice),
     .class_init    = spapr_vlan_class_init,
+    .instance_init = spapr_vlan_instance_init,
 };
 
 static void spapr_vlan_register_types(void)
diff --git a/hw/net/stellaris_enet.c b/hw/net/stellaris_enet.c
index c9ee5d3f1..c07e5137c 100644
--- a/hw/net/stellaris_enet.c
+++ b/hw/net/stellaris_enet.c
@@ -485,13 +485,6 @@ static int stellaris_enet_init(SysBusDevice *sbd)
     return 0;
 }
 
-static void stellaris_enet_unrealize(DeviceState *dev, Error **errp)
-{
-    stellaris_enet_state *s = STELLARIS_ENET(dev);
-
-    memory_region_destroy(&s->mmio);
-}
-
 static Property stellaris_enet_properties[] = {
     DEFINE_NIC_PROPERTIES(stellaris_enet_state, conf),
     DEFINE_PROP_END_OF_LIST(),
@@ -503,7 +496,6 @@ static void stellaris_enet_class_init(ObjectClass *klass, void *data)
     SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
     k->init = stellaris_enet_init;
-    dc->unrealize = stellaris_enet_unrealize;
     dc->props = stellaris_enet_properties;
     dc->vmsd = &vmstate_stellaris_enet;
 }
diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c
index f87c79824..4e3a06162 100644
--- a/hw/net/vhost_net.c
+++ b/hw/net/vhost_net.c
@@ -28,7 +28,6 @@
 #include <sys/socket.h>
 #include <linux/kvm.h>
 #include <fcntl.h>
-#include <linux/virtio_ring.h>
 #include <netpacket/packet.h>
 #include <net/ethernet.h>
 #include <net/if.h>
@@ -36,6 +35,7 @@
 
 #include <stdio.h>
 
+#include "hw/virtio/virtio_ring.h"
 #include "hw/virtio/vhost.h"
 #include "hw/virtio/virtio-bus.h"
 
@@ -115,6 +115,7 @@ unsigned vhost_net_get_features(struct vhost_net *net, unsigned features)
 
 void vhost_net_ack_features(struct vhost_net *net, unsigned features)
 {
+    net->dev.acked_features = net->dev.backend_features;
     vhost_ack_features(&net->dev, vhost_net_get_feature_bits(net), features);
 }
 
@@ -162,11 +163,11 @@ struct vhost_net *vhost_net_init(VhostNetOptions *options)
     if (r < 0) {
         goto fail;
     }
-    if (!qemu_has_vnet_hdr_len(options->net_backend,
-                               sizeof(struct virtio_net_hdr_mrg_rxbuf))) {
-        net->dev.features &= ~(1 << VIRTIO_NET_F_MRG_RXBUF);
-    }
     if (backend_kernel) {
+        if (!qemu_has_vnet_hdr_len(options->net_backend,
+                               sizeof(struct virtio_net_hdr_mrg_rxbuf))) {
+            net->dev.features &= ~(1 << VIRTIO_NET_F_MRG_RXBUF);
+        }
         if (~net->dev.features & net->dev.backend_features) {
             fprintf(stderr, "vhost lacks feature mask %" PRIu64
                    " for backend\n",
@@ -188,20 +189,19 @@ bool vhost_net_query(VHostNetState *net, VirtIODevice *dev)
     return vhost_dev_query(&net->dev, dev);
 }
 
+static void vhost_net_set_vq_index(struct vhost_net *net, int vq_index)
+{
+    net->dev.vq_index = vq_index;
+}
+
 static int vhost_net_start_one(struct vhost_net *net,
-                               VirtIODevice *dev,
-                               int vq_index)
+                               VirtIODevice *dev)
 {
     struct vhost_vring_file file = { };
     int r;
 
-    if (net->dev.started) {
-        return 0;
-    }
-
     net->dev.nvqs = 2;
     net->dev.vqs = net->vqs;
-    net->dev.vq_index = vq_index;
 
     r = vhost_dev_enable_notifiers(&net->dev, dev);
     if (r < 0) {
@@ -256,10 +256,6 @@ static void vhost_net_stop_one(struct vhost_net *net,
 {
     struct vhost_vring_file file = { .fd = -1 };
 
-    if (!net->dev.started) {
-        return;
-    }
-
     if (net->nc->info->type == NET_CLIENT_OPTIONS_KIND_TAP) {
         for (file.index = 0; file.index < net->dev.nvqs; ++file.index) {
             const VhostOps *vhost_ops = net->dev.vhost_ops;
@@ -294,7 +290,7 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(dev)));
     VirtioBusState *vbus = VIRTIO_BUS(qbus);
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
-    int r, i = 0;
+    int r, e, i;
 
     if (!vhost_net_device_endian_ok(dev)) {
         error_report("vhost-net does not support cross-endian");
@@ -309,11 +305,7 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
     }
 
     for (i = 0; i < total_queues; i++) {
-        r = vhost_net_start_one(get_vhost_net(ncs[i].peer), dev, i * 2);
-
-        if (r < 0) {
-            goto err;
-        }
+        vhost_net_set_vq_index(get_vhost_net(ncs[i].peer), i * 2);
     }
 
     r = k->set_guest_notifiers(qbus->parent, total_queues * 2, true);
@@ -322,12 +314,26 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
         goto err;
     }
 
+    for (i = 0; i < total_queues; i++) {
+        r = vhost_net_start_one(get_vhost_net(ncs[i].peer), dev);
+
+        if (r < 0) {
+            goto err_start;
+        }
+    }
+
     return 0;
 
-err:
+err_start:
     while (--i >= 0) {
         vhost_net_stop_one(get_vhost_net(ncs[i].peer), dev);
     }
+    e = k->set_guest_notifiers(qbus->parent, total_queues * 2, false);
+    if (e < 0) {
+        fprintf(stderr, "vhost guest notifier cleanup failed: %d\n", e);
+        fflush(stderr);
+    }
+err:
     return r;
 }
 
@@ -339,16 +345,16 @@ void vhost_net_stop(VirtIODevice *dev, NetClientState *ncs,
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
     int i, r;
 
+    for (i = 0; i < total_queues; i++) {
+        vhost_net_stop_one(get_vhost_net(ncs[i].peer), dev);
+    }
+
     r = k->set_guest_notifiers(qbus->parent, total_queues * 2, false);
     if (r < 0) {
         fprintf(stderr, "vhost guest notifier cleanup failed: %d\n", r);
         fflush(stderr);
     }
     assert(r >= 0);
-
-    for (i = 0; i < total_queues; i++) {
-        vhost_net_stop_one(get_vhost_net(ncs[i].peer), dev);
-    }
 }
 
 void vhost_net_cleanup(struct vhost_net *net)
diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c
index 268eff9df..e574bd432 100644
--- a/hw/net/virtio-net.c
+++ b/hw/net/virtio-net.c
@@ -125,10 +125,23 @@ static void virtio_net_vhost_status(VirtIONet *n, uint8_t status)
         return;
     }
     if (!n->vhost_started) {
-        int r;
+        int r, i;
+
         if (!vhost_net_query(get_vhost_net(nc->peer), vdev)) {
             return;
         }
+
+        /* Any packets outstanding? Purge them to avoid touching rings
+         * when vhost is running.
+         */
+        for (i = 0;  i < queues; i++) {
+            NetClientState *qnc = qemu_get_subqueue(n->nic, i);
+
+            /* Purge both directions: TX and RX. */
+            qemu_net_queue_purge(qnc->peer->incoming_queue, qnc);
+            qemu_net_queue_purge(qnc->incoming_queue, qnc->peer);
+        }
+
         n->vhost_started = 1;
         r = vhost_net_start(vdev, n->nic->ncs, queues);
         if (r < 0) {
@@ -785,7 +798,7 @@ static void virtio_net_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
     virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
     VirtQueueElement elem;
     size_t s;
-    struct iovec *iov;
+    struct iovec *iov, *iov2;
     unsigned int iov_cnt;
 
     while (virtqueue_pop(vq, &elem)) {
@@ -795,8 +808,8 @@ static void virtio_net_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
             exit(1);
         }
 
-        iov = elem.out_sg;
         iov_cnt = elem.out_num;
+        iov2 = iov = g_memdup(elem.out_sg, sizeof(struct iovec) * elem.out_num);
         s = iov_to_buf(iov, iov_cnt, 0, &ctrl, sizeof(ctrl));
         iov_discard_front(&iov, &iov_cnt, sizeof(ctrl));
         if (s != sizeof(ctrl)) {
@@ -820,6 +833,7 @@ static void virtio_net_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
 
         virtqueue_push(vq, &elem, sizeof(status));
         virtio_notify(vdev, vq);
+        g_free(iov2);
     }
 }
 
@@ -1112,8 +1126,6 @@ static int32_t virtio_net_flush_tx(VirtIONetQueue *q)
         return num_packets;
     }
 
-    assert(vdev->vm_running);
-
     if (q->async_tx.elem.out_num) {
         virtio_queue_set_notification(q->tx_vq, 0);
         return num_packets;
@@ -1224,7 +1236,12 @@ static void virtio_net_tx_timer(void *opaque)
     VirtIONetQueue *q = opaque;
     VirtIONet *n = q->n;
     VirtIODevice *vdev = VIRTIO_DEVICE(n);
-    assert(vdev->vm_running);
+    /* This happens when device was stopped but BH wasn't. */
+    if (!vdev->vm_running) {
+        /* Make sure tx waiting is set, so we'll run when restarted. */
+        assert(q->tx_waiting);
+        return;
+    }
 
     q->tx_waiting = 0;
 
@@ -1244,7 +1261,12 @@ static void virtio_net_tx_bh(void *opaque)
     VirtIODevice *vdev = VIRTIO_DEVICE(n);
     int32_t ret;
 
-    assert(vdev->vm_running);
+    /* This happens when device was stopped but BH wasn't. */
+    if (!vdev->vm_running) {
+        /* Make sure tx waiting is set, so we'll run when restarted. */
+        assert(q->tx_waiting);
+        return;
+    }
 
     q->tx_waiting = 0;
 
@@ -1640,8 +1662,6 @@ static void virtio_net_device_realize(DeviceState *dev, Error **errp)
     n->qdev = dev;
     register_savevm(dev, "virtio-net", -1, VIRTIO_NET_VM_VERSION,
                     virtio_net_save, virtio_net_load, n);
-
-    add_boot_device_path(n->nic_conf.bootindex, dev, "/ethernet-phy@0");
 }
 
 static void virtio_net_device_unrealize(DeviceState *dev, Error **errp)
@@ -1693,6 +1713,9 @@ static void virtio_net_instance_init(Object *obj)
      * Can be overriden with virtio_net_set_config_size.
      */
     n->config_size = sizeof(struct virtio_net_config);
+    device_add_bootindex_property(obj, &n->nic_conf.bootindex,
+                                  "bootindex", "/ethernet-phy@0",
+                                  DEVICE(n), NULL);
 }
 
 static Property virtio_net_properties[] = {
diff --git a/hw/net/vmxnet3.c b/hw/net/vmxnet3.c
index 77bea6f89..8eea58989 100644
--- a/hw/net/vmxnet3.c
+++ b/hw/net/vmxnet3.c
@@ -34,6 +34,7 @@
 
 #define PCI_DEVICE_ID_VMWARE_VMXNET3_REVISION 0x1
 #define VMXNET3_MSIX_BAR_SIZE 0x2000
+#define MIN_BUF_SIZE 60
 
 #define VMXNET3_BAR0_IDX      (0)
 #define VMXNET3_BAR1_IDX      (1)
@@ -1009,7 +1010,7 @@ vmxnet3_indicate_packet(VMXNET3State *s)
 
         vmxnet3_dump_rx_descr(&rxd);
 
-        if (0 != ready_rxcd_pa) {
+        if (ready_rxcd_pa != 0) {
             cpu_physical_memory_write(ready_rxcd_pa, &rxcd, sizeof(rxcd));
         }
 
@@ -1020,7 +1021,7 @@ vmxnet3_indicate_packet(VMXNET3State *s)
         rxcd.gen = new_rxcd_gen;
         rxcd.rqID = RXQ_IDX + rx_ridx * s->rxq_num;
 
-        if (0 == bytes_left) {
+        if (bytes_left == 0) {
             vmxnet3_rx_update_descr(s->rx_pkt, &rxcd);
         }
 
@@ -1038,16 +1039,16 @@ vmxnet3_indicate_packet(VMXNET3State *s)
         num_frags++;
     }
 
-    if (0 != ready_rxcd_pa) {
+    if (ready_rxcd_pa != 0) {
         rxcd.eop = 1;
-        rxcd.err = (0 != bytes_left);
+        rxcd.err = (bytes_left != 0);
         cpu_physical_memory_write(ready_rxcd_pa, &rxcd, sizeof(rxcd));
 
         /* Flush RX descriptor changes */
         smp_wmb();
     }
 
-    if (0 != new_rxcd_pa) {
+    if (new_rxcd_pa != 0) {
         vmxnet3_revert_rxc_descr(s, RXQ_IDX);
     }
 
@@ -1190,8 +1191,8 @@ static void vmxnet3_update_mcast_filters(VMXNET3State *s)
     s->mcast_list_len = list_bytes / sizeof(s->mcast_list[0]);
 
     s->mcast_list = g_realloc(s->mcast_list, list_bytes);
-    if (NULL == s->mcast_list) {
-        if (0 == s->mcast_list_len) {
+    if (!s->mcast_list) {
+        if (s->mcast_list_len == 0) {
             VMW_CFPRN("Current multicast list is empty");
         } else {
             VMW_ERPRN("Failed to allocate multicast list of %d elements",
@@ -1667,7 +1668,7 @@ vmxnet3_io_bar1_write(void *opaque,
          * memory address. We save it to temp variable and set the
          * shared address only after we get the high part
          */
-        if (0 == val) {
+        if (val == 0) {
             s->device_active = false;
         }
         s->temp_shared_guest_driver_memory = val;
@@ -1871,12 +1872,21 @@ vmxnet3_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
     VMXNET3State *s = qemu_get_nic_opaque(nc);
     size_t bytes_indicated;
+    uint8_t min_buf[MIN_BUF_SIZE];
 
     if (!vmxnet3_can_receive(nc)) {
         VMW_PKPRN("Cannot receive now");
         return -1;
     }
 
+    /* Pad to minimum Ethernet frame length */
+    if (size < sizeof(min_buf)) {
+        memcpy(min_buf, buf, size);
+        memset(&min_buf[size], 0, sizeof(min_buf) - size);
+        buf = min_buf;
+        size = sizeof(min_buf);
+    }
+
     if (s->peer_has_vhdr) {
         vmxnet_rx_pkt_set_vhdr(s->rx_pkt, (struct virtio_net_hdr *)buf);
         buf += sizeof(struct virtio_net_hdr);
@@ -2162,11 +2172,16 @@ static int vmxnet3_pci_init(PCIDevice *pci_dev)
     register_savevm(dev, "vmxnet3-msix", -1, 1,
                     vmxnet3_msix_save, vmxnet3_msix_load, s);
 
-    add_boot_device_path(s->conf.bootindex, dev, "/ethernet-phy@0");
-
     return 0;
 }
 
+static void vmxnet3_instance_init(Object *obj)
+{
+    VMXNET3State *s = VMXNET3(obj);
+    device_add_bootindex_property(obj, &s->conf.bootindex,
+                                  "bootindex", "/ethernet-phy@0",
+                                  DEVICE(obj), NULL);
+}
 
 static void vmxnet3_pci_uninit(PCIDevice *pci_dev)
 {
@@ -2182,10 +2197,6 @@ static void vmxnet3_pci_uninit(PCIDevice *pci_dev)
     vmxnet3_cleanup_msix(s);
 
     vmxnet3_cleanup_msi(s);
-
-    memory_region_destroy(&s->bar0);
-    memory_region_destroy(&s->bar1);
-    memory_region_destroy(&s->msix_bar);
 }
 
 static void vmxnet3_qdev_reset(DeviceState *dev)
@@ -2518,6 +2529,7 @@ static const TypeInfo vmxnet3_info = {
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VMXNET3State),
     .class_init    = vmxnet3_class_init,
+    .instance_init = vmxnet3_instance_init,
 };
 
 static void vmxnet3_register_types(void)
diff --git a/hw/nvram/fw_cfg.c b/hw/nvram/fw_cfg.c
index b71d25156..a7122ee56 100644
--- a/hw/nvram/fw_cfg.c
+++ b/hw/nvram/fw_cfg.c
@@ -402,6 +402,26 @@ static void fw_cfg_add_bytes_read_callback(FWCfgState *s, uint16_t key,
     s->entries[arch][key].callback_opaque = callback_opaque;
 }
 
+static void *fw_cfg_modify_bytes_read(FWCfgState *s, uint16_t key,
+                                              void *data, size_t len)
+{
+    void *ptr;
+    int arch = !!(key & FW_CFG_ARCH_LOCAL);
+
+    key &= FW_CFG_ENTRY_MASK;
+
+    assert(key < FW_CFG_MAX_ENTRY && len < UINT32_MAX);
+
+    /* return the old data to the function caller, avoid memory leak */
+    ptr = s->entries[arch][key].data;
+    s->entries[arch][key].data = data;
+    s->entries[arch][key].len = len;
+    s->entries[arch][key].callback_opaque = NULL;
+    s->entries[arch][key].callback = NULL;
+
+    return ptr;
+}
+
 void fw_cfg_add_bytes(FWCfgState *s, uint16_t key, void *data, size_t len)
 {
     fw_cfg_add_bytes_read_callback(s, key, NULL, NULL, data, len);
@@ -499,13 +519,45 @@ void fw_cfg_add_file(FWCfgState *s,  const char *filename,
     fw_cfg_add_file_callback(s, filename, NULL, NULL, data, len);
 }
 
-static void fw_cfg_machine_ready(struct Notifier *n, void *data)
+void *fw_cfg_modify_file(FWCfgState *s, const char *filename,
+                        void *data, size_t len)
+{
+    int i, index;
+    void *ptr = NULL;
+
+    assert(s->files);
+
+    index = be32_to_cpu(s->files->count);
+    assert(index < FW_CFG_FILE_SLOTS);
+
+    for (i = 0; i < index; i++) {
+        if (strcmp(filename, s->files->f[i].name) == 0) {
+            ptr = fw_cfg_modify_bytes_read(s, FW_CFG_FILE_FIRST + i,
+                                           data, len);
+            s->files->f[i].size   = cpu_to_be32(len);
+            return ptr;
+        }
+    }
+    /* add new one */
+    fw_cfg_add_file_callback(s, filename, NULL, NULL, data, len);
+    return NULL;
+}
+
+static void fw_cfg_machine_reset(void *opaque)
 {
+    void *ptr;
     size_t len;
-    FWCfgState *s = container_of(n, FWCfgState, machine_ready);
+    FWCfgState *s = opaque;
     char *bootindex = get_boot_devices_list(&len, false);
 
-    fw_cfg_add_file(s, "bootorder", (uint8_t*)bootindex, len);
+    ptr = fw_cfg_modify_file(s, "bootorder", (uint8_t *)bootindex, len);
+    g_free(ptr);
+}
+
+static void fw_cfg_machine_ready(struct Notifier *n, void *data)
+{
+    FWCfgState *s = container_of(n, FWCfgState, machine_ready);
+    qemu_register_reset(fw_cfg_machine_reset, s);
 }
 
 FWCfgState *fw_cfg_init(uint32_t ctl_port, uint32_t data_port,
diff --git a/hw/nvram/mac_nvram.c b/hw/nvram/mac_nvram.c
index 170b10b76..d35f8a312 100644
--- a/hw/nvram/mac_nvram.c
+++ b/hw/nvram/mac_nvram.c
@@ -26,6 +26,7 @@
 #include "hw/nvram/openbios_firmware_abi.h"
 #include "sysemu/sysemu.h"
 #include "hw/ppc/mac.h"
+#include <zlib.h>
 
 /* debug NVR */
 //#define DEBUG_NVR
@@ -39,29 +40,6 @@
 
 #define DEF_SYSTEM_SIZE 0xc10
 
-/* Direct access to NVRAM */
-uint8_t macio_nvram_read(MacIONVRAMState *s, uint32_t addr)
-{
-    uint32_t ret;
-
-    if (addr < s->size) {
-        ret = s->data[addr];
-    } else {
-        ret = -1;
-    }
-    NVR_DPRINTF("read addr %04" PRIx32 " val %" PRIx8 "\n", addr, ret);
-
-    return ret;
-}
-
-void macio_nvram_write(MacIONVRAMState *s, uint32_t addr, uint8_t val)
-{
-    NVR_DPRINTF("write addr %04" PRIx32 " val %" PRIx8 "\n", addr, val);
-    if (addr < s->size) {
-        s->data[addr] = val;
-    }
-}
-
 /* macio style NVRAM device */
 static void macio_nvram_writeb(void *opaque, hwaddr addr,
                                uint64_t value, unsigned size)
@@ -89,6 +67,10 @@ static uint64_t macio_nvram_readb(void *opaque, hwaddr addr,
 static const MemoryRegionOps macio_nvram_ops = {
     .read = macio_nvram_readb,
     .write = macio_nvram_writeb,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .impl.min_access_size = 1,
+    .impl.max_access_size = 1,
     .endianness = DEVICE_BIG_ENDIAN,
 };
 
@@ -156,15 +138,16 @@ static void macio_nvram_register_types(void)
 }
 
 /* Set up a system OpenBIOS NVRAM partition */
-void pmac_format_nvram_partition (MacIONVRAMState *nvr, int len)
+static void pmac_format_nvram_partition_of(MacIONVRAMState *nvr, int off,
+                                           int len)
 {
     unsigned int i;
-    uint32_t start = 0, end;
+    uint32_t start = off, end;
     struct OpenBIOS_nvpart_v1 *part_header;
 
     // OpenBIOS nvram variables
     // Variable partition
-    part_header = (struct OpenBIOS_nvpart_v1 *)nvr->data;
+    part_header = (struct OpenBIOS_nvpart_v1 *)&nvr->data[start];
     part_header->signature = OPENBIOS_PART_SYSTEM;
     pstrcpy(part_header->name, sizeof(part_header->name), "system");
 
@@ -192,4 +175,39 @@ void pmac_format_nvram_partition (MacIONVRAMState *nvr, int len)
     OpenBIOS_finish_partition(part_header, end - start);
 }
 
+#define OSX_NVRAM_SIGNATURE     (0x5A)
+
+/* Set up a Mac OS X NVRAM partition */
+static void pmac_format_nvram_partition_osx(MacIONVRAMState *nvr, int off,
+                                            int len)
+{
+    uint32_t start = off;
+    struct OpenBIOS_nvpart_v1 *part_header;
+    unsigned char *data = &nvr->data[start];
+
+    /* empty partition */
+    part_header = (struct OpenBIOS_nvpart_v1 *)data;
+    part_header->signature = OSX_NVRAM_SIGNATURE;
+    pstrcpy(part_header->name, sizeof(part_header->name), "wwwwwwwwwwww");
+
+    OpenBIOS_finish_partition(part_header, len);
+
+    /* Generation */
+    stl_be_p(&data[20], 2);
+
+    /* Adler32 checksum */
+    stl_be_p(&data[16], adler32(0, &data[20], len - 20));
+}
+
+/* Set up NVRAM with OF and OSX partitions */
+void pmac_format_nvram_partition(MacIONVRAMState *nvr, int len)
+{
+    /*
+     * Mac OS X expects side "B" of the flash at the second half of NVRAM,
+     * so we use half of the chip for OF and the other half for a free OSX
+     * partition.
+     */
+    pmac_format_nvram_partition_of(nvr, 0, len / 2);
+    pmac_format_nvram_partition_osx(nvr, len / 2, len / 2);
+}
 type_init(macio_nvram_register_types)
diff --git a/hw/nvram/spapr_nvram.c b/hw/nvram/spapr_nvram.c
index 6a72ef481..35dc6d568 100644
--- a/hw/nvram/spapr_nvram.c
+++ b/hw/nvram/spapr_nvram.c
@@ -24,6 +24,7 @@
 
 #include <libfdt.h>
 
+#include "sysemu/block-backend.h"
 #include "sysemu/device_tree.h"
 #include "hw/sysbus.h"
 #include "hw/ppc/spapr.h"
@@ -33,7 +34,7 @@ typedef struct sPAPRNVRAM {
     VIOsPAPRDevice sdev;
     uint32_t size;
     uint8_t *buf;
-    BlockDriverState *drive;
+    BlockBackend *blk;
 } sPAPRNVRAM;
 
 #define TYPE_VIO_SPAPR_NVRAM "spapr-nvram"
@@ -51,7 +52,6 @@ static void rtas_nvram_fetch(PowerPCCPU *cpu, sPAPREnvironment *spapr,
 {
     sPAPRNVRAM *nvram = spapr->nvram;
     hwaddr offset, buffer, len;
-    int alen;
     void *membuf;
 
     if ((nargs != 3) || (nret != 2)) {
@@ -76,19 +76,14 @@ static void rtas_nvram_fetch(PowerPCCPU *cpu, sPAPREnvironment *spapr,
         return;
     }
 
-    membuf = cpu_physical_memory_map(buffer, &len, 1);
-    if (nvram->drive) {
-        alen = bdrv_pread(nvram->drive, offset, membuf, len);
-    } else {
-        assert(nvram->buf);
+    assert(nvram->buf);
 
-        memcpy(membuf, nvram->buf + offset, len);
-        alen = len;
-    }
+    membuf = cpu_physical_memory_map(buffer, &len, 1);
+    memcpy(membuf, nvram->buf + offset, len);
     cpu_physical_memory_unmap(membuf, len, 1, len);
 
-    rtas_st(rets, 0, (alen < len) ? RTAS_OUT_HW_ERROR : RTAS_OUT_SUCCESS);
-    rtas_st(rets, 1, (alen < 0) ? 0 : alen);
+    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
+    rtas_st(rets, 1, len);
 }
 
 static void rtas_nvram_store(PowerPCCPU *cpu, sPAPREnvironment *spapr,
@@ -122,14 +117,15 @@ static void rtas_nvram_store(PowerPCCPU *cpu, sPAPREnvironment *spapr,
     }
 
     membuf = cpu_physical_memory_map(buffer, &len, 0);
-    if (nvram->drive) {
-        alen = bdrv_pwrite(nvram->drive, offset, membuf, len);
-    } else {
-        assert(nvram->buf);
 
-        memcpy(nvram->buf + offset, membuf, len);
-        alen = len;
+    alen = len;
+    if (nvram->blk) {
+        alen = blk_pwrite(nvram->blk, offset, membuf, len);
     }
+
+    assert(nvram->buf);
+    memcpy(nvram->buf + offset, membuf, len);
+
     cpu_physical_memory_unmap(membuf, len, 0, len);
 
     rtas_st(rets, 0, (alen < len) ? RTAS_OUT_HW_ERROR : RTAS_OUT_SUCCESS);
@@ -140,19 +136,28 @@ static int spapr_nvram_init(VIOsPAPRDevice *dev)
 {
     sPAPRNVRAM *nvram = VIO_SPAPR_NVRAM(dev);
 
-    if (nvram->drive) {
-        nvram->size = bdrv_getlength(nvram->drive);
+    if (nvram->blk) {
+        nvram->size = blk_getlength(nvram->blk);
     } else {
         nvram->size = DEFAULT_NVRAM_SIZE;
-        nvram->buf = g_malloc0(nvram->size);
     }
 
+    nvram->buf = g_malloc0(nvram->size);
+
     if ((nvram->size < MIN_NVRAM_SIZE) || (nvram->size > MAX_NVRAM_SIZE)) {
         fprintf(stderr, "spapr-nvram must be between %d and %d bytes in size\n",
                 MIN_NVRAM_SIZE, MAX_NVRAM_SIZE);
         return -1;
     }
 
+    if (nvram->blk) {
+        int alen = blk_pread(nvram->blk, 0, nvram->buf, nvram->size);
+
+        if (alen != nvram->size) {
+            return -1;
+        }
+    }
+
     spapr_rtas_register(RTAS_NVRAM_FETCH, "nvram-fetch", rtas_nvram_fetch);
     spapr_rtas_register(RTAS_NVRAM_STORE, "nvram-store", rtas_nvram_store);
 
@@ -166,9 +171,51 @@ static int spapr_nvram_devnode(VIOsPAPRDevice *dev, void *fdt, int node_off)
     return fdt_setprop_cell(fdt, node_off, "#bytes", nvram->size);
 }
 
+static int spapr_nvram_pre_load(void *opaque)
+{
+    sPAPRNVRAM *nvram = VIO_SPAPR_NVRAM(opaque);
+
+    g_free(nvram->buf);
+    nvram->buf = NULL;
+    nvram->size = 0;
+
+    return 0;
+}
+
+static int spapr_nvram_post_load(void *opaque, int version_id)
+{
+    sPAPRNVRAM *nvram = VIO_SPAPR_NVRAM(opaque);
+
+    if (nvram->blk) {
+        int alen = blk_pwrite(nvram->blk, 0, nvram->buf, nvram->size);
+
+        if (alen < 0) {
+            return alen;
+        }
+        if (alen != nvram->size) {
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+static const VMStateDescription vmstate_spapr_nvram = {
+    .name = "spapr_nvram",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .pre_load = spapr_nvram_pre_load,
+    .post_load = spapr_nvram_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(size, sPAPRNVRAM),
+        VMSTATE_VBUFFER_ALLOC_UINT32(buf, sPAPRNVRAM, 1, NULL, 0, size),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
 static Property spapr_nvram_properties[] = {
     DEFINE_SPAPR_PROPERTIES(sPAPRNVRAM, sdev),
-    DEFINE_PROP_DRIVE("drive", sPAPRNVRAM, drive),
+    DEFINE_PROP_DRIVE("drive", sPAPRNVRAM, blk),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -184,6 +231,7 @@ static void spapr_nvram_class_init(ObjectClass *klass, void *data)
     k->dt_compatible = "qemu,spapr-nvram";
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
     dc->props = spapr_nvram_properties;
+    dc->vmsd = &vmstate_spapr_nvram;
 }
 
 static const TypeInfo spapr_nvram_type_info = {
diff --git a/hw/openrisc/openrisc_sim.c b/hw/openrisc/openrisc_sim.c
index b2b4f9b86..1da0657dd 100644
--- a/hw/openrisc/openrisc_sim.c
+++ b/hw/openrisc/openrisc_sim.c
@@ -72,7 +72,7 @@ static void cpu_openrisc_load_kernel(ram_addr_t ram_size,
         entry = elf_entry;
         if (kernel_size < 0) {
             kernel_size = load_uimage(kernel_filename,
-                                      &entry, NULL, NULL);
+                                      &entry, NULL, NULL, NULL, NULL);
         }
         if (kernel_size < 0) {
             kernel_size = load_image_targphys(kernel_filename,
@@ -114,7 +114,7 @@ static void openrisc_sim_init(MachineState *machine)
     }
 
     ram = g_malloc(sizeof(*ram));
-    memory_region_init_ram(ram, NULL, "openrisc.ram", ram_size);
+    memory_region_init_ram(ram, NULL, "openrisc.ram", ram_size, &error_abort);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(get_system_memory(), 0, ram);
 
diff --git a/hw/pci-bridge/ioh3420.c b/hw/pci-bridge/ioh3420.c
index 7cd87fcbb..cce2fdd8e 100644
--- a/hw/pci-bridge/ioh3420.c
+++ b/hw/pci-bridge/ioh3420.c
@@ -85,6 +85,7 @@ static void ioh3420_reset(DeviceState *qdev)
     pcie_cap_root_reset(d);
     pcie_cap_deverr_reset(d);
     pcie_cap_slot_reset(d);
+    pcie_cap_arifwd_reset(d);
     pcie_aer_root_reset(d);
     pci_bridge_reset(qdev);
     pci_bridge_disable_base_limit(d);
@@ -118,6 +119,8 @@ static int ioh3420_initfn(PCIDevice *d)
     if (rc < 0) {
         goto err_msi;
     }
+
+    pcie_cap_arifwd_init(d);
     pcie_cap_deverr_init(d);
     pcie_cap_slot_init(d, s->slot);
     pcie_chassis_create(s->chassis);
@@ -156,30 +159,6 @@ static void ioh3420_exitfn(PCIDevice *d)
     pci_bridge_exitfn(d);
 }
 
-PCIESlot *ioh3420_init(PCIBus *bus, int devfn, bool multifunction,
-                         const char *bus_name, pci_map_irq_fn map_irq,
-                         uint8_t port, uint8_t chassis, uint16_t slot)
-{
-    PCIDevice *d;
-    PCIBridge *br;
-    DeviceState *qdev;
-
-    d = pci_create_multifunction(bus, devfn, multifunction, "ioh3420");
-    if (!d) {
-        return NULL;
-    }
-    br = PCI_BRIDGE(d);
-
-    qdev = DEVICE(d);
-    pci_bridge_map_irq(br, bus_name, map_irq);
-    qdev_prop_set_uint8(qdev, "port", port);
-    qdev_prop_set_uint8(qdev, "chassis", chassis);
-    qdev_prop_set_uint16(qdev, "slot", slot);
-    qdev_init_nofail(qdev);
-
-    return PCIE_SLOT(d);
-}
-
 static Property ioh3420_props[] = {
     DEFINE_PROP_BIT(COMPAT_PROP_PCP, PCIDevice, cap_present,
                     QEMU_PCIE_SLTCAP_PCP_BITNR, true),
diff --git a/hw/pci-bridge/ioh3420.h b/hw/pci-bridge/ioh3420.h
index 7776e5b02..ea423cb99 100644
--- a/hw/pci-bridge/ioh3420.h
+++ b/hw/pci-bridge/ioh3420.h
@@ -3,8 +3,4 @@
 
 #include "hw/pci/pcie_port.h"
 
-PCIESlot *ioh3420_init(PCIBus *bus, int devfn, bool multifunction,
-                       const char *bus_name, pci_map_irq_fn map_irq,
-                       uint8_t port, uint8_t chassis, uint16_t slot);
-
 #endif /* QEMU_IOH3420_H */
diff --git a/hw/pci-bridge/pci_bridge_dev.c b/hw/pci-bridge/pci_bridge_dev.c
index e68145c52..252ea5eb5 100644
--- a/hw/pci-bridge/pci_bridge_dev.c
+++ b/hw/pci-bridge/pci_bridge_dev.c
@@ -81,7 +81,6 @@ msi_error:
 slotid_error:
     shpc_cleanup(dev, &bridge_dev->bar);
 shpc_error:
-    memory_region_destroy(&bridge_dev->bar);
     pci_bridge_exitfn(dev);
 bridge_error:
     return err;
@@ -95,7 +94,6 @@ static void pci_bridge_dev_exitfn(PCIDevice *dev)
     }
     slotid_cap_cleanup(dev);
     shpc_cleanup(dev, &bridge_dev->bar);
-    memory_region_destroy(&bridge_dev->bar);
     pci_bridge_exitfn(dev);
 }
 
@@ -152,7 +150,7 @@ static void pci_bridge_dev_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &pci_bridge_dev_vmstate;
     set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
     hc->plug = shpc_device_hotplug_cb;
-    hc->unplug = shpc_device_hot_unplug_cb;
+    hc->unplug_request = shpc_device_hot_unplug_request_cb;
 }
 
 static const TypeInfo pci_bridge_dev_info = {
diff --git a/hw/pci-bridge/xio3130_downstream.c b/hw/pci-bridge/xio3130_downstream.c
index 51f20d746..b3a647926 100644
--- a/hw/pci-bridge/xio3130_downstream.c
+++ b/hw/pci-bridge/xio3130_downstream.c
@@ -50,7 +50,7 @@ static void xio3130_downstream_reset(DeviceState *qdev)
 
     pcie_cap_deverr_reset(d);
     pcie_cap_slot_reset(d);
-    pcie_cap_ari_reset(d);
+    pcie_cap_arifwd_reset(d);
     pci_bridge_reset(qdev);
 }
 
@@ -91,7 +91,7 @@ static int xio3130_downstream_initfn(PCIDevice *d)
     if (rc < 0) {
         goto err_pcie_cap;
     }
-    pcie_cap_ari_init(d);
+    pcie_cap_arifwd_init(d);
     rc = pcie_aer_init(d, XIO3130_AER_OFFSET);
     if (rc < 0) {
         goto err;
diff --git a/hw/pci-host/apb.c b/hw/pci-host/apb.c
index d238a84f9..f573875ba 100644
--- a/hw/pci-host/apb.c
+++ b/hw/pci-host/apb.c
@@ -94,6 +94,7 @@ do { printf("IOMMU: " fmt , ## __VA_ARGS__); } while (0)
 #define IOMMU_CTRL_TSB_SHIFT    16
 
 #define IOMMU_BASE              0x8
+#define IOMMU_FLUSH             0x10
 
 #define IOMMU_TTE_DATA_V        (1ULL << 63)
 #define IOMMU_TTE_DATA_SIZE     (1ULL << 61)
@@ -141,6 +142,7 @@ typedef struct APBState {
     IOMMUState iommu;
     uint32_t pci_control[16];
     uint32_t pci_irq_map[8];
+    uint32_t pci_err_irq_map[4];
     uint32_t obio_irq_map[32];
     qemu_irq *pbm_irqs;
     qemu_irq *ivec_irqs;
@@ -203,7 +205,8 @@ static AddressSpace *pbm_pci_dma_iommu(PCIBus *bus, void *opaque, int devfn)
     return &is->iommu_as;
 }
 
-static IOMMUTLBEntry pbm_translate_iommu(MemoryRegion *iommu, hwaddr addr)
+static IOMMUTLBEntry pbm_translate_iommu(MemoryRegion *iommu, hwaddr addr,
+                                         bool is_write)
 {
     IOMMUState *is = container_of(iommu, IOMMUState, iommu);
     hwaddr baseaddr, offset;
@@ -352,6 +355,9 @@ static void iommu_config_write(void *opaque, hwaddr addr,
         is->regs[IOMMU_BASE >> 3] &= 0xffffffff00000000ULL;
         is->regs[IOMMU_BASE >> 3] |= val & 0xffffffffULL;
         break;
+    case IOMMU_FLUSH:
+    case IOMMU_FLUSH + 0x4:
+        break;
     default:
         qemu_log_mask(LOG_UNIMP,
                   "apb iommu: Unimplemented register write "
@@ -387,6 +393,10 @@ static uint64_t iommu_config_read(void *opaque, hwaddr addr, unsigned size)
     case IOMMU_BASE + 0x4:
         val = is->regs[IOMMU_BASE >> 3] & 0xffffffffULL;
         break;
+    case IOMMU_FLUSH:
+    case IOMMU_FLUSH + 0x4:
+        val = 0;
+        break;
     default:
         qemu_log_mask(LOG_UNIMP,
                       "apb iommu: Unimplemented register read "
@@ -415,7 +425,7 @@ static void apb_config_writel (void *opaque, hwaddr addr,
         /* XXX: not implemented yet */
         break;
     case 0x200 ... 0x217: /* IOMMU */
-        iommu_config_write(is, (addr & 0xf), val, size);
+        iommu_config_write(is, (addr & 0x1f), val, size);
         break;
     case 0xc00 ... 0xc3f: /* PCI interrupt control */
         if (addr & 4) {
@@ -428,7 +438,7 @@ static void apb_config_writel (void *opaque, hwaddr addr,
             pbm_check_irqs(s);
         }
         break;
-    case 0x1000 ... 0x1080: /* OBIO interrupt control */
+    case 0x1000 ... 0x107f: /* OBIO interrupt control */
         if (addr & 4) {
             unsigned int ino = ((addr & 0xff) >> 3);
             s->obio_irq_map[ino] &= PBM_PCI_IMR_MASK;
@@ -497,7 +507,7 @@ static uint64_t apb_config_readl (void *opaque,
         /* XXX: not implemented yet */
         break;
     case 0x200 ... 0x217: /* IOMMU */
-        val = iommu_config_read(is, (addr & 0xf), size);
+        val = iommu_config_read(is, (addr & 0x1f), size);
         break;
     case 0xc00 ... 0xc3f: /* PCI interrupt control */
         if (addr & 4) {
@@ -506,13 +516,20 @@ static uint64_t apb_config_readl (void *opaque,
             val = 0;
         }
         break;
-    case 0x1000 ... 0x1080: /* OBIO interrupt control */
+    case 0x1000 ... 0x107f: /* OBIO interrupt control */
         if (addr & 4) {
             val = s->obio_irq_map[(addr & 0xff) >> 3];
         } else {
             val = 0;
         }
         break;
+    case 0x1080 ... 0x108f: /* PCI bus error */
+        if (addr & 4) {
+            val = s->pci_err_irq_map[(addr & 0xf) >> 3];
+        } else {
+            val = 0;
+        }
+        break;
     case 0x2000 ... 0x202f: /* PCI control */
         val = s->pci_control[(addr & 0x3f) >> 2];
         break;
@@ -744,6 +761,9 @@ static int pci_pbm_init_device(SysBusDevice *dev)
     for (i = 0; i < 8; i++) {
         s->pci_irq_map[i] = (0x1f << 6) | (i << 2);
     }
+    for (i = 0; i < 2; i++) {
+        s->pci_err_irq_map[i] = (0x1f << 6) | 0x30;
+    }
     for (i = 0; i < 32; i++) {
         s->obio_irq_map[i] = ((0x1f << 6) | 0x20) + i;
     }
diff --git a/hw/pci-host/pam.c b/hw/pci-host/pam.c
index e1e95aabc..8272de3f2 100644
--- a/hw/pci-host/pam.c
+++ b/hw/pci-host/pam.c
@@ -1,12 +1,12 @@
 /*
- * QEMU i440FX/PIIX3 PCI Bridge Emulation
+ * QEMU Smram/pam logic implementation
  *
  * Copyright (c) 2006 Fabrice Bellard
  * Copyright (c) 2011 Isaku Yamahata <yamahata at valinux co jp>
  *                    VA Linux Systems Japan K.K.
  * Copyright (c) 2012 Jason Baron <jbaron@redhat.com>
  *
- * Split out from piix_pci.c
+ * Split out from piix.c
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
diff --git a/hw/pci-host/piix.c b/hw/pci-host/piix.c
index e0e094609..1530038cb 100644
--- a/hw/pci-host/piix.c
+++ b/hw/pci-host/piix.c
@@ -409,7 +409,7 @@ static void piix3_set_irq_pic(PIIX3State *piix3, int pic_irq)
                      (pic_irq * PIIX_NUM_PIRQS))));
 }
 
-static void piix3_set_irq_level(PIIX3State *piix3, int pirq, int level)
+static void piix3_set_irq_level_internal(PIIX3State *piix3, int pirq, int level)
 {
     int pic_irq;
     uint64_t mask;
@@ -422,6 +422,18 @@ static void piix3_set_irq_level(PIIX3State *piix3, int pirq, int level)
     mask = 1ULL << ((pic_irq * PIIX_NUM_PIRQS) + pirq);
     piix3->pic_levels &= ~mask;
     piix3->pic_levels |= mask * !!level;
+}
+
+static void piix3_set_irq_level(PIIX3State *piix3, int pirq, int level)
+{
+    int pic_irq;
+
+    pic_irq = piix3->dev.config[PIIX_PIRQC + pirq];
+    if (pic_irq >= PIIX_NUM_PIC_IRQS) {
+        return;
+    }
+
+    piix3_set_irq_level_internal(piix3, pirq, level);
 
     piix3_set_irq_pic(piix3, pic_irq);
 }
@@ -527,7 +539,21 @@ static void piix3_reset(void *opaque)
 static int piix3_post_load(void *opaque, int version_id)
 {
     PIIX3State *piix3 = opaque;
-    piix3_update_irq_levels(piix3);
+    int pirq;
+
+    /* Because the i8259 has not been deserialized yet, qemu_irq_raise
+     * might bring the system to a different state than the saved one;
+     * for example, the interrupt could be masked but the i8259 would
+     * not know that yet and would trigger an interrupt in the CPU.
+     *
+     * Here, we update irq levels without raising the interrupt.
+     * Interrupt state will be deserialized separately through the i8259.
+     */
+    piix3->pic_levels = 0;
+    for (pirq = 0; pirq < PIIX_NUM_PIRQS; pirq++) {
+        piix3_set_irq_level_internal(piix3, pirq,
+                            pci_bus_get_irq_level(piix3->dev.bus, pirq));
+    }
     return 0;
 }
 
diff --git a/hw/pci-host/prep.c b/hw/pci-host/prep.c
index ec6f18625..1de3681db 100644
--- a/hw/pci-host/prep.c
+++ b/hw/pci-host/prep.c
@@ -299,7 +299,8 @@ static int raven_init(PCIDevice *d)
     d->config[0x0D] = 0x10; // latency_timer
     d->config[0x34] = 0x00; // capabilities_pointer
 
-    memory_region_init_ram(&s->bios, OBJECT(s), "bios", BIOS_SIZE);
+    memory_region_init_ram(&s->bios, OBJECT(s), "bios", BIOS_SIZE,
+                           &error_abort);
     memory_region_set_readonly(&s->bios, true);
     memory_region_add_subregion(get_system_memory(), (uint32_t)(-BIOS_SIZE),
                                 &s->bios);
diff --git a/hw/pci-host/q35.c b/hw/pci-host/q35.c
index a0a3068dd..b20bad837 100644
--- a/hw/pci-host/q35.c
+++ b/hw/pci-host/q35.c
@@ -7,7 +7,7 @@
  *               VA Linux Systems Japan K.K.
  * Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
  *
- * This is based on piix_pci.c, but heavily modified.
+ * This is based on piix.c, but heavily modified.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -347,6 +347,49 @@ static void mch_reset(DeviceState *qdev)
     mch_update(mch);
 }
 
+static AddressSpace *q35_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
+{
+    IntelIOMMUState *s = opaque;
+    VTDAddressSpace **pvtd_as;
+    int bus_num = pci_bus_num(bus);
+
+    assert(0 <= bus_num && bus_num <= VTD_PCI_BUS_MAX);
+    assert(0 <= devfn && devfn <= VTD_PCI_DEVFN_MAX);
+
+    pvtd_as = s->address_spaces[bus_num];
+    if (!pvtd_as) {
+        /* No corresponding free() */
+        pvtd_as = g_malloc0(sizeof(VTDAddressSpace *) * VTD_PCI_DEVFN_MAX);
+        s->address_spaces[bus_num] = pvtd_as;
+    }
+    if (!pvtd_as[devfn]) {
+        pvtd_as[devfn] = g_malloc0(sizeof(VTDAddressSpace));
+
+        pvtd_as[devfn]->bus_num = (uint8_t)bus_num;
+        pvtd_as[devfn]->devfn = (uint8_t)devfn;
+        pvtd_as[devfn]->iommu_state = s;
+        pvtd_as[devfn]->context_cache_entry.context_cache_gen = 0;
+        memory_region_init_iommu(&pvtd_as[devfn]->iommu, OBJECT(s),
+                                 &s->iommu_ops, "intel_iommu", UINT64_MAX);
+        address_space_init(&pvtd_as[devfn]->as,
+                           &pvtd_as[devfn]->iommu, "intel_iommu");
+    }
+    return &pvtd_as[devfn]->as;
+}
+
+static void mch_init_dmar(MCHPCIState *mch)
+{
+    PCIBus *pci_bus = PCI_BUS(qdev_get_parent_bus(DEVICE(mch)));
+
+    mch->iommu = INTEL_IOMMU_DEVICE(qdev_create(NULL, TYPE_INTEL_IOMMU_DEVICE));
+    object_property_add_child(OBJECT(mch), "intel-iommu",
+                              OBJECT(mch->iommu), NULL);
+    qdev_init_nofail(DEVICE(mch->iommu));
+    sysbus_mmio_map(SYS_BUS_DEVICE(mch->iommu), 0, Q35_HOST_BRIDGE_IOMMU_ADDR);
+
+    pci_setup_iommu(pci_bus, q35_host_dma_iommu, mch->iommu);
+}
+
 static int mch_init(PCIDevice *d)
 {
     int i;
@@ -363,12 +406,17 @@ static int mch_init(PCIDevice *d)
     memory_region_add_subregion_overlap(mch->system_memory, 0xa0000,
                                         &mch->smram_region, 1);
     memory_region_set_enabled(&mch->smram_region, false);
-    init_pam(DEVICE(mch), mch->ram_memory, mch->system_memory, mch->pci_address_space,
-             &mch->pam_regions[0], PAM_BIOS_BASE, PAM_BIOS_SIZE);
+    init_pam(DEVICE(mch), mch->ram_memory, mch->system_memory,
+             mch->pci_address_space, &mch->pam_regions[0],
+             PAM_BIOS_BASE, PAM_BIOS_SIZE);
     for (i = 0; i < 12; ++i) {
-        init_pam(DEVICE(mch), mch->ram_memory, mch->system_memory, mch->pci_address_space,
-                 &mch->pam_regions[i+1], PAM_EXPAN_BASE + i * PAM_EXPAN_SIZE,
-                 PAM_EXPAN_SIZE);
+        init_pam(DEVICE(mch), mch->ram_memory, mch->system_memory,
+                 mch->pci_address_space, &mch->pam_regions[i+1],
+                 PAM_EXPAN_BASE + i * PAM_EXPAN_SIZE, PAM_EXPAN_SIZE);
+    }
+    /* Intel IOMMU (VT-d) */
+    if (qemu_opt_get_bool(qemu_get_machine_opts(), "iommu", false)) {
+        mch_init_dmar(mch);
     }
     return 0;
 }
diff --git a/hw/pci/msi.c b/hw/pci/msi.c
index a4a3040d4..52d23130d 100644
--- a/hw/pci/msi.c
+++ b/hw/pci/msi.c
@@ -291,7 +291,7 @@ void msi_notify(PCIDevice *dev, unsigned int vector)
                    "notify vector 0x%x"
                    " address: 0x%"PRIx64" data: 0x%"PRIx32"\n",
                    vector, msg.address, msg.data);
-    stl_le_phys(&address_space_memory, msg.address, msg.data);
+    stl_le_phys(&dev->bus_master_as, msg.address, msg.data);
 }
 
 /* Normally called by pci_default_write_config(). */
diff --git a/hw/pci/msix.c b/hw/pci/msix.c
index 5c49bfc30..24de2605f 100644
--- a/hw/pci/msix.c
+++ b/hw/pci/msix.c
@@ -319,7 +319,6 @@ int msix_init_exclusive_bar(PCIDevice *dev, unsigned short nentries,
                     bar_nr, MSIX_EXCLUSIVE_BAR_PBA_OFFSET,
                     MSIX_EXCLUSIVE_CAP_OFFSET);
     if (ret) {
-        memory_region_destroy(&dev->msix_exclusive_bar);
         return ret;
     }
 
@@ -359,11 +358,9 @@ void msix_uninit(PCIDevice *dev, MemoryRegion *table_bar, MemoryRegion *pba_bar)
     msix_free_irq_entries(dev);
     dev->msix_entries_nr = 0;
     memory_region_del_subregion(pba_bar, &dev->msix_pba_mmio);
-    memory_region_destroy(&dev->msix_pba_mmio);
     g_free(dev->msix_pba);
     dev->msix_pba = NULL;
     memory_region_del_subregion(table_bar, &dev->msix_table_mmio);
-    memory_region_destroy(&dev->msix_table_mmio);
     g_free(dev->msix_table);
     dev->msix_table = NULL;
     g_free(dev->msix_entry_used);
@@ -375,7 +372,6 @@ void msix_uninit_exclusive_bar(PCIDevice *dev)
 {
     if (msix_present(dev)) {
         msix_uninit(dev, &dev->msix_exclusive_bar, &dev->msix_exclusive_bar);
-        memory_region_destroy(&dev->msix_exclusive_bar);
     }
 }
 
@@ -439,7 +435,7 @@ void msix_notify(PCIDevice *dev, unsigned vector)
 
     msg = msix_get_message(dev, vector);
 
-    stl_le_phys(&address_space_memory, msg.address, msg.data);
+    stl_le_phys(&dev->bus_master_as, msg.address, msg.data);
 }
 
 void msix_reset(PCIDevice *dev)
diff --git a/hw/pci/pci-hotplug-old.c b/hw/pci/pci-hotplug-old.c
index cf2caebfb..0c09c72ac 100644
--- a/hw/pci/pci-hotplug-old.c
+++ b/hw/pci/pci-hotplug-old.c
@@ -33,7 +33,7 @@
 #include "hw/scsi/scsi.h"
 #include "hw/virtio/virtio-blk.h"
 #include "qemu/config-file.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "qapi/error.h"
 
 static int pci_read_devaddr(Monitor *mon, const char *addr,
@@ -77,7 +77,7 @@ static PCIDevice *qemu_pci_hot_add_nic(Monitor *mon,
         monitor_printf(mon, "Invalid PCI device address %s\n", devaddr);
         return NULL;
     }
-    if (!((BusState*)bus)->allow_hotplug) {
+    if (!qbus_is_hotpluggable(BUS(bus))) {
         monitor_printf(mon, "PCI bus doesn't support hotplug\n");
         return NULL;
     }
@@ -107,6 +107,7 @@ static int scsi_hot_add(Monitor *mon, DeviceState *adapter,
 {
     SCSIBus *scsibus;
     SCSIDevice *scsidev;
+    Error *local_err = NULL;
 
     scsibus = (SCSIBus *)
         object_dynamic_cast(OBJECT(QLIST_FIRST(&adapter->child_bus)),
@@ -126,9 +127,13 @@ static int scsi_hot_add(Monitor *mon, DeviceState *adapter,
      */
     dinfo->unit = qemu_opt_get_number(dinfo->opts, "unit", -1);
     dinfo->bus = scsibus->busnr;
-    scsidev = scsi_bus_legacy_add_drive(scsibus, dinfo->bdrv, dinfo->unit,
-                                        false, -1, NULL, NULL);
+    scsidev = scsi_bus_legacy_add_drive(scsibus,
+                                        blk_by_legacy_dinfo(dinfo),
+                                        dinfo->unit, false, -1, NULL,
+                                        &local_err);
     if (!scsidev) {
+        error_report("%s", error_get_pretty(local_err));
+        error_free(local_err);
         return -1;
     }
     dinfo->unit = scsidev->id;
@@ -224,7 +229,7 @@ static PCIDevice *qemu_pci_hot_add_storage(Monitor *mon,
         monitor_printf(mon, "Invalid PCI device address %s\n", devaddr);
         return NULL;
     }
-    if (!((BusState*)bus)->allow_hotplug) {
+    if (!qbus_is_hotpluggable(BUS(bus))) {
         monitor_printf(mon, "PCI bus doesn't support hotplug\n");
         return NULL;
     }
@@ -247,7 +252,8 @@ static PCIDevice *qemu_pci_hot_add_storage(Monitor *mon,
             return NULL;
         }
         dev = pci_create(bus, devfn, "virtio-blk-pci");
-        if (qdev_prop_set_drive(&dev->qdev, "drive", dinfo->bdrv) < 0) {
+        if (qdev_prop_set_drive(&dev->qdev, "drive",
+                                blk_by_legacy_dinfo(dinfo)) < 0) {
             object_unparent(OBJECT(dev));
             dev = NULL;
             break;
diff --git a/hw/pci/pci.c b/hw/pci/pci.c
index 351d32047..371699cf8 100644
--- a/hw/pci/pci.c
+++ b/hw/pci/pci.c
@@ -799,7 +799,6 @@ static void do_pci_unregister_device(PCIDevice *pci_dev)
     pci_config_free(pci_dev);
 
     address_space_destroy(&pci_dev->bus_master_as);
-    memory_region_destroy(&pci_dev->bus_master_enable_region);
 }
 
 /* -1 for devfn means auto assign */
@@ -1147,9 +1146,10 @@ uint32_t pci_default_read_config(PCIDevice *d,
     return le32_to_cpu(val);
 }
 
-void pci_default_write_config(PCIDevice *d, uint32_t addr, uint32_t val, int l)
+void pci_default_write_config(PCIDevice *d, uint32_t addr, uint32_t val_in, int l)
 {
     int i, was_irq_disabled = pci_irq_disabled(d);
+    uint32_t val = val_in;
 
     for (i = 0; i < l; val >>= 8, ++i) {
         uint8_t wmask = d->wmask[addr + i];
@@ -1171,8 +1171,8 @@ void pci_default_write_config(PCIDevice *d, uint32_t addr, uint32_t val, int l)
                                     & PCI_COMMAND_MASTER);
     }
 
-    msi_write_config(d, addr, val, l);
-    msix_write_config(d, addr, val, l);
+    msi_write_config(d, addr, val_in, l);
+    msix_write_config(d, addr, val_in, l);
 }
 
 /***********************************************************/
@@ -1776,7 +1776,12 @@ static int pci_qdev_init(DeviceState *qdev)
         pci_dev->romfile = g_strdup(pc->romfile);
         is_default_rom = true;
     }
-    pci_add_option_rom(pci_dev, is_default_rom);
+
+    rc = pci_add_option_rom(pci_dev, is_default_rom);
+    if (rc != 0) {
+        pci_unregister_device(DEVICE(pci_dev));
+        return rc;
+    }
 
     return 0;
 }
@@ -1937,6 +1942,15 @@ static int pci_add_option_rom(PCIDevice *pdev, bool is_default_rom)
          * for 0.11 compatibility.
          */
         int class = pci_get_word(pdev->config + PCI_CLASS_DEVICE);
+
+        /*
+         * Hot-plugged devices can't use the option ROM
+         * if the rom bar is disabled.
+         */
+        if (DEVICE(pdev)->hotplugged) {
+            return -1;
+        }
+
         if (class == 0x0300) {
             rom_add_vga(pdev->romfile);
         } else {
@@ -1974,7 +1988,7 @@ static int pci_add_option_rom(PCIDevice *pdev, bool is_default_rom)
         snprintf(name, sizeof(name), "%s.rom", object_get_typename(OBJECT(pdev)));
     }
     pdev->has_rom = true;
-    memory_region_init_ram(&pdev->rom, OBJECT(pdev), name, size);
+    memory_region_init_ram(&pdev->rom, OBJECT(pdev), name, size, &error_abort);
     vmstate_register_ram(&pdev->rom, &pdev->qdev);
     ptr = memory_region_get_ram_ptr(&pdev->rom);
     load_image(path, ptr);
@@ -1996,7 +2010,6 @@ static void pci_del_option_rom(PCIDevice *pdev)
         return;
 
     vmstate_unregister_ram(&pdev->rom, &pdev->qdev);
-    memory_region_destroy(&pdev->rom);
     pdev->has_rom = false;
 }
 
diff --git a/hw/pci/pci_bridge.c b/hw/pci/pci_bridge.c
index 4becdc14b..40c97b155 100644
--- a/hw/pci/pci_bridge.c
+++ b/hw/pci/pci_bridge.c
@@ -219,12 +219,12 @@ static void pci_bridge_region_del(PCIBridge *br, PCIBridgeWindows *w)
 
 static void pci_bridge_region_cleanup(PCIBridge *br, PCIBridgeWindows *w)
 {
-    memory_region_destroy(&w->alias_io);
-    memory_region_destroy(&w->alias_mem);
-    memory_region_destroy(&w->alias_pref_mem);
-    memory_region_destroy(&w->alias_vga[QEMU_PCI_VGA_IO_LO]);
-    memory_region_destroy(&w->alias_vga[QEMU_PCI_VGA_IO_HI]);
-    memory_region_destroy(&w->alias_vga[QEMU_PCI_VGA_MEM]);
+    object_unparent(OBJECT(&w->alias_io));
+    object_unparent(OBJECT(&w->alias_mem));
+    object_unparent(OBJECT(&w->alias_pref_mem));
+    object_unparent(OBJECT(&w->alias_vga[QEMU_PCI_VGA_IO_LO]));
+    object_unparent(OBJECT(&w->alias_vga[QEMU_PCI_VGA_IO_HI]));
+    object_unparent(OBJECT(&w->alias_vga[QEMU_PCI_VGA_MEM]));
     g_free(w);
 }
 
@@ -389,8 +389,6 @@ void pci_bridge_exitfn(PCIDevice *pci_dev)
     QLIST_REMOVE(&s->sec_bus, sibling);
     pci_bridge_region_del(s, s->windows);
     pci_bridge_region_cleanup(s, s->windows);
-    memory_region_destroy(&s->address_space_mem);
-    memory_region_destroy(&s->address_space_io);
     /* object_unparent() is called automatically during device deletion */
 }
 
diff --git a/hw/pci/pcie.c b/hw/pci/pcie.c
index a123c01ef..1abbbb192 100644
--- a/hw/pci/pcie.c
+++ b/hw/pci/pcie.c
@@ -145,7 +145,7 @@ void pcie_cap_deverr_init(PCIDevice *dev)
                                PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE);
     pci_long_test_and_set_mask(dev->w1cmask + pos + PCI_EXP_DEVSTA,
                                PCI_EXP_DEVSTA_CED | PCI_EXP_DEVSTA_NFED |
-                               PCI_EXP_DEVSTA_URD | PCI_EXP_DEVSTA_URD);
+                               PCI_EXP_DEVSTA_FED | PCI_EXP_DEVSTA_URD);
 }
 
 void pcie_cap_deverr_reset(PCIDevice *dev)
@@ -229,7 +229,7 @@ static void pcie_cap_slot_hotplug_common(PCIDevice *hotplug_dev,
         /* the slot is electromechanically locked.
          * This error is propagated up to qdev and then to HMP/QMP.
          */
-        error_setg_errno(errp, -EBUSY, "slot is electromechanically locked");
+        error_setg_errno(errp, EBUSY, "slot is electromechanically locked");
     }
 }
 
@@ -262,8 +262,8 @@ void pcie_cap_slot_hotplug_cb(HotplugHandler *hotplug_dev, DeviceState *dev,
                         PCI_EXP_HP_EV_PDC | PCI_EXP_HP_EV_ABP);
 }
 
-void pcie_cap_slot_hot_unplug_cb(HotplugHandler *hotplug_dev, DeviceState *dev,
-                                 Error **errp)
+void pcie_cap_slot_hot_unplug_request_cb(HotplugHandler *hotplug_dev,
+                                         DeviceState *dev, Error **errp)
 {
     uint8_t *exp_cap;
 
@@ -497,9 +497,10 @@ void pcie_cap_flr_write_config(PCIDevice *dev,
     }
 }
 
-/* Alternative Routing-ID Interpretation (ARI) */
-/* ari forwarding support for down stream port */
-void pcie_cap_ari_init(PCIDevice *dev)
+/* Alternative Routing-ID Interpretation (ARI)
+ * forwarding support for root and downstream ports
+ */
+void pcie_cap_arifwd_init(PCIDevice *dev)
 {
     uint32_t pos = dev->exp.exp_cap;
     pci_long_test_and_set_mask(dev->config + pos + PCI_EXP_DEVCAP2,
@@ -508,13 +509,13 @@ void pcie_cap_ari_init(PCIDevice *dev)
                                PCI_EXP_DEVCTL2_ARI);
 }
 
-void pcie_cap_ari_reset(PCIDevice *dev)
+void pcie_cap_arifwd_reset(PCIDevice *dev)
 {
     uint8_t *devctl2 = dev->config + dev->exp.exp_cap + PCI_EXP_DEVCTL2;
     pci_long_test_and_clear_mask(devctl2, PCI_EXP_DEVCTL2_ARI);
 }
 
-bool pcie_cap_is_ari_enabled(const PCIDevice *dev)
+bool pcie_cap_is_arifwd_enabled(const PCIDevice *dev)
 {
     if (!pci_is_express(dev)) {
         return false;
@@ -528,7 +529,7 @@ bool pcie_cap_is_ari_enabled(const PCIDevice *dev)
 }
 
 /**************************************************************************
- * pci express extended capability allocation functions
+ * pci express extended capability list management functions
  * uint16_t ext_cap_id (16 bit)
  * uint8_t cap_ver (4 bit)
  * uint16_t cap_offset (12 bit)
@@ -630,5 +631,5 @@ void pcie_ari_init(PCIDevice *dev, uint16_t offset, uint16_t nextfn)
 {
     pcie_add_capability(dev, PCI_EXT_CAP_ID_ARI, PCI_ARI_VER,
                         offset, PCI_ARI_SIZEOF);
-    pci_set_long(dev->config + offset + PCI_ARI_CAP, PCI_ARI_CAP_NFN(nextfn));
+    pci_set_long(dev->config + offset + PCI_ARI_CAP, (nextfn & 0xff) << 8);
 }
diff --git a/hw/pci/pcie_host.c b/hw/pci/pcie_host.c
index 7c88a1d09..3db038fc7 100644
--- a/hw/pci/pcie_host.c
+++ b/hw/pci/pcie_host.c
@@ -94,7 +94,6 @@ void pcie_host_mmcfg_unmap(PCIExpressHost *e)
 {
     if (e->base_addr != PCIE_BASE_ADDR_UNMAPPED) {
         memory_region_del_subregion(get_system_memory(), &e->mmio);
-        memory_region_destroy(&e->mmio);
         e->base_addr = PCIE_BASE_ADDR_UNMAPPED;
     }
 }
diff --git a/hw/pci/pcie_port.c b/hw/pci/pcie_port.c
index fa2487795..40ca8d5d1 100644
--- a/hw/pci/pcie_port.c
+++ b/hw/pci/pcie_port.c
@@ -154,7 +154,7 @@ static void pcie_slot_class_init(ObjectClass *oc, void *data)
 
     dc->props = pcie_slot_props;
     hc->plug = pcie_cap_slot_hotplug_cb;
-    hc->unplug = pcie_cap_slot_hot_unplug_cb;
+    hc->unplug_request = pcie_cap_slot_hot_unplug_request_cb;
 }
 
 static const TypeInfo pcie_slot_type_info = {
diff --git a/hw/pci/shpc.c b/hw/pci/shpc.c
index 180faa7ad..27c496e8c 100644
--- a/hw/pci/shpc.c
+++ b/hw/pci/shpc.c
@@ -549,8 +549,8 @@ void shpc_device_hotplug_cb(HotplugHandler *hotplug_dev, DeviceState *dev,
     shpc_interrupt_update(pci_hotplug_dev);
 }
 
-void shpc_device_hot_unplug_cb(HotplugHandler *hotplug_dev, DeviceState *dev,
-                               Error **errp)
+void shpc_device_hot_unplug_request_cb(HotplugHandler *hotplug_dev,
+                                       DeviceState *dev, Error **errp)
 {
     Error *local_err = NULL;
     PCIDevice *pci_hotplug_dev = PCI_DEVICE(hotplug_dev);
@@ -559,8 +559,9 @@ void shpc_device_hot_unplug_cb(HotplugHandler *hotplug_dev, DeviceState *dev,
     uint8_t led;
     int slot;
 
-    shpc_device_hotplug_common(PCI_DEVICE(dev), &slot, shpc, errp);
+    shpc_device_hotplug_common(PCI_DEVICE(dev), &slot, shpc, &local_err);
     if (local_err) {
+        error_propagate(errp, local_err);
         return;
     }
 
@@ -662,12 +663,12 @@ void shpc_cleanup(PCIDevice *d, MemoryRegion *bar)
     SHPCDevice *shpc = d->shpc;
     d->cap_present &= ~QEMU_PCI_CAP_SHPC;
     memory_region_del_subregion(bar, &shpc->mmio);
+    object_unparent(OBJECT(&shpc->mmio));
     /* TODO: cleanup config space changes? */
     g_free(shpc->config);
     g_free(shpc->cmask);
     g_free(shpc->wmask);
     g_free(shpc->w1cmask);
-    memory_region_destroy(&shpc->mmio);
     g_free(shpc);
 }
 
diff --git a/hw/pcmcia/pxa2xx.c b/hw/pcmcia/pxa2xx.c
index 55e8a2a62..a7e187743 100644
--- a/hw/pcmcia/pxa2xx.c
+++ b/hw/pcmcia/pxa2xx.c
@@ -149,24 +149,11 @@ PXA2xxPCMCIAState *pxa2xx_pcmcia_init(MemoryRegion *sysmem,
     sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
     s = PXA2XX_PCMCIA(dev);
 
-    if (base == 0x30000000) {
-        s->slot.slot_string = "PXA PC Card Socket 1";
-    } else {
-        s->slot.slot_string = "PXA PC Card Socket 0";
-    }
-
     qdev_init_nofail(dev);
 
     return s;
 }
 
-static void pxa2xx_pcmcia_realize(DeviceState *dev, Error **errp)
-{
-    PXA2xxPCMCIAState *s = PXA2XX_PCMCIA(dev);
-
-    pcmcia_socket_register(&s->slot);
-}
-
 static void pxa2xx_pcmcia_initfn(Object *obj)
 {
     SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
@@ -262,19 +249,11 @@ void pxa2xx_pcmcia_set_irq_cb(void *opaque, qemu_irq irq, qemu_irq cd_irq)
     s->cd_irq = cd_irq;
 }
 
-static void pxa2xx_pcmcia_class_init(ObjectClass *oc, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(oc);
-
-    dc->realize = pxa2xx_pcmcia_realize;
-}
-
 static const TypeInfo pxa2xx_pcmcia_type_info = {
     .name = TYPE_PXA2XX_PCMCIA,
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxPCMCIAState),
     .instance_init = pxa2xx_pcmcia_initfn,
-    .class_init = pxa2xx_pcmcia_class_init,
 };
 
 static void pxa2xx_pcmcia_register_types(void)
diff --git a/hw/ppc/Makefile.objs b/hw/ppc/Makefile.objs
index edd44d03e..19d99200a 100644
--- a/hw/ppc/Makefile.objs
+++ b/hw/ppc/Makefile.objs
@@ -20,4 +20,4 @@ obj-$(CONFIG_MAC) += mac_newworld.o
 obj-$(CONFIG_E500) += e500.o mpc8544ds.o e500plat.o
 obj-$(CONFIG_E500) += mpc8544_guts.o ppce500_spin.o
 # PowerPC 440 Xilinx ML507 reference board.
-obj-y += virtex_ml507.o
+obj-$(CONFIG_XILINX) += virtex_ml507.o
diff --git a/hw/ppc/e500.c b/hw/ppc/e500.c
index 1a5b30d3c..2832fc0da 100644
--- a/hw/ppc/e500.c
+++ b/hw/ppc/e500.c
@@ -36,6 +36,9 @@
 #include "exec/address-spaces.h"
 #include "qemu/host-utils.h"
 #include "hw/pci-host/ppce500.h"
+#include "qemu/error-report.h"
+#include "hw/platform-bus.h"
+#include "hw/net/fsl_etsec/etsec.h"
 
 #define EPAPR_MAGIC                (0x45504150)
 #define BINARY_DEVICE_TREE_FILE    "mpc8544ds.dtb"
@@ -61,6 +64,8 @@
 #define MPC8544_PCI_IO             0xE1000000ULL
 #define MPC8544_UTIL_OFFSET        0xe0000ULL
 #define MPC8544_SPIN_BASE          0xEF000000ULL
+#define MPC8XXX_GPIO_OFFSET        0x000FF000ULL
+#define MPC8XXX_GPIO_IRQ           43
 
 struct boot_info
 {
@@ -122,6 +127,142 @@ static void dt_serial_create(void *fdt, unsigned long long offset,
     }
 }
 
+static void create_dt_mpc8xxx_gpio(void *fdt, const char *soc, const char *mpic)
+{
+    hwaddr mmio0 = MPC8XXX_GPIO_OFFSET;
+    int irq0 = MPC8XXX_GPIO_IRQ;
+    gchar *node = g_strdup_printf("%s/gpio@%"PRIx64, soc, mmio0);
+    gchar *poweroff = g_strdup_printf("%s/power-off", soc);
+    int gpio_ph;
+
+    qemu_fdt_add_subnode(fdt, node);
+    qemu_fdt_setprop_string(fdt, node, "compatible", "fsl,qoriq-gpio");
+    qemu_fdt_setprop_cells(fdt, node, "reg", mmio0, 0x1000);
+    qemu_fdt_setprop_cells(fdt, node, "interrupts", irq0, 0x2);
+    qemu_fdt_setprop_phandle(fdt, node, "interrupt-parent", mpic);
+    qemu_fdt_setprop_cells(fdt, node, "#gpio-cells", 2);
+    qemu_fdt_setprop(fdt, node, "gpio-controller", NULL, 0);
+    gpio_ph = qemu_fdt_alloc_phandle(fdt);
+    qemu_fdt_setprop_cell(fdt, node, "phandle", gpio_ph);
+    qemu_fdt_setprop_cell(fdt, node, "linux,phandle", gpio_ph);
+
+    /* Power Off Pin */
+    qemu_fdt_add_subnode(fdt, poweroff);
+    qemu_fdt_setprop_string(fdt, poweroff, "compatible", "gpio-poweroff");
+    qemu_fdt_setprop_cells(fdt, poweroff, "gpios", gpio_ph, 0, 0);
+
+    g_free(node);
+    g_free(poweroff);
+}
+
+typedef struct PlatformDevtreeData {
+    void *fdt;
+    const char *mpic;
+    int irq_start;
+    const char *node;
+    PlatformBusDevice *pbus;
+} PlatformDevtreeData;
+
+static int create_devtree_etsec(SysBusDevice *sbdev, PlatformDevtreeData *data)
+{
+    eTSEC *etsec = ETSEC_COMMON(sbdev);
+    PlatformBusDevice *pbus = data->pbus;
+    hwaddr mmio0 = platform_bus_get_mmio_addr(pbus, sbdev, 0);
+    int irq0 = platform_bus_get_irqn(pbus, sbdev, 0);
+    int irq1 = platform_bus_get_irqn(pbus, sbdev, 1);
+    int irq2 = platform_bus_get_irqn(pbus, sbdev, 2);
+    gchar *node = g_strdup_printf("/platform/ethernet@%"PRIx64, mmio0);
+    gchar *group = g_strdup_printf("%s/queue-group", node);
+    void *fdt = data->fdt;
+
+    assert((int64_t)mmio0 >= 0);
+    assert(irq0 >= 0);
+    assert(irq1 >= 0);
+    assert(irq2 >= 0);
+
+    qemu_fdt_add_subnode(fdt, node);
+    qemu_fdt_setprop_string(fdt, node, "device_type", "network");
+    qemu_fdt_setprop_string(fdt, node, "compatible", "fsl,etsec2");
+    qemu_fdt_setprop_string(fdt, node, "model", "eTSEC");
+    qemu_fdt_setprop(fdt, node, "local-mac-address", etsec->conf.macaddr.a, 6);
+    qemu_fdt_setprop_cells(fdt, node, "fixed-link", 0, 1, 1000, 0, 0);
+
+    qemu_fdt_add_subnode(fdt, group);
+    qemu_fdt_setprop_cells(fdt, group, "reg", mmio0, 0x1000);
+    qemu_fdt_setprop_cells(fdt, group, "interrupts",
+        data->irq_start + irq0, 0x2,
+        data->irq_start + irq1, 0x2,
+        data->irq_start + irq2, 0x2);
+
+    g_free(node);
+    g_free(group);
+
+    return 0;
+}
+
+static int sysbus_device_create_devtree(SysBusDevice *sbdev, void *opaque)
+{
+    PlatformDevtreeData *data = opaque;
+    bool matched = false;
+
+    if (object_dynamic_cast(OBJECT(sbdev), TYPE_ETSEC_COMMON)) {
+        create_devtree_etsec(sbdev, data);
+        matched = true;
+    }
+
+    if (!matched) {
+        error_report("Device %s is not supported by this machine yet.",
+                     qdev_fw_name(DEVICE(sbdev)));
+        exit(1);
+    }
+
+    return 0;
+}
+
+static void platform_bus_create_devtree(PPCE500Params *params, void *fdt,
+                                        const char *mpic)
+{
+    gchar *node = g_strdup_printf("/platform@%"PRIx64, params->platform_bus_base);
+    const char platcomp[] = "qemu,platform\0simple-bus";
+    uint64_t addr = params->platform_bus_base;
+    uint64_t size = params->platform_bus_size;
+    int irq_start = params->platform_bus_first_irq;
+    PlatformBusDevice *pbus;
+    DeviceState *dev;
+
+    /* Create a /platform node that we can put all devices into */
+
+    qemu_fdt_add_subnode(fdt, node);
+    qemu_fdt_setprop(fdt, node, "compatible", platcomp, sizeof(platcomp));
+
+    /* Our platform bus region is less than 32bit big, so 1 cell is enough for
+       address and size */
+    qemu_fdt_setprop_cells(fdt, node, "#size-cells", 1);
+    qemu_fdt_setprop_cells(fdt, node, "#address-cells", 1);
+    qemu_fdt_setprop_cells(fdt, node, "ranges", 0, addr >> 32, addr, size);
+
+    qemu_fdt_setprop_phandle(fdt, node, "interrupt-parent", mpic);
+
+    dev = qdev_find_recursive(sysbus_get_default(), TYPE_PLATFORM_BUS_DEVICE);
+    pbus = PLATFORM_BUS_DEVICE(dev);
+
+    /* We can only create dt nodes for dynamic devices when they're ready */
+    if (pbus->done_gathering) {
+        PlatformDevtreeData data = {
+            .fdt = fdt,
+            .mpic = mpic,
+            .irq_start = irq_start,
+            .node = node,
+            .pbus = pbus,
+        };
+
+        /* Loop through all dynamic sysbus devices and create nodes for them */
+        foreach_dynamic_sysbus_device(sysbus_device_create_devtree, &data);
+    }
+
+    g_free(node);
+}
+
 static int ppce500_load_device_tree(MachineState *machine,
                                     PPCE500Params *params,
                                     hwaddr addr,
@@ -379,6 +520,14 @@ static int ppce500_load_device_tree(MachineState *machine,
     qemu_fdt_setprop_cell(fdt, pci, "#address-cells", 3);
     qemu_fdt_setprop_string(fdt, "/aliases", "pci0", pci);
 
+    if (params->has_mpc8xxx_gpio) {
+        create_dt_mpc8xxx_gpio(fdt, soc, mpic);
+    }
+
+    if (params->has_platform_bus) {
+        platform_bus_create_devtree(params, fdt, mpic);
+    }
+
     params->fixup_devtree(params, fdt);
 
     if (toplevel_compat) {
@@ -407,6 +556,7 @@ typedef struct DeviceTreeParams {
     hwaddr initrd_size;
     hwaddr kernel_base;
     hwaddr kernel_size;
+    Notifier notifier;
 } DeviceTreeParams;
 
 static void ppce500_reset_device_tree(void *opaque)
@@ -417,6 +567,12 @@ static void ppce500_reset_device_tree(void *opaque)
                              false);
 }
 
+static void ppce500_init_notify(Notifier *notifier, void *data)
+{
+    DeviceTreeParams *p = container_of(notifier, DeviceTreeParams, notifier);
+    ppce500_reset_device_tree(p);
+}
+
 static int ppce500_prep_device_tree(MachineState *machine,
                                     PPCE500Params *params,
                                     hwaddr addr,
@@ -435,6 +591,8 @@ static int ppce500_prep_device_tree(MachineState *machine,
     p->kernel_size = kernel_size;
 
     qemu_register_reset(ppce500_reset_device_tree, p);
+    p->notifier.notify = ppce500_init_notify;
+    qemu_add_machine_init_done_notifier(&p->notifier);
 
     /* Issue the device tree loader once, so that we get the size of the blob */
     return ppce500_load_device_tree(machine, params, addr, initrd_base,
@@ -583,7 +741,7 @@ static qemu_irq *ppce500_init_mpic(PPCE500Params *params, MemoryRegion *ccsr,
     SysBusDevice *s;
     int i;
 
-    mpic = g_new(qemu_irq, 256);
+    mpic = g_new0(qemu_irq, 256);
 
     if (kvm_enabled()) {
         QemuOpts *machine_opts = qemu_get_machine_opts();
@@ -618,6 +776,13 @@ static qemu_irq *ppce500_init_mpic(PPCE500Params *params, MemoryRegion *ccsr,
     return mpic;
 }
 
+static void ppce500_power_off(void *opaque, int line, int on)
+{
+    if (on) {
+        qemu_system_shutdown_request();
+    }
+}
+
 void ppce500_init(MachineState *machine, PPCE500Params *params)
 {
     MemoryRegion *address_space_mem = get_system_memory();
@@ -769,6 +934,40 @@ void ppce500_init(MachineState *machine, PPCE500Params *params)
         cur_base = (32 * 1024 * 1024);
     }
 
+    if (params->has_mpc8xxx_gpio) {
+        qemu_irq poweroff_irq;
+
+        dev = qdev_create(NULL, "mpc8xxx_gpio");
+        s = SYS_BUS_DEVICE(dev);
+        qdev_init_nofail(dev);
+        sysbus_connect_irq(s, 0, mpic[MPC8XXX_GPIO_IRQ]);
+        memory_region_add_subregion(ccsr_addr_space, MPC8XXX_GPIO_OFFSET,
+                                    sysbus_mmio_get_region(s, 0));
+
+        /* Power Off GPIO at Pin 0 */
+        poweroff_irq = qemu_allocate_irq(ppce500_power_off, NULL, 0);
+        qdev_connect_gpio_out(dev, 0, poweroff_irq);
+    }
+
+    /* Platform Bus Device */
+    if (params->has_platform_bus) {
+        dev = qdev_create(NULL, TYPE_PLATFORM_BUS_DEVICE);
+        dev->id = TYPE_PLATFORM_BUS_DEVICE;
+        qdev_prop_set_uint32(dev, "num_irqs", params->platform_bus_num_irqs);
+        qdev_prop_set_uint32(dev, "mmio_size", params->platform_bus_size);
+        qdev_init_nofail(dev);
+        s = SYS_BUS_DEVICE(dev);
+
+        for (i = 0; i < params->platform_bus_num_irqs; i++) {
+            int irqn = params->platform_bus_first_irq + i;
+            sysbus_connect_irq(s, i, mpic[irqn]);
+        }
+
+        memory_region_add_subregion(address_space_mem,
+                                    params->platform_bus_base,
+                                    sysbus_mmio_get_region(s, 0));
+    }
+
     /* Load kernel. */
     if (machine->kernel_filename) {
         kernel_base = cur_base;
@@ -830,7 +1029,8 @@ void ppce500_init(MachineState *machine, PPCE500Params *params)
          * Hrm. No ELF image? Try a uImage, maybe someone is giving us an
          * ePAPR compliant kernel
          */
-        kernel_size = load_uimage(filename, &bios_entry, &loadaddr, NULL);
+        kernel_size = load_uimage(filename, &bios_entry, &loadaddr, NULL,
+                                  NULL, NULL);
         if (kernel_size < 0) {
             fprintf(stderr, "qemu: could not load firmware '%s'\n", filename);
             exit(1);
diff --git a/hw/ppc/e500.h b/hw/ppc/e500.h
index 08b25fab4..9f61ab2b1 100644
--- a/hw/ppc/e500.h
+++ b/hw/ppc/e500.h
@@ -11,6 +11,12 @@ typedef struct PPCE500Params {
     void (*fixup_devtree)(struct PPCE500Params *params, void *fdt);
 
     int mpic_version;
+    bool has_mpc8xxx_gpio;
+    bool has_platform_bus;
+    hwaddr platform_bus_base;
+    hwaddr platform_bus_size;
+    int platform_bus_first_irq;
+    int platform_bus_num_irqs;
 } PPCE500Params;
 
 void ppce500_init(MachineState *machine, PPCE500Params *params);
diff --git a/hw/ppc/e500plat.c b/hw/ppc/e500plat.c
index 27df31ddb..d50ae000e 100644
--- a/hw/ppc/e500plat.c
+++ b/hw/ppc/e500plat.c
@@ -35,6 +35,12 @@ static void e500plat_init(MachineState *machine)
         .pci_nr_slots = PCI_SLOT_MAX - 1,
         .fixup_devtree = e500plat_fixup_devtree,
         .mpic_version = OPENPIC_MODEL_FSL_MPIC_42,
+        .has_mpc8xxx_gpio = true,
+        .has_platform_bus = true,
+        .platform_bus_base = 0xf00000000ULL,
+        .platform_bus_size = (128ULL * 1024 * 1024),
+        .platform_bus_first_irq = 5,
+        .platform_bus_num_irqs = 10,
     };
 
     /* Older KVM versions don't support EPR which breaks guests when we announce
@@ -51,6 +57,7 @@ static QEMUMachine e500plat_machine = {
     .desc = "generic paravirt e500 platform",
     .init = e500plat_init,
     .max_cpus = 32,
+    .has_dynamic_sysbus = true,
 };
 
 static void e500plat_machine_init(void)
diff --git a/hw/ppc/mac.h b/hw/ppc/mac.h
index c1faf9ce2..8bdba30c1 100644
--- a/hw/ppc/mac.h
+++ b/hw/ppc/mac.h
@@ -57,6 +57,7 @@ typedef struct CUDATimer {
     uint16_t counter_value;
     int64_t load_time;
     int64_t next_irq_time;
+    uint64_t frequency;
     QEMUTimer *timer;
 } CUDATimer;
 
@@ -97,6 +98,7 @@ typedef struct CUDAState {
     CUDATimer timers[2];
 
     uint32_t tick_offset;
+    uint64_t frequency;
 
     uint8_t last_b;
     uint8_t last_acr;
@@ -129,7 +131,7 @@ typedef struct MACIOIDEState {
 
     MemoryRegion mem;
     IDEBus bus;
-    BlockDriverAIOCB *aiocb;
+    BlockAIOCB *aiocb;
     IDEDMA dma;
     void *dbdma;
     bool dma_active;
@@ -178,6 +180,4 @@ typedef struct MacIONVRAMState {
 } MacIONVRAMState;
 
 void pmac_format_nvram_partition (MacIONVRAMState *nvr, int len);
-uint8_t macio_nvram_read(MacIONVRAMState *s, uint32_t addr);
-void macio_nvram_write(MacIONVRAMState *s, uint32_t addr, uint8_t val);
 #endif /* !defined(__PPC_MAC_H__) */
diff --git a/hw/ppc/mac_newworld.c b/hw/ppc/mac_newworld.c
index f5bccd2ad..89aee716d 100644
--- a/hw/ppc/mac_newworld.c
+++ b/hw/ppc/mac_newworld.c
@@ -65,7 +65,7 @@
 #include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 #include "hw/usb.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 #include "hw/sysbus.h"
 
@@ -176,6 +176,8 @@ static void ppc_core99_init(MachineState *machine)
     SysBusDevice *s;
     DeviceState *dev;
     int *token = g_new(int, 1);
+    hwaddr nvram_addr = 0xFFF04000;
+    uint64_t tbfreq;
 
     linux_boot = (kernel_filename != NULL);
 
@@ -204,7 +206,8 @@ static void ppc_core99_init(MachineState *machine)
     memory_region_add_subregion(get_system_memory(), 0, ram);
 
     /* allocate and load BIOS */
-    memory_region_init_ram(bios, NULL, "ppc_core99.bios", BIOS_SIZE);
+    memory_region_init_ram(bios, NULL, "ppc_core99.bios", BIOS_SIZE,
+                           &error_abort);
     vmstate_register_ram_global(bios);
 
     if (bios_name == NULL)
@@ -346,7 +349,7 @@ static void ppc_core99_init(MachineState *machine)
         }
     }
 
-    pic = g_new(qemu_irq, 64);
+    pic = g_new0(qemu_irq, 64);
 
     dev = qdev_create(NULL, TYPE_OPENPIC);
     qdev_prop_set_uint32(dev, "model", OPENPIC_MODEL_RAVEN);
@@ -372,6 +375,14 @@ static void ppc_core99_init(MachineState *machine)
         pci_bus = pci_pmac_init(pic, get_system_memory(), get_system_io());
         machine_arch = ARCH_MAC99;
     }
+
+    /* Timebase Frequency */
+    if (kvm_enabled()) {
+        tbfreq = kvmppc_get_tbfreq();
+    } else {
+        tbfreq = TBFREQ;
+    }
+
     /* init basic PC hardware */
     escc_mem = escc_init(0, pic[0x25], pic[0x24],
                          serial_hds[0], serial_hds[1], ESCC_CLOCK, 4);
@@ -385,10 +396,11 @@ static void ppc_core99_init(MachineState *machine)
     qdev_connect_gpio_out(dev, 2, pic[0x02]); /* IDE DMA */
     qdev_connect_gpio_out(dev, 3, pic[0x0e]); /* IDE */
     qdev_connect_gpio_out(dev, 4, pic[0x03]); /* IDE DMA */
+    qdev_prop_set_uint64(dev, "frequency", tbfreq);
     macio_init(macio, pic_mem, escc_bar);
 
     /* We only emulate 2 out of 3 IDE controllers for now */
-    ide_drive_get(hd, MAX_IDE_BUS);
+    ide_drive_get(hd, ARRAY_SIZE(hd));
 
     macio_ide = MACIO_IDE(object_resolve_path_component(OBJECT(macio),
                                                         "ide[0]"));
@@ -426,11 +438,18 @@ static void ppc_core99_init(MachineState *machine)
     }
 
     /* The NewWorld NVRAM is not located in the MacIO device */
+#ifdef CONFIG_KVM
+    if (kvm_enabled() && getpagesize() > 4096) {
+        /* We can't combine read-write and read-only in a single page, so
+           move the NVRAM out of ROM again for KVM */
+        nvram_addr = 0xFFE00000;
+    }
+#endif
     dev = qdev_create(NULL, TYPE_MACIO_NVRAM);
     qdev_prop_set_uint32(dev, "size", 0x2000);
     qdev_prop_set_uint32(dev, "it_shift", 1);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0xFFF04000);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, nvram_addr);
     nvr = MACIO_NVRAM(dev);
     pmac_format_nvram_partition(nvr, 0x2000);
     /* No PCI init: the BIOS will do it */
@@ -461,28 +480,34 @@ static void ppc_core99_init(MachineState *machine)
 #ifdef CONFIG_KVM
         uint8_t *hypercall;
 
-        fw_cfg_add_i32(fw_cfg, FW_CFG_PPC_TBFREQ, kvmppc_get_tbfreq());
         hypercall = g_malloc(16);
         kvmppc_get_hypercall(env, hypercall, 16);
         fw_cfg_add_bytes(fw_cfg, FW_CFG_PPC_KVM_HC, hypercall, 16);
         fw_cfg_add_i32(fw_cfg, FW_CFG_PPC_KVM_PID, getpid());
 #endif
-    } else {
-        fw_cfg_add_i32(fw_cfg, FW_CFG_PPC_TBFREQ, TBFREQ);
     }
+    fw_cfg_add_i32(fw_cfg, FW_CFG_PPC_TBFREQ, tbfreq);
     /* Mac OS X requires a "known good" clock-frequency value; pass it one. */
     fw_cfg_add_i32(fw_cfg, FW_CFG_PPC_CLOCKFREQ, CLOCKFREQ);
     fw_cfg_add_i32(fw_cfg, FW_CFG_PPC_BUSFREQ, BUSFREQ);
+    fw_cfg_add_i32(fw_cfg, FW_CFG_PPC_NVRAM_ADDR, nvram_addr);
 
     qemu_register_boot_set(fw_cfg_boot_set, fw_cfg);
 }
 
+static int core99_kvm_type(const char *arg)
+{
+    /* Always force PR KVM */
+    return 2;
+}
+
 static QEMUMachine core99_machine = {
     .name = "mac99",
     .desc = "Mac99 based PowerMAC",
     .init = ppc_core99_init,
     .max_cpus = MAX_CPUS,
     .default_boot_order = "cd",
+    .kvm_type = core99_kvm_type,
 };
 
 static void core99_machine_init(void)
diff --git a/hw/ppc/mac_oldworld.c b/hw/ppc/mac_oldworld.c
index cd9bdbc53..32c21a428 100644
--- a/hw/ppc/mac_oldworld.c
+++ b/hw/ppc/mac_oldworld.c
@@ -40,7 +40,7 @@
 #include "elf.h"
 #include "sysemu/kvm.h"
 #include "kvm_ppc.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 
 #define MAX_IDE_BUS 2
@@ -103,6 +103,7 @@ static void ppc_heathrow_init(MachineState *machine)
     uint16_t ppc_boot_device;
     DriveInfo *hd[MAX_IDE_BUS * MAX_IDE_DEVS];
     void *fw_cfg;
+    uint64_t tbfreq;
 
     linux_boot = (kernel_filename != NULL);
 
@@ -135,7 +136,8 @@ static void ppc_heathrow_init(MachineState *machine)
     memory_region_add_subregion(sysmem, 0, ram);
 
     /* allocate and load BIOS */
-    memory_region_init_ram(bios, NULL, "ppc_heathrow.bios", BIOS_SIZE);
+    memory_region_init_ram(bios, NULL, "ppc_heathrow.bios", BIOS_SIZE,
+                           &error_abort);
     vmstate_register_ram_global(bios);
 
     if (bios_name == NULL)
@@ -250,6 +252,13 @@ static void ppc_heathrow_init(MachineState *machine)
         }
     }
 
+    /* Timebase Frequency */
+    if (kvm_enabled()) {
+        tbfreq = kvmppc_get_tbfreq();
+    } else {
+        tbfreq = TBFREQ;
+    }
+
     /* init basic PC hardware */
     if (PPC_INPUT(env) != PPC_FLAGS_INPUT_6xx) {
         hw_error("Only 6xx bus is supported on heathrow machine\n");
@@ -269,7 +278,7 @@ static void ppc_heathrow_init(MachineState *machine)
         pci_nic_init_nofail(&nd_table[i], pci_bus, "ne2k_pci", NULL);
 
 
-    ide_drive_get(hd, MAX_IDE_BUS);
+    ide_drive_get(hd, ARRAY_SIZE(hd));
 
     macio = pci_create(pci_bus, -1, TYPE_OLDWORLD_MACIO);
     dev = DEVICE(macio);
@@ -278,6 +287,7 @@ static void ppc_heathrow_init(MachineState *machine)
     qdev_connect_gpio_out(dev, 2, pic[0x02]); /* IDE-0 DMA */
     qdev_connect_gpio_out(dev, 3, pic[0x0E]); /* IDE-1 */
     qdev_connect_gpio_out(dev, 4, pic[0x03]); /* IDE-1 DMA */
+    qdev_prop_set_uint64(dev, "frequency", tbfreq);
     macio_init(macio, pic_mem, escc_bar);
 
     macio_ide = MACIO_IDE(object_resolve_path_component(OBJECT(macio),
@@ -330,15 +340,13 @@ static void ppc_heathrow_init(MachineState *machine)
 #ifdef CONFIG_KVM
         uint8_t *hypercall;
 
-        fw_cfg_add_i32(fw_cfg, FW_CFG_PPC_TBFREQ, kvmppc_get_tbfreq());
         hypercall = g_malloc(16);
         kvmppc_get_hypercall(env, hypercall, 16);
         fw_cfg_add_bytes(fw_cfg, FW_CFG_PPC_KVM_HC, hypercall, 16);
         fw_cfg_add_i32(fw_cfg, FW_CFG_PPC_KVM_PID, getpid());
 #endif
-    } else {
-        fw_cfg_add_i32(fw_cfg, FW_CFG_PPC_TBFREQ, TBFREQ);
     }
+    fw_cfg_add_i32(fw_cfg, FW_CFG_PPC_TBFREQ, tbfreq);
     /* Mac OS X requires a "known good" clock-frequency value; pass it one. */
     fw_cfg_add_i32(fw_cfg, FW_CFG_PPC_CLOCKFREQ, CLOCKFREQ);
     fw_cfg_add_i32(fw_cfg, FW_CFG_PPC_BUSFREQ, BUSFREQ);
@@ -346,6 +354,12 @@ static void ppc_heathrow_init(MachineState *machine)
     qemu_register_boot_set(fw_cfg_boot_set, fw_cfg);
 }
 
+static int heathrow_kvm_type(const char *arg)
+{
+    /* Always force PR KVM */
+    return 2;
+}
+
 static QEMUMachine heathrow_machine = {
     .name = "g3beige",
     .desc = "Heathrow based PowerMAC",
@@ -355,6 +369,7 @@ static QEMUMachine heathrow_machine = {
     .is_default = 1,
 #endif
     .default_boot_order = "cd", /* TOFIX "cad" when Mac floppy is implemented */
+    .kvm_type = heathrow_kvm_type,
 };
 
 static void heathrow_machine_init(void)
diff --git a/hw/ppc/ppc405_boards.c b/hw/ppc/ppc405_boards.c
index 11d33792f..1dcea7730 100644
--- a/hw/ppc/ppc405_boards.c
+++ b/hw/ppc/ppc405_boards.c
@@ -28,11 +28,12 @@
 #include "hw/block/flash.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/qtest.h"
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "hw/boards.h"
 #include "qemu/log.h"
 #include "qemu/error-report.h"
 #include "hw/loader.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "exec/address-spaces.h"
 
@@ -214,7 +215,7 @@ static void ref405ep_init(MachineState *machine)
                         33333333, &pic, kernel_filename == NULL ? 0 : 1);
     /* allocate SRAM */
     sram_size = 512 * 1024;
-    memory_region_init_ram(sram, NULL, "ef405ep.sram", sram_size);
+    memory_region_init_ram(sram, NULL, "ef405ep.sram", sram_size, &error_abort);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(sysmem, 0xFFF00000, sram);
     /* allocate and load BIOS */
@@ -225,17 +226,19 @@ static void ref405ep_init(MachineState *machine)
 #ifdef USE_FLASH_BIOS
     dinfo = drive_get(IF_PFLASH, 0, fl_idx);
     if (dinfo) {
-        bios_size = bdrv_getlength(dinfo->bdrv);
+        BlockBackend *blk = blk_by_legacy_dinfo(dinfo);
+
+        bios_size = blk_getlength(blk);
         fl_sectors = (bios_size + 65535) >> 16;
 #ifdef DEBUG_BOARD_INIT
         printf("Register parallel flash %d size %lx"
                " at addr %lx '%s' %d\n",
                fl_idx, bios_size, -bios_size,
-               bdrv_get_device_name(dinfo->bdrv), fl_sectors);
+               blk_name(blk), fl_sectors);
 #endif
         pflash_cfi02_register((uint32_t)(-bios_size),
                               NULL, "ef405ep.bios", bios_size,
-                              dinfo->bdrv, 65536, fl_sectors, 1,
+                              blk, 65536, fl_sectors, 1,
                               2, 0x0001, 0x22DA, 0x0000, 0x0000, 0x555, 0x2AA,
                               1);
         fl_idx++;
@@ -246,7 +249,8 @@ static void ref405ep_init(MachineState *machine)
         printf("Load BIOS from file\n");
 #endif
         bios = g_new(MemoryRegion, 1);
-        memory_region_init_ram(bios, NULL, "ef405ep.bios", BIOS_SIZE);
+        memory_region_init_ram(bios, NULL, "ef405ep.bios", BIOS_SIZE,
+                               &error_abort);
         vmstate_register_ram_global(bios);
 
         if (bios_name == NULL)
@@ -547,7 +551,9 @@ static void taihu_405ep_init(MachineState *machine)
 #if defined(USE_FLASH_BIOS)
     dinfo = drive_get(IF_PFLASH, 0, fl_idx);
     if (dinfo) {
-        bios_size = bdrv_getlength(dinfo->bdrv);
+        BlockBackend *blk = blk_by_legacy_dinfo(dinfo);
+
+        bios_size = blk_getlength(blk);
         /* XXX: should check that size is 2MB */
         //        bios_size = 2 * 1024 * 1024;
         fl_sectors = (bios_size + 65535) >> 16;
@@ -555,11 +561,11 @@ static void taihu_405ep_init(MachineState *machine)
         printf("Register parallel flash %d size %lx"
                " at addr %lx '%s' %d\n",
                fl_idx, bios_size, -bios_size,
-               bdrv_get_device_name(dinfo->bdrv), fl_sectors);
+               blk_name(blk), fl_sectors);
 #endif
         pflash_cfi02_register((uint32_t)(-bios_size),
                               NULL, "taihu_405ep.bios", bios_size,
-                              dinfo->bdrv, 65536, fl_sectors, 1,
+                              blk, 65536, fl_sectors, 1,
                               4, 0x0001, 0x22DA, 0x0000, 0x0000, 0x555, 0x2AA,
                               1);
         fl_idx++;
@@ -572,7 +578,8 @@ static void taihu_405ep_init(MachineState *machine)
         if (bios_name == NULL)
             bios_name = BIOS_FILENAME;
         bios = g_new(MemoryRegion, 1);
-        memory_region_init_ram(bios, NULL, "taihu_405ep.bios", BIOS_SIZE);
+        memory_region_init_ram(bios, NULL, "taihu_405ep.bios", BIOS_SIZE,
+                               &error_abort);
         vmstate_register_ram_global(bios);
         filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
         if (filename) {
@@ -593,7 +600,9 @@ static void taihu_405ep_init(MachineState *machine)
     /* Register Linux flash */
     dinfo = drive_get(IF_PFLASH, 0, fl_idx);
     if (dinfo) {
-        bios_size = bdrv_getlength(dinfo->bdrv);
+        BlockBackend *blk = blk_by_legacy_dinfo(dinfo);
+
+        bios_size = blk_getlength(blk);
         /* XXX: should check that size is 32MB */
         bios_size = 32 * 1024 * 1024;
         fl_sectors = (bios_size + 65535) >> 16;
@@ -601,10 +610,10 @@ static void taihu_405ep_init(MachineState *machine)
         printf("Register parallel flash %d size %lx"
                " at addr " TARGET_FMT_lx " '%s'\n",
                fl_idx, bios_size, (target_ulong)0xfc000000,
-               bdrv_get_device_name(dinfo->bdrv));
+               blk_name(blk));
 #endif
         pflash_cfi02_register(0xfc000000, NULL, "taihu_405ep.flash", bios_size,
-                              dinfo->bdrv, 65536, fl_sectors, 1,
+                              blk, 65536, fl_sectors, 1,
                               4, 0x0001, 0x22DA, 0x0000, 0x0000, 0x555, 0x2AA,
                               1);
         fl_idx++;
diff --git a/hw/ppc/ppc405_uc.c b/hw/ppc/ppc405_uc.c
index a73e918a1..c77434ae0 100644
--- a/hw/ppc/ppc405_uc.c
+++ b/hw/ppc/ppc405_uc.c
@@ -974,7 +974,8 @@ static void ppc405_ocm_init(CPUPPCState *env)
 
     ocm = g_malloc0(sizeof(ppc405_ocm_t));
     /* XXX: Size is 4096 or 0x04000000 */
-    memory_region_init_ram(&ocm->isarc_ram, NULL, "ppc405.ocm", 4096);
+    memory_region_init_ram(&ocm->isarc_ram, NULL, "ppc405.ocm", 4096,
+                           &error_abort);
     vmstate_register_ram_global(&ocm->isarc_ram);
     memory_region_init_alias(&ocm->dsarc_ram, NULL, "ppc405.dsarc", &ocm->isarc_ram,
                              0, 4096);
diff --git a/hw/ppc/ppc440_bamboo.c b/hw/ppc/ppc440_bamboo.c
index 81a06d310..778970aa9 100644
--- a/hw/ppc/ppc440_bamboo.c
+++ b/hw/ppc/ppc440_bamboo.c
@@ -253,7 +253,8 @@ static void bamboo_init(MachineState *machine)
 
     /* Load kernel. */
     if (kernel_filename) {
-        success = load_uimage(kernel_filename, &entry, &loadaddr, NULL);
+        success = load_uimage(kernel_filename, &entry, &loadaddr, NULL,
+                              NULL, NULL);
         if (success < 0) {
             success = load_elf(kernel_filename, NULL, NULL, &elf_entry,
                                &elf_lowaddr, NULL, 1, ELF_MACHINE, 0);
diff --git a/hw/ppc/ppc4xx_devs.c b/hw/ppc/ppc4xx_devs.c
index 2b5d2cd6c..2f38ff7d2 100644
--- a/hw/ppc/ppc4xx_devs.c
+++ b/hw/ppc/ppc4xx_devs.c
@@ -422,7 +422,7 @@ static void sdram_set_bcr(ppc4xx_sdram_t *sdram,
                                     &sdram->containers[n]);
         memory_region_del_subregion(&sdram->containers[n],
                                     &sdram->ram_memories[n]);
-        memory_region_destroy(&sdram->containers[n]);
+        object_unparent(OBJECT(&sdram->containers[n]));
     }
     *bcrp = bcr & 0xFFDEE001;
     if (enabled && (bcr & 0x00000001)) {
diff --git a/hw/ppc/ppc4xx_pci.c b/hw/ppc/ppc4xx_pci.c
index 55a3cabee..0bb3cdb46 100644
--- a/hw/ppc/ppc4xx_pci.c
+++ b/hw/ppc/ppc4xx_pci.c
@@ -92,30 +92,6 @@ typedef struct PPC4xxPCIState PPC4xxPCIState;
 
 #define PCI_ALL_SIZE        (PCI_REG_BASE + PCI_REG_SIZE)
 
-static uint64_t pci4xx_cfgaddr_read(void *opaque, hwaddr addr,
-                                    unsigned size)
-{
-    PPC4xxPCIState *ppc4xx_pci = opaque;
-    PCIHostState *phb = PCI_HOST_BRIDGE(ppc4xx_pci);
-
-    return phb->config_reg;
-}
-
-static void pci4xx_cfgaddr_write(void *opaque, hwaddr addr,
-                                  uint64_t value, unsigned size)
-{
-    PPC4xxPCIState *ppc4xx_pci = opaque;
-    PCIHostState *phb = PCI_HOST_BRIDGE(ppc4xx_pci);
-
-    phb->config_reg = value & ~0x3;
-}
-
-static const MemoryRegionOps pci4xx_cfgaddr_ops = {
-    .read = pci4xx_cfgaddr_read,
-    .write = pci4xx_cfgaddr_write,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
 static void ppc4xx_pci_reg_write4(void *opaque, hwaddr offset,
                                   uint64_t value, unsigned size)
 {
diff --git a/hw/ppc/prep.c b/hw/ppc/prep.c
index f0ef1af11..dd8433d0c 100644
--- a/hw/ppc/prep.c
+++ b/hw/ppc/prep.c
@@ -38,7 +38,7 @@
 #include "hw/loader.h"
 #include "hw/timer/mc146818rtc.h"
 #include "hw/isa/pc87312.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/arch_init.h"
 #include "sysemu/qtest.h"
 #include "exec/address-spaces.h"
@@ -519,7 +519,7 @@ static void ppc_prep_init(MachineState *machine)
         }
     }
 
-    ide_drive_get(hd, MAX_IDE_BUS);
+    ide_drive_get(hd, ARRAY_SIZE(hd));
     for(i = 0; i < MAX_IDE_BUS; i++) {
         isa_ide_init(isa_bus, ide_iobase[i], ide_iobase2[i], ide_irq[i],
                      hd[2 * i],
diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
index d01978f3d..30de25de5 100644
--- a/hw/ppc/spapr.c
+++ b/hw/ppc/spapr.c
@@ -29,7 +29,7 @@
 #include "hw/fw-path-provider.h"
 #include "elf.h"
 #include "net/net.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/cpus.h"
 #include "sysemu/kvm.h"
 #include "kvm_ppc.h"
@@ -55,6 +55,9 @@
 #include "qemu/config-file.h"
 #include "qemu/error-report.h"
 #include "trace.h"
+#include "hw/nmi.h"
+
+#include "hw/compat.h"
 
 #include <libfdt.h>
 
@@ -70,6 +73,7 @@
  */
 #define FDT_MAX_SIZE            0x40000
 #define RTAS_MAX_SIZE           0x10000
+#define RTAS_MAX_ADDR           0x80000000 /* RTAS must stay below that */
 #define FW_MAX_SIZE             0x400000
 #define FW_FILE_NAME            "slof.bin"
 #define FW_OVERHEAD             0x2800000
@@ -79,7 +83,7 @@
 
 #define TIMEBASE_FREQ           512000000ULL
 
-#define MAX_CPUS                256
+#define MAX_CPUS                255
 
 #define PHANDLE_XICP            0x00001111
 
@@ -282,6 +286,19 @@ static size_t create_page_sizes_prop(CPUPPCState *env, uint32_t *prop,
     return (p - prop) * sizeof(uint32_t);
 }
 
+static hwaddr spapr_node0_size(void)
+{
+    if (nb_numa_nodes) {
+        int i;
+        for (i = 0; i < nb_numa_nodes; ++i) {
+            if (numa_info[i].node_mem) {
+                return MIN(pow2floor(numa_info[i].node_mem), ram_size);
+            }
+        }
+    }
+    return ram_size;
+}
+
 #define _FDT(exp) \
     do { \
         int ret = (exp);                                           \
@@ -318,6 +335,7 @@ static void *spapr_create_fdt_skel(hwaddr initrd_base,
     QemuOpts *opts = qemu_opts_find(qemu_find_opts("smp-opts"), NULL);
     unsigned sockets = opts ? qemu_opt_get_number(opts, "sockets", 0) : 0;
     uint32_t cpus_per_socket = sockets ? (smp_cpus / sockets) : 1;
+    char *buf;
 
     add_str(hypertas, "hcall-pft");
     add_str(hypertas, "hcall-term");
@@ -347,6 +365,29 @@ static void *spapr_create_fdt_skel(hwaddr initrd_base,
     _FDT((fdt_property_string(fdt, "model", "IBM pSeries (emulated by qemu)")));
     _FDT((fdt_property_string(fdt, "compatible", "qemu,pseries")));
 
+    /*
+     * Add info to guest to indentify which host is it being run on
+     * and what is the uuid of the guest
+     */
+    if (kvmppc_get_host_model(&buf)) {
+        _FDT((fdt_property_string(fdt, "host-model", buf)));
+        g_free(buf);
+    }
+    if (kvmppc_get_host_serial(&buf)) {
+        _FDT((fdt_property_string(fdt, "host-serial", buf)));
+        g_free(buf);
+    }
+
+    buf = g_strdup_printf(UUID_FMT, qemu_uuid[0], qemu_uuid[1],
+                          qemu_uuid[2], qemu_uuid[3], qemu_uuid[4],
+                          qemu_uuid[5], qemu_uuid[6], qemu_uuid[7],
+                          qemu_uuid[8], qemu_uuid[9], qemu_uuid[10],
+                          qemu_uuid[11], qemu_uuid[12], qemu_uuid[13],
+                          qemu_uuid[14], qemu_uuid[15]);
+
+    _FDT((fdt_property_string(fdt, "vm,uuid", buf)));
+    g_free(buf);
+
     _FDT((fdt_property_cell(fdt, "#address-cells", 0x2)));
     _FDT((fdt_property_cell(fdt, "#size-cells", 0x2)));
 
@@ -501,6 +542,15 @@ static void *spapr_create_fdt_skel(hwaddr initrd_base,
 
     _FDT((fdt_property_cell(fdt, "rtas-error-log-max", RTAS_ERROR_LOG_MAX)));
 
+    /*
+     * According to PAPR, rtas ibm,os-term does not guarantee a return
+     * back to the guest cpu.
+     *
+     * While an additional ibm,extended-os-term property indicates that
+     * rtas call return will always occur. Set this property.
+     */
+    _FDT((fdt_property(fdt, "ibm,extended-os-term", NULL, 0)));
+
     _FDT((fdt_end_node(fdt)));
 
     /* interrupt controller */
@@ -596,72 +646,75 @@ int spapr_h_cas_compose_response(target_ulong addr, target_ulong size)
     return 0;
 }
 
-static int spapr_populate_memory(sPAPREnvironment *spapr, void *fdt)
+static void spapr_populate_memory_node(void *fdt, int nodeid, hwaddr start,
+                                       hwaddr size)
 {
-    uint32_t associativity[] = {cpu_to_be32(0x4), cpu_to_be32(0x0),
-                                cpu_to_be32(0x0), cpu_to_be32(0x0),
-                                cpu_to_be32(0x0)};
+    uint32_t associativity[] = {
+        cpu_to_be32(0x4), /* length */
+        cpu_to_be32(0x0), cpu_to_be32(0x0),
+        cpu_to_be32(0x0), cpu_to_be32(nodeid)
+    };
     char mem_name[32];
-    hwaddr node0_size, mem_start, node_size;
     uint64_t mem_reg_property[2];
-    int i, off;
+    int off;
 
-    /* memory node(s) */
-    if (nb_numa_nodes > 1 && numa_info[0].node_mem < ram_size) {
-        node0_size = numa_info[0].node_mem;
-    } else {
-        node0_size = ram_size;
-    }
+    mem_reg_property[0] = cpu_to_be64(start);
+    mem_reg_property[1] = cpu_to_be64(size);
 
-    /* RMA */
-    mem_reg_property[0] = 0;
-    mem_reg_property[1] = cpu_to_be64(spapr->rma_size);
-    off = fdt_add_subnode(fdt, 0, "memory@0");
+    sprintf(mem_name, "memory@" TARGET_FMT_lx, start);
+    off = fdt_add_subnode(fdt, 0, mem_name);
     _FDT(off);
     _FDT((fdt_setprop_string(fdt, off, "device_type", "memory")));
     _FDT((fdt_setprop(fdt, off, "reg", mem_reg_property,
                       sizeof(mem_reg_property))));
     _FDT((fdt_setprop(fdt, off, "ibm,associativity", associativity,
                       sizeof(associativity))));
+}
+
+static int spapr_populate_memory(sPAPREnvironment *spapr, void *fdt)
+{
+    hwaddr mem_start, node_size;
+    int i, nb_nodes = nb_numa_nodes;
+    NodeInfo *nodes = numa_info;
+    NodeInfo ramnode;
 
-    /* RAM: Node 0 */
-    if (node0_size > spapr->rma_size) {
-        mem_reg_property[0] = cpu_to_be64(spapr->rma_size);
-        mem_reg_property[1] = cpu_to_be64(node0_size - spapr->rma_size);
-
-        sprintf(mem_name, "memory@" TARGET_FMT_lx, spapr->rma_size);
-        off = fdt_add_subnode(fdt, 0, mem_name);
-        _FDT(off);
-        _FDT((fdt_setprop_string(fdt, off, "device_type", "memory")));
-        _FDT((fdt_setprop(fdt, off, "reg", mem_reg_property,
-                          sizeof(mem_reg_property))));
-        _FDT((fdt_setprop(fdt, off, "ibm,associativity", associativity,
-                          sizeof(associativity))));
-    }
-
-    /* RAM: Node 1 and beyond */
-    mem_start = node0_size;
-    for (i = 1; i < nb_numa_nodes; i++) {
-        mem_reg_property[0] = cpu_to_be64(mem_start);
+    /* No NUMA nodes, assume there is just one node with whole RAM */
+    if (!nb_numa_nodes) {
+        nb_nodes = 1;
+        ramnode.node_mem = ram_size;
+        nodes = &ramnode;
+    }
+
+    for (i = 0, mem_start = 0; i < nb_nodes; ++i) {
+        if (!nodes[i].node_mem) {
+            continue;
+        }
         if (mem_start >= ram_size) {
             node_size = 0;
         } else {
-            node_size = numa_info[i].node_mem;
+            node_size = nodes[i].node_mem;
             if (node_size > ram_size - mem_start) {
                 node_size = ram_size - mem_start;
             }
         }
-        mem_reg_property[1] = cpu_to_be64(node_size);
-        associativity[3] = associativity[4] = cpu_to_be32(i);
-        sprintf(mem_name, "memory@" TARGET_FMT_lx, mem_start);
-        off = fdt_add_subnode(fdt, 0, mem_name);
-        _FDT(off);
-        _FDT((fdt_setprop_string(fdt, off, "device_type", "memory")));
-        _FDT((fdt_setprop(fdt, off, "reg", mem_reg_property,
-                          sizeof(mem_reg_property))));
-        _FDT((fdt_setprop(fdt, off, "ibm,associativity", associativity,
-                          sizeof(associativity))));
-        mem_start += node_size;
+        if (!mem_start) {
+            /* ppc_spapr_init() checks for rma_size <= node0_size already */
+            spapr_populate_memory_node(fdt, i, 0, spapr->rma_size);
+            mem_start += spapr->rma_size;
+            node_size -= spapr->rma_size;
+        }
+        for ( ; node_size; ) {
+            hwaddr sizetmp = pow2floor(node_size);
+
+            /* mem_start != 0 here */
+            if (ctzl(mem_start) < ctzl(sizetmp)) {
+                sizetmp = 1ULL << ctzl(mem_start);
+            }
+
+            spapr_populate_memory_node(fdt, i, mem_start, sizetmp);
+            node_size -= sizetmp;
+            mem_start += sizetmp;
+        }
     }
 
     return 0;
@@ -745,6 +798,7 @@ static void spapr_finalize_fdt(sPAPREnvironment *spapr,
 
     cpu_physical_memory_write(fdt_addr, fdt, fdt_totalsize(fdt));
 
+    g_free(bootlist);
     g_free(fdt);
 }
 
@@ -791,25 +845,58 @@ static void spapr_reset_htab(sPAPREnvironment *spapr)
 
     /* Update the RMA size if necessary */
     if (spapr->vrma_adjust) {
-        hwaddr node0_size = (nb_numa_nodes > 1) ?
-            numa_info[0].node_mem : ram_size;
-        spapr->rma_size = kvmppc_rma_size(node0_size, spapr->htab_shift);
+        spapr->rma_size = kvmppc_rma_size(spapr_node0_size(),
+                                          spapr->htab_shift);
     }
 }
 
+static int find_unknown_sysbus_device(SysBusDevice *sbdev, void *opaque)
+{
+    bool matched = false;
+
+    if (object_dynamic_cast(OBJECT(sbdev), TYPE_SPAPR_PCI_HOST_BRIDGE)) {
+        matched = true;
+    }
+
+    if (!matched) {
+        error_report("Device %s is not supported by this machine yet.",
+                     qdev_fw_name(DEVICE(sbdev)));
+        exit(1);
+    }
+
+    return 0;
+}
+
 static void ppc_spapr_reset(void)
 {
     PowerPCCPU *first_ppc_cpu;
+    uint32_t rtas_limit;
+
+    /* Check for unknown sysbus devices */
+    foreach_dynamic_sysbus_device(find_unknown_sysbus_device, NULL);
 
     /* Reset the hash table & recalc the RMA */
     spapr_reset_htab(spapr);
 
     qemu_devices_reset();
 
+    /*
+     * We place the device tree and RTAS just below either the top of the RMA,
+     * or just below 2GB, whichever is lowere, so that it can be
+     * processed with 32-bit real mode code if necessary
+     */
+    rtas_limit = MIN(spapr->rma_size, RTAS_MAX_ADDR);
+    spapr->rtas_addr = rtas_limit - RTAS_MAX_SIZE;
+    spapr->fdt_addr = spapr->rtas_addr - FDT_MAX_SIZE;
+
     /* Load the fdt */
     spapr_finalize_fdt(spapr, spapr->fdt_addr, spapr->rtas_addr,
                        spapr->rtas_size);
 
+    /* Copy RTAS over */
+    cpu_physical_memory_write(spapr->rtas_addr, spapr->rtas_blob,
+                              spapr->rtas_size);
+
     /* Set up the entry state */
     first_ppc_cpu = POWERPC_CPU(first_cpu);
     first_ppc_cpu->env.gpr[3] = spapr->fdt_addr;
@@ -860,7 +947,7 @@ static void spapr_create_nvram(sPAPREnvironment *spapr)
     DriveInfo *dinfo = drive_get(IF_PFLASH, 0, 0);
 
     if (dinfo) {
-        qdev_prop_set_drive_nofail(dev, "drive", dinfo->bdrv);
+        qdev_prop_set_drive_nofail(dev, "drive", blk_by_legacy_dinfo(dinfo));
     }
 
     qdev_init_nofail(dev);
@@ -1226,10 +1313,10 @@ static void ppc_spapr_init(MachineState *machine)
     MemoryRegion *rma_region;
     void *rma = NULL;
     hwaddr rma_alloc_size;
-    hwaddr node0_size = (nb_numa_nodes > 1) ? numa_info[0].node_mem : ram_size;
+    hwaddr node0_size = spapr_node0_size();
     uint32_t initrd_base = 0;
     long kernel_size = 0, initrd_size = 0;
-    long load_limit, rtas_limit, fw_size;
+    long load_limit, fw_size;
     bool kernel_le = false;
     char *filename;
 
@@ -1274,13 +1361,8 @@ static void ppc_spapr_init(MachineState *machine)
         exit(1);
     }
 
-    /* We place the device tree and RTAS just below either the top of the RMA,
-     * or just below 2GB, whichever is lowere, so that it can be
-     * processed with 32-bit real mode code if necessary */
-    rtas_limit = MIN(spapr->rma_size, 0x80000000);
-    spapr->rtas_addr = rtas_limit - RTAS_MAX_SIZE;
-    spapr->fdt_addr = spapr->rtas_addr - FDT_MAX_SIZE;
-    load_limit = spapr->fdt_addr - FW_OVERHEAD;
+    /* Setup a load limit for the ramdisk leaving room for SLOF and FDT */
+    load_limit = MIN(spapr->rma_size, RTAS_MAX_ADDR) - FW_OVERHEAD;
 
     /* We aim for a hash table of size 1/128 the size of RAM.  The
      * normal rule of thumb is 1/64 the size of RAM, but that's much
@@ -1348,14 +1430,14 @@ static void ppc_spapr_init(MachineState *machine)
     }
 
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, "spapr-rtas.bin");
-    spapr->rtas_size = load_image_targphys(filename, spapr->rtas_addr,
-                                           rtas_limit - spapr->rtas_addr);
-    if (spapr->rtas_size < 0) {
+    spapr->rtas_size = get_image_size(filename);
+    spapr->rtas_blob = g_malloc(spapr->rtas_size);
+    if (load_image_size(filename, spapr->rtas_blob, spapr->rtas_size) < 0) {
         hw_error("qemu: could not load LPAR rtas '%s'\n", filename);
         exit(1);
     }
     if (spapr->rtas_size > RTAS_MAX_SIZE) {
-        hw_error("RTAS too big ! 0x%lx bytes (max is 0x%x)\n",
+        hw_error("RTAS too big ! 0x%zx bytes (max is 0x%x)\n",
                  spapr->rtas_size, RTAS_MAX_SIZE);
         exit(1);
     }
@@ -1377,7 +1459,6 @@ static void ppc_spapr_init(MachineState *machine)
     spapr_create_nvram(spapr);
 
     /* Set up PCI */
-    spapr_pci_msi_init(spapr, SPAPR_PCI_MSI_WINDOW);
     spapr_pci_rtas_init();
 
     phb = spapr_create_phb(spapr, 0);
@@ -1576,14 +1657,29 @@ static void spapr_machine_initfn(Object *obj)
                             spapr_get_kvm_type, spapr_set_kvm_type, NULL);
 }
 
+static void ppc_cpu_do_nmi_on_cpu(void *arg)
+{
+    CPUState *cs = arg;
+
+    cpu_synchronize_state(cs);
+    ppc_cpu_do_system_reset(cs);
+}
+
+static void spapr_nmi(NMIState *n, int cpu_index, Error **errp)
+{
+    CPUState *cs;
+
+    CPU_FOREACH(cs) {
+        async_run_on_cpu(cs, ppc_cpu_do_nmi_on_cpu, cs);
+    }
+}
+
 static void spapr_machine_class_init(ObjectClass *oc, void *data)
 {
     MachineClass *mc = MACHINE_CLASS(oc);
     FWPathProviderClass *fwc = FW_PATH_PROVIDER_CLASS(oc);
+    NMIClass *nc = NMI_CLASS(oc);
 
-    mc->name = "pseries";
-    mc->desc = "pSeries Logical Partition (PAPR compliant)";
-    mc->is_default = 1;
     mc->init = ppc_spapr_init;
     mc->reset = ppc_spapr_reset;
     mc->block_default_type = IF_SCSI;
@@ -1591,18 +1687,22 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data)
     mc->no_parallel = 1;
     mc->default_boot_order = NULL;
     mc->kvm_type = spapr_kvm_type;
+    mc->has_dynamic_sysbus = true;
 
     fwc->get_dev_path = spapr_get_fw_dev_path;
+    nc->nmi_monitor_handler = spapr_nmi;
 }
 
 static const TypeInfo spapr_machine_info = {
     .name          = TYPE_SPAPR_MACHINE,
     .parent        = TYPE_MACHINE,
+    .abstract      = true,
     .instance_size = sizeof(sPAPRMachineState),
     .instance_init = spapr_machine_initfn,
     .class_init    = spapr_machine_class_init,
     .interfaces = (InterfaceInfo[]) {
         { TYPE_FW_PATH_PROVIDER },
+        { TYPE_NMI },
         { }
     },
 };
@@ -1610,10 +1710,14 @@ static const TypeInfo spapr_machine_info = {
 static void spapr_machine_2_1_class_init(ObjectClass *oc, void *data)
 {
     MachineClass *mc = MACHINE_CLASS(oc);
+    static GlobalProperty compat_props[] = {
+        HW_COMPAT_2_1,
+        { /* end of list */ }
+    };
 
     mc->name = "pseries-2.1";
     mc->desc = "pSeries Logical Partition (PAPR compliant) v2.1";
-    mc->is_default = 0;
+    mc->compat_props = compat_props;
 }
 
 static const TypeInfo spapr_machine_2_1_info = {
@@ -1622,10 +1726,27 @@ static const TypeInfo spapr_machine_2_1_info = {
     .class_init    = spapr_machine_2_1_class_init,
 };
 
+static void spapr_machine_2_2_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->name = "pseries-2.2";
+    mc->desc = "pSeries Logical Partition (PAPR compliant) v2.2";
+    mc->alias = "pseries";
+    mc->is_default = 1;
+}
+
+static const TypeInfo spapr_machine_2_2_info = {
+    .name          = TYPE_SPAPR_MACHINE "2.2",
+    .parent        = TYPE_SPAPR_MACHINE,
+    .class_init    = spapr_machine_2_2_class_init,
+};
+
 static void spapr_machine_register_types(void)
 {
     type_register_static(&spapr_machine_info);
     type_register_static(&spapr_machine_2_1_info);
+    type_register_static(&spapr_machine_2_2_info);
 }
 
 type_init(spapr_machine_register_types)
diff --git a/hw/ppc/spapr_hcall.c b/hw/ppc/spapr_hcall.c
index 467858ce0..86514472f 100644
--- a/hw/ppc/spapr_hcall.c
+++ b/hw/ppc/spapr_hcall.c
@@ -712,10 +712,10 @@ static target_ulong h_logical_dcbf(PowerPCCPU *cpu, sPAPREnvironment *spapr,
     return H_SUCCESS;
 }
 
-static target_ulong h_set_mode_resouce_le(PowerPCCPU *cpu,
-                                          target_ulong mflags,
-                                          target_ulong value1,
-                                          target_ulong value2)
+static target_ulong h_set_mode_resource_le(PowerPCCPU *cpu,
+                                           target_ulong mflags,
+                                           target_ulong value1,
+                                           target_ulong value2)
 {
     CPUState *cs;
 
@@ -743,10 +743,10 @@ static target_ulong h_set_mode_resouce_le(PowerPCCPU *cpu,
     return H_UNSUPPORTED_FLAG;
 }
 
-static target_ulong h_set_mode_resouce_addr_trans_mode(PowerPCCPU *cpu,
-                                                       target_ulong mflags,
-                                                       target_ulong value1,
-                                                       target_ulong value2)
+static target_ulong h_set_mode_resource_addr_trans_mode(PowerPCCPU *cpu,
+                                                        target_ulong mflags,
+                                                        target_ulong value1,
+                                                        target_ulong value2)
 {
     CPUState *cs;
     PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(cpu);
@@ -794,11 +794,11 @@ static target_ulong h_set_mode(PowerPCCPU *cpu, sPAPREnvironment *spapr,
 
     switch (resource) {
     case H_SET_MODE_RESOURCE_LE:
-        ret = h_set_mode_resouce_le(cpu, args[0], args[2], args[3]);
+        ret = h_set_mode_resource_le(cpu, args[0], args[2], args[3]);
         break;
     case H_SET_MODE_RESOURCE_ADDR_TRANS_MODE:
-        ret = h_set_mode_resouce_addr_trans_mode(cpu, args[0],
-                                                 args[2], args[3]);
+        ret = h_set_mode_resource_addr_trans_mode(cpu, args[0],
+                                                  args[2], args[3]);
         break;
     }
 
diff --git a/hw/ppc/spapr_iommu.c b/hw/ppc/spapr_iommu.c
index f6e32a48a..6c91d8edd 100644
--- a/hw/ppc/spapr_iommu.c
+++ b/hw/ppc/spapr_iommu.c
@@ -59,7 +59,8 @@ static sPAPRTCETable *spapr_tce_find_by_liobn(uint32_t liobn)
     return NULL;
 }
 
-static IOMMUTLBEntry spapr_tce_translate_iommu(MemoryRegion *iommu, hwaddr addr)
+static IOMMUTLBEntry spapr_tce_translate_iommu(MemoryRegion *iommu, hwaddr addr,
+                                               bool is_write)
 {
     sPAPRTCETable *tcet = container_of(iommu, sPAPRTCETable, iommu);
     uint64_t tce;
diff --git a/hw/ppc/spapr_pci.c b/hw/ppc/spapr_pci.c
index 9ed39a93b..21b95b342 100644
--- a/hw/ppc/spapr_pci.c
+++ b/hw/ppc/spapr_pci.c
@@ -262,7 +262,6 @@ static void rtas_ibm_change_msi(PowerPCCPU *cpu, sPAPREnvironment *spapr,
     unsigned int irq, max_irqs = 0, num = 0;
     sPAPRPHBState *phb = NULL;
     PCIDevice *pdev = NULL;
-    bool msix = false;
     spapr_pci_msi *msi;
     int *config_addr_key;
 
@@ -300,7 +299,12 @@ static void rtas_ibm_change_msi(PowerPCCPU *cpu, sPAPREnvironment *spapr,
         }
 
         xics_free(spapr->icp, msi->first_irq, msi->num);
-        spapr_msi_setmsg(pdev, 0, msix, 0, num);
+        if (msi_present(pdev)) {
+            spapr_msi_setmsg(pdev, 0, false, 0, num);
+        }
+        if (msix_present(pdev)) {
+            spapr_msi_setmsg(pdev, 0, true, 0, num);
+        }
         g_hash_table_remove(phb->msi, &config_addr);
 
         trace_spapr_pci_msi("Released MSIs", config_addr);
@@ -341,7 +345,7 @@ static void rtas_ibm_change_msi(PowerPCCPU *cpu, sPAPREnvironment *spapr,
     }
 
     /* Setup MSI/MSIX vectors in the device (via cfgspace or MSIX BAR) */
-    spapr_msi_setmsg(pdev, spapr->msi_win_addr, ret_intr_type == RTAS_TYPE_MSIX,
+    spapr_msi_setmsg(pdev, SPAPR_PCI_MSI_WINDOW, ret_intr_type == RTAS_TYPE_MSIX,
                      irq, req_num);
 
     /* Add MSI device to cache */
@@ -465,34 +469,6 @@ static const MemoryRegionOps spapr_msi_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN
 };
 
-void spapr_pci_msi_init(sPAPREnvironment *spapr, hwaddr addr)
-{
-    uint64_t window_size = 4096;
-
-    /*
-     * As MSI/MSIX interrupts trigger by writing at MSI/MSIX vectors,
-     * we need to allocate some memory to catch those writes coming
-     * from msi_notify()/msix_notify().
-     * As MSIMessage:addr is going to be the same and MSIMessage:data
-     * is going to be a VIRQ number, 4 bytes of the MSI MR will only
-     * be used.
-     *
-     * For KVM we want to ensure that this memory is a full page so that
-     * our memory slot is of page size granularity.
-     */
-#ifdef CONFIG_KVM
-    if (kvm_enabled()) {
-        window_size = getpagesize();
-    }
-#endif
-
-    spapr->msi_win_addr = addr;
-    memory_region_init_io(&spapr->msiwindow, NULL, &spapr_msi_ops, spapr,
-                          "msi", window_size);
-    memory_region_add_subregion(get_system_memory(), spapr->msi_win_addr,
-                                &spapr->msiwindow);
-}
-
 /*
  * PHB PCI device
  */
@@ -512,6 +488,7 @@ static void spapr_phb_realize(DeviceState *dev, Error **errp)
     char *namebuf;
     int i;
     PCIBus *bus;
+    uint64_t msi_window_size = 4096;
 
     if (sphb->index != -1) {
         hwaddr windows_base;
@@ -604,6 +581,28 @@ static void spapr_phb_realize(DeviceState *dev, Error **errp)
     address_space_init(&sphb->iommu_as, &sphb->iommu_root,
                        sphb->dtbusname);
 
+    /*
+     * As MSI/MSIX interrupts trigger by writing at MSI/MSIX vectors,
+     * we need to allocate some memory to catch those writes coming
+     * from msi_notify()/msix_notify().
+     * As MSIMessage:addr is going to be the same and MSIMessage:data
+     * is going to be a VIRQ number, 4 bytes of the MSI MR will only
+     * be used.
+     *
+     * For KVM we want to ensure that this memory is a full page so that
+     * our memory slot is of page size granularity.
+     */
+#ifdef CONFIG_KVM
+    if (kvm_enabled()) {
+        msi_window_size = getpagesize();
+    }
+#endif
+
+    memory_region_init_io(&sphb->msiwindow, NULL, &spapr_msi_ops, spapr,
+                          "msi", msi_window_size);
+    memory_region_add_subregion(&sphb->iommu_root, SPAPR_PCI_MSI_WINDOW,
+                                &sphb->msiwindow);
+
     pci_setup_iommu(bus, spapr_pci_dma_iommu, sphb);
 
     pci_bus_set_route_irq_fn(bus, spapr_route_intx_pin_to_irq);
@@ -705,28 +704,34 @@ static const VMStateDescription vmstate_spapr_pci_msi = {
     },
 };
 
+static void spapr_pci_fill_msi_devs(gpointer key, gpointer value,
+                                    gpointer opaque)
+{
+    sPAPRPHBState *sphb = opaque;
+
+    sphb->msi_devs[sphb->msi_devs_num].key = *(uint32_t *)key;
+    sphb->msi_devs[sphb->msi_devs_num].value = *(spapr_pci_msi *)value;
+    sphb->msi_devs_num++;
+}
+
 static void spapr_pci_pre_save(void *opaque)
 {
     sPAPRPHBState *sphb = opaque;
-    GHashTableIter iter;
-    gpointer key, value;
-    int i;
+    int msi_devs_num;
 
     if (sphb->msi_devs) {
         g_free(sphb->msi_devs);
         sphb->msi_devs = NULL;
     }
-    sphb->msi_devs_num = g_hash_table_size(sphb->msi);
-    if (!sphb->msi_devs_num) {
+    sphb->msi_devs_num = 0;
+    msi_devs_num = g_hash_table_size(sphb->msi);
+    if (!msi_devs_num) {
         return;
     }
-    sphb->msi_devs = g_malloc(sphb->msi_devs_num * sizeof(spapr_pci_msi_mig));
+    sphb->msi_devs = g_malloc(msi_devs_num * sizeof(spapr_pci_msi_mig));
 
-    g_hash_table_iter_init(&iter, sphb->msi);
-    for (i = 0; g_hash_table_iter_next(&iter, &key, &value); ++i) {
-        sphb->msi_devs[i].key = *(uint32_t *) key;
-        sphb->msi_devs[i].value = *(spapr_pci_msi *) value;
-    }
+    g_hash_table_foreach(sphb->msi, spapr_pci_fill_msi_devs, sphb);
+    assert(sphb->msi_devs_num == msi_devs_num);
 }
 
 static int spapr_pci_post_load(void *opaque, int version_id)
diff --git a/hw/ppc/spapr_rtas.c b/hw/ppc/spapr_rtas.c
index 9ba1ba69f..2ec2a8e4d 100644
--- a/hw/ppc/spapr_rtas.c
+++ b/hw/ppc/spapr_rtas.c
@@ -277,6 +277,19 @@ static void rtas_ibm_set_system_parameter(PowerPCCPU *cpu,
     rtas_st(rets, 0, ret);
 }
 
+static void rtas_ibm_os_term(PowerPCCPU *cpu,
+                            sPAPREnvironment *spapr,
+                            uint32_t token, uint32_t nargs,
+                            target_ulong args,
+                            uint32_t nret, target_ulong rets)
+{
+    target_ulong ret = 0;
+
+    qapi_event_send_guest_panicked(GUEST_PANIC_ACTION_PAUSE, &error_abort);
+
+    rtas_st(rets, 0, ret);
+}
+
 static struct rtas_call {
     const char *name;
     spapr_rtas_fn fn;
@@ -404,6 +417,8 @@ static void core_rtas_register_types(void)
     spapr_rtas_register(RTAS_IBM_SET_SYSTEM_PARAMETER,
                         "ibm,set-system-parameter",
                         rtas_ibm_set_system_parameter);
+    spapr_rtas_register(RTAS_IBM_OS_TERM, "ibm,os-term",
+                        rtas_ibm_os_term);
 }
 
 type_init(core_rtas_register_types)
diff --git a/hw/ppc/virtex_ml507.c b/hw/ppc/virtex_ml507.c
index 0de51481f..6ebd5bee8 100644
--- a/hw/ppc/virtex_ml507.c
+++ b/hw/ppc/virtex_ml507.c
@@ -39,7 +39,7 @@
 #include "hw/ppc/ppc4xx.h"
 #include "ppc405.h"
 
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "qapi/qmp/qerror.h"
 
 #define EPAPR_MAGIC    (0x45504150)
@@ -227,8 +227,8 @@ static void virtex_init(MachineState *machine)
 
     dinfo = drive_get(IF_PFLASH, 0, 0);
     pflash_cfi01_register(PFLASH_BASEADDR, NULL, "virtex.flash", FLASH_SIZE,
-                          dinfo ? dinfo->bdrv : NULL, (64 * 1024),
-                          FLASH_SIZE >> 16,
+                          dinfo ? blk_by_legacy_dinfo(dinfo) : NULL,
+                          (64 * 1024), FLASH_SIZE >> 16,
                           1, 0x89, 0x18, 0x0000, 0x0, 1);
 
     cpu_irq = (qemu_irq *) &env->irq_inputs[PPC40x_INPUT_INT];
diff --git a/hw/s390x/css.c b/hw/s390x/css.c
index 49c2aaff1..b67c039a7 100644
--- a/hw/s390x/css.c
+++ b/hw/s390x/css.c
@@ -243,17 +243,25 @@ static void copy_sense_id_to_guest(SenseId *dest, SenseId *src)
     }
 }
 
-static CCW1 copy_ccw_from_guest(hwaddr addr)
+static CCW1 copy_ccw_from_guest(hwaddr addr, bool fmt1)
 {
-    CCW1 tmp;
+    CCW0 tmp0;
+    CCW1 tmp1;
     CCW1 ret;
 
-    cpu_physical_memory_read(addr, &tmp, sizeof(tmp));
-    ret.cmd_code = tmp.cmd_code;
-    ret.flags = tmp.flags;
-    ret.count = be16_to_cpu(tmp.count);
-    ret.cda = be32_to_cpu(tmp.cda);
-
+    if (fmt1) {
+        cpu_physical_memory_read(addr, &tmp1, sizeof(tmp1));
+        ret.cmd_code = tmp1.cmd_code;
+        ret.flags = tmp1.flags;
+        ret.count = be16_to_cpu(tmp1.count);
+        ret.cda = be32_to_cpu(tmp1.cda);
+    } else {
+        cpu_physical_memory_read(addr, &tmp0, sizeof(tmp0));
+        ret.cmd_code = tmp0.cmd_code;
+        ret.flags = tmp0.flags;
+        ret.count = be16_to_cpu(tmp0.count);
+        ret.cda = be16_to_cpu(tmp0.cda1) | (tmp0.cda0 << 16);
+    }
     return ret;
 }
 
@@ -268,7 +276,8 @@ static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr)
         return -EIO;
     }
 
-    ccw = copy_ccw_from_guest(ccw_addr);
+    /* Translate everything to format-1 ccws - the information is the same. */
+    ccw = copy_ccw_from_guest(ccw_addr, sch->ccw_fmt_1);
 
     /* Check for invalid command codes. */
     if ((ccw.cmd_code & 0x0f) == 0) {
@@ -285,6 +294,13 @@ static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr)
 
     check_len = !((ccw.flags & CCW_FLAG_SLI) && !(ccw.flags & CCW_FLAG_DC));
 
+    if (!ccw.cda) {
+        if (sch->ccw_no_data_cnt == 255) {
+            return -EINVAL;
+        }
+        sch->ccw_no_data_cnt++;
+    }
+
     /* Look at the command. */
     switch (ccw.cmd_code) {
     case CCW_CMD_NOOP:
@@ -386,6 +402,8 @@ static void sch_handle_start_func(SubchDev *sch, ORB *orb)
             s->ctrl |= (SCSW_STCTL_ALERT | SCSW_STCTL_STATUS_PEND);
             return;
         }
+        sch->ccw_fmt_1 = !!(orb->ctrl0 & ORB_CTRL0_MASK_FMT);
+        sch->ccw_no_data_cnt = 0;
     } else {
         s->ctrl &= ~(SCSW_ACTL_SUSP | SCSW_ACTL_RESUME_PEND);
     }
@@ -1347,6 +1365,8 @@ void subch_device_save(SubchDev *s, QEMUFile *f)
         qemu_put_byte(f, s->id.ciw[i].command);
         qemu_put_be16(f, s->id.ciw[i].count);
     }
+    qemu_put_byte(f, s->ccw_fmt_1);
+    qemu_put_byte(f, s->ccw_no_data_cnt);
     return;
 }
 
@@ -1402,6 +1422,8 @@ int subch_device_load(SubchDev *s, QEMUFile *f)
         s->id.ciw[i].command = qemu_get_byte(f);
         s->id.ciw[i].count = qemu_get_be16(f);
     }
+    s->ccw_fmt_1 = qemu_get_byte(f);
+    s->ccw_no_data_cnt = qemu_get_byte(f);
     return 0;
 }
 
diff --git a/hw/s390x/css.h b/hw/s390x/css.h
index c864ea765..33104ac58 100644
--- a/hw/s390x/css.h
+++ b/hw/s390x/css.h
@@ -76,7 +76,9 @@ struct SubchDev {
     hwaddr channel_prog;
     CCW1 last_cmd;
     bool last_cmd_valid;
+    bool ccw_fmt_1;
     bool thinint_active;
+    uint8_t ccw_no_data_cnt;
     /* transport-provided data: */
     int (*ccw_cb) (SubchDev *, CCW1);
     SenseId id;
diff --git a/hw/s390x/event-facility.c b/hw/s390x/event-facility.c
index 597db3401..78da71836 100644
--- a/hw/s390x/event-facility.c
+++ b/hw/s390x/event-facility.c
@@ -333,7 +333,6 @@ static int init_event_facility(SCLPEventFacility *event_facility)
     /* Spawn a new bus for SCLP events */
     qbus_create_inplace(&event_facility->sbus, sizeof(event_facility->sbus),
                         TYPE_SCLP_EVENTS_BUS, sdev, NULL);
-    event_facility->sbus.qbus.allow_hotplug = 0;
 
     quiesce = qdev_create(&event_facility->sbus.qbus, "sclpquiesce");
     if (!quiesce) {
@@ -408,7 +407,6 @@ static void event_class_init(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
 
     dc->bus_type = TYPE_SCLP_EVENTS_BUS;
-    dc->unplug = qdev_simple_unplug_cb;
     dc->realize = event_realize;
     dc->unrealize = event_unrealize;
 }
diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c
index 4fa9cffde..3b77c9a22 100644
--- a/hw/s390x/ipl.c
+++ b/hw/s390x/ipl.c
@@ -176,7 +176,7 @@ static void s390_ipl_reset(DeviceState *dev)
         }
     }
 
-    s390_add_running_cpu(cpu);
+    s390_cpu_set_state(CPU_STATE_OPERATING, cpu);
 }
 
 static void s390_ipl_class_init(ObjectClass *klass, void *data)
diff --git a/hw/s390x/s390-virtio-bus.c b/hw/s390x/s390-virtio-bus.c
index 6b6fb61c4..39dc2011b 100644
--- a/hw/s390x/s390-virtio-bus.c
+++ b/hw/s390x/s390-virtio-bus.c
@@ -18,7 +18,7 @@
  */
 
 #include "hw/hw.h"
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/sysemu.h"
 #include "hw/boards.h"
 #include "monitor/monitor.h"
@@ -102,7 +102,7 @@ VirtIOS390Bus *s390_virtio_bus_init(ram_addr_t *ram_size)
     bus->next_ring = bus->dev_page + TARGET_PAGE_SIZE;
 
     /* Enable hotplugging */
-    _bus->allow_hotplug = 1;
+    qbus_set_hotplug_handler(_bus, dev, &error_abort);
 
     /* Allocate RAM for VirtIO device pages (descriptors, queues, rings) */
     *ram_size += S390_DEVICE_PAGES * TARGET_PAGE_SIZE;
@@ -159,8 +159,11 @@ static int s390_virtio_net_init(VirtIOS390Device *s390_dev)
 static void s390_virtio_net_instance_init(Object *obj)
 {
     VirtIONetS390 *dev = VIRTIO_NET_S390(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_NET);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_NET);
+    object_property_add_alias(obj, "bootindex", OBJECT(&dev->vdev),
+                              "bootindex", &error_abort);
 }
 
 static int s390_virtio_blk_init(VirtIOS390Device *s390_dev)
@@ -177,12 +180,13 @@ static int s390_virtio_blk_init(VirtIOS390Device *s390_dev)
 static void s390_virtio_blk_instance_init(Object *obj)
 {
     VirtIOBlkS390 *dev = VIRTIO_BLK_S390(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_BLK);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
-    object_unref(OBJECT(&dev->vdev));
-    qdev_alias_all_properties(DEVICE(&dev->vdev), obj);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_BLK);
     object_property_add_alias(obj, "iothread", OBJECT(&dev->vdev),"iothread",
                               &error_abort);
+    object_property_add_alias(obj, "bootindex", OBJECT(&dev->vdev),
+                              "bootindex", &error_abort);
 }
 
 static int s390_virtio_serial_init(VirtIOS390Device *s390_dev)
@@ -222,8 +226,9 @@ static int s390_virtio_serial_init(VirtIOS390Device *s390_dev)
 static void s390_virtio_serial_instance_init(Object *obj)
 {
     VirtIOSerialS390 *dev = VIRTIO_SERIAL_S390(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_SERIAL);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_SERIAL);
 }
 
 static int s390_virtio_scsi_init(VirtIOS390Device *s390_dev)
@@ -254,8 +259,9 @@ static int s390_virtio_scsi_init(VirtIOS390Device *s390_dev)
 static void s390_virtio_scsi_instance_init(Object *obj)
 {
     VirtIOSCSIS390 *dev = VIRTIO_SCSI_S390(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_SCSI);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_SCSI);
 }
 
 #ifdef CONFIG_VHOST_SCSI
@@ -275,8 +281,9 @@ static int s390_vhost_scsi_init(VirtIOS390Device *s390_dev)
 static void s390_vhost_scsi_instance_init(Object *obj)
 {
     VHostSCSIS390 *dev = VHOST_SCSI_S390(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VHOST_SCSI);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VHOST_SCSI);
 }
 #endif
 
@@ -301,12 +308,11 @@ static int s390_virtio_rng_init(VirtIOS390Device *s390_dev)
 static void s390_virtio_rng_instance_init(Object *obj)
 {
     VirtIORNGS390 *dev = VIRTIO_RNG_S390(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_RNG);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
-    object_property_add_link(obj, "rng", TYPE_RNG_BACKEND,
-                             (Object **)&dev->vdev.conf.rng,
-                             qdev_prop_allow_set_link_before_realize,
-                             OBJ_PROP_LINK_UNREF_ON_RELEASE, NULL);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_RNG);
+    object_property_add_alias(obj, "rng", OBJECT(&dev->vdev),
+                              "rng", &error_abort);
 }
 
 static uint64_t s390_virtio_device_vq_token(VirtIOS390Device *dev, int vq)
@@ -493,10 +499,8 @@ static unsigned virtio_s390_get_features(DeviceState *d)
 /**************** S390 Virtio Bus Device Descriptions *******************/
 
 static Property s390_virtio_net_properties[] = {
-    DEFINE_NIC_PROPERTIES(VirtIONetS390, vdev.nic_conf),
     DEFINE_VIRTIO_COMMON_FEATURES(VirtIOS390Device, host_features),
     DEFINE_VIRTIO_NET_FEATURES(VirtIOS390Device, host_features),
-    DEFINE_VIRTIO_NET_PROPERTIES(VirtIONetS390, vdev.net_conf),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -533,7 +537,6 @@ static const TypeInfo s390_virtio_blk = {
 };
 
 static Property s390_virtio_serial_properties[] = {
-    DEFINE_VIRTIO_SERIAL_PROPERTIES(VirtIOSerialS390, vdev.serial),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -556,7 +559,6 @@ static const TypeInfo s390_virtio_serial = {
 
 static Property s390_virtio_rng_properties[] = {
     DEFINE_VIRTIO_COMMON_FEATURES(VirtIOS390Device, host_features),
-    DEFINE_VIRTIO_RNG_PROPERTIES(VirtIORNGS390, vdev.conf),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -600,7 +602,6 @@ static void virtio_s390_device_class_init(ObjectClass *klass, void *data)
 
     dc->init = s390_virtio_busdev_init;
     dc->bus_type = TYPE_S390_VIRTIO_BUS;
-    dc->unplug = qdev_simple_unplug_cb;
     dc->reset = s390_virtio_busdev_reset;
 }
 
@@ -614,7 +615,6 @@ static const TypeInfo virtio_s390_device_info = {
 };
 
 static Property s390_virtio_scsi_properties[] = {
-    DEFINE_VIRTIO_SCSI_PROPERTIES(VirtIOSCSIS390, vdev.parent_obj.conf),
     DEFINE_VIRTIO_COMMON_FEATURES(VirtIOS390Device, host_features),
     DEFINE_VIRTIO_SCSI_FEATURES(VirtIOS390Device, host_features),
     DEFINE_PROP_END_OF_LIST(),
@@ -640,7 +640,6 @@ static const TypeInfo s390_virtio_scsi = {
 #ifdef CONFIG_VHOST_SCSI
 static Property s390_vhost_scsi_properties[] = {
     DEFINE_VIRTIO_COMMON_FEATURES(VirtIOS390Device, host_features),
-    DEFINE_VHOST_SCSI_PROPERTIES(VHostSCSIS390, vdev.parent_obj.conf),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -683,6 +682,10 @@ static const TypeInfo s390_virtio_bridge_info = {
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysBusDevice),
     .class_init    = s390_virtio_bridge_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_HOTPLUG_HANDLER },
+        { }
+    }
 };
 
 /* virtio-s390-bus */
@@ -691,13 +694,10 @@ static void virtio_s390_bus_new(VirtioBusState *bus, size_t bus_size,
                                 VirtIOS390Device *dev)
 {
     DeviceState *qdev = DEVICE(dev);
-    BusState *qbus;
     char virtio_bus_name[] = "virtio-bus";
 
     qbus_create_inplace(bus, bus_size, TYPE_VIRTIO_S390_BUS,
                         qdev, virtio_bus_name);
-    qbus = BUS(bus);
-    qbus->allow_hotplug = 1;
 }
 
 static void virtio_s390_bus_class_init(ObjectClass *klass, void *data)
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index 42f5cec4c..bc4dc2ae8 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -17,6 +17,9 @@
 #include "ioinst.h"
 #include "css.h"
 #include "virtio-ccw.h"
+#include "qemu/config-file.h"
+
+#define TYPE_S390_CCW_MACHINE               "s390-ccw-machine"
 
 void io_subsystem_reset(void)
 {
@@ -84,17 +87,35 @@ static void ccw_init(MachineState *machine)
     ram_addr_t my_ram_size = machine->ram_size;
     MemoryRegion *sysmem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
-    int shift = 0;
+    sclpMemoryHotplugDev *mhd = init_sclp_memory_hotplug_dev();
     uint8_t *storage_keys;
     int ret;
     VirtualCssBus *css_bus;
-
-    /* s390x ram size detection needs a 16bit multiplier + an increment. So
-       guests > 64GB can be specified in 2MB steps etc. */
-    while ((my_ram_size >> (20 + shift)) > 65535) {
-        shift++;
+    QemuOpts *opts = qemu_opts_find(qemu_find_opts("memory"), NULL);
+    ram_addr_t pad_size = 0;
+    ram_addr_t maxmem = qemu_opt_get_size(opts, "maxmem", my_ram_size);
+    ram_addr_t standby_mem_size = maxmem - my_ram_size;
+
+    /* The storage increment size is a multiple of 1M and is a power of 2.
+     * The number of storage increments must be MAX_STORAGE_INCREMENTS or fewer.
+     * The variable 'mhd->increment_size' is an exponent of 2 that can be
+     * used to calculate the size (in bytes) of an increment. */
+    mhd->increment_size = 20;
+    while ((my_ram_size >> mhd->increment_size) > MAX_STORAGE_INCREMENTS) {
+        mhd->increment_size++;
+    }
+    while ((standby_mem_size >> mhd->increment_size) > MAX_STORAGE_INCREMENTS) {
+        mhd->increment_size++;
     }
-    my_ram_size = my_ram_size >> (20 + shift) << (20 + shift);
+
+    /* The core and standby memory areas need to be aligned with
+     * the increment size.  In effect, this can cause the
+     * user-specified memory size to be rounded down to align
+     * with the nearest increment boundary. */
+    standby_mem_size = standby_mem_size >> mhd->increment_size
+                                        << mhd->increment_size;
+    my_ram_size = my_ram_size >> mhd->increment_size
+                              << mhd->increment_size;
 
     /* let's propagate the changed ram size into the global variable. */
     ram_size = my_ram_size;
@@ -109,11 +130,22 @@ static void ccw_init(MachineState *machine)
     /* register hypercalls */
     virtio_ccw_register_hcalls();
 
-    /* allocate RAM */
-    memory_region_init_ram(ram, NULL, "s390.ram", my_ram_size);
+    /* allocate RAM for core */
+    memory_region_init_ram(ram, NULL, "s390.ram", my_ram_size, &error_abort);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(sysmem, 0, ram);
 
+    /* If the size of ram is not on a MEM_SECTION_SIZE boundary,
+       calculate the pad size necessary to force this boundary. */
+    if (standby_mem_size) {
+        if (my_ram_size % MEM_SECTION_SIZE) {
+            pad_size = MEM_SECTION_SIZE - my_ram_size % MEM_SECTION_SIZE;
+        }
+        my_ram_size += standby_mem_size + pad_size;
+        mhd->pad_size = pad_size;
+        mhd->standby_mem_size = standby_mem_size;
+    }
+
     /* allocate storage keys */
     storage_keys = g_malloc0(my_ram_size / TARGET_PAGE_SIZE);
 
@@ -134,24 +166,39 @@ static void ccw_init(MachineState *machine)
     s390_create_virtio_net(BUS(css_bus), "virtio-net-ccw");
 }
 
-static QEMUMachine ccw_machine = {
-    .name = "s390-ccw-virtio",
-    .alias = "s390-ccw",
-    .desc = "VirtIO-ccw based S390 machine",
-    .init = ccw_init,
-    .block_default_type = IF_VIRTIO,
-    .no_cdrom = 1,
-    .no_floppy = 1,
-    .no_serial = 1,
-    .no_parallel = 1,
-    .no_sdcard = 1,
-    .use_sclp = 1,
-    .max_cpus = 255,
+static void ccw_machine_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+    NMIClass *nc = NMI_CLASS(oc);
+
+    mc->name = "s390-ccw-virtio";
+    mc->alias = "s390-ccw";
+    mc->desc = "VirtIO-ccw based S390 machine";
+    mc->init = ccw_init;
+    mc->block_default_type = IF_VIRTIO;
+    mc->no_cdrom = 1;
+    mc->no_floppy = 1;
+    mc->no_serial = 1;
+    mc->no_parallel = 1;
+    mc->no_sdcard = 1;
+    mc->use_sclp = 1,
+    mc->max_cpus = 255;
+    nc->nmi_monitor_handler = s390_nmi;
+}
+
+static const TypeInfo ccw_machine_info = {
+    .name          = TYPE_S390_CCW_MACHINE,
+    .parent        = TYPE_MACHINE,
+    .class_init    = ccw_machine_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_NMI },
+        { }
+    },
 };
 
-static void ccw_machine_init(void)
+static void ccw_machine_register_types(void)
 {
-    qemu_register_machine(&ccw_machine);
+    type_register_static(&ccw_machine_info);
 }
 
-machine_init(ccw_machine_init)
+type_init(ccw_machine_register_types)
diff --git a/hw/s390x/s390-virtio.c b/hw/s390x/s390-virtio.c
index 93c7acea7..c215cd80e 100644
--- a/hw/s390x/s390-virtio.c
+++ b/hw/s390x/s390-virtio.c
@@ -22,7 +22,7 @@
  */
 
 #include "hw/hw.h"
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/sysemu.h"
 #include "net/net.h"
@@ -51,6 +51,7 @@
 
 #define MAX_BLK_DEVS                    10
 #define ZIPL_FILENAME                   "s390-zipl.rom"
+#define TYPE_S390_MACHINE               "s390-machine"
 
 static VirtIOS390Bus *s390_bus;
 static S390CPU **ipi_states;
@@ -124,38 +125,6 @@ static void s390_virtio_register_hcalls(void)
                                    s390_virtio_hcall_set_status);
 }
 
-/*
- * The number of running CPUs. On s390 a shutdown is the state of all CPUs
- * being either stopped or disabled (for interrupts) waiting. We have to
- * track this number to call the shutdown sequence accordingly. This
- * number is modified either on startup or while holding the big qemu lock.
- */
-static unsigned s390_running_cpus;
-
-void s390_add_running_cpu(S390CPU *cpu)
-{
-    CPUState *cs = CPU(cpu);
-
-    if (cs->halted) {
-        s390_running_cpus++;
-        cs->halted = 0;
-        cs->exception_index = -1;
-    }
-}
-
-unsigned s390_del_running_cpu(S390CPU *cpu)
-{
-    CPUState *cs = CPU(cpu);
-
-    if (cs->halted == 0) {
-        assert(s390_running_cpus >= 1);
-        s390_running_cpus--;
-        cs->halted = 1;
-        cs->exception_index = EXCP_HLT;
-    }
-    return s390_running_cpus;
-}
-
 void s390_init_ipl_dev(const char *kernel_filename,
                        const char *kernel_cmdline,
                        const char *initrd_filename,
@@ -229,18 +198,21 @@ static void s390_init(MachineState *machine)
     ram_addr_t my_ram_size = machine->ram_size;
     MemoryRegion *sysmem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
-    int shift = 0;
+    int increment_size = 20;
     uint8_t *storage_keys;
     void *virtio_region;
     hwaddr virtio_region_len;
     hwaddr virtio_region_start;
 
-    /* s390x ram size detection needs a 16bit multiplier + an increment. So
-       guests > 64GB can be specified in 2MB steps etc. */
-    while ((my_ram_size >> (20 + shift)) > 65535) {
-        shift++;
+    /*
+     * The storage increment size is a multiple of 1M and is a power of 2.
+     * The number of storage increments must be MAX_STORAGE_INCREMENTS or
+     * fewer.
+     */
+    while ((my_ram_size >> increment_size) > MAX_STORAGE_INCREMENTS) {
+        increment_size++;
     }
-    my_ram_size = my_ram_size >> (20 + shift) << (20 + shift);
+    my_ram_size = my_ram_size >> increment_size << increment_size;
 
     /* let's propagate the changed ram size into the global variable. */
     ram_size = my_ram_size;
@@ -256,7 +228,7 @@ static void s390_init(MachineState *machine)
     s390_virtio_register_hcalls();
 
     /* allocate RAM */
-    memory_region_init_ram(ram, NULL, "s390.ram", my_ram_size);
+    memory_region_init_ram(ram, NULL, "s390.ram", my_ram_size, &error_abort);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(sysmem, 0, ram);
 
@@ -279,25 +251,49 @@ static void s390_init(MachineState *machine)
     s390_create_virtio_net((BusState *)s390_bus, "virtio-net-s390");
 }
 
-static QEMUMachine s390_machine = {
-    .name = "s390-virtio",
-    .alias = "s390",
-    .desc = "VirtIO based S390 machine",
-    .init = s390_init,
-    .block_default_type = IF_VIRTIO,
-    .no_cdrom = 1,
-    .no_floppy = 1,
-    .no_serial = 1,
-    .no_parallel = 1,
-    .no_sdcard = 1,
-    .use_virtcon = 1,
-    .max_cpus = 255,
-    .is_default = 1,
+void s390_nmi(NMIState *n, int cpu_index, Error **errp)
+{
+    CPUState *cs = qemu_get_cpu(cpu_index);
+
+    if (s390_cpu_restart(S390_CPU(cs))) {
+        error_set(errp, QERR_UNSUPPORTED);
+    }
+}
+
+static void s390_machine_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+    NMIClass *nc = NMI_CLASS(oc);
+
+    mc->name = "s390-virtio";
+    mc->alias = "s390";
+    mc->desc = "VirtIO based S390 machine";
+    mc->init = s390_init;
+    mc->block_default_type = IF_VIRTIO;
+    mc->max_cpus = 255;
+    mc->no_serial = 1;
+    mc->no_parallel = 1;
+    mc->use_virtcon = 1;
+    mc->no_floppy = 1;
+    mc->no_cdrom = 1;
+    mc->no_sdcard = 1;
+    mc->is_default = 1;
+    nc->nmi_monitor_handler = s390_nmi;
+}
+
+static const TypeInfo s390_machine_info = {
+    .name          = TYPE_S390_MACHINE,
+    .parent        = TYPE_MACHINE,
+    .class_init    = s390_machine_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_NMI },
+        { }
+    },
 };
 
-static void s390_machine_init(void)
+static void s390_machine_register_types(void)
 {
-    qemu_register_machine(&s390_machine);
+    type_register_static(&s390_machine_info);
 }
 
-machine_init(s390_machine_init);
+type_init(s390_machine_register_types)
diff --git a/hw/s390x/s390-virtio.h b/hw/s390x/s390-virtio.h
index 5c405e775..33847aefc 100644
--- a/hw/s390x/s390-virtio.h
+++ b/hw/s390x/s390-virtio.h
@@ -12,6 +12,8 @@
 #ifndef HW_S390_VIRTIO_H
 #define HW_S390_VIRTIO_H 1
 
+#include "hw/nmi.h"
+
 #define KVM_S390_VIRTIO_NOTIFY          0
 #define KVM_S390_VIRTIO_RESET           1
 #define KVM_S390_VIRTIO_SET_STATUS      2
@@ -26,4 +28,5 @@ void s390_init_ipl_dev(const char *kernel_filename,
                        const char *initrd_filename,
                        const char *firmware);
 void s390_create_virtio_net(BusState *bus, const char *name);
+void s390_nmi(NMIState *n, int cpu_index, Error **errp);
 #endif
diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c
index d8ddf35e5..a759da7f3 100644
--- a/hw/s390x/sclp.c
+++ b/hw/s390x/sclp.c
@@ -16,7 +16,8 @@
 #include "sysemu/kvm.h"
 #include "exec/memory.h"
 #include "sysemu/sysemu.h"
-
+#include "exec/address-spaces.h"
+#include "qemu/config-file.h"
 #include "hw/s390x/sclp.h"
 #include "hw/s390x/event-facility.h"
 
@@ -33,10 +34,19 @@ static inline SCLPEventFacility *get_event_facility(void)
 static void read_SCP_info(SCCB *sccb)
 {
     ReadInfo *read_info = (ReadInfo *) sccb;
+    sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
     CPUState *cpu;
-    int shift = 0;
     int cpu_count = 0;
     int i = 0;
+    int increment_size = 20;
+    int rnsize, rnmax;
+    QemuOpts *opts = qemu_opts_find(qemu_find_opts("memory"), NULL);
+    int slots = qemu_opt_get_number(opts, "slots", 0);
+    int max_avail_slots = s390_get_memslot_count(kvm_state);
+
+    if (slots > max_avail_slots) {
+        slots = max_avail_slots;
+    }
 
     CPU_FOREACH(cpu) {
         cpu_count++;
@@ -54,14 +64,235 @@ static void read_SCP_info(SCCB *sccb)
 
     read_info->facilities = cpu_to_be64(SCLP_HAS_CPU_INFO);
 
-    while ((ram_size >> (20 + shift)) > 65535) {
-        shift++;
+    /*
+     * The storage increment size is a multiple of 1M and is a power of 2.
+     * The number of storage increments must be MAX_STORAGE_INCREMENTS or fewer.
+     */
+    while ((ram_size >> increment_size) > MAX_STORAGE_INCREMENTS) {
+        increment_size++;
+    }
+    rnmax = ram_size >> increment_size;
+
+    /* Memory Hotplug is only supported for the ccw machine type */
+    if (mhd) {
+        while ((mhd->standby_mem_size >> increment_size) >
+               MAX_STORAGE_INCREMENTS) {
+            increment_size++;
+        }
+        assert(increment_size == mhd->increment_size);
+
+        mhd->standby_subregion_size = MEM_SECTION_SIZE;
+        /* Deduct the memory slot already used for core */
+        if (slots > 0) {
+            while ((mhd->standby_subregion_size * (slots - 1)
+                    < mhd->standby_mem_size)) {
+                mhd->standby_subregion_size = mhd->standby_subregion_size << 1;
+            }
+        }
+        /*
+         * Initialize mapping of guest standby memory sections indicating which
+         * are and are not online. Assume all standby memory begins offline.
+         */
+        if (mhd->standby_state_map == 0) {
+            if (mhd->standby_mem_size % mhd->standby_subregion_size) {
+                mhd->standby_state_map = g_malloc0((mhd->standby_mem_size /
+                                             mhd->standby_subregion_size + 1) *
+                                             (mhd->standby_subregion_size /
+                                             MEM_SECTION_SIZE));
+            } else {
+                mhd->standby_state_map = g_malloc0(mhd->standby_mem_size /
+                                                   MEM_SECTION_SIZE);
+            }
+        }
+        mhd->padded_ram_size = ram_size + mhd->pad_size;
+        mhd->rzm = 1 << mhd->increment_size;
+        rnmax = ((ram_size + mhd->standby_mem_size + mhd->pad_size)
+             >> mhd->increment_size);
+
+        read_info->facilities |= cpu_to_be64(SCLP_FC_ASSIGN_ATTACH_READ_STOR);
+    }
+
+    rnsize = 1 << (increment_size - 20);
+    if (rnsize <= 128) {
+        read_info->rnsize = rnsize;
+    } else {
+        read_info->rnsize = 0;
+        read_info->rnsize2 = cpu_to_be32(rnsize);
+    }
+
+    if (rnmax < 0x10000) {
+        read_info->rnmax = cpu_to_be16(rnmax);
+    } else {
+        read_info->rnmax = cpu_to_be16(0);
+        read_info->rnmax2 = cpu_to_be64(rnmax);
     }
-    read_info->rnmax = cpu_to_be16(ram_size >> (20 + shift));
-    read_info->rnsize = 1 << shift;
+
     sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_READ_COMPLETION);
 }
 
+static void read_storage_element0_info(SCCB *sccb)
+{
+    int i, assigned;
+    int subincrement_id = SCLP_STARTING_SUBINCREMENT_ID;
+    ReadStorageElementInfo *storage_info = (ReadStorageElementInfo *) sccb;
+    sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
+
+    assert(mhd);
+
+    if ((ram_size >> mhd->increment_size) >= 0x10000) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_SCCB_BOUNDARY_VIOLATION);
+        return;
+    }
+
+    /* Return information regarding core memory */
+    storage_info->max_id = cpu_to_be16(mhd->standby_mem_size ? 1 : 0);
+    assigned = ram_size >> mhd->increment_size;
+    storage_info->assigned = cpu_to_be16(assigned);
+
+    for (i = 0; i < assigned; i++) {
+        storage_info->entries[i] = cpu_to_be32(subincrement_id);
+        subincrement_id += SCLP_INCREMENT_UNIT;
+    }
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_READ_COMPLETION);
+}
+
+static void read_storage_element1_info(SCCB *sccb)
+{
+    ReadStorageElementInfo *storage_info = (ReadStorageElementInfo *) sccb;
+    sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
+
+    assert(mhd);
+
+    if ((mhd->standby_mem_size >> mhd->increment_size) >= 0x10000) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_SCCB_BOUNDARY_VIOLATION);
+        return;
+    }
+
+    /* Return information regarding standby memory */
+    storage_info->max_id = cpu_to_be16(mhd->standby_mem_size ? 1 : 0);
+    storage_info->assigned = cpu_to_be16(mhd->standby_mem_size >>
+                                         mhd->increment_size);
+    storage_info->standby = cpu_to_be16(mhd->standby_mem_size >>
+                                        mhd->increment_size);
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_STANDBY_READ_COMPLETION);
+}
+
+static void attach_storage_element(SCCB *sccb, uint16_t element)
+{
+    int i, assigned, subincrement_id;
+    AttachStorageElement *attach_info = (AttachStorageElement *) sccb;
+    sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
+
+    assert(mhd);
+
+    if (element != 1) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        return;
+    }
+
+    assigned = mhd->standby_mem_size >> mhd->increment_size;
+    attach_info->assigned = cpu_to_be16(assigned);
+    subincrement_id = ((ram_size >> mhd->increment_size) << 16)
+                      + SCLP_STARTING_SUBINCREMENT_ID;
+    for (i = 0; i < assigned; i++) {
+        attach_info->entries[i] = cpu_to_be32(subincrement_id);
+        subincrement_id += SCLP_INCREMENT_UNIT;
+    }
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_COMPLETION);
+}
+
+static void assign_storage(SCCB *sccb)
+{
+    MemoryRegion *mr = NULL;
+    uint64_t this_subregion_size;
+    AssignStorage *assign_info = (AssignStorage *) sccb;
+    sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
+    assert(mhd);
+    ram_addr_t assign_addr = (assign_info->rn - 1) * mhd->rzm;
+    MemoryRegion *sysmem = get_system_memory();
+
+    if ((assign_addr % MEM_SECTION_SIZE == 0) &&
+        (assign_addr >= mhd->padded_ram_size)) {
+        /* Re-use existing memory region if found */
+        mr = memory_region_find(sysmem, assign_addr, 1).mr;
+        if (!mr) {
+
+            MemoryRegion *standby_ram = g_new(MemoryRegion, 1);
+
+            /* offset to align to standby_subregion_size for allocation */
+            ram_addr_t offset = assign_addr -
+                                (assign_addr - mhd->padded_ram_size)
+                                % mhd->standby_subregion_size;
+
+            /* strlen("standby.ram") + 4 (Max of KVM_MEMORY_SLOTS) +  NULL */
+            char id[16];
+            snprintf(id, 16, "standby.ram%d",
+                     (int)((offset - mhd->padded_ram_size) /
+                     mhd->standby_subregion_size) + 1);
+
+            /* Allocate a subregion of the calculated standby_subregion_size */
+            if (offset + mhd->standby_subregion_size >
+                mhd->padded_ram_size + mhd->standby_mem_size) {
+                this_subregion_size = mhd->padded_ram_size +
+                  mhd->standby_mem_size - offset;
+            } else {
+                this_subregion_size = mhd->standby_subregion_size;
+            }
+
+            memory_region_init_ram(standby_ram, NULL, id, this_subregion_size, &error_abort);
+            vmstate_register_ram_global(standby_ram);
+            memory_region_add_subregion(sysmem, offset, standby_ram);
+        }
+        /* The specified subregion is no longer in standby */
+        mhd->standby_state_map[(assign_addr - mhd->padded_ram_size)
+                               / MEM_SECTION_SIZE] = 1;
+    }
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_COMPLETION);
+}
+
+static void unassign_storage(SCCB *sccb)
+{
+    MemoryRegion *mr = NULL;
+    AssignStorage *assign_info = (AssignStorage *) sccb;
+    sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
+    assert(mhd);
+    ram_addr_t unassign_addr = (assign_info->rn - 1) * mhd->rzm;
+    MemoryRegion *sysmem = get_system_memory();
+
+    /* if the addr is a multiple of 256 MB */
+    if ((unassign_addr % MEM_SECTION_SIZE == 0) &&
+        (unassign_addr >= mhd->padded_ram_size)) {
+        mhd->standby_state_map[(unassign_addr -
+                           mhd->padded_ram_size) / MEM_SECTION_SIZE] = 0;
+
+        /* find the specified memory region and destroy it */
+        mr = memory_region_find(sysmem, unassign_addr, 1).mr;
+        if (mr) {
+            int i;
+            int is_removable = 1;
+            ram_addr_t map_offset = (unassign_addr - mhd->padded_ram_size -
+                                     (unassign_addr - mhd->padded_ram_size)
+                                     % mhd->standby_subregion_size);
+            /* Mark all affected subregions as 'standby' once again */
+            for (i = 0;
+                 i < (mhd->standby_subregion_size / MEM_SECTION_SIZE);
+                 i++) {
+
+                if (mhd->standby_state_map[i + map_offset / MEM_SECTION_SIZE]) {
+                    is_removable = 0;
+                    break;
+                }
+            }
+            if (is_removable) {
+                memory_region_del_subregion(sysmem, mr);
+                object_unparent(OBJECT(mr));
+                g_free(mr);
+            }
+        }
+    }
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_COMPLETION);
+}
+
 /* Provide information about the CPU */
 static void sclp_read_cpu_info(SCCB *sccb)
 {
@@ -103,6 +334,22 @@ static void sclp_execute(SCCB *sccb, uint32_t code)
     case SCLP_CMDW_READ_CPU_INFO:
         sclp_read_cpu_info(sccb);
         break;
+    case SCLP_READ_STORAGE_ELEMENT_INFO:
+        if (code & 0xff00) {
+            read_storage_element1_info(sccb);
+        } else {
+            read_storage_element0_info(sccb);
+        }
+        break;
+    case SCLP_ATTACH_STORAGE_ELEMENT:
+        attach_storage_element(sccb, (code & 0xff00) >> 8);
+        break;
+    case SCLP_ASSIGN_STORAGE:
+        assign_storage(sccb);
+        break;
+    case SCLP_UNASSIGN_STORAGE:
+        unassign_storage(sccb);
+        break;
     default:
         efc->command_handler(ef, sccb, code);
         break;
@@ -183,3 +430,33 @@ void s390_sclp_init(void)
                               OBJECT(dev), NULL);
     qdev_init_nofail(dev);
 }
+
+sclpMemoryHotplugDev *init_sclp_memory_hotplug_dev(void)
+{
+    DeviceState *dev;
+    dev = qdev_create(NULL, TYPE_SCLP_MEMORY_HOTPLUG_DEV);
+    object_property_add_child(qdev_get_machine(),
+                              TYPE_SCLP_MEMORY_HOTPLUG_DEV,
+                              OBJECT(dev), NULL);
+    qdev_init_nofail(dev);
+    return SCLP_MEMORY_HOTPLUG_DEV(object_resolve_path(
+                                   TYPE_SCLP_MEMORY_HOTPLUG_DEV, NULL));
+}
+
+sclpMemoryHotplugDev *get_sclp_memory_hotplug_dev(void)
+{
+    return SCLP_MEMORY_HOTPLUG_DEV(object_resolve_path(
+                                   TYPE_SCLP_MEMORY_HOTPLUG_DEV, NULL));
+}
+
+static TypeInfo sclp_memory_hotplug_dev_info = {
+    .name = TYPE_SCLP_MEMORY_HOTPLUG_DEV,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(sclpMemoryHotplugDev),
+};
+
+static void register_types(void)
+{
+    type_register_static(&sclp_memory_hotplug_dev_info);
+}
+type_init(register_types);
diff --git a/hw/s390x/virtio-ccw.c b/hw/s390x/virtio-ccw.c
index 33a1d863b..ea236c9b0 100644
--- a/hw/s390x/virtio-ccw.c
+++ b/hw/s390x/virtio-ccw.c
@@ -10,7 +10,7 @@
  */
 
 #include "hw/hw.h"
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/sysemu.h"
 #include "net/net.h"
@@ -230,7 +230,7 @@ VirtualCssBus *virtual_css_bus_init(void)
     cbus = VIRTUAL_CSS_BUS(bus);
 
     /* Enable hotplugging */
-    bus->allow_hotplug = 1;
+    qbus_set_hotplug_handler(bus, dev, &error_abort);
 
     return cbus;
 }
@@ -792,8 +792,11 @@ static int virtio_ccw_net_init(VirtioCcwDevice *ccw_dev)
 static void virtio_ccw_net_instance_init(Object *obj)
 {
     VirtIONetCcw *dev = VIRTIO_NET_CCW(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_NET);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_NET);
+    object_property_add_alias(obj, "bootindex", OBJECT(&dev->vdev),
+                              "bootindex", &error_abort);
 }
 
 static int virtio_ccw_blk_init(VirtioCcwDevice *ccw_dev)
@@ -811,12 +814,13 @@ static int virtio_ccw_blk_init(VirtioCcwDevice *ccw_dev)
 static void virtio_ccw_blk_instance_init(Object *obj)
 {
     VirtIOBlkCcw *dev = VIRTIO_BLK_CCW(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_BLK);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
-    object_unref(OBJECT(&dev->vdev));
-    qdev_alias_all_properties(DEVICE(&dev->vdev), obj);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_BLK);
     object_property_add_alias(obj, "iothread", OBJECT(&dev->vdev),"iothread",
                               &error_abort);
+    object_property_add_alias(obj, "bootindex", OBJECT(&dev->vdev),
+                              "bootindex", &error_abort);
 }
 
 static int virtio_ccw_serial_init(VirtioCcwDevice *ccw_dev)
@@ -848,8 +852,9 @@ static int virtio_ccw_serial_init(VirtioCcwDevice *ccw_dev)
 static void virtio_ccw_serial_instance_init(Object *obj)
 {
     VirtioSerialCcw *dev = VIRTIO_SERIAL_CCW(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_SERIAL);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_SERIAL);
 }
 
 static int virtio_ccw_balloon_init(VirtioCcwDevice *ccw_dev)
@@ -896,7 +901,7 @@ static void virtio_ccw_balloon_instance_init(Object *obj)
     VirtIOBalloonCcw *dev = VIRTIO_BALLOON_CCW(obj);
     object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_BALLOON);
     object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
-
+    object_unref(OBJECT(&dev->vdev));
     object_property_add(obj, "guest-stats", "guest statistics",
                         balloon_ccw_stats_get_all, NULL, NULL, dev, NULL);
 
@@ -934,8 +939,11 @@ static int virtio_ccw_scsi_init(VirtioCcwDevice *ccw_dev)
 static void virtio_ccw_scsi_instance_init(Object *obj)
 {
     VirtIOSCSICcw *dev = VIRTIO_SCSI_CCW(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_SCSI);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_SCSI);
+    object_property_add_alias(obj, "iothread", OBJECT(&dev->vdev), "iothread",
+                              &error_abort);
 }
 
 #ifdef CONFIG_VHOST_SCSI
@@ -955,8 +963,9 @@ static int vhost_ccw_scsi_init(VirtioCcwDevice *ccw_dev)
 static void vhost_ccw_scsi_instance_init(Object *obj)
 {
     VHostSCSICcw *dev = VHOST_SCSI_CCW(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VHOST_SCSI);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VHOST_SCSI);
 }
 #endif
 
@@ -1374,8 +1383,6 @@ static int virtio_ccw_load_config(DeviceState *d, QEMUFile *f)
 static Property virtio_ccw_net_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
     DEFINE_VIRTIO_NET_FEATURES(VirtioCcwDevice, host_features[0]),
-    DEFINE_VIRTIO_NET_PROPERTIES(VirtIONetCcw, vdev.net_conf),
-    DEFINE_NIC_PROPERTIES(VirtIONetCcw, vdev.nic_conf),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
     DEFINE_PROP_END_OF_LIST(),
@@ -1428,7 +1435,6 @@ static const TypeInfo virtio_ccw_blk = {
 
 static Property virtio_ccw_serial_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
-    DEFINE_VIRTIO_SERIAL_PROPERTIES(VirtioSerialCcw, vdev.serial),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
     DEFINE_PROP_END_OF_LIST(),
@@ -1481,7 +1487,6 @@ static const TypeInfo virtio_ccw_balloon = {
 
 static Property virtio_ccw_scsi_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
-    DEFINE_VIRTIO_SCSI_PROPERTIES(VirtIOSCSICcw, vdev.parent_obj.conf),
     DEFINE_VIRTIO_SCSI_FEATURES(VirtioCcwDevice, host_features[0]),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
@@ -1510,7 +1515,6 @@ static const TypeInfo virtio_ccw_scsi = {
 #ifdef CONFIG_VHOST_SCSI
 static Property vhost_ccw_scsi_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
-    DEFINE_VHOST_SCSI_PROPERTIES(VirtIOSCSICcw, vdev.parent_obj.conf),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1528,7 +1532,7 @@ static void vhost_ccw_scsi_class_init(ObjectClass *klass, void *data)
 static const TypeInfo vhost_ccw_scsi = {
     .name          = TYPE_VHOST_SCSI_CCW,
     .parent        = TYPE_VIRTIO_CCW_DEVICE,
-    .instance_size = sizeof(VirtIOSCSICcw),
+    .instance_size = sizeof(VHostSCSICcw),
     .instance_init = vhost_ccw_scsi_instance_init,
     .class_init    = vhost_ccw_scsi_class_init,
 };
@@ -1537,17 +1541,15 @@ static const TypeInfo vhost_ccw_scsi = {
 static void virtio_ccw_rng_instance_init(Object *obj)
 {
     VirtIORNGCcw *dev = VIRTIO_RNG_CCW(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_RNG);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
-    object_property_add_link(obj, "rng", TYPE_RNG_BACKEND,
-                             (Object **)&dev->vdev.conf.rng,
-                             qdev_prop_allow_set_link_before_realize,
-                             OBJ_PROP_LINK_UNREF_ON_RELEASE, NULL);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_RNG);
+    object_property_add_alias(obj, "rng", OBJECT(&dev->vdev),
+                              "rng", &error_abort);
 }
 
 static Property virtio_ccw_rng_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
-    DEFINE_VIRTIO_RNG_PROPERTIES(VirtIORNGCcw, vdev.conf),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
     DEFINE_PROP_END_OF_LIST(),
@@ -1590,7 +1592,8 @@ static int virtio_ccw_busdev_exit(DeviceState *dev)
     return _info->exit(_dev);
 }
 
-static int virtio_ccw_busdev_unplug(DeviceState *dev)
+static void virtio_ccw_busdev_unplug(HotplugHandler *hotplug_dev,
+                                     DeviceState *dev, Error **errp)
 {
     VirtioCcwDevice *_dev = (VirtioCcwDevice *)dev;
     SubchDev *sch = _dev->sch;
@@ -1609,7 +1612,6 @@ static int virtio_ccw_busdev_unplug(DeviceState *dev)
     css_generate_sch_crws(sch->cssid, sch->ssid, sch->schid, 1, 0);
 
     object_unparent(OBJECT(dev));
-    return 0;
 }
 
 static Property virtio_ccw_properties[] = {
@@ -1624,9 +1626,7 @@ static void virtio_ccw_device_class_init(ObjectClass *klass, void *data)
     dc->props = virtio_ccw_properties;
     dc->init = virtio_ccw_busdev_init;
     dc->exit = virtio_ccw_busdev_exit;
-    dc->unplug = virtio_ccw_busdev_unplug;
     dc->bus_type = TYPE_VIRTUAL_CSS_BUS;
-
 }
 
 static const TypeInfo virtio_ccw_device_info = {
@@ -1650,8 +1650,10 @@ static int virtual_css_bridge_init(SysBusDevice *dev)
 static void virtual_css_bridge_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+    HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass);
 
     k->init = virtual_css_bridge_init;
+    hc->unplug = virtio_ccw_busdev_unplug;
 }
 
 static const TypeInfo virtual_css_bridge_info = {
@@ -1659,6 +1661,10 @@ static const TypeInfo virtual_css_bridge_info = {
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysBusDevice),
     .class_init    = virtual_css_bridge_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_HOTPLUG_HANDLER },
+        { }
+    }
 };
 
 /* virtio-ccw-bus */
@@ -1667,13 +1673,10 @@ static void virtio_ccw_bus_new(VirtioBusState *bus, size_t bus_size,
                                VirtioCcwDevice *dev)
 {
     DeviceState *qdev = DEVICE(dev);
-    BusState *qbus;
     char virtio_bus_name[] = "virtio-bus";
 
     qbus_create_inplace(bus, bus_size, TYPE_VIRTIO_CCW_BUS,
                         qdev, virtio_bus_name);
-    qbus = BUS(bus);
-    qbus->allow_hotplug = 1;
 }
 
 static void virtio_ccw_bus_class_init(ObjectClass *klass, void *data)
diff --git a/hw/scsi/Makefile.objs b/hw/scsi/Makefile.objs
index 121ddc5cf..40c79d34c 100644
--- a/hw/scsi/Makefile.objs
+++ b/hw/scsi/Makefile.objs
@@ -8,6 +8,6 @@ common-obj-$(CONFIG_ESP_PCI) += esp-pci.o
 obj-$(CONFIG_PSERIES) += spapr_vscsi.o
 
 ifeq ($(CONFIG_VIRTIO),y)
-obj-y += virtio-scsi.o
+obj-y += virtio-scsi.o virtio-scsi-dataplane.o
 obj-$(CONFIG_VHOST_SCSI) += vhost-scsi.o
 endif
diff --git a/hw/scsi/esp-pci.c b/hw/scsi/esp-pci.c
index 9971bbf92..00b729735 100644
--- a/hw/scsi/esp-pci.c
+++ b/hw/scsi/esp-pci.c
@@ -268,6 +268,9 @@ static void esp_pci_dma_memory_rw(PCIESPState *pci, uint8_t *buf, int len,
     /* update status registers */
     pci->dma_regs[DMA_WBC] -= len;
     pci->dma_regs[DMA_WAC] += len;
+    if (pci->dma_regs[DMA_WBC] == 0) {
+        pci->dma_regs[DMA_STAT] |= DMA_STAT_DONE;
+    }
 }
 
 static void esp_pci_dma_memory_read(void *opaque, uint8_t *buf, int len)
@@ -378,7 +381,6 @@ static void esp_pci_scsi_uninit(PCIDevice *d)
     PCIESPState *pci = PCI_ESP(d);
 
     qemu_free_irq(pci->esp.irq);
-    memory_region_destroy(&pci->io);
 }
 
 static void esp_pci_class_init(ObjectClass *klass, void *data)
diff --git a/hw/scsi/esp.c b/hw/scsi/esp.c
index 5ab44d860..272d13d63 100644
--- a/hw/scsi/esp.c
+++ b/hw/scsi/esp.c
@@ -364,7 +364,7 @@ void esp_hard_reset(ESPState *s)
 {
     memset(s->rregs, 0, ESP_REGS);
     memset(s->wregs, 0, ESP_REGS);
-    s->rregs[ESP_TCHI] = s->chip_id;
+    s->tchi_written = 0;
     s->ti_size = 0;
     s->ti_rptr = 0;
     s->ti_wptr = 0;
@@ -422,6 +422,11 @@ uint64_t esp_reg_read(ESPState *s, uint32_t saddr)
         esp_lower_irq(s);
 
         return old_val;
+    case ESP_TCHI:
+        /* Return the unique id if the value has never been written */
+        if (!s->tchi_written) {
+            return s->chip_id;
+        }
     default:
         break;
     }
@@ -432,9 +437,11 @@ void esp_reg_write(ESPState *s, uint32_t saddr, uint64_t val)
 {
     trace_esp_mem_writeb(saddr, s->wregs[saddr], val);
     switch (saddr) {
+    case ESP_TCHI:
+        s->tchi_written = true;
+        /* fall through */
     case ESP_TCLO:
     case ESP_TCMID:
-    case ESP_TCHI:
         s->rregs[ESP_RSTAT] &= ~STAT_TC;
         break;
     case ESP_FIFO:
diff --git a/hw/scsi/lsi53c895a.c b/hw/scsi/lsi53c895a.c
index 786d8483e..d9b4c7ea3 100644
--- a/hw/scsi/lsi53c895a.c
+++ b/hw/scsi/lsi53c895a.c
@@ -19,6 +19,7 @@
 #include "hw/pci/pci.h"
 #include "hw/scsi/scsi.h"
 #include "sysemu/dma.h"
+#include "qemu/error-report.h"
 
 //#define DEBUG_LSI
 //#define DEBUG_LSI_REG
@@ -2072,15 +2073,6 @@ static const VMStateDescription vmstate_lsi_scsi = {
     }
 };
 
-static void lsi_scsi_uninit(PCIDevice *d)
-{
-    LSIState *s = LSI53C895A(d);
-
-    memory_region_destroy(&s->mmio_io);
-    memory_region_destroy(&s->ram_io);
-    memory_region_destroy(&s->io_io);
-}
-
 static const struct SCSIBusInfo lsi_scsi_info = {
     .tcq = true,
     .max_target = LSI_MAX_DEVS,
@@ -2134,7 +2126,6 @@ static void lsi_class_init(ObjectClass *klass, void *data)
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
     k->init = lsi_scsi_init;
-    k->exit = lsi_scsi_uninit;
     k->vendor_id = PCI_VENDOR_ID_LSI_LOGIC;
     k->device_id = PCI_DEVICE_ID_LSI_53C895A;
     k->class_id = PCI_CLASS_STORAGE_SCSI;
diff --git a/hw/scsi/megasas.c b/hw/scsi/megasas.c
index c68a873f1..604252a19 100644
--- a/hw/scsi/megasas.c
+++ b/hw/scsi/megasas.c
@@ -21,6 +21,7 @@
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "sysemu/dma.h"
+#include "sysemu/block-backend.h"
 #include "hw/pci/msi.h"
 #include "hw/pci/msix.h"
 #include "qemu/iov.h"
@@ -30,9 +31,11 @@
 
 #include "mfi.h"
 
-#define MEGASAS_VERSION "1.70"
+#define MEGASAS_VERSION_GEN1 "1.70"
+#define MEGASAS_VERSION_GEN2 "1.80"
 #define MEGASAS_MAX_FRAMES 2048         /* Firmware limit at 65535 */
 #define MEGASAS_DEFAULT_FRAMES 1000     /* Windows requires this */
+#define MEGASAS_GEN2_DEFAULT_FRAMES 1008     /* Windows requires this */
 #define MEGASAS_MAX_SGE 128             /* Firmware limit */
 #define MEGASAS_DEFAULT_SGE 80
 #define MEGASAS_MAX_SECTORS 0xFFFF      /* No real limit */
@@ -90,6 +93,8 @@ typedef struct MegasasState {
     int intr_mask;
     int doorbell;
     int busy;
+    int diag;
+    int adp_reset;
 
     MegasasCmd *event_cmd;
     int event_locale;
@@ -110,14 +115,30 @@ typedef struct MegasasState {
     uint64_t producer_pa;
 
     MegasasCmd frames[MEGASAS_MAX_FRAMES];
-
+    DECLARE_BITMAP(frame_map, MEGASAS_MAX_FRAMES);
     SCSIBus bus;
 } MegasasState;
 
-#define TYPE_MEGASAS "megasas"
+typedef struct MegasasBaseClass {
+    PCIDeviceClass parent_class;
+    const char *product_name;
+    const char *product_version;
+    int mmio_bar;
+    int ioport_bar;
+    int osts;
+} MegasasBaseClass;
+
+#define TYPE_MEGASAS_BASE "megasas-base"
+#define TYPE_MEGASAS_GEN1 "megasas"
+#define TYPE_MEGASAS_GEN2 "megasas-gen2"
 
 #define MEGASAS(obj) \
-    OBJECT_CHECK(MegasasState, (obj), TYPE_MEGASAS)
+    OBJECT_CHECK(MegasasState, (obj), TYPE_MEGASAS_BASE)
+
+#define MEGASAS_DEVICE_CLASS(oc) \
+    OBJECT_CLASS_CHECK(MegasasBaseClass, (oc), TYPE_MEGASAS_BASE)
+#define MEGASAS_DEVICE_GET_CLASS(oc) \
+    OBJECT_GET_CLASS(MegasasBaseClass, (oc), TYPE_MEGASAS_BASE)
 
 #define MEGASAS_INTR_DISABLED_MASK 0xFFFFFFFF
 
@@ -442,34 +463,20 @@ static MegasasCmd *megasas_lookup_frame(MegasasState *s,
     return cmd;
 }
 
-static MegasasCmd *megasas_next_frame(MegasasState *s,
-    hwaddr frame)
+static void megasas_unmap_frame(MegasasState *s, MegasasCmd *cmd)
 {
-    MegasasCmd *cmd = NULL;
-    int num = 0, index;
+    PCIDevice *p = PCI_DEVICE(s);
 
-    cmd = megasas_lookup_frame(s, frame);
-    if (cmd) {
-        trace_megasas_qf_found(cmd->index, cmd->pa);
-        return cmd;
-    }
-    index = s->reply_queue_head;
-    num = 0;
-    while (num < s->fw_cmds) {
-        if (!s->frames[index].pa) {
-            cmd = &s->frames[index];
-            break;
-        }
-        index = megasas_next_index(s, index, s->fw_cmds);
-        num++;
-    }
-    if (!cmd) {
-        trace_megasas_qf_failed(frame);
-    }
-    trace_megasas_qf_new(index, cmd);
-    return cmd;
+    pci_dma_unmap(p, cmd->frame, cmd->pa_size, 0, 0);
+    cmd->frame = NULL;
+    cmd->pa = 0;
+    clear_bit(cmd->index, s->frame_map);
 }
 
+/*
+ * This absolutely needs to be locked if
+ * qemu ever goes multithreaded.
+ */
 static MegasasCmd *megasas_enqueue_frame(MegasasState *s,
     hwaddr frame, uint64_t context, int count)
 {
@@ -477,37 +484,50 @@ static MegasasCmd *megasas_enqueue_frame(MegasasState *s,
     MegasasCmd *cmd = NULL;
     int frame_size = MFI_FRAME_SIZE * 16;
     hwaddr frame_size_p = frame_size;
+    unsigned long index;
 
-    cmd = megasas_next_frame(s, frame);
-    /* All frames busy */
-    if (!cmd) {
+    index = 0;
+    while (index < s->fw_cmds) {
+        index = find_next_zero_bit(s->frame_map, s->fw_cmds, index);
+        if (!s->frames[index].pa)
+            break;
+        /* Busy frame found */
+        trace_megasas_qf_mapped(index);
+    }
+    if (index >= s->fw_cmds) {
+        /* All frames busy */
+        trace_megasas_qf_busy(frame);
         return NULL;
     }
-    if (!cmd->pa) {
-        cmd->pa = frame;
-        /* Map all possible frames */
-        cmd->frame = pci_dma_map(pcid, frame, &frame_size_p, 0);
-        if (frame_size_p != frame_size) {
-            trace_megasas_qf_map_failed(cmd->index, (unsigned long)frame);
-            if (cmd->frame) {
-                pci_dma_unmap(pcid, cmd->frame, frame_size_p, 0, 0);
-                cmd->frame = NULL;
-                cmd->pa = 0;
-            }
-            s->event_count++;
-            return NULL;
-        }
-        cmd->pa_size = frame_size_p;
-        cmd->context = context;
-        if (!megasas_use_queue64(s)) {
-            cmd->context &= (uint64_t)0xFFFFFFFF;
+    cmd = &s->frames[index];
+    set_bit(index, s->frame_map);
+    trace_megasas_qf_new(index, frame);
+
+    cmd->pa = frame;
+    /* Map all possible frames */
+    cmd->frame = pci_dma_map(pcid, frame, &frame_size_p, 0);
+    if (frame_size_p != frame_size) {
+        trace_megasas_qf_map_failed(cmd->index, (unsigned long)frame);
+        if (cmd->frame) {
+            megasas_unmap_frame(s, cmd);
         }
+        s->event_count++;
+        return NULL;
+    }
+    cmd->pa_size = frame_size_p;
+    cmd->context = context;
+    if (!megasas_use_queue64(s)) {
+        cmd->context &= (uint64_t)0xFFFFFFFF;
     }
     cmd->count = count;
     s->busy++;
 
+    if (s->consumer_pa) {
+        s->reply_queue_tail = ldl_le_phys(&address_space_memory,
+                                          s->consumer_pa);
+    }
     trace_megasas_qf_enqueue(cmd->index, cmd->count, cmd->context,
-                             s->reply_queue_head, s->busy);
+                             s->reply_queue_head, s->reply_queue_tail, s->busy);
 
     return cmd;
 }
@@ -519,39 +539,47 @@ static void megasas_complete_frame(MegasasState *s, uint64_t context)
 
     /* Decrement busy count */
     s->busy--;
-
     if (s->reply_queue_pa) {
         /*
          * Put command on the reply queue.
          * Context is opaque, but emulation is running in
          * little endian. So convert it.
          */
-        tail = s->reply_queue_head;
         if (megasas_use_queue64(s)) {
-            queue_offset = tail * sizeof(uint64_t);
+            queue_offset = s->reply_queue_head * sizeof(uint64_t);
             stq_le_phys(&address_space_memory,
                         s->reply_queue_pa + queue_offset, context);
         } else {
-            queue_offset = tail * sizeof(uint32_t);
+            queue_offset = s->reply_queue_head * sizeof(uint32_t);
             stl_le_phys(&address_space_memory,
                         s->reply_queue_pa + queue_offset, context);
         }
-        s->reply_queue_head = megasas_next_index(s, tail, s->fw_cmds);
-        trace_megasas_qf_complete(context, tail, queue_offset,
-                                  s->busy, s->doorbell);
+        s->reply_queue_tail = ldl_le_phys(&address_space_memory,
+                                          s->consumer_pa);
+        trace_megasas_qf_complete(context, s->reply_queue_head,
+                                  s->reply_queue_tail, s->busy);
     }
 
     if (megasas_intr_enabled(s)) {
+        /* Update reply queue pointer */
+        s->reply_queue_tail = ldl_le_phys(&address_space_memory,
+                                          s->consumer_pa);
+        tail = s->reply_queue_head;
+        s->reply_queue_head = megasas_next_index(s, tail, s->fw_cmds);
+        trace_megasas_qf_update(s->reply_queue_head, s->reply_queue_tail,
+                                s->busy);
+        stl_le_phys(&address_space_memory,
+                    s->producer_pa, s->reply_queue_head);
         /* Notify HBA */
-        s->doorbell++;
-        if (s->doorbell == 1) {
-            if (msix_enabled(pci_dev)) {
-                trace_megasas_msix_raise(0);
-                msix_notify(pci_dev, 0);
-            } else if (msi_enabled(pci_dev)) {
-                trace_megasas_msi_raise(0);
-                msi_notify(pci_dev, 0);
-            } else {
+        if (msix_enabled(pci_dev)) {
+            trace_megasas_msix_raise(0);
+            msix_notify(pci_dev, 0);
+        } else if (msi_enabled(pci_dev)) {
+            trace_megasas_msi_raise(0);
+            msi_notify(pci_dev, 0);
+        } else {
+            s->doorbell++;
+            if (s->doorbell == 1) {
                 trace_megasas_irq_raise();
                 pci_irq_assert(pci_dev);
             }
@@ -563,18 +591,16 @@ static void megasas_complete_frame(MegasasState *s, uint64_t context)
 
 static void megasas_reset_frames(MegasasState *s)
 {
-    PCIDevice *pcid = PCI_DEVICE(s);
     int i;
     MegasasCmd *cmd;
 
     for (i = 0; i < s->fw_cmds; i++) {
         cmd = &s->frames[i];
         if (cmd->pa) {
-            pci_dma_unmap(pcid, cmd->frame, cmd->pa_size, 0, 0);
-            cmd->frame = NULL;
-            cmd->pa = 0;
+            megasas_unmap_frame(s, cmd);
         }
     }
+    bitmap_zero(s->frame_map, MEGASAS_MAX_FRAMES);
 }
 
 static void megasas_abort_command(MegasasCmd *cmd)
@@ -589,16 +615,19 @@ static int megasas_init_firmware(MegasasState *s, MegasasCmd *cmd)
 {
     PCIDevice *pcid = PCI_DEVICE(s);
     uint32_t pa_hi, pa_lo;
-    hwaddr iq_pa, initq_size;
-    struct mfi_init_qinfo *initq;
+    hwaddr iq_pa, initq_size = sizeof(struct mfi_init_qinfo);
+    struct mfi_init_qinfo *initq = NULL;
     uint32_t flags;
     int ret = MFI_STAT_OK;
 
+    if (s->reply_queue_pa) {
+        trace_megasas_initq_mapped(s->reply_queue_pa);
+        goto out;
+    }
     pa_lo = le32_to_cpu(cmd->frame->init.qinfo_new_addr_lo);
     pa_hi = le32_to_cpu(cmd->frame->init.qinfo_new_addr_hi);
     iq_pa = (((uint64_t) pa_hi << 32) | pa_lo);
     trace_megasas_init_firmware((uint64_t)iq_pa);
-    initq_size = sizeof(*initq);
     initq = pci_dma_map(pcid, iq_pa, &initq_size, 0);
     if (!initq || initq_size != sizeof(*initq)) {
         trace_megasas_initq_map_failed(cmd->index);
@@ -685,11 +714,12 @@ static void megasas_finish_dcmd(MegasasCmd *cmd, uint32_t iov_size)
 static int megasas_ctrl_get_info(MegasasState *s, MegasasCmd *cmd)
 {
     PCIDevice *pci_dev = PCI_DEVICE(s);
+    PCIDeviceClass *pci_class = PCI_DEVICE_GET_CLASS(pci_dev);
+    MegasasBaseClass *base_class = MEGASAS_DEVICE_GET_CLASS(s);
     struct mfi_ctrl_info info;
     size_t dcmd_size = sizeof(info);
     BusChild *kid;
-    int num_ld_disks = 0;
-    uint16_t sdev_id;
+    int num_pd_disks = 0;
 
     memset(&info, 0x0, cmd->iov_size);
     if (cmd->iov_size < dcmd_size) {
@@ -698,10 +728,10 @@ static int megasas_ctrl_get_info(MegasasState *s, MegasasCmd *cmd)
         return MFI_STAT_INVALID_PARAMETER;
     }
 
-    info.pci.vendor = cpu_to_le16(PCI_VENDOR_ID_LSI_LOGIC);
-    info.pci.device = cpu_to_le16(PCI_DEVICE_ID_LSI_SAS1078);
-    info.pci.subvendor = cpu_to_le16(PCI_VENDOR_ID_LSI_LOGIC);
-    info.pci.subdevice = cpu_to_le16(0x1013);
+    info.pci.vendor = cpu_to_le16(pci_class->vendor_id);
+    info.pci.device = cpu_to_le16(pci_class->device_id);
+    info.pci.subvendor = cpu_to_le16(pci_class->subsystem_vendor_id);
+    info.pci.subdevice = cpu_to_le16(pci_class->subsystem_id);
 
     /*
      * For some reason the firmware supports
@@ -718,20 +748,22 @@ static int megasas_ctrl_get_info(MegasasState *s, MegasasCmd *cmd)
     info.device.port_count = 8;
     QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
         SCSIDevice *sdev = DO_UPCAST(SCSIDevice, qdev, kid->child);
+        uint16_t pd_id;
 
-        if (num_ld_disks < 8) {
-            sdev_id = ((sdev->id & 0xFF) >> 8) | (sdev->lun & 0xFF);
-            info.device.port_addr[num_ld_disks] =
-                cpu_to_le64(megasas_get_sata_addr(sdev_id));
+        if (num_pd_disks < 8) {
+            pd_id = ((sdev->id & 0xFF) << 8) | (sdev->lun & 0xFF);
+            info.device.port_addr[num_pd_disks] =
+                cpu_to_le64(megasas_get_sata_addr(pd_id));
         }
-        num_ld_disks++;
+        num_pd_disks++;
     }
 
-    memcpy(info.product_name, "MegaRAID SAS 8708EM2", 20);
+    memcpy(info.product_name, base_class->product_name, 24);
     snprintf(info.serial_number, 32, "%s", s->hba_serial);
     snprintf(info.package_version, 0x60, "%s-QEMU", QEMU_VERSION);
     memcpy(info.image_component[0].name, "APP", 3);
-    memcpy(info.image_component[0].version, MEGASAS_VERSION "-QEMU", 9);
+    snprintf(info.image_component[0].version, 10, "%s-QEMU",
+             base_class->product_version);
     memcpy(info.image_component[0].build_date, "Apr  1 2014", 11);
     memcpy(info.image_component[0].build_time, "12:34:56", 8);
     info.image_component_count = 1;
@@ -750,13 +782,14 @@ static int megasas_ctrl_get_info(MegasasState *s, MegasasCmd *cmd)
     info.max_arms = 32;
     info.max_spans = 8;
     info.max_arrays = MEGASAS_MAX_ARRAYS;
-    info.max_lds = s->fw_luns;
+    info.max_lds = MFI_MAX_LD;
     info.max_cmds = cpu_to_le16(s->fw_cmds);
     info.max_sg_elements = cpu_to_le16(s->fw_sge);
     info.max_request_size = cpu_to_le32(MEGASAS_MAX_SECTORS);
-    info.lds_present = cpu_to_le16(num_ld_disks);
-    info.pd_present = cpu_to_le16(num_ld_disks);
-    info.pd_disks_present = cpu_to_le16(num_ld_disks);
+    if (!megasas_is_jbod(s))
+        info.lds_present = cpu_to_le16(num_pd_disks);
+    info.pd_present = cpu_to_le16(num_pd_disks);
+    info.pd_disks_present = cpu_to_le16(num_pd_disks);
     info.hw_present = cpu_to_le32(MFI_INFO_HW_NVRAM |
                                    MFI_INFO_HW_MEM |
                                    MFI_INFO_HW_FLASH);
@@ -915,7 +948,6 @@ static int megasas_dcmd_pd_get_list(MegasasState *s, MegasasCmd *cmd)
     size_t dcmd_size = sizeof(info);
     BusChild *kid;
     uint32_t offset, dcmd_limit, num_pd_disks = 0, max_pd_disks;
-    uint16_t sdev_id;
 
     memset(&info, 0, dcmd_size);
     offset = 8;
@@ -927,22 +959,25 @@ static int megasas_dcmd_pd_get_list(MegasasState *s, MegasasCmd *cmd)
     }
 
     max_pd_disks = (cmd->iov_size - offset) / sizeof(struct mfi_pd_address);
-    if (max_pd_disks > s->fw_luns) {
-        max_pd_disks = s->fw_luns;
+    if (max_pd_disks > MFI_MAX_SYS_PDS) {
+        max_pd_disks = MFI_MAX_SYS_PDS;
     }
-
     QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
         SCSIDevice *sdev = DO_UPCAST(SCSIDevice, qdev, kid->child);
+        uint16_t pd_id;
+
+        if (num_pd_disks >= max_pd_disks)
+            break;
 
-        sdev_id = ((sdev->id & 0xFF) >> 8) | (sdev->lun & 0xFF);
-        info.addr[num_pd_disks].device_id = cpu_to_le16(sdev_id);
+        pd_id = ((sdev->id & 0xFF) << 8) | (sdev->lun & 0xFF);
+        info.addr[num_pd_disks].device_id = cpu_to_le16(pd_id);
         info.addr[num_pd_disks].encl_device_id = 0xFFFF;
         info.addr[num_pd_disks].encl_index = 0;
-        info.addr[num_pd_disks].slot_number = (sdev->id & 0xFF);
+        info.addr[num_pd_disks].slot_number = sdev->id & 0xFF;
         info.addr[num_pd_disks].scsi_dev_type = sdev->type;
         info.addr[num_pd_disks].connect_port_bitmap = 0x1;
         info.addr[num_pd_disks].sas_addr[0] =
-            cpu_to_le64(megasas_get_sata_addr(sdev_id));
+            cpu_to_le64(megasas_get_sata_addr(pd_id));
         num_pd_disks++;
         offset += sizeof(struct mfi_pd_address);
     }
@@ -976,9 +1011,8 @@ static int megasas_pd_get_info_submit(SCSIDevice *sdev, int lun,
 {
     struct mfi_pd_info *info = cmd->iov_buf;
     size_t dcmd_size = sizeof(struct mfi_pd_info);
-    BlockConf *conf = &sdev->conf;
     uint64_t pd_size;
-    uint16_t sdev_id = ((sdev->id & 0xFF) >> 8) | (lun & 0xFF);
+    uint16_t pd_id = ((sdev->id & 0xFF) << 8) | (lun & 0xFF);
     uint8_t cmdbuf[6];
     SCSIRequest *req;
     size_t len, resid;
@@ -1034,10 +1068,10 @@ static int megasas_pd_get_info_submit(SCSIDevice *sdev, int lun,
         info->fw_state = cpu_to_le16(MFI_PD_STATE_OFFLINE);
     }
 
-    info->ref.v.device_id = cpu_to_le16(sdev_id);
+    info->ref.v.device_id = cpu_to_le16(pd_id);
     info->state.ddf.pd_type = cpu_to_le16(MFI_PD_DDF_TYPE_IN_VD|
                                           MFI_PD_DDF_TYPE_INTF_SAS);
-    bdrv_get_geometry(conf->bs, &pd_size);
+    blk_get_geometry(sdev->conf.blk, &pd_size);
     info->raw_size = cpu_to_le64(pd_size);
     info->non_coerced_size = cpu_to_le64(pd_size);
     info->coerced_size = cpu_to_le64(pd_size);
@@ -1045,7 +1079,7 @@ static int megasas_pd_get_info_submit(SCSIDevice *sdev, int lun,
     info->slot_number = (sdev->id & 0xFF);
     info->path_info.count = 1;
     info->path_info.sas_addr[0] =
-        cpu_to_le64(megasas_get_sata_addr(sdev_id));
+        cpu_to_le64(megasas_get_sata_addr(pd_id));
     info->connected_port_bitmap = 0x1;
     info->device_speed = 1;
     info->link_speed = 1;
@@ -1060,6 +1094,7 @@ static int megasas_dcmd_pd_get_info(MegasasState *s, MegasasCmd *cmd)
 {
     size_t dcmd_size = sizeof(struct mfi_pd_info);
     uint16_t pd_id;
+    uint8_t target_id, lun_id;
     SCSIDevice *sdev = NULL;
     int retval = MFI_STAT_DEVICE_NOT_FOUND;
 
@@ -1069,7 +1104,9 @@ static int megasas_dcmd_pd_get_info(MegasasState *s, MegasasCmd *cmd)
 
     /* mbox0 has the ID */
     pd_id = le16_to_cpu(cmd->frame->dcmd.mbox[0]);
-    sdev = scsi_device_find(&s->bus, 0, pd_id, 0);
+    target_id = (pd_id >> 8) & 0xFF;
+    lun_id = pd_id & 0xFF;
+    sdev = scsi_device_find(&s->bus, 0, target_id, lun_id);
     trace_megasas_dcmd_pd_get_info(cmd->index, pd_id);
 
     if (sdev) {
@@ -1084,31 +1121,33 @@ static int megasas_dcmd_ld_get_list(MegasasState *s, MegasasCmd *cmd)
 {
     struct mfi_ld_list info;
     size_t dcmd_size = sizeof(info), resid;
-    uint32_t num_ld_disks = 0, max_ld_disks = s->fw_luns;
+    uint32_t num_ld_disks = 0, max_ld_disks;
     uint64_t ld_size;
     BusChild *kid;
 
     memset(&info, 0, dcmd_size);
-    if (cmd->iov_size < dcmd_size) {
+    if (cmd->iov_size > dcmd_size) {
         trace_megasas_dcmd_invalid_xfer_len(cmd->index, cmd->iov_size,
                                             dcmd_size);
         return MFI_STAT_INVALID_PARAMETER;
     }
 
+    max_ld_disks = (cmd->iov_size - 8) / 16;
     if (megasas_is_jbod(s)) {
         max_ld_disks = 0;
     }
+    if (max_ld_disks > MFI_MAX_LD) {
+        max_ld_disks = MFI_MAX_LD;
+    }
     QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
         SCSIDevice *sdev = DO_UPCAST(SCSIDevice, qdev, kid->child);
-        BlockConf *conf = &sdev->conf;
 
         if (num_ld_disks >= max_ld_disks) {
             break;
         }
         /* Logical device size is in blocks */
-        bdrv_get_geometry(conf->bs, &ld_size);
+        blk_get_geometry(sdev->conf.blk, &ld_size);
         info.ld_list[num_ld_disks].ld.v.target_id = sdev->id;
-        info.ld_list[num_ld_disks].ld.v.lun_id = sdev->lun;
         info.ld_list[num_ld_disks].state = MFI_LD_STATE_OPTIMAL;
         info.ld_list[num_ld_disks].size = cpu_to_le64(ld_size);
         num_ld_disks++;
@@ -1124,15 +1163,49 @@ static int megasas_dcmd_ld_get_list(MegasasState *s, MegasasCmd *cmd)
 static int megasas_dcmd_ld_list_query(MegasasState *s, MegasasCmd *cmd)
 {
     uint16_t flags;
+    struct mfi_ld_targetid_list info;
+    size_t dcmd_size = sizeof(info), resid;
+    uint32_t num_ld_disks = 0, max_ld_disks = s->fw_luns;
+    BusChild *kid;
 
     /* mbox0 contains flags */
     flags = le16_to_cpu(cmd->frame->dcmd.mbox[0]);
     trace_megasas_dcmd_ld_list_query(cmd->index, flags);
-    if (flags == MR_LD_QUERY_TYPE_ALL ||
-        flags == MR_LD_QUERY_TYPE_EXPOSED_TO_HOST) {
-        return megasas_dcmd_ld_get_list(s, cmd);
+    if (flags != MR_LD_QUERY_TYPE_ALL &&
+        flags != MR_LD_QUERY_TYPE_EXPOSED_TO_HOST) {
+        max_ld_disks = 0;
     }
 
+    memset(&info, 0, dcmd_size);
+    if (cmd->iov_size < 12) {
+        trace_megasas_dcmd_invalid_xfer_len(cmd->index, cmd->iov_size,
+                                            dcmd_size);
+        return MFI_STAT_INVALID_PARAMETER;
+    }
+    dcmd_size = sizeof(uint32_t) * 2 + 3;
+    max_ld_disks = cmd->iov_size - dcmd_size;
+    if (megasas_is_jbod(s)) {
+        max_ld_disks = 0;
+    }
+    if (max_ld_disks > MFI_MAX_LD) {
+        max_ld_disks = MFI_MAX_LD;
+    }
+    QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
+        SCSIDevice *sdev = DO_UPCAST(SCSIDevice, qdev, kid->child);
+
+        if (num_ld_disks >= max_ld_disks) {
+            break;
+        }
+        info.targetid[num_ld_disks] = sdev->lun;
+        num_ld_disks++;
+        dcmd_size++;
+    }
+    info.ld_count = cpu_to_le32(num_ld_disks);
+    info.size = dcmd_size;
+    trace_megasas_dcmd_ld_get_list(cmd->index, num_ld_disks, max_ld_disks);
+
+    resid = dma_buf_read((uint8_t *)&info, dcmd_size, &cmd->qsg);
+    cmd->iov_size = dcmd_size - resid;
     return MFI_STAT_OK;
 }
 
@@ -1144,8 +1217,7 @@ static int megasas_ld_get_info_submit(SCSIDevice *sdev, int lun,
     uint8_t cdb[6];
     SCSIRequest *req;
     ssize_t len, resid;
-    BlockConf *conf = &sdev->conf;
-    uint16_t sdev_id = ((sdev->id & 0xFF) >> 8) | (lun & 0xFF);
+    uint16_t sdev_id = ((sdev->id & 0xFF) << 8) | (lun & 0xFF);
     uint64_t ld_size;
 
     if (!cmd->iov_buf) {
@@ -1177,7 +1249,7 @@ static int megasas_ld_get_info_submit(SCSIDevice *sdev, int lun,
     info->ld_config.params.num_drives = 1;
     info->ld_config.params.is_consistent = 1;
     /* Logical device size is in blocks */
-    bdrv_get_geometry(conf->bs, &ld_size);
+    blk_get_geometry(sdev->conf.blk, &ld_size);
     info->size = cpu_to_le64(ld_size);
     memset(info->ld_config.span, 0, sizeof(info->ld_config.span));
     info->ld_config.span[0].start_block = 0;
@@ -1261,15 +1333,14 @@ static int megasas_dcmd_cfg_read(MegasasState *s, MegasasCmd *cmd)
 
     QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
         SCSIDevice *sdev = DO_UPCAST(SCSIDevice, qdev, kid->child);
-        BlockConf *conf = &sdev->conf;
-        uint16_t sdev_id = ((sdev->id & 0xFF) >> 8) | (sdev->lun & 0xFF);
+        uint16_t sdev_id = ((sdev->id & 0xFF) << 8) | (sdev->lun & 0xFF);
         struct mfi_array *array;
         struct mfi_ld_config *ld;
         uint64_t pd_size;
         int i;
 
         array = (struct mfi_array *)(data + array_offset);
-        bdrv_get_geometry(conf->bs, &pd_size);
+        blk_get_geometry(sdev->conf.blk, &pd_size);
         array->size = cpu_to_le64(pd_size);
         array->num_drives = 1;
         array->array_ref = cpu_to_le16(sdev_id);
@@ -1288,7 +1359,7 @@ static int megasas_dcmd_cfg_read(MegasasState *s, MegasasCmd *cmd)
         array_offset += sizeof(struct mfi_array);
         ld = (struct mfi_ld_config *)(data + ld_offset);
         memset(ld, 0, sizeof(struct mfi_ld_config));
-        ld->properties.ld.v.target_id = (sdev->id & 0xFF);
+        ld->properties.ld.v.target_id = sdev->id;
         ld->properties.default_cache_policy = MR_LD_CACHE_READ_AHEAD |
             MR_LD_CACHE_READ_ADAPTIVE;
         ld->properties.current_cache_policy = MR_LD_CACHE_READ_AHEAD |
@@ -1340,7 +1411,7 @@ static int megasas_dcmd_get_properties(MegasasState *s, MegasasCmd *cmd)
 
 static int megasas_cache_flush(MegasasState *s, MegasasCmd *cmd)
 {
-    bdrv_drain_all();
+    blk_drain_all();
     return MFI_STAT_OK;
 }
 
@@ -1350,9 +1421,23 @@ static int megasas_ctrl_shutdown(MegasasState *s, MegasasCmd *cmd)
     return MFI_STAT_OK;
 }
 
+/* Some implementations use CLUSTER RESET LD to simulate a device reset */
 static int megasas_cluster_reset_ld(MegasasState *s, MegasasCmd *cmd)
 {
-    return MFI_STAT_INVALID_DCMD;
+    uint16_t target_id;
+    int i;
+
+    /* mbox0 contains the device index */
+    target_id = le16_to_cpu(cmd->frame->dcmd.mbox[0]);
+    trace_megasas_dcmd_reset_ld(cmd->index, target_id);
+    for (i = 0; i < s->fw_cmds; i++) {
+        MegasasCmd *tmp_cmd = &s->frames[i];
+        if (tmp_cmd->req && tmp_cmd->req->dev->id == target_id) {
+            SCSIDevice *d = tmp_cmd->req->dev;
+            qdev_reset_all(&d->qdev);
+        }
+    }
+    return MFI_STAT_OK;
 }
 
 static int megasas_dcmd_set_properties(MegasasState *s, MegasasCmd *cmd)
@@ -1569,16 +1654,23 @@ static int megasas_handle_scsi(MegasasState *s, MegasasCmd *cmd,
                                bool is_logical)
 {
     uint8_t *cdb;
-    int len;
     bool is_write;
     struct SCSIDevice *sdev = NULL;
 
     cdb = cmd->frame->pass.cdb;
 
-    if (cmd->frame->header.target_id < s->fw_luns) {
-        sdev = scsi_device_find(&s->bus, 0, cmd->frame->header.target_id,
-                                cmd->frame->header.lun_id);
+    if (is_logical) {
+        if (cmd->frame->header.target_id >= MFI_MAX_LD ||
+            cmd->frame->header.lun_id != 0) {
+            trace_megasas_scsi_target_not_present(
+                mfi_frame_desc[cmd->frame->header.frame_cmd], is_logical,
+                cmd->frame->header.target_id, cmd->frame->header.lun_id);
+            return MFI_STAT_DEVICE_NOT_FOUND;
+        }
     }
+    sdev = scsi_device_find(&s->bus, 0, cmd->frame->header.target_id,
+                            cmd->frame->header.lun_id);
+
     cmd->iov_size = le32_to_cpu(cmd->frame->header.data_len);
     trace_megasas_handle_scsi(mfi_frame_desc[cmd->frame->header.frame_cmd],
                               is_logical, cmd->frame->header.target_id,
@@ -1622,16 +1714,16 @@ static int megasas_handle_scsi(MegasasState *s, MegasasCmd *cmd,
     }
 
     is_write = (cmd->req->cmd.mode == SCSI_XFER_TO_DEV);
-    len = megasas_enqueue_req(cmd, is_write);
-    if (len > 0) {
+    if (cmd->iov_size) {
         if (is_write) {
-            trace_megasas_scsi_write_start(cmd->index, len);
+            trace_megasas_scsi_write_start(cmd->index, cmd->iov_size);
         } else {
-            trace_megasas_scsi_read_start(cmd->index, len);
+            trace_megasas_scsi_read_start(cmd->index, cmd->iov_size);
         }
     } else {
         trace_megasas_scsi_nodata(cmd->index);
     }
+    megasas_enqueue_req(cmd, is_write);
     return MFI_STAT_INVALID_STATUS;
 }
 
@@ -1649,7 +1741,8 @@ static int megasas_handle_io(MegasasState *s, MegasasCmd *cmd)
     lba_start_hi = le32_to_cpu(cmd->frame->io.lba_hi);
     lba_start = ((uint64_t)lba_start_hi << 32) | lba_start_lo;
 
-    if (cmd->frame->header.target_id < s->fw_luns) {
+    if (cmd->frame->header.target_id < MFI_MAX_LD &&
+        cmd->frame->header.lun_id == 0) {
         sdev = scsi_device_find(&s->bus, 0, cmd->frame->header.target_id,
                                 cmd->frame->header.lun_id);
     }
@@ -1800,6 +1893,7 @@ static void megasas_command_complete(SCSIRequest *req, uint32_t status,
         cmd->req = NULL;
     }
     cmd->frame->header.cmd_status = cmd_status;
+    megasas_unmap_frame(cmd->state, cmd);
     megasas_complete_frame(cmd->state, cmd->context);
 }
 
@@ -1903,6 +1997,7 @@ static void megasas_handle_frame(MegasasState *s, uint64_t frame_addr,
         } else {
             megasas_frame_set_cmd_status(frame_addr, frame_status);
         }
+        megasas_unmap_frame(s, cmd);
         megasas_complete_frame(s, cmd->context);
     }
 }
@@ -1911,38 +2006,55 @@ static uint64_t megasas_mmio_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     MegasasState *s = opaque;
+    PCIDevice *pci_dev = PCI_DEVICE(s);
+    MegasasBaseClass *base_class = MEGASAS_DEVICE_GET_CLASS(s);
     uint32_t retval = 0;
 
     switch (addr) {
     case MFI_IDB:
         retval = 0;
+        trace_megasas_mmio_readl("MFI_IDB", retval);
         break;
     case MFI_OMSG0:
     case MFI_OSP0:
-        retval = (megasas_use_msix(s) ? MFI_FWSTATE_MSIX_SUPPORTED : 0) |
+        retval = (msix_present(pci_dev) ? MFI_FWSTATE_MSIX_SUPPORTED : 0) |
             (s->fw_state & MFI_FWSTATE_MASK) |
             ((s->fw_sge & 0xff) << 16) |
             (s->fw_cmds & 0xFFFF);
+        trace_megasas_mmio_readl(addr == MFI_OMSG0 ? "MFI_OMSG0" : "MFI_OSP0",
+                                 retval);
         break;
     case MFI_OSTS:
         if (megasas_intr_enabled(s) && s->doorbell) {
-            retval = MFI_1078_RM | 1;
+            retval = base_class->osts;
         }
+        trace_megasas_mmio_readl("MFI_OSTS", retval);
         break;
     case MFI_OMSK:
         retval = s->intr_mask;
+        trace_megasas_mmio_readl("MFI_OMSK", retval);
         break;
     case MFI_ODCR0:
-        retval = s->doorbell;
+        retval = s->doorbell ? 1 : 0;
+        trace_megasas_mmio_readl("MFI_ODCR0", retval);
+        break;
+    case MFI_DIAG:
+        retval = s->diag;
+        trace_megasas_mmio_readl("MFI_DIAG", retval);
+        break;
+    case MFI_OSP1:
+        retval = 15;
+        trace_megasas_mmio_readl("MFI_OSP1", retval);
         break;
     default:
         trace_megasas_mmio_invalid_readl(addr);
         break;
     }
-    trace_megasas_mmio_readl(addr, retval);
     return retval;
 }
 
+static int adp_reset_seq[] = {0x00, 0x04, 0x0b, 0x02, 0x07, 0x0d};
+
 static void megasas_mmio_write(void *opaque, hwaddr addr,
                                uint64_t val, unsigned size)
 {
@@ -1952,9 +2064,9 @@ static void megasas_mmio_write(void *opaque, hwaddr addr,
     uint32_t frame_count;
     int i;
 
-    trace_megasas_mmio_writel(addr, val);
     switch (addr) {
     case MFI_IDB:
+        trace_megasas_mmio_writel("MFI_IDB", val);
         if (val & MFI_FWINIT_ABORT) {
             /* Abort all pending cmds */
             for (i = 0; i < s->fw_cmds; i++) {
@@ -1968,8 +2080,13 @@ static void megasas_mmio_write(void *opaque, hwaddr addr,
         if (val & MFI_FWINIT_MFIMODE) {
             /* discard MFIs */
         }
+        if (val & MFI_FWINIT_STOP_ADP) {
+            /* Terminal error, stop processing */
+            s->fw_state = MFI_FWSTATE_FAULT;
+        }
         break;
     case MFI_OMSK:
+        trace_megasas_mmio_writel("MFI_OMSK", val);
         s->intr_mask = val;
         if (!megasas_intr_enabled(s) &&
             !msi_enabled(pci_dev) &&
@@ -1987,29 +2104,34 @@ static void megasas_mmio_write(void *opaque, hwaddr addr,
             }
         } else {
             trace_megasas_intr_disabled();
+            megasas_soft_reset(s);
         }
         break;
     case MFI_ODCR0:
+        trace_megasas_mmio_writel("MFI_ODCR0", val);
         s->doorbell = 0;
-        if (s->producer_pa && megasas_intr_enabled(s)) {
-            /* Update reply queue pointer */
-            trace_megasas_qf_update(s->reply_queue_head, s->busy);
-            stl_le_phys(&address_space_memory,
-                        s->producer_pa, s->reply_queue_head);
-            if (!msix_enabled(pci_dev)) {
+        if (megasas_intr_enabled(s)) {
+            if (!msix_enabled(pci_dev) && !msi_enabled(pci_dev)) {
                 trace_megasas_irq_lower();
                 pci_irq_deassert(pci_dev);
             }
         }
         break;
     case MFI_IQPH:
+        trace_megasas_mmio_writel("MFI_IQPH", val);
         /* Received high 32 bits of a 64 bit MFI frame address */
         s->frame_hi = val;
         break;
     case MFI_IQPL:
+        trace_megasas_mmio_writel("MFI_IQPL", val);
         /* Received low 32 bits of a 64 bit MFI frame address */
+        /* Fallthrough */
     case MFI_IQP:
-        /* Received 32 bit MFI frame address */
+        if (addr == MFI_IQP) {
+            trace_megasas_mmio_writel("MFI_IQP", val);
+            /* Received 64 bit MFI frame address */
+            s->frame_hi = 0;
+        }
         frame_addr = (val & ~0x1F);
         /* Add possible 64 bit offset */
         frame_addr |= ((uint64_t)s->frame_hi << 32);
@@ -2017,6 +2139,30 @@ static void megasas_mmio_write(void *opaque, hwaddr addr,
         frame_count = (val >> 1) & 0xF;
         megasas_handle_frame(s, frame_addr, frame_count);
         break;
+    case MFI_SEQ:
+        trace_megasas_mmio_writel("MFI_SEQ", val);
+        /* Magic sequence to start ADP reset */
+        if (adp_reset_seq[s->adp_reset] == val) {
+            s->adp_reset++;
+        } else {
+            s->adp_reset = 0;
+            s->diag = 0;
+        }
+        if (s->adp_reset == 6) {
+            s->diag = MFI_DIAG_WRITE_ENABLE;
+        }
+        break;
+    case MFI_DIAG:
+        trace_megasas_mmio_writel("MFI_DIAG", val);
+        /* ADP reset */
+        if ((s->diag & MFI_DIAG_WRITE_ENABLE) &&
+            (val & MFI_DIAG_RESET_ADP)) {
+            s->diag |= MFI_DIAG_RESET_ADP;
+            megasas_soft_reset(s);
+            s->adp_reset = 0;
+            s->diag = 0;
+        }
+        break;
     default:
         trace_megasas_mmio_invalid_writel(addr, val);
         break;
@@ -2075,11 +2221,26 @@ static void megasas_soft_reset(MegasasState *s)
     int i;
     MegasasCmd *cmd;
 
-    trace_megasas_reset();
+    trace_megasas_reset(s->fw_state);
     for (i = 0; i < s->fw_cmds; i++) {
         cmd = &s->frames[i];
         megasas_abort_command(cmd);
     }
+    if (s->fw_state == MFI_FWSTATE_READY) {
+        BusChild *kid;
+
+        /*
+         * The EFI firmware doesn't handle UA,
+         * so we need to clear the Power On/Reset UA
+         * after the initial reset.
+         */
+        QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
+            SCSIDevice *sdev = DO_UPCAST(SCSIDevice, qdev, kid->child);
+
+            sdev->unit_attention = SENSE_CODE(NO_SENSE);
+            scsi_device_unit_attention_reported(sdev);
+        }
+    }
     megasas_reset_frames(s);
     s->reply_queue_len = s->fw_cmds;
     s->reply_queue_pa = 0;
@@ -2101,7 +2262,7 @@ static void megasas_scsi_reset(DeviceState *dev)
     megasas_soft_reset(s);
 }
 
-static const VMStateDescription vmstate_megasas = {
+static const VMStateDescription vmstate_megasas_gen1 = {
     .name = "megasas",
     .version_id = 0,
     .minimum_version_id = 0,
@@ -2119,6 +2280,25 @@ static const VMStateDescription vmstate_megasas = {
     }
 };
 
+static const VMStateDescription vmstate_megasas_gen2 = {
+    .name = "megasas-gen2",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .minimum_version_id_old = 0,
+    .fields      = (VMStateField[]) {
+        VMSTATE_PCIE_DEVICE(parent_obj, MegasasState),
+        VMSTATE_MSIX(parent_obj, MegasasState),
+
+        VMSTATE_INT32(fw_state, MegasasState),
+        VMSTATE_INT32(intr_mask, MegasasState),
+        VMSTATE_INT32(doorbell, MegasasState),
+        VMSTATE_UINT64(reply_queue_pa, MegasasState),
+        VMSTATE_UINT64(consumer_pa, MegasasState),
+        VMSTATE_UINT64(producer_pa, MegasasState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static void megasas_scsi_uninit(PCIDevice *d)
 {
     MegasasState *s = MEGASAS(d);
@@ -2129,9 +2309,6 @@ static void megasas_scsi_uninit(PCIDevice *d)
     if (megasas_use_msi(s)) {
         msi_uninit(d);
     }
-    memory_region_destroy(&s->mmio_io);
-    memory_region_destroy(&s->port_io);
-    memory_region_destroy(&s->queue_io);
 }
 
 static const struct SCSIBusInfo megasas_scsi_info = {
@@ -2149,6 +2326,7 @@ static int megasas_scsi_init(PCIDevice *dev)
 {
     DeviceState *d = DEVICE(dev);
     MegasasState *s = MEGASAS(dev);
+    MegasasBaseClass *b = MEGASAS_DEVICE_GET_CLASS(s);
     uint8_t *pci_conf;
     int i, bar_type;
     Error *err = NULL;
@@ -2172,20 +2350,25 @@ static int megasas_scsi_init(PCIDevice *dev)
         s->flags &= ~MEGASAS_MASK_USE_MSI;
     }
     if (megasas_use_msix(s) &&
-        msix_init(dev, 15, &s->mmio_io, 0, 0x2000,
-                  &s->mmio_io, 0, 0x3800, 0x68)) {
+        msix_init(dev, 15, &s->mmio_io, b->mmio_bar, 0x2000,
+                  &s->mmio_io, b->mmio_bar, 0x3800, 0x68)) {
         s->flags &= ~MEGASAS_MASK_USE_MSIX;
     }
+    if (pci_is_express(dev)) {
+        pcie_endpoint_cap_init(dev, 0xa0);
+    }
 
     bar_type = PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_64;
-    pci_register_bar(dev, 0, bar_type, &s->mmio_io);
-    pci_register_bar(dev, 2, PCI_BASE_ADDRESS_SPACE_IO, &s->port_io);
+    pci_register_bar(dev, b->ioport_bar,
+                     PCI_BASE_ADDRESS_SPACE_IO, &s->port_io);
+    pci_register_bar(dev, b->mmio_bar, bar_type, &s->mmio_io);
     pci_register_bar(dev, 3, bar_type, &s->queue_io);
 
     if (megasas_use_msix(s)) {
         msix_vector_use(dev, 0);
     }
 
+    s->fw_state = MFI_FWSTATE_READY;
     if (!s->sas_addr) {
         s->sas_addr = ((NAA_LOCALLY_ASSIGNED_ID << 24) |
                        IEEE_COMPANY_LOCALLY_ASSIGNED) << 36;
@@ -2208,8 +2391,12 @@ static int megasas_scsi_init(PCIDevice *dev)
     }
     trace_megasas_init(s->fw_sge, s->fw_cmds,
                        megasas_is_jbod(s) ? "jbod" : "raid");
-    s->fw_luns = (MFI_MAX_LD > MAX_SCSI_DEVS) ?
-        MAX_SCSI_DEVS : MFI_MAX_LD;
+
+    if (megasas_is_jbod(s)) {
+        s->fw_luns = MFI_MAX_SYS_PDS;
+    } else {
+        s->fw_luns = MFI_MAX_LD;
+    }
     s->producer_pa = 0;
     s->consumer_pa = 0;
     for (i = 0; i < s->fw_cmds; i++) {
@@ -2238,7 +2425,7 @@ megasas_write_config(PCIDevice *pci, uint32_t addr, uint32_t val, int len)
     msi_write_config(pci, addr, val, len);
 }
 
-static Property megasas_properties[] = {
+static Property megasas_properties_gen1[] = {
     DEFINE_PROP_UINT32("max_sge", MegasasState, fw_sge,
                        MEGASAS_DEFAULT_SGE),
     DEFINE_PROP_UINT32("max_cmds", MegasasState, fw_cmds,
@@ -2254,36 +2441,119 @@ static Property megasas_properties[] = {
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static Property megasas_properties_gen2[] = {
+    DEFINE_PROP_UINT32("max_sge", MegasasState, fw_sge,
+                       MEGASAS_DEFAULT_SGE),
+    DEFINE_PROP_UINT32("max_cmds", MegasasState, fw_cmds,
+                       MEGASAS_GEN2_DEFAULT_FRAMES),
+    DEFINE_PROP_STRING("hba_serial", MegasasState, hba_serial),
+    DEFINE_PROP_UINT64("sas_address", MegasasState, sas_addr, 0),
+    DEFINE_PROP_BIT("use_msi", MegasasState, flags,
+                    MEGASAS_FLAG_USE_MSI, true),
+    DEFINE_PROP_BIT("use_msix", MegasasState, flags,
+                    MEGASAS_FLAG_USE_MSIX, true),
+    DEFINE_PROP_BIT("use_jbod", MegasasState, flags,
+                    MEGASAS_FLAG_USE_JBOD, false),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+typedef struct MegasasInfo {
+    const char *name;
+    const char *desc;
+    const char *product_name;
+    const char *product_version;
+    uint16_t device_id;
+    uint16_t subsystem_id;
+    int ioport_bar;
+    int mmio_bar;
+    bool is_express;
+    int osts;
+    const VMStateDescription *vmsd;
+    Property *props;
+} MegasasInfo;
+
+static struct MegasasInfo megasas_devices[] = {
+    {
+        .name = TYPE_MEGASAS_GEN1,
+        .desc = "LSI MegaRAID SAS 1078",
+        .product_name = "LSI MegaRAID SAS 8708EM2",
+        .product_version = MEGASAS_VERSION_GEN1,
+        .device_id = PCI_DEVICE_ID_LSI_SAS1078,
+        .subsystem_id = 0x1013,
+        .ioport_bar = 2,
+        .mmio_bar = 0,
+        .osts = MFI_1078_RM | 1,
+        .is_express = false,
+        .vmsd = &vmstate_megasas_gen1,
+        .props = megasas_properties_gen1,
+    },{
+        .name = TYPE_MEGASAS_GEN2,
+        .desc = "LSI MegaRAID SAS 2108",
+        .product_name = "LSI MegaRAID SAS 9260-8i",
+        .product_version = MEGASAS_VERSION_GEN2,
+        .device_id = PCI_DEVICE_ID_LSI_SAS0079,
+        .subsystem_id = 0x9261,
+        .ioport_bar = 0,
+        .mmio_bar = 1,
+        .osts = MFI_GEN2_RM,
+        .is_express = true,
+        .vmsd = &vmstate_megasas_gen2,
+        .props = megasas_properties_gen2,
+    }
+};
+
 static void megasas_class_init(ObjectClass *oc, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(oc);
     PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
+    MegasasBaseClass *e = MEGASAS_DEVICE_CLASS(oc);
+    const MegasasInfo *info = data;
 
     pc->init = megasas_scsi_init;
     pc->exit = megasas_scsi_uninit;
     pc->vendor_id = PCI_VENDOR_ID_LSI_LOGIC;
-    pc->device_id = PCI_DEVICE_ID_LSI_SAS1078;
+    pc->device_id = info->device_id;
     pc->subsystem_vendor_id = PCI_VENDOR_ID_LSI_LOGIC;
-    pc->subsystem_id = 0x1013;
+    pc->subsystem_id = info->subsystem_id;
     pc->class_id = PCI_CLASS_STORAGE_RAID;
-    dc->props = megasas_properties;
+    pc->is_express = info->is_express;
+    e->mmio_bar = info->mmio_bar;
+    e->ioport_bar = info->ioport_bar;
+    e->osts = info->osts;
+    e->product_name = info->product_name;
+    e->product_version = info->product_version;
+    dc->props = info->props;
     dc->reset = megasas_scsi_reset;
-    dc->vmsd = &vmstate_megasas;
+    dc->vmsd = info->vmsd;
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
-    dc->desc = "LSI MegaRAID SAS 1078";
+    dc->desc = info->desc;
     pc->config_write = megasas_write_config;
 }
 
 static const TypeInfo megasas_info = {
-    .name  = TYPE_MEGASAS,
+    .name  = TYPE_MEGASAS_BASE,
     .parent = TYPE_PCI_DEVICE,
     .instance_size = sizeof(MegasasState),
-    .class_init = megasas_class_init,
+    .class_size = sizeof(MegasasBaseClass),
+    .abstract = true,
 };
 
 static void megasas_register_types(void)
 {
+    int i;
+
     type_register_static(&megasas_info);
+    for (i = 0; i < ARRAY_SIZE(megasas_devices); i++) {
+        const MegasasInfo *info = &megasas_devices[i];
+        TypeInfo type_info = {};
+
+        type_info.name = info->name;
+        type_info.parent = TYPE_MEGASAS_BASE;
+        type_info.class_data = (void *)info;
+        type_info.class_init = megasas_class_init;
+
+        type_register(&type_info);
+    }
 }
 
 type_init(megasas_register_types)
diff --git a/hw/scsi/mfi.h b/hw/scsi/mfi.h
index a3034f623..29d41775d 100644
--- a/hw/scsi/mfi.h
+++ b/hw/scsi/mfi.h
@@ -60,6 +60,7 @@
 #define MFI_ODR0        0x9c            /* outbound doorbell register0 */
 #define MFI_ODCR0       0xa0            /* outbound doorbell clear register0  */
 #define MFI_OSP0        0xb0            /* outbound scratch pad0  */
+#define MFI_OSP1        0xb4            /* outbound scratch pad1  */
 #define MFI_IQPL        0xc0            /* Inbound queue port (low bytes)  */
 #define MFI_IQPH        0xc4            /* Inbound queue port (high bytes)  */
 #define MFI_DIAG        0xf8            /* Host diag */
@@ -116,6 +117,12 @@
 #define MFI_FWINIT_STOP_ADP     0x00000020 /* Move to operational, stop */
 #define MFI_FWINIT_ADP_RESET    0x00000040 /* Reset ADP */
 
+/*
+ * Control bits for the DIAG register
+ */
+#define MFI_DIAG_WRITE_ENABLE 0x00000080
+#define MFI_DIAG_RESET_ADP    0x00000004
+
 /* MFI Commands */
 typedef enum {
     MFI_CMD_INIT = 0x00,
@@ -1094,7 +1101,7 @@ struct mfi_pd_list {
 union mfi_ld_ref {
     struct {
         uint8_t target_id;
-        uint8_t lun_id;
+        uint8_t reserved;
         uint16_t seq;
     } v;
     uint32_t ref;
@@ -1111,6 +1118,13 @@ struct mfi_ld_list {
     } ld_list[MFI_MAX_LD];
 } QEMU_PACKED;
 
+struct mfi_ld_targetid_list {
+    uint32_t size;
+    uint32_t ld_count;
+    uint8_t pad[3];
+    uint8_t targetid[MFI_MAX_LD];
+} QEMU_PACKED;
+
 enum mfi_ld_access {
     MFI_LD_ACCESS_RW =          0,
     MFI_LD_ACCSSS_RO =          2,
diff --git a/hw/scsi/scsi-bus.c b/hw/scsi/scsi-bus.c
index 434175425..24f7b74ae 100644
--- a/hw/scsi/scsi-bus.c
+++ b/hw/scsi/scsi-bus.c
@@ -3,13 +3,13 @@
 #include "hw/scsi/scsi.h"
 #include "block/scsi.h"
 #include "hw/qdev.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "trace.h"
 #include "sysemu/dma.h"
 
 static char *scsibus_get_dev_path(DeviceState *dev);
 static char *scsibus_get_fw_dev_path(DeviceState *dev);
-static int scsi_req_parse(SCSICommand *cmd, SCSIDevice *dev, uint8_t *buf);
 static void scsi_req_dequeue(SCSIRequest *req);
 static uint8_t *scsi_target_alloc_buf(SCSIRequest *req, size_t len);
 static void scsi_target_free_buf(SCSIRequest *req);
@@ -24,9 +24,11 @@ static Property scsi_props[] = {
 static void scsi_bus_class_init(ObjectClass *klass, void *data)
 {
     BusClass *k = BUS_CLASS(klass);
+    HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass);
 
     k->get_dev_path = scsibus_get_dev_path;
     k->get_fw_dev_path = scsibus_get_fw_dev_path;
+    hc->unplug = qdev_simple_device_unplug_cb;
 }
 
 static const TypeInfo scsi_bus_info = {
@@ -34,24 +36,33 @@ static const TypeInfo scsi_bus_info = {
     .parent = TYPE_BUS,
     .instance_size = sizeof(SCSIBus),
     .class_init = scsi_bus_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_HOTPLUG_HANDLER },
+        { }
+    }
 };
 static int next_scsi_bus;
 
-static int scsi_device_init(SCSIDevice *s)
+static void scsi_device_realize(SCSIDevice *s, Error **errp)
 {
     SCSIDeviceClass *sc = SCSI_DEVICE_GET_CLASS(s);
-    if (sc->init) {
-        return sc->init(s);
+    if (sc->realize) {
+        sc->realize(s, errp);
     }
-    return 0;
 }
 
-static void scsi_device_destroy(SCSIDevice *s)
+int scsi_bus_parse_cdb(SCSIDevice *dev, SCSICommand *cmd, uint8_t *buf,
+                       void *hba_private)
 {
-    SCSIDeviceClass *sc = SCSI_DEVICE_GET_CLASS(s);
-    if (sc->destroy) {
-        sc->destroy(s);
+    SCSIBus *bus = DO_UPCAST(SCSIBus, qbus, dev->qdev.parent_bus);
+    int rc;
+
+    assert(cmd->len == 0);
+    rc = scsi_req_parse_cdb(dev, cmd, buf);
+    if (bus->info->parse_cdb) {
+        rc = bus->info->parse_cdb(dev, cmd, buf, hba_private);
     }
+    return rc;
 }
 
 static SCSIRequest *scsi_device_alloc_req(SCSIDevice *s, uint32_t tag, uint32_t lun,
@@ -65,7 +76,7 @@ static SCSIRequest *scsi_device_alloc_req(SCSIDevice *s, uint32_t tag, uint32_t
     return NULL;
 }
 
-static void scsi_device_unit_attention_reported(SCSIDevice *s)
+void scsi_device_unit_attention_reported(SCSIDevice *s)
 {
     SCSIDeviceClass *sc = SCSI_DEVICE_GET_CLASS(s);
     if (sc->unit_attention_reported) {
@@ -80,7 +91,7 @@ void scsi_bus_new(SCSIBus *bus, size_t bus_size, DeviceState *host,
     qbus_create_inplace(bus, bus_size, TYPE_SCSI_BUS, host, bus_name);
     bus->busnr = next_scsi_bus++;
     bus->info = info;
-    bus->qbus.allow_hotplug = 1;
+    qbus_set_bus_hotplug_handler(BUS(bus), &error_abort);
 }
 
 static void scsi_dma_restart_bh(void *opaque)
@@ -130,24 +141,24 @@ static void scsi_dma_restart_cb(void *opaque, int running, RunState state)
     }
 }
 
-static int scsi_qdev_init(DeviceState *qdev)
+static void scsi_qdev_realize(DeviceState *qdev, Error **errp)
 {
     SCSIDevice *dev = SCSI_DEVICE(qdev);
     SCSIBus *bus = DO_UPCAST(SCSIBus, qbus, dev->qdev.parent_bus);
     SCSIDevice *d;
-    int rc = -1;
+    Error *local_err = NULL;
 
     if (dev->channel > bus->info->max_channel) {
-        error_report("bad scsi channel id: %d", dev->channel);
-        goto err;
+        error_setg(errp, "bad scsi channel id: %d", dev->channel);
+        return;
     }
     if (dev->id != -1 && dev->id > bus->info->max_target) {
-        error_report("bad scsi device id: %d", dev->id);
-        goto err;
+        error_setg(errp, "bad scsi device id: %d", dev->id);
+        return;
     }
     if (dev->lun != -1 && dev->lun > bus->info->max_lun) {
-        error_report("bad scsi device lun: %d", dev->lun);
-        goto err;
+        error_setg(errp, "bad scsi device lun: %d", dev->lun);
+        return;
     }
 
     if (dev->id == -1) {
@@ -159,8 +170,8 @@ static int scsi_qdev_init(DeviceState *qdev)
             d = scsi_device_find(bus, dev->channel, ++id, dev->lun);
         } while (d && d->lun == dev->lun && id < bus->info->max_target);
         if (d && d->lun == dev->lun) {
-            error_report("no free target");
-            goto err;
+            error_setg(errp, "no free target");
+            return;
         }
         dev->id = id;
     } else if (dev->lun == -1) {
@@ -169,47 +180,43 @@ static int scsi_qdev_init(DeviceState *qdev)
             d = scsi_device_find(bus, dev->channel, dev->id, ++lun);
         } while (d && d->lun == lun && lun < bus->info->max_lun);
         if (d && d->lun == lun) {
-            error_report("no free lun");
-            goto err;
+            error_setg(errp, "no free lun");
+            return;
         }
         dev->lun = lun;
     } else {
         d = scsi_device_find(bus, dev->channel, dev->id, dev->lun);
         assert(d);
         if (d->lun == dev->lun && dev != d) {
-            error_report("lun already used by '%s'", d->qdev.id);
-            goto err;
+            error_setg(errp, "lun already used by '%s'", d->qdev.id);
+            return;
         }
     }
 
     QTAILQ_INIT(&dev->requests);
-    rc = scsi_device_init(dev);
-    if (rc == 0) {
-        dev->vmsentry = qemu_add_vm_change_state_handler(scsi_dma_restart_cb,
-                                                         dev);
-    }
-
-    if (bus->info->hotplug) {
-        bus->info->hotplug(bus, dev);
+    scsi_device_realize(dev, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
     }
-
-err:
-    return rc;
+    dev->vmsentry = qemu_add_vm_change_state_handler(scsi_dma_restart_cb,
+                                                     dev);
 }
 
-static int scsi_qdev_exit(DeviceState *qdev)
+static void scsi_qdev_unrealize(DeviceState *qdev, Error **errp)
 {
     SCSIDevice *dev = SCSI_DEVICE(qdev);
 
     if (dev->vmsentry) {
         qemu_del_vm_change_state_handler(dev->vmsentry);
     }
-    scsi_device_destroy(dev);
-    return 0;
+
+    scsi_device_purge_requests(dev, SENSE_CODE(NO_SENSE));
+    blockdev_mark_auto_del(dev->conf.blk);
 }
 
 /* handle legacy '-drive if=scsi,...' cmd line args */
-SCSIDevice *scsi_bus_legacy_add_drive(SCSIBus *bus, BlockDriverState *bdrv,
+SCSIDevice *scsi_bus_legacy_add_drive(SCSIBus *bus, BlockBackend *blk,
                                       int unit, bool removable, int bootindex,
                                       const char *serial, Error **errp)
 {
@@ -217,11 +224,12 @@ SCSIDevice *scsi_bus_legacy_add_drive(SCSIBus *bus, BlockDriverState *bdrv,
     DeviceState *dev;
     Error *err = NULL;
 
-    driver = bdrv_is_sg(bdrv) ? "scsi-generic" : "scsi-disk";
+    driver = blk_is_sg(blk) ? "scsi-generic" : "scsi-disk";
     dev = qdev_create(&bus->qbus, driver);
     qdev_prop_set_uint32(dev, "scsi-id", unit);
     if (bootindex >= 0) {
-        qdev_prop_set_int32(dev, "bootindex", bootindex);
+        object_property_set_int(OBJECT(dev), bootindex, "bootindex",
+                                &error_abort);
     }
     if (object_property_find(OBJECT(dev), "removable", NULL)) {
         qdev_prop_set_bit(dev, "removable", removable);
@@ -229,7 +237,7 @@ SCSIDevice *scsi_bus_legacy_add_drive(SCSIBus *bus, BlockDriverState *bdrv,
     if (serial && object_property_find(OBJECT(dev), "serial", NULL)) {
         qdev_prop_set_string(dev, "serial", serial);
     }
-    if (qdev_prop_set_drive(dev, "drive", bdrv) < 0) {
+    if (qdev_prop_set_drive(dev, "drive", blk) < 0) {
         error_setg(errp, "Setting drive property failed");
         object_unparent(OBJECT(dev));
         return NULL;
@@ -257,9 +265,10 @@ void scsi_bus_legacy_handle_cmdline(SCSIBus *bus, Error **errp)
             continue;
         }
         qemu_opts_loc_restore(dinfo->opts);
-        scsi_bus_legacy_add_drive(bus, dinfo->bdrv, unit, false, -1, NULL,
-                                  &err);
+        scsi_bus_legacy_add_drive(bus, blk_by_legacy_dinfo(dinfo),
+                                  unit, false, -1, NULL, &err);
         if (err != NULL) {
+            error_report("%s", error_get_pretty(err));
             error_propagate(errp, err);
             break;
         }
@@ -540,8 +549,11 @@ SCSIRequest *scsi_req_alloc(const SCSIReqOps *reqops, SCSIDevice *d,
     SCSIRequest *req;
     SCSIBus *bus = scsi_bus_from_device(d);
     BusState *qbus = BUS(bus);
+    const int memset_off = offsetof(SCSIRequest, sense)
+                           + sizeof(req->sense);
 
-    req = g_malloc0(reqops->size);
+    req = g_slice_alloc(reqops->size);
+    memset((uint8_t *)req + memset_off, 0, reqops->size - memset_off);
     req->refcount = 1;
     req->bus = bus;
     req->dev = d;
@@ -549,10 +561,10 @@ SCSIRequest *scsi_req_alloc(const SCSIReqOps *reqops, SCSIDevice *d,
     req->lun = lun;
     req->hba_private = hba_private;
     req->status = -1;
-    req->sense_len = 0;
     req->ops = reqops;
     object_ref(OBJECT(d));
     object_ref(OBJECT(qbus->parent));
+    notifier_list_init(&req->cancel_notifiers);
     trace_scsi_req_alloc(req->dev->id, req->lun, req->tag);
     return req;
 }
@@ -561,13 +573,44 @@ SCSIRequest *scsi_req_new(SCSIDevice *d, uint32_t tag, uint32_t lun,
                           uint8_t *buf, void *hba_private)
 {
     SCSIBus *bus = DO_UPCAST(SCSIBus, qbus, d->qdev.parent_bus);
+    const SCSIReqOps *ops;
+    SCSIDeviceClass *sc = SCSI_DEVICE_GET_CLASS(d);
     SCSIRequest *req;
-    SCSICommand cmd;
+    SCSICommand cmd = { .len = 0 };
+    int ret;
+
+    if ((d->unit_attention.key == UNIT_ATTENTION ||
+         bus->unit_attention.key == UNIT_ATTENTION) &&
+        (buf[0] != INQUIRY &&
+         buf[0] != REPORT_LUNS &&
+         buf[0] != GET_CONFIGURATION &&
+         buf[0] != GET_EVENT_STATUS_NOTIFICATION &&
+
+         /*
+          * If we already have a pending unit attention condition,
+          * report this one before triggering another one.
+          */
+         !(buf[0] == REQUEST_SENSE && d->sense_is_ua))) {
+        ops = &reqops_unit_attention;
+    } else if (lun != d->lun ||
+               buf[0] == REPORT_LUNS ||
+               (buf[0] == REQUEST_SENSE && d->sense_len)) {
+        ops = &reqops_target_command;
+    } else {
+        ops = NULL;
+    }
+
+    if (ops != NULL || !sc->parse_cdb) {
+        ret = scsi_req_parse_cdb(d, &cmd, buf);
+    } else {
+        ret = sc->parse_cdb(d, &cmd, buf, hba_private);
+    }
 
-    if (scsi_req_parse(&cmd, d, buf) != 0) {
+    if (ret != 0) {
         trace_scsi_req_parse_bad(d->id, lun, tag, buf[0]);
         req = scsi_req_alloc(&reqops_invalid_opcode, d, tag, lun, hba_private);
     } else {
+        assert(cmd.len != 0);
         trace_scsi_req_parsed(d->id, lun, tag, buf[0],
                               cmd.mode, cmd.xfer);
         if (cmd.lba != -1) {
@@ -577,25 +620,8 @@ SCSIRequest *scsi_req_new(SCSIDevice *d, uint32_t tag, uint32_t lun,
 
         if (cmd.xfer > INT32_MAX) {
             req = scsi_req_alloc(&reqops_invalid_field, d, tag, lun, hba_private);
-        } else if ((d->unit_attention.key == UNIT_ATTENTION ||
-                   bus->unit_attention.key == UNIT_ATTENTION) &&
-                  (buf[0] != INQUIRY &&
-                   buf[0] != REPORT_LUNS &&
-                   buf[0] != GET_CONFIGURATION &&
-                   buf[0] != GET_EVENT_STATUS_NOTIFICATION &&
-
-                   /*
-                    * If we already have a pending unit attention condition,
-                    * report this one before triggering another one.
-                    */
-                   !(buf[0] == REQUEST_SENSE && d->sense_is_ua))) {
-            req = scsi_req_alloc(&reqops_unit_attention, d, tag, lun,
-                                 hba_private);
-        } else if (lun != d->lun ||
-                   buf[0] == REPORT_LUNS ||
-                   (buf[0] == REQUEST_SENSE && d->sense_len)) {
-            req = scsi_req_alloc(&reqops_target_command, d, tag, lun,
-                                 hba_private);
+        } else if (ops) {
+            req = scsi_req_alloc(ops, d, tag, lun, hba_private);
         } else {
             req = scsi_device_alloc_req(d, tag, lun, buf, hba_private);
         }
@@ -794,7 +820,7 @@ static int ata_passthrough_xfer_unit(SCSIDevice *dev, uint8_t *buf)
     return xfer_unit;
 }
 
-static int ata_passthrough_12_xfer_size(SCSIDevice *dev, uint8_t *buf)
+static int ata_passthrough_12_xfer(SCSIDevice *dev, uint8_t *buf)
 {
     int length = buf[2] & 0x3;
     int xfer;
@@ -817,7 +843,7 @@ static int ata_passthrough_12_xfer_size(SCSIDevice *dev, uint8_t *buf)
     return xfer * unit;
 }
 
-static int ata_passthrough_16_xfer_size(SCSIDevice *dev, uint8_t *buf)
+static int ata_passthrough_16_xfer(SCSIDevice *dev, uint8_t *buf)
 {
     int extend = buf[1] & 0x1;
     int length = buf[2] & 0x3;
@@ -843,16 +869,16 @@ static int ata_passthrough_16_xfer_size(SCSIDevice *dev, uint8_t *buf)
     return xfer * unit;
 }
 
-uint32_t scsi_data_cdb_length(uint8_t *buf)
+uint32_t scsi_data_cdb_xfer(uint8_t *buf)
 {
     if ((buf[0] >> 5) == 0 && buf[4] == 0) {
         return 256;
     } else {
-        return scsi_cdb_length(buf);
+        return scsi_cdb_xfer(buf);
     }
 }
 
-uint32_t scsi_cdb_length(uint8_t *buf)
+uint32_t scsi_cdb_xfer(uint8_t *buf)
 {
     switch (buf[0] >> 5) {
     case 0:
@@ -873,9 +899,9 @@ uint32_t scsi_cdb_length(uint8_t *buf)
     }
 }
 
-static int scsi_req_length(SCSICommand *cmd, SCSIDevice *dev, uint8_t *buf)
+static int scsi_req_xfer(SCSICommand *cmd, SCSIDevice *dev, uint8_t *buf)
 {
-    cmd->xfer = scsi_cdb_length(buf);
+    cmd->xfer = scsi_cdb_xfer(buf);
     switch (buf[0]) {
     case TEST_UNIT_READY:
     case REWIND:
@@ -1006,17 +1032,17 @@ static int scsi_req_length(SCSICommand *cmd, SCSIDevice *dev, uint8_t *buf)
             /* BLANK command of MMC */
             cmd->xfer = 0;
         } else {
-            cmd->xfer = ata_passthrough_12_xfer_size(dev, buf);
+            cmd->xfer = ata_passthrough_12_xfer(dev, buf);
         }
         break;
     case ATA_PASSTHROUGH_16:
-        cmd->xfer = ata_passthrough_16_xfer_size(dev, buf);
+        cmd->xfer = ata_passthrough_16_xfer(dev, buf);
         break;
     }
     return 0;
 }
 
-static int scsi_req_stream_length(SCSICommand *cmd, SCSIDevice *dev, uint8_t *buf)
+static int scsi_req_stream_xfer(SCSICommand *cmd, SCSIDevice *dev, uint8_t *buf)
 {
     switch (buf[0]) {
     /* stream commands */
@@ -1071,12 +1097,12 @@ static int scsi_req_stream_length(SCSICommand *cmd, SCSIDevice *dev, uint8_t *bu
         break;
     /* generic commands */
     default:
-        return scsi_req_length(cmd, dev, buf);
+        return scsi_req_xfer(cmd, dev, buf);
     }
     return 0;
 }
 
-static int scsi_req_medium_changer_length(SCSICommand *cmd, SCSIDevice *dev, uint8_t *buf)
+static int scsi_req_medium_changer_xfer(SCSICommand *cmd, SCSIDevice *dev, uint8_t *buf)
 {
     switch (buf[0]) {
     /* medium changer commands */
@@ -1093,7 +1119,7 @@ static int scsi_req_medium_changer_length(SCSICommand *cmd, SCSIDevice *dev, uin
 
     /* generic commands */
     default:
-        return scsi_req_length(cmd, dev, buf);
+        return scsi_req_xfer(cmd, dev, buf);
     }
     return 0;
 }
@@ -1182,37 +1208,45 @@ static uint64_t scsi_cmd_lba(SCSICommand *cmd)
     return lba;
 }
 
-static int scsi_req_parse(SCSICommand *cmd, SCSIDevice *dev, uint8_t *buf)
-{
-    int rc;
+int scsi_cdb_length(uint8_t *buf) {
+    int cdb_len;
 
     switch (buf[0] >> 5) {
     case 0:
-        cmd->len = 6;
+        cdb_len = 6;
         break;
     case 1:
     case 2:
-        cmd->len = 10;
+        cdb_len = 10;
         break;
     case 4:
-        cmd->len = 16;
+        cdb_len = 16;
         break;
     case 5:
-        cmd->len = 12;
+        cdb_len = 12;
         break;
     default:
-        return -1;
+        cdb_len = -1;
     }
+    return cdb_len;
+}
+
+int scsi_req_parse_cdb(SCSIDevice *dev, SCSICommand *cmd, uint8_t *buf)
+{
+    int rc;
+
+    cmd->lba = -1;
+    cmd->len = scsi_cdb_length(buf);
 
     switch (dev->type) {
     case TYPE_TAPE:
-        rc = scsi_req_stream_length(cmd, dev, buf);
+        rc = scsi_req_stream_xfer(cmd, dev, buf);
         break;
     case TYPE_MEDIUM_CHANGER:
-        rc = scsi_req_medium_changer_length(cmd, dev, buf);
+        rc = scsi_req_medium_changer_xfer(cmd, dev, buf);
         break;
     default:
-        rc = scsi_req_length(cmd, dev, buf);
+        rc = scsi_req_xfer(cmd, dev, buf);
         break;
     }
 
@@ -1577,7 +1611,7 @@ void scsi_req_unref(SCSIRequest *req)
         }
         object_unref(OBJECT(req->dev));
         object_unref(OBJECT(qbus->parent));
-        g_free(req);
+        g_slice_free1(req->ops->size, req);
     }
 }
 
@@ -1687,40 +1721,56 @@ void scsi_req_complete(SCSIRequest *req, int status)
     scsi_req_ref(req);
     scsi_req_dequeue(req);
     req->bus->info->complete(req, req->status, req->resid);
+
+    /* Cancelled requests might end up being completed instead of cancelled */
+    notifier_list_notify(&req->cancel_notifiers, req);
     scsi_req_unref(req);
 }
 
-void scsi_req_cancel(SCSIRequest *req)
+/* Called by the devices when the request is canceled. */
+void scsi_req_cancel_complete(SCSIRequest *req)
+{
+    assert(req->io_canceled);
+    if (req->bus->info->cancel) {
+        req->bus->info->cancel(req);
+    }
+    notifier_list_notify(&req->cancel_notifiers, req);
+    scsi_req_unref(req);
+}
+
+/* Cancel @req asynchronously. @notifier is added to @req's cancellation
+ * notifier list, the bus will be notified the requests cancellation is
+ * completed.
+ * */
+void scsi_req_cancel_async(SCSIRequest *req, Notifier *notifier)
 {
     trace_scsi_req_cancel(req->dev->id, req->lun, req->tag);
-    if (!req->enqueued) {
+    if (notifier) {
+        notifier_list_add(&req->cancel_notifiers, notifier);
+    }
+    if (req->io_canceled) {
         return;
     }
     scsi_req_ref(req);
     scsi_req_dequeue(req);
     req->io_canceled = true;
-    if (req->ops->cancel_io) {
-        req->ops->cancel_io(req);
-    }
-    if (req->bus->info->cancel) {
-        req->bus->info->cancel(req);
+    if (req->aiocb) {
+        blk_aio_cancel_async(req->aiocb);
     }
-    scsi_req_unref(req);
 }
 
-void scsi_req_abort(SCSIRequest *req, int status)
+void scsi_req_cancel(SCSIRequest *req)
 {
+    trace_scsi_req_cancel(req->dev->id, req->lun, req->tag);
     if (!req->enqueued) {
         return;
     }
     scsi_req_ref(req);
     scsi_req_dequeue(req);
     req->io_canceled = true;
-    if (req->ops->cancel_io) {
-        req->ops->cancel_io(req);
+    if (req->aiocb) {
+        blk_aio_cancel(req->aiocb);
     }
-    scsi_req_complete(req, status);
-    scsi_req_unref(req);
 }
 
 static int scsi_ua_precedence(SCSISense sense)
@@ -1892,17 +1942,6 @@ static int get_scsi_requests(QEMUFile *f, void *pv, size_t size)
     return 0;
 }
 
-static int scsi_qdev_unplug(DeviceState *qdev)
-{
-    SCSIDevice *dev = SCSI_DEVICE(qdev);
-    SCSIBus *bus = DO_UPCAST(SCSIBus, qbus, dev->qdev.parent_bus);
-
-    if (bus->info->hot_unplug) {
-        bus->info->hot_unplug(bus, dev);
-    }
-    return qdev_simple_unplug_cb(qdev);
-}
-
 static const VMStateInfo vmstate_info_scsi_requests = {
     .name = "scsi-requests",
     .get  = get_scsi_requests,
@@ -1964,11 +2003,20 @@ static void scsi_device_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *k = DEVICE_CLASS(klass);
     set_bit(DEVICE_CATEGORY_STORAGE, k->categories);
-    k->bus_type = TYPE_SCSI_BUS;
-    k->init     = scsi_qdev_init;
-    k->unplug   = scsi_qdev_unplug;
-    k->exit     = scsi_qdev_exit;
-    k->props    = scsi_props;
+    k->bus_type  = TYPE_SCSI_BUS;
+    k->realize   = scsi_qdev_realize;
+    k->unrealize = scsi_qdev_unrealize;
+    k->props     = scsi_props;
+}
+
+static void scsi_dev_instance_init(Object *obj)
+{
+    DeviceState *dev = DEVICE(obj);
+    SCSIDevice *s = DO_UPCAST(SCSIDevice, qdev, dev);
+
+    device_add_bootindex_property(obj, &s->conf.bootindex,
+                                  "bootindex", NULL,
+                                  &s->qdev, NULL);
 }
 
 static const TypeInfo scsi_device_type_info = {
@@ -1978,6 +2026,7 @@ static const TypeInfo scsi_device_type_info = {
     .abstract = true,
     .class_size = sizeof(SCSIDeviceClass),
     .class_init = scsi_device_class_init,
+    .instance_init = scsi_dev_instance_init,
 };
 
 static void scsi_register_types(void)
diff --git a/hw/scsi/scsi-disk.c b/hw/scsi/scsi-disk.c
index d47ecd6ab..2f75d7d51 100644
--- a/hw/scsi/scsi-disk.c
+++ b/hw/scsi/scsi-disk.c
@@ -33,6 +33,7 @@ do { printf("scsi-disk: " fmt , ## __VA_ARGS__); } while (0)
 #include "hw/scsi/scsi.h"
 #include "block/scsi.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "hw/block/block.h"
 #include "sysemu/dma.h"
@@ -105,30 +106,13 @@ static void scsi_check_condition(SCSIDiskReq *r, SCSISense sense)
     scsi_req_complete(&r->req, CHECK_CONDITION);
 }
 
-/* Cancel a pending data transfer.  */
-static void scsi_cancel_io(SCSIRequest *req)
-{
-    SCSIDiskReq *r = DO_UPCAST(SCSIDiskReq, req, req);
-
-    DPRINTF("Cancel tag=0x%x\n", req->tag);
-    if (r->req.aiocb) {
-        bdrv_aio_cancel(r->req.aiocb);
-
-        /* This reference was left in by scsi_*_data.  We take ownership of
-         * it the moment scsi_req_cancel is called, independent of whether
-         * bdrv_aio_cancel completes the request or not.  */
-        scsi_req_unref(&r->req);
-    }
-    r->req.aiocb = NULL;
-}
-
 static uint32_t scsi_init_iovec(SCSIDiskReq *r, size_t size)
 {
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
 
     if (!r->iov.iov_base) {
         r->buflen = size;
-        r->iov.iov_base = qemu_blockalign(s->qdev.conf.bs, r->buflen);
+        r->iov.iov_base = blk_blockalign(s->qdev.conf.blk, r->buflen);
     }
     r->iov.iov_len = MIN(r->sector_count * 512, r->buflen);
     qemu_iovec_init_external(&r->qiov, &r->iov, 1);
@@ -183,8 +167,9 @@ static void scsi_aio_complete(void *opaque, int ret)
 
     assert(r->req.aiocb != NULL);
     r->req.aiocb = NULL;
-    bdrv_acct_done(s->qdev.conf.bs, &r->acct);
+    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
     if (r->req.io_canceled) {
+        scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
@@ -197,9 +182,7 @@ static void scsi_aio_complete(void *opaque, int ret)
     scsi_req_complete(&r->req, GOOD);
 
 done:
-    if (!r->req.io_canceled) {
-        scsi_req_unref(&r->req);
-    }
+    scsi_req_unref(&r->req);
 }
 
 static bool scsi_is_cmd_fua(SCSICommand *cmd)
@@ -233,21 +216,21 @@ static void scsi_write_do_fua(SCSIDiskReq *r)
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
 
     if (r->req.io_canceled) {
+        scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
     if (scsi_is_cmd_fua(&r->req.cmd)) {
-        bdrv_acct_start(s->qdev.conf.bs, &r->acct, 0, BDRV_ACCT_FLUSH);
-        r->req.aiocb = bdrv_aio_flush(s->qdev.conf.bs, scsi_aio_complete, r);
+        block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct, 0,
+                         BLOCK_ACCT_FLUSH);
+        r->req.aiocb = blk_aio_flush(s->qdev.conf.blk, scsi_aio_complete, r);
         return;
     }
 
     scsi_req_complete(&r->req, GOOD);
 
 done:
-    if (!r->req.io_canceled) {
-        scsi_req_unref(&r->req);
-    }
+    scsi_req_unref(&r->req);
 }
 
 static void scsi_dma_complete_noio(void *opaque, int ret)
@@ -257,9 +240,10 @@ static void scsi_dma_complete_noio(void *opaque, int ret)
 
     if (r->req.aiocb != NULL) {
         r->req.aiocb = NULL;
-        bdrv_acct_done(s->qdev.conf.bs, &r->acct);
+        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
     }
     if (r->req.io_canceled) {
+        scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
@@ -279,9 +263,7 @@ static void scsi_dma_complete_noio(void *opaque, int ret)
     }
 
 done:
-    if (!r->req.io_canceled) {
-        scsi_req_unref(&r->req);
-    }
+    scsi_req_unref(&r->req);
 }
 
 static void scsi_dma_complete(void *opaque, int ret)
@@ -300,8 +282,9 @@ static void scsi_read_complete(void * opaque, int ret)
 
     assert(r->req.aiocb != NULL);
     r->req.aiocb = NULL;
-    bdrv_acct_done(s->qdev.conf.bs, &r->acct);
+    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
     if (r->req.io_canceled) {
+        scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
@@ -319,9 +302,7 @@ static void scsi_read_complete(void * opaque, int ret)
     scsi_req_data(&r->req, r->qiov.size);
 
 done:
-    if (!r->req.io_canceled) {
-        scsi_req_unref(&r->req);
-    }
+    scsi_req_unref(&r->req);
 }
 
 /* Actually issue a read to the block device.  */
@@ -333,9 +314,10 @@ static void scsi_do_read(void *opaque, int ret)
 
     if (r->req.aiocb != NULL) {
         r->req.aiocb = NULL;
-        bdrv_acct_done(s->qdev.conf.bs, &r->acct);
+        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
     }
     if (r->req.io_canceled) {
+        scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
@@ -349,21 +331,20 @@ static void scsi_do_read(void *opaque, int ret)
     scsi_req_ref(&r->req);
 
     if (r->req.sg) {
-        dma_acct_start(s->qdev.conf.bs, &r->acct, r->req.sg, BDRV_ACCT_READ);
+        dma_acct_start(s->qdev.conf.blk, &r->acct, r->req.sg, BLOCK_ACCT_READ);
         r->req.resid -= r->req.sg->size;
-        r->req.aiocb = dma_bdrv_read(s->qdev.conf.bs, r->req.sg, r->sector,
-                                     scsi_dma_complete, r);
+        r->req.aiocb = dma_blk_read(s->qdev.conf.blk, r->req.sg, r->sector,
+                                    scsi_dma_complete, r);
     } else {
         n = scsi_init_iovec(r, SCSI_DMA_BUF_SIZE);
-        bdrv_acct_start(s->qdev.conf.bs, &r->acct, n * BDRV_SECTOR_SIZE, BDRV_ACCT_READ);
-        r->req.aiocb = bdrv_aio_readv(s->qdev.conf.bs, r->sector, &r->qiov, n,
-                                      scsi_read_complete, r);
+        block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct,
+                         n * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
+        r->req.aiocb = blk_aio_readv(s->qdev.conf.blk, r->sector, &r->qiov, n,
+                                     scsi_read_complete, r);
     }
 
 done:
-    if (!r->req.io_canceled) {
-        scsi_req_unref(&r->req);
-    }
+    scsi_req_unref(&r->req);
 }
 
 /* Read more data from scsi device into buffer.  */
@@ -399,8 +380,9 @@ static void scsi_read_data(SCSIRequest *req)
     first = !r->started;
     r->started = true;
     if (first && scsi_is_cmd_fua(&r->req.cmd)) {
-        bdrv_acct_start(s->qdev.conf.bs, &r->acct, 0, BDRV_ACCT_FLUSH);
-        r->req.aiocb = bdrv_aio_flush(s->qdev.conf.bs, scsi_do_read, r);
+        block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct, 0,
+                         BLOCK_ACCT_FLUSH);
+        r->req.aiocb = blk_aio_flush(s->qdev.conf.blk, scsi_do_read, r);
     } else {
         scsi_do_read(r, 0);
     }
@@ -417,7 +399,8 @@ static int scsi_handle_rw_error(SCSIDiskReq *r, int error)
 {
     bool is_read = (r->req.cmd.xfer == SCSI_XFER_FROM_DEV);
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
-    BlockErrorAction action = bdrv_get_error_action(s->qdev.conf.bs, is_read, error);
+    BlockErrorAction action = blk_get_error_action(s->qdev.conf.blk,
+                                                   is_read, error);
 
     if (action == BLOCK_ERROR_ACTION_REPORT) {
         switch (error) {
@@ -438,7 +421,7 @@ static int scsi_handle_rw_error(SCSIDiskReq *r, int error)
             break;
         }
     }
-    bdrv_error_action(s->qdev.conf.bs, action, is_read, error);
+    blk_error_action(s->qdev.conf.blk, action, is_read, error);
     if (action == BLOCK_ERROR_ACTION_STOP) {
         scsi_req_retry(&r->req);
     }
@@ -453,9 +436,10 @@ static void scsi_write_complete(void * opaque, int ret)
 
     if (r->req.aiocb != NULL) {
         r->req.aiocb = NULL;
-        bdrv_acct_done(s->qdev.conf.bs, &r->acct);
+        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
     }
     if (r->req.io_canceled) {
+        scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
@@ -478,9 +462,7 @@ static void scsi_write_complete(void * opaque, int ret)
     }
 
 done:
-    if (!r->req.io_canceled) {
-        scsi_req_unref(&r->req);
-    }
+    scsi_req_unref(&r->req);
 }
 
 static void scsi_write_data(SCSIRequest *req)
@@ -522,15 +504,16 @@ static void scsi_write_data(SCSIRequest *req)
     }
 
     if (r->req.sg) {
-        dma_acct_start(s->qdev.conf.bs, &r->acct, r->req.sg, BDRV_ACCT_WRITE);
+        dma_acct_start(s->qdev.conf.blk, &r->acct, r->req.sg, BLOCK_ACCT_WRITE);
         r->req.resid -= r->req.sg->size;
-        r->req.aiocb = dma_bdrv_write(s->qdev.conf.bs, r->req.sg, r->sector,
-                                      scsi_dma_complete, r);
+        r->req.aiocb = dma_blk_write(s->qdev.conf.blk, r->req.sg, r->sector,
+                                     scsi_dma_complete, r);
     } else {
         n = r->qiov.size / 512;
-        bdrv_acct_start(s->qdev.conf.bs, &r->acct, n * BDRV_SECTOR_SIZE, BDRV_ACCT_WRITE);
-        r->req.aiocb = bdrv_aio_writev(s->qdev.conf.bs, r->sector, &r->qiov, n,
-                                       scsi_write_complete, r);
+        block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct,
+                         n * BDRV_SECTOR_SIZE, BLOCK_ACCT_WRITE);
+        r->req.aiocb = blk_aio_writev(s->qdev.conf.blk, r->sector, &r->qiov, n,
+                                      scsi_write_complete, r);
     }
 }
 
@@ -597,7 +580,7 @@ static int scsi_disk_emulate_inquiry(SCSIRequest *req, uint8_t *outbuf)
 
         case 0x83: /* Device identification page, mandatory */
         {
-            const char *str = s->serial ?: bdrv_get_device_name(s->qdev.conf.bs);
+            const char *str = s->serial ?: blk_name(s->qdev.conf.blk);
             int max_len = s->serial ? 20 : 255 - 8;
             int id_len = strlen(str);
 
@@ -758,10 +741,10 @@ static inline bool media_is_dvd(SCSIDiskState *s)
     if (s->qdev.type != TYPE_ROM) {
         return false;
     }
-    if (!bdrv_is_inserted(s->qdev.conf.bs)) {
+    if (!blk_is_inserted(s->qdev.conf.blk)) {
         return false;
     }
-    bdrv_get_geometry(s->qdev.conf.bs, &nb_sectors);
+    blk_get_geometry(s->qdev.conf.blk, &nb_sectors);
     return nb_sectors > CD_MAX_SECTORS;
 }
 
@@ -771,10 +754,10 @@ static inline bool media_is_cd(SCSIDiskState *s)
     if (s->qdev.type != TYPE_ROM) {
         return false;
     }
-    if (!bdrv_is_inserted(s->qdev.conf.bs)) {
+    if (!blk_is_inserted(s->qdev.conf.blk)) {
         return false;
     }
-    bdrv_get_geometry(s->qdev.conf.bs, &nb_sectors);
+    blk_get_geometry(s->qdev.conf.blk, &nb_sectors);
     return nb_sectors <= CD_MAX_SECTORS;
 }
 
@@ -835,7 +818,7 @@ static int scsi_read_dvd_structure(SCSIDiskState *s, SCSIDiskReq *r,
     }
 
     if (format != 0xff) {
-        if (s->tray_open || !bdrv_is_inserted(s->qdev.conf.bs)) {
+        if (s->tray_open || !blk_is_inserted(s->qdev.conf.blk)) {
             scsi_check_condition(r, SENSE_CODE(NO_MEDIUM));
             return -1;
         }
@@ -857,7 +840,7 @@ static int scsi_read_dvd_structure(SCSIDiskState *s, SCSIDiskReq *r,
         if (layer != 0) {
             goto fail;
         }
-        bdrv_get_geometry(s->qdev.conf.bs, &nb_sectors);
+        blk_get_geometry(s->qdev.conf.blk, &nb_sectors);
 
         outbuf[4] = 1;   /* DVD-ROM, part version 1 */
         outbuf[5] = 0xf; /* 120mm disc, minimum rate unspecified */
@@ -912,7 +895,7 @@ static int scsi_event_status_media(SCSIDiskState *s, uint8_t *outbuf)
     media_status = 0;
     if (s->tray_open) {
         media_status = MS_TRAY_OPEN;
-    } else if (bdrv_is_inserted(s->qdev.conf.bs)) {
+    } else if (blk_is_inserted(s->qdev.conf.blk)) {
         media_status = MS_MEDIA_PRESENT;
     }
 
@@ -1110,7 +1093,7 @@ static int mode_sense_page(SCSIDiskState *s, int page, uint8_t **p_outbuf,
     case MODE_PAGE_CACHING:
         length = 0x12;
         if (page_control == 1 || /* Changeable Values */
-            bdrv_enable_write_cache(s->qdev.conf.bs)) {
+            blk_enable_write_cache(s->qdev.conf.blk)) {
             p[0] = 4; /* WCE */
         }
         break;
@@ -1191,7 +1174,7 @@ static int scsi_disk_emulate_mode_sense(SCSIDiskReq *r, uint8_t *outbuf)
 
     if (s->qdev.type == TYPE_DISK) {
         dev_specific_param = s->features & (1 << SCSI_DISK_F_DPOFUA) ? 0x10 : 0;
-        if (bdrv_is_read_only(s->qdev.conf.bs)) {
+        if (blk_is_read_only(s->qdev.conf.blk)) {
             dev_specific_param |= 0x80; /* Readonly.  */
         }
     } else {
@@ -1213,7 +1196,7 @@ static int scsi_disk_emulate_mode_sense(SCSIDiskReq *r, uint8_t *outbuf)
         p += 8;
     }
 
-    bdrv_get_geometry(s->qdev.conf.bs, &nb_sectors);
+    blk_get_geometry(s->qdev.conf.blk, &nb_sectors);
     if (!dbd && nb_sectors) {
         if (r->req.cmd.buf[0] == MODE_SENSE) {
             outbuf[3] = 8; /* Block descriptor length  */
@@ -1276,7 +1259,7 @@ static int scsi_disk_emulate_read_toc(SCSIRequest *req, uint8_t *outbuf)
     msf = req->cmd.buf[1] & 2;
     format = req->cmd.buf[2] & 0xf;
     start_track = req->cmd.buf[6];
-    bdrv_get_geometry(s->qdev.conf.bs, &nb_sectors);
+    blk_get_geometry(s->qdev.conf.blk, &nb_sectors);
     DPRINTF("Read TOC (track %d format %d msf %d)\n", start_track, format, msf >> 1);
     nb_sectors /= s->qdev.blocksize / 512;
     switch (format) {
@@ -1316,14 +1299,14 @@ static int scsi_disk_emulate_start_stop(SCSIDiskReq *r)
     if ((s->features & (1 << SCSI_DISK_F_REMOVABLE)) && loej) {
         if (!start && !s->tray_open && s->tray_locked) {
             scsi_check_condition(r,
-                                 bdrv_is_inserted(s->qdev.conf.bs)
+                                 blk_is_inserted(s->qdev.conf.blk)
                                  ? SENSE_CODE(ILLEGAL_REQ_REMOVAL_PREVENTED)
                                  : SENSE_CODE(NOT_READY_REMOVAL_PREVENTED));
             return -1;
         }
 
         if (s->tray_open != !start) {
-            bdrv_eject(s->qdev.conf.bs, !start);
+            blk_eject(s->qdev.conf.blk, !start);
             s->tray_open = !start;
         }
     }
@@ -1390,7 +1373,7 @@ static void scsi_disk_apply_mode_select(SCSIDiskState *s, int page, uint8_t *p)
 {
     switch (page) {
     case MODE_PAGE_CACHING:
-        bdrv_set_enable_write_cache(s->qdev.conf.bs, (p[0] & 4) != 0);
+        blk_set_enable_write_cache(s->qdev.conf.blk, (p[0] & 4) != 0);
         break;
 
     default:
@@ -1493,11 +1476,12 @@ static void scsi_disk_emulate_mode_select(SCSIDiskReq *r, uint8_t *inbuf)
             return;
         }
     }
-    if (!bdrv_enable_write_cache(s->qdev.conf.bs)) {
+    if (!blk_enable_write_cache(s->qdev.conf.blk)) {
         /* The request is used as the AIO opaque value, so add a ref.  */
         scsi_req_ref(&r->req);
-        bdrv_acct_start(s->qdev.conf.bs, &r->acct, 0, BDRV_ACCT_FLUSH);
-        r->req.aiocb = bdrv_aio_flush(s->qdev.conf.bs, scsi_aio_complete, r);
+        block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct, 0,
+                         BLOCK_ACCT_FLUSH);
+        r->req.aiocb = blk_aio_flush(s->qdev.conf.blk, scsi_aio_complete, r);
         return;
     }
 
@@ -1548,6 +1532,7 @@ static void scsi_unmap_complete(void *opaque, int ret)
 
     r->req.aiocb = NULL;
     if (r->req.io_canceled) {
+        scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
@@ -1565,10 +1550,10 @@ static void scsi_unmap_complete(void *opaque, int ret)
             goto done;
         }
 
-        r->req.aiocb = bdrv_aio_discard(s->qdev.conf.bs,
-                                        sector_num * (s->qdev.blocksize / 512),
-                                        nb_sectors * (s->qdev.blocksize / 512),
-                                        scsi_unmap_complete, data);
+        r->req.aiocb = blk_aio_discard(s->qdev.conf.blk,
+                                       sector_num * (s->qdev.blocksize / 512),
+                                       nb_sectors * (s->qdev.blocksize / 512),
+                                       scsi_unmap_complete, data);
         data->count--;
         data->inbuf += 16;
         return;
@@ -1577,9 +1562,7 @@ static void scsi_unmap_complete(void *opaque, int ret)
     scsi_req_complete(&r->req, GOOD);
 
 done:
-    if (!r->req.io_canceled) {
-        scsi_req_unref(&r->req);
-    }
+    scsi_req_unref(&r->req);
     g_free(data);
 }
 
@@ -1608,7 +1591,7 @@ static void scsi_disk_emulate_unmap(SCSIDiskReq *r, uint8_t *inbuf)
         goto invalid_param_len;
     }
 
-    if (bdrv_is_read_only(s->qdev.conf.bs)) {
+    if (blk_is_read_only(s->qdev.conf.blk)) {
         scsi_check_condition(r, SENSE_CODE(WRITE_PROTECTED));
         return;
     }
@@ -1647,8 +1630,9 @@ static void scsi_write_same_complete(void *opaque, int ret)
 
     assert(r->req.aiocb != NULL);
     r->req.aiocb = NULL;
-    bdrv_acct_done(s->qdev.conf.bs, &r->acct);
+    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
     if (r->req.io_canceled) {
+        scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
@@ -1662,19 +1646,18 @@ static void scsi_write_same_complete(void *opaque, int ret)
     data->sector += data->iov.iov_len / 512;
     data->iov.iov_len = MIN(data->nb_sectors * 512, data->iov.iov_len);
     if (data->iov.iov_len) {
-        bdrv_acct_start(s->qdev.conf.bs, &r->acct, data->iov.iov_len, BDRV_ACCT_WRITE);
-        r->req.aiocb = bdrv_aio_writev(s->qdev.conf.bs, data->sector,
-                                       &data->qiov, data->iov.iov_len / 512,
-                                       scsi_write_same_complete, data);
+        block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct,
+                         data->iov.iov_len, BLOCK_ACCT_WRITE);
+        r->req.aiocb = blk_aio_writev(s->qdev.conf.blk, data->sector,
+                                      &data->qiov, data->iov.iov_len / 512,
+                                      scsi_write_same_complete, data);
         return;
     }
 
     scsi_req_complete(&r->req, GOOD);
 
 done:
-    if (!r->req.io_canceled) {
-        scsi_req_unref(&r->req);
-    }
+    scsi_req_unref(&r->req);
     qemu_vfree(data->iov.iov_base);
     g_free(data);
 }
@@ -1683,7 +1666,7 @@ static void scsi_disk_emulate_write_same(SCSIDiskReq *r, uint8_t *inbuf)
 {
     SCSIRequest *req = &r->req;
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, req->dev);
-    uint32_t nb_sectors = scsi_data_cdb_length(r->req.cmd.buf);
+    uint32_t nb_sectors = scsi_data_cdb_xfer(r->req.cmd.buf);
     WriteSameCBData *data;
     uint8_t *buf;
     int i;
@@ -1694,7 +1677,7 @@ static void scsi_disk_emulate_write_same(SCSIDiskReq *r, uint8_t *inbuf)
         return;
     }
 
-    if (bdrv_is_read_only(s->qdev.conf.bs)) {
+    if (blk_is_read_only(s->qdev.conf.blk)) {
         scsi_check_condition(r, SENSE_CODE(WRITE_PROTECTED));
         return;
     }
@@ -1708,12 +1691,13 @@ static void scsi_disk_emulate_write_same(SCSIDiskReq *r, uint8_t *inbuf)
 
         /* The request is used as the AIO opaque value, so add a ref.  */
         scsi_req_ref(&r->req);
-        bdrv_acct_start(s->qdev.conf.bs, &r->acct, nb_sectors * s->qdev.blocksize,
-                        BDRV_ACCT_WRITE);
-        r->req.aiocb = bdrv_aio_write_zeroes(s->qdev.conf.bs,
-                                             r->req.cmd.lba * (s->qdev.blocksize / 512),
-                                             nb_sectors * (s->qdev.blocksize / 512),
-                                             flags, scsi_aio_complete, r);
+        block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct,
+                         nb_sectors * s->qdev.blocksize,
+                        BLOCK_ACCT_WRITE);
+        r->req.aiocb = blk_aio_write_zeroes(s->qdev.conf.blk,
+                                r->req.cmd.lba * (s->qdev.blocksize / 512),
+                                nb_sectors * (s->qdev.blocksize / 512),
+                                flags, scsi_aio_complete, r);
         return;
     }
 
@@ -1722,7 +1706,8 @@ static void scsi_disk_emulate_write_same(SCSIDiskReq *r, uint8_t *inbuf)
     data->sector = r->req.cmd.lba * (s->qdev.blocksize / 512);
     data->nb_sectors = nb_sectors * (s->qdev.blocksize / 512);
     data->iov.iov_len = MIN(data->nb_sectors * 512, SCSI_WRITE_SAME_MAX);
-    data->iov.iov_base = buf = qemu_blockalign(s->qdev.conf.bs, data->iov.iov_len);
+    data->iov.iov_base = buf = blk_blockalign(s->qdev.conf.blk,
+                                              data->iov.iov_len);
     qemu_iovec_init_external(&data->qiov, &data->iov, 1);
 
     for (i = 0; i < data->iov.iov_len; i += s->qdev.blocksize) {
@@ -1730,10 +1715,11 @@ static void scsi_disk_emulate_write_same(SCSIDiskReq *r, uint8_t *inbuf)
     }
 
     scsi_req_ref(&r->req);
-    bdrv_acct_start(s->qdev.conf.bs, &r->acct, data->iov.iov_len, BDRV_ACCT_WRITE);
-    r->req.aiocb = bdrv_aio_writev(s->qdev.conf.bs, data->sector,
-                                   &data->qiov, data->iov.iov_len / 512,
-                                   scsi_write_same_complete, data);
+    block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct,
+                     data->iov.iov_len, BLOCK_ACCT_WRITE);
+    r->req.aiocb = blk_aio_writev(s->qdev.conf.blk, data->sector,
+                                  &data->qiov, data->iov.iov_len / 512,
+                                  scsi_write_same_complete, data);
 }
 
 static void scsi_disk_emulate_write_data(SCSIRequest *req)
@@ -1802,7 +1788,7 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
         break;
 
     default:
-        if (s->tray_open || !bdrv_is_inserted(s->qdev.conf.bs)) {
+        if (s->tray_open || !blk_is_inserted(s->qdev.conf.blk)) {
             scsi_check_condition(r, SENSE_CODE(NO_MEDIUM));
             return 0;
         }
@@ -1823,7 +1809,7 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
     r->buflen = MAX(4096, req->cmd.xfer);
 
     if (!r->iov.iov_base) {
-        r->iov.iov_base = qemu_blockalign(s->qdev.conf.bs, r->buflen);
+        r->iov.iov_base = blk_blockalign(s->qdev.conf.blk, r->buflen);
     }
 
     buflen = req->cmd.xfer;
@@ -1831,7 +1817,7 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
     memset(outbuf, 0, r->buflen);
     switch (req->cmd.buf[0]) {
     case TEST_UNIT_READY:
-        assert(!s->tray_open && bdrv_is_inserted(s->qdev.conf.bs));
+        assert(!s->tray_open && blk_is_inserted(s->qdev.conf.blk));
         break;
     case INQUIRY:
         buflen = scsi_disk_emulate_inquiry(req, outbuf);
@@ -1879,12 +1865,12 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
         break;
     case ALLOW_MEDIUM_REMOVAL:
         s->tray_locked = req->cmd.buf[4] & 1;
-        bdrv_lock_medium(s->qdev.conf.bs, req->cmd.buf[4] & 1);
+        blk_lock_medium(s->qdev.conf.blk, req->cmd.buf[4] & 1);
         break;
     case READ_CAPACITY_10:
         /* The normal LEN field for this command is zero.  */
         memset(outbuf, 0, 8);
-        bdrv_get_geometry(s->qdev.conf.bs, &nb_sectors);
+        blk_get_geometry(s->qdev.conf.blk, &nb_sectors);
         if (!nb_sectors) {
             scsi_check_condition(r, SENSE_CODE(LUN_NOT_READY));
             return 0;
@@ -1953,7 +1939,7 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
         if ((req->cmd.buf[1] & 31) == SAI_READ_CAPACITY_16) {
             DPRINTF("SAI READ CAPACITY(16)\n");
             memset(outbuf, 0, req->cmd.xfer);
-            bdrv_get_geometry(s->qdev.conf.bs, &nb_sectors);
+            blk_get_geometry(s->qdev.conf.blk, &nb_sectors);
             if (!nb_sectors) {
                 scsi_check_condition(r, SENSE_CODE(LUN_NOT_READY));
                 return 0;
@@ -1994,8 +1980,9 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
     case SYNCHRONIZE_CACHE:
         /* The request is used as the AIO opaque value, so add a ref.  */
         scsi_req_ref(&r->req);
-        bdrv_acct_start(s->qdev.conf.bs, &r->acct, 0, BDRV_ACCT_FLUSH);
-        r->req.aiocb = bdrv_aio_flush(s->qdev.conf.bs, scsi_aio_complete, r);
+        block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct, 0,
+                         BLOCK_ACCT_FLUSH);
+        r->req.aiocb = blk_aio_flush(s->qdev.conf.blk, scsi_aio_complete, r);
         return 0;
     case SEEK_10:
         DPRINTF("Seek(10) (sector %" PRId64 ")\n", r->req.cmd.lba);
@@ -2069,12 +2056,12 @@ static int32_t scsi_disk_dma_command(SCSIRequest *req, uint8_t *buf)
 
     command = buf[0];
 
-    if (s->tray_open || !bdrv_is_inserted(s->qdev.conf.bs)) {
+    if (s->tray_open || !blk_is_inserted(s->qdev.conf.blk)) {
         scsi_check_condition(r, SENSE_CODE(NO_MEDIUM));
         return 0;
     }
 
-    len = scsi_data_cdb_length(r->req.cmd.buf);
+    len = scsi_data_cdb_xfer(r->req.cmd.buf);
     switch (command) {
     case READ_6:
     case READ_10:
@@ -2097,7 +2084,7 @@ static int32_t scsi_disk_dma_command(SCSIRequest *req, uint8_t *buf)
     case WRITE_VERIFY_10:
     case WRITE_VERIFY_12:
     case WRITE_VERIFY_16:
-        if (bdrv_is_read_only(s->qdev.conf.bs)) {
+        if (blk_is_read_only(s->qdev.conf.blk)) {
             scsi_check_condition(r, SENSE_CODE(WRITE_PROTECTED));
             return 0;
         }
@@ -2140,7 +2127,7 @@ static void scsi_disk_reset(DeviceState *dev)
 
     scsi_device_purge_requests(&s->qdev, SENSE_CODE(RESET));
 
-    bdrv_get_geometry(s->qdev.conf.bs, &nb_sectors);
+    blk_get_geometry(s->qdev.conf.blk, &nb_sectors);
     nb_sectors /= s->qdev.blocksize / 512;
     if (nb_sectors) {
         nb_sectors--;
@@ -2151,14 +2138,6 @@ static void scsi_disk_reset(DeviceState *dev)
     s->tray_open = 0;
 }
 
-static void scsi_destroy(SCSIDevice *dev)
-{
-    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, dev);
-
-    scsi_device_purge_requests(&s->qdev, SENSE_CODE(NO_SENSE));
-    blockdev_mark_auto_del(s->qdev.conf.bs);
-}
-
 static void scsi_disk_resize_cb(void *opaque)
 {
     SCSIDiskState *s = opaque;
@@ -2234,25 +2213,29 @@ static void scsi_disk_unit_attention_reported(SCSIDevice *dev)
     }
 }
 
-static int scsi_initfn(SCSIDevice *dev)
+static void scsi_realize(SCSIDevice *dev, Error **errp)
 {
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, dev);
+    Error *err = NULL;
 
-    if (!s->qdev.conf.bs) {
-        error_report("drive property not set");
-        return -1;
+    if (!s->qdev.conf.blk) {
+        error_setg(errp, "drive property not set");
+        return;
     }
 
     if (!(s->features & (1 << SCSI_DISK_F_REMOVABLE)) &&
-        !bdrv_is_inserted(s->qdev.conf.bs)) {
-        error_report("Device needs media, but drive is empty");
-        return -1;
+        !blk_is_inserted(s->qdev.conf.blk)) {
+        error_setg(errp, "Device needs media, but drive is empty");
+        return;
     }
 
     blkconf_serial(&s->qdev.conf, &s->serial);
-    if (dev->type == TYPE_DISK
-        && blkconf_geometry(&dev->conf, NULL, 65535, 255, 255) < 0) {
-        return -1;
+    if (dev->type == TYPE_DISK) {
+        blkconf_geometry(&dev->conf, NULL, 65535, 255, 255, &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
     }
 
     if (s->qdev.conf.discard_granularity == -1) {
@@ -2267,25 +2250,23 @@ static int scsi_initfn(SCSIDevice *dev)
         s->vendor = g_strdup("QEMU");
     }
 
-    if (bdrv_is_sg(s->qdev.conf.bs)) {
-        error_report("unwanted /dev/sg*");
-        return -1;
+    if (blk_is_sg(s->qdev.conf.blk)) {
+        error_setg(errp, "unwanted /dev/sg*");
+        return;
     }
 
     if ((s->features & (1 << SCSI_DISK_F_REMOVABLE)) &&
             !(s->features & (1 << SCSI_DISK_F_NO_REMOVABLE_DEVOPS))) {
-        bdrv_set_dev_ops(s->qdev.conf.bs, &scsi_disk_removable_block_ops, s);
+        blk_set_dev_ops(s->qdev.conf.blk, &scsi_disk_removable_block_ops, s);
     } else {
-        bdrv_set_dev_ops(s->qdev.conf.bs, &scsi_disk_block_ops, s);
+        blk_set_dev_ops(s->qdev.conf.blk, &scsi_disk_block_ops, s);
     }
-    bdrv_set_guest_block_size(s->qdev.conf.bs, s->qdev.blocksize);
+    blk_set_guest_block_size(s->qdev.conf.blk, s->qdev.blocksize);
 
-    bdrv_iostatus_enable(s->qdev.conf.bs);
-    add_boot_device_path(s->qdev.conf.bootindex, &dev->qdev, NULL);
-    return 0;
+    blk_iostatus_enable(s->qdev.conf.blk);
 }
 
-static int scsi_hd_initfn(SCSIDevice *dev)
+static void scsi_hd_realize(SCSIDevice *dev, Error **errp)
 {
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, dev);
     s->qdev.blocksize = s->qdev.conf.logical_block_size;
@@ -2293,10 +2274,10 @@ static int scsi_hd_initfn(SCSIDevice *dev)
     if (!s->product) {
         s->product = g_strdup("QEMU HARDDISK");
     }
-    return scsi_initfn(&s->qdev);
+    scsi_realize(&s->qdev, errp);
 }
 
-static int scsi_cd_initfn(SCSIDevice *dev)
+static void scsi_cd_realize(SCSIDevice *dev, Error **errp)
 {
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, dev);
     s->qdev.blocksize = 2048;
@@ -2305,22 +2286,26 @@ static int scsi_cd_initfn(SCSIDevice *dev)
     if (!s->product) {
         s->product = g_strdup("QEMU CD-ROM");
     }
-    return scsi_initfn(&s->qdev);
+    scsi_realize(&s->qdev, errp);
 }
 
-static int scsi_disk_initfn(SCSIDevice *dev)
+static void scsi_disk_realize(SCSIDevice *dev, Error **errp)
 {
     DriveInfo *dinfo;
+    Error *local_err = NULL;
 
-    if (!dev->conf.bs) {
-        return scsi_initfn(dev);  /* ... and die there */
+    if (!dev->conf.blk) {
+        scsi_realize(dev, &local_err);
+        assert(local_err);
+        error_propagate(errp, local_err);
+        return;
     }
 
-    dinfo = drive_get_by_blockdev(dev->conf.bs);
-    if (dinfo->media_cd) {
-        return scsi_cd_initfn(dev);
+    dinfo = blk_legacy_dinfo(dev->conf.blk);
+    if (dinfo && dinfo->media_cd) {
+        scsi_cd_realize(dev, errp);
     } else {
-        return scsi_hd_initfn(dev);
+        scsi_hd_realize(dev, errp);
     }
 }
 
@@ -2330,7 +2315,6 @@ static const SCSIReqOps scsi_disk_emulate_reqops = {
     .send_command = scsi_disk_emulate_command,
     .read_data    = scsi_disk_emulate_read_data,
     .write_data   = scsi_disk_emulate_write_data,
-    .cancel_io    = scsi_cancel_io,
     .get_buf      = scsi_get_buf,
 };
 
@@ -2340,7 +2324,6 @@ static const SCSIReqOps scsi_disk_dma_reqops = {
     .send_command = scsi_disk_dma_command,
     .read_data    = scsi_read_data,
     .write_data   = scsi_write_data,
-    .cancel_io    = scsi_cancel_io,
     .get_buf      = scsi_get_buf,
     .load_request = scsi_disk_load_request,
     .save_request = scsi_disk_save_request,
@@ -2405,7 +2388,7 @@ static SCSIRequest *scsi_new_request(SCSIDevice *d, uint32_t tag, uint32_t lun,
     DPRINTF("Command: lun=%d tag=0x%x data=0x%02x", lun, tag, buf[0]);
     {
         int i;
-        for (i = 1; i < req->cmd.len; i++) {
+        for (i = 1; i < scsi_cdb_length(buf); i++) {
             printf(" 0x%02x", buf[i]);
         }
         printf("\n");
@@ -2418,7 +2401,6 @@ static SCSIRequest *scsi_new_request(SCSIDevice *d, uint32_t tag, uint32_t lun,
 #ifdef __linux__
 static int get_device_type(SCSIDiskState *s)
 {
-    BlockDriverState *bdrv = s->qdev.conf.bs;
     uint8_t cmd[16];
     uint8_t buf[36];
     uint8_t sensebuf[8];
@@ -2441,7 +2423,7 @@ static int get_device_type(SCSIDiskState *s)
     io_header.sbp = sensebuf;
     io_header.timeout = 6000; /* XXX */
 
-    ret = bdrv_ioctl(bdrv, SG_IO, &io_header);
+    ret = blk_ioctl(s->qdev.conf.blk, SG_IO, &io_header);
     if (ret < 0 || io_header.driver_status || io_header.host_status) {
         return -1;
     }
@@ -2452,35 +2434,35 @@ static int get_device_type(SCSIDiskState *s)
     return 0;
 }
 
-static int scsi_block_initfn(SCSIDevice *dev)
+static void scsi_block_realize(SCSIDevice *dev, Error **errp)
 {
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, dev);
     int sg_version;
     int rc;
 
-    if (!s->qdev.conf.bs) {
-        error_report("drive property not set");
-        return -1;
+    if (!s->qdev.conf.blk) {
+        error_setg(errp, "drive property not set");
+        return;
     }
 
     /* check we are using a driver managing SG_IO (version 3 and after) */
-    rc = bdrv_ioctl(s->qdev.conf.bs, SG_GET_VERSION_NUM, &sg_version);
+    rc = blk_ioctl(s->qdev.conf.blk, SG_GET_VERSION_NUM, &sg_version);
     if (rc < 0) {
-        error_report("cannot get SG_IO version number: %s.  "
+        error_setg(errp, "cannot get SG_IO version number: %s.  "
                      "Is this a SCSI device?",
                      strerror(-rc));
-        return -1;
+        return;
     }
     if (sg_version < 30000) {
-        error_report("scsi generic interface too old");
-        return -1;
+        error_setg(errp, "scsi generic interface too old");
+        return;
     }
 
     /* get device type from INQUIRY data */
     rc = get_device_type(s);
     if (rc < 0) {
-        error_report("INQUIRY failed");
-        return -1;
+        error_setg(errp, "INQUIRY failed");
+        return;
     }
 
     /* Make a guess for the block size, we'll fix it when the guest sends.
@@ -2498,15 +2480,11 @@ static int scsi_block_initfn(SCSIDevice *dev)
      */
     s->features |= (1 << SCSI_DISK_F_NO_REMOVABLE_DEVOPS);
 
-    return scsi_initfn(&s->qdev);
+    scsi_realize(&s->qdev, errp);
 }
 
-static SCSIRequest *scsi_block_new_request(SCSIDevice *d, uint32_t tag,
-                                           uint32_t lun, uint8_t *buf,
-                                           void *hba_private)
+static bool scsi_block_is_passthrough(SCSIDiskState *s, uint8_t *buf)
 {
-    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, d);
-
     switch (buf[0]) {
     case READ_6:
     case READ_10:
@@ -2523,11 +2501,11 @@ static SCSIRequest *scsi_block_new_request(SCSIDevice *d, uint32_t tag,
     case WRITE_VERIFY_12:
     case WRITE_VERIFY_16:
         /* If we are not using O_DIRECT, we might read stale data from the
-	 * host cache if writes were made using other commands than these
-	 * ones (such as WRITE SAME or EXTENDED COPY, etc.).  So, without
-	 * O_DIRECT everything must go through SG_IO.
+         * host cache if writes were made using other commands than these
+         * ones (such as WRITE SAME or EXTENDED COPY, etc.).  So, without
+         * O_DIRECT everything must go through SG_IO.
          */
-        if (!(bdrv_get_flags(s->qdev.conf.bs) & BDRV_O_NOCACHE)) {
+        if (!(blk_get_flags(s->qdev.conf.blk) & BDRV_O_NOCACHE)) {
             break;
         }
 
@@ -2542,14 +2520,45 @@ static SCSIRequest *scsi_block_new_request(SCSIDevice *d, uint32_t tag,
          * just make scsi-block operate the same as scsi-generic for them.
          */
         if (s->qdev.type != TYPE_ROM) {
-            return scsi_req_alloc(&scsi_disk_dma_reqops, &s->qdev, tag, lun,
-                                  hba_private);
+            return false;
         }
+        break;
+
+    default:
+        break;
     }
 
-    return scsi_req_alloc(&scsi_generic_req_ops, &s->qdev, tag, lun,
-                          hba_private);
+    return true;
 }
+
+
+static SCSIRequest *scsi_block_new_request(SCSIDevice *d, uint32_t tag,
+                                           uint32_t lun, uint8_t *buf,
+                                           void *hba_private)
+{
+    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, d);
+
+    if (scsi_block_is_passthrough(s, buf)) {
+        return scsi_req_alloc(&scsi_generic_req_ops, &s->qdev, tag, lun,
+                              hba_private);
+    } else {
+        return scsi_req_alloc(&scsi_disk_dma_reqops, &s->qdev, tag, lun,
+                              hba_private);
+    }
+}
+
+static int scsi_block_parse_cdb(SCSIDevice *d, SCSICommand *cmd,
+                                  uint8_t *buf, void *hba_private)
+{
+    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, d);
+
+    if (scsi_block_is_passthrough(s, buf)) {
+        return scsi_bus_parse_cdb(&s->qdev, cmd, buf, hba_private);
+    } else {
+        return scsi_req_parse_cdb(&s->qdev, cmd, buf);
+    }
+}
+
 #endif
 
 #define DEFINE_SCSI_DISK_PROPERTIES()                                \
@@ -2594,8 +2603,7 @@ static void scsi_hd_class_initfn(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     SCSIDeviceClass *sc = SCSI_DEVICE_CLASS(klass);
 
-    sc->init         = scsi_hd_initfn;
-    sc->destroy      = scsi_destroy;
+    sc->realize      = scsi_hd_realize;
     sc->alloc_req    = scsi_new_request;
     sc->unit_attention_reported = scsi_disk_unit_attention_reported;
     dc->fw_name = "disk";
@@ -2625,8 +2633,7 @@ static void scsi_cd_class_initfn(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     SCSIDeviceClass *sc = SCSI_DEVICE_CLASS(klass);
 
-    sc->init         = scsi_cd_initfn;
-    sc->destroy      = scsi_destroy;
+    sc->realize      = scsi_cd_realize;
     sc->alloc_req    = scsi_new_request;
     sc->unit_attention_reported = scsi_disk_unit_attention_reported;
     dc->fw_name = "disk";
@@ -2645,8 +2652,7 @@ static const TypeInfo scsi_cd_info = {
 
 #ifdef __linux__
 static Property scsi_block_properties[] = {
-    DEFINE_PROP_DRIVE("drive", SCSIDiskState, qdev.conf.bs),
-    DEFINE_PROP_INT32("bootindex", SCSIDiskState, qdev.conf.bootindex, -1),
+    DEFINE_PROP_DRIVE("drive", SCSIDiskState, qdev.conf.blk),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -2655,9 +2661,9 @@ static void scsi_block_class_initfn(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     SCSIDeviceClass *sc = SCSI_DEVICE_CLASS(klass);
 
-    sc->init         = scsi_block_initfn;
-    sc->destroy      = scsi_destroy;
+    sc->realize      = scsi_block_realize;
     sc->alloc_req    = scsi_block_new_request;
+    sc->parse_cdb    = scsi_block_parse_cdb;
     dc->fw_name = "disk";
     dc->desc = "SCSI block device passthrough";
     dc->reset = scsi_disk_reset;
@@ -2692,8 +2698,7 @@ static void scsi_disk_class_initfn(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     SCSIDeviceClass *sc = SCSI_DEVICE_CLASS(klass);
 
-    sc->init         = scsi_disk_initfn;
-    sc->destroy      = scsi_destroy;
+    sc->realize      = scsi_disk_realize;
     sc->alloc_req    = scsi_new_request;
     sc->unit_attention_reported = scsi_disk_unit_attention_reported;
     dc->fw_name = "disk";
diff --git a/hw/scsi/scsi-generic.c b/hw/scsi/scsi-generic.c
index 3733d2c36..6b9e4e1ef 100644
--- a/hw/scsi/scsi-generic.c
+++ b/hw/scsi/scsi-generic.c
@@ -14,6 +14,7 @@
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "hw/scsi/scsi.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 
 #ifdef __linux__
@@ -93,6 +94,10 @@ static void scsi_command_complete(void *opaque, int ret)
     SCSIGenericReq *r = (SCSIGenericReq *)opaque;
 
     r->req.aiocb = NULL;
+    if (r->req.io_canceled) {
+        scsi_req_cancel_complete(&r->req);
+        goto done;
+    }
     if (r->io_header.driver_status & SG_ERR_DRIVER_SENSE) {
         r->req.sense_len = r->io_header.sb_len_wr;
     }
@@ -133,31 +138,13 @@ static void scsi_command_complete(void *opaque, int ret)
             r, r->req.tag, status);
 
     scsi_req_complete(&r->req, status);
-    if (!r->req.io_canceled) {
-        scsi_req_unref(&r->req);
-    }
-}
-
-/* Cancel a pending data transfer.  */
-static void scsi_cancel_io(SCSIRequest *req)
-{
-    SCSIGenericReq *r = DO_UPCAST(SCSIGenericReq, req, req);
-
-    DPRINTF("Cancel tag=0x%x\n", req->tag);
-    if (r->req.aiocb) {
-        bdrv_aio_cancel(r->req.aiocb);
-
-        /* This reference was left in by scsi_*_data.  We take ownership of
-         * it independent of whether bdrv_aio_cancel completes the request
-         * or not.  */
-        scsi_req_unref(&r->req);
-    }
-    r->req.aiocb = NULL;
+done:
+    scsi_req_unref(&r->req);
 }
 
-static int execute_command(BlockDriverState *bdrv,
+static int execute_command(BlockBackend *blk,
                            SCSIGenericReq *r, int direction,
-			   BlockDriverCompletionFunc *complete)
+                           BlockCompletionFunc *complete)
 {
     r->io_header.interface_id = 'S';
     r->io_header.dxfer_direction = direction;
@@ -171,7 +158,7 @@ static int execute_command(BlockDriverState *bdrv,
     r->io_header.usr_ptr = r;
     r->io_header.flags |= SG_FLAG_DIRECT_IO;
 
-    r->req.aiocb = bdrv_aio_ioctl(bdrv, SG_IO, &r->io_header, complete, r);
+    r->req.aiocb = blk_aio_ioctl(blk, SG_IO, &r->io_header, complete, r);
     if (r->req.aiocb == NULL) {
         return -EIO;
     }
@@ -186,8 +173,7 @@ static void scsi_read_complete(void * opaque, int ret)
     int len;
 
     r->req.aiocb = NULL;
-    if (ret) {
-        DPRINTF("IO error ret %d\n", ret);
+    if (ret || r->req.io_canceled) {
         scsi_command_complete(r, ret);
         return;
     }
@@ -208,12 +194,10 @@ static void scsi_read_complete(void * opaque, int ret)
             s->blocksize = ldl_be_p(&r->buf[8]);
             s->max_lba = ldq_be_p(&r->buf[0]);
         }
-        bdrv_set_guest_block_size(s->conf.bs, s->blocksize);
+        blk_set_guest_block_size(s->conf.blk, s->blocksize);
 
         scsi_req_data(&r->req, len);
-        if (!r->req.io_canceled) {
-            scsi_req_unref(&r->req);
-        }
+        scsi_req_unref(&r->req);
     }
 }
 
@@ -233,7 +217,8 @@ static void scsi_read_data(SCSIRequest *req)
         return;
     }
 
-    ret = execute_command(s->conf.bs, r, SG_DXFER_FROM_DEV, scsi_read_complete);
+    ret = execute_command(s->conf.blk, r, SG_DXFER_FROM_DEV,
+                          scsi_read_complete);
     if (ret < 0) {
         scsi_command_complete(r, ret);
     }
@@ -246,8 +231,7 @@ static void scsi_write_complete(void * opaque, int ret)
 
     DPRINTF("scsi_write_complete() ret = %d\n", ret);
     r->req.aiocb = NULL;
-    if (ret) {
-        DPRINTF("IO error\n");
+    if (ret || r->req.io_canceled) {
         scsi_command_complete(r, ret);
         return;
     }
@@ -278,7 +262,7 @@ static void scsi_write_data(SCSIRequest *req)
 
     /* The request is used as the AIO opaque value, so add a ref.  */
     scsi_req_ref(&r->req);
-    ret = execute_command(s->conf.bs, r, SG_DXFER_TO_DEV, scsi_write_complete);
+    ret = execute_command(s->conf.blk, r, SG_DXFER_TO_DEV, scsi_write_complete);
     if (ret < 0) {
         scsi_command_complete(r, ret);
     }
@@ -303,9 +287,6 @@ static int32_t scsi_send_command(SCSIRequest *req, uint8_t *cmd)
     SCSIDevice *s = r->req.dev;
     int ret;
 
-    DPRINTF("Command: lun=%d tag=0x%x len %zd data=0x%02x", lun, tag,
-            r->req.cmd.xfer, cmd[0]);
-
 #ifdef DEBUG_SCSI
     {
         int i;
@@ -323,7 +304,8 @@ static int32_t scsi_send_command(SCSIRequest *req, uint8_t *cmd)
         r->buf = NULL;
         /* The request is used as the AIO opaque value, so add a ref.  */
         scsi_req_ref(&r->req);
-        ret = execute_command(s->conf.bs, r, SG_DXFER_NONE, scsi_command_complete);
+        ret = execute_command(s->conf.blk, r, SG_DXFER_NONE,
+                              scsi_command_complete);
         if (ret < 0) {
             scsi_command_complete(r, ret);
             return 0;
@@ -348,7 +330,7 @@ static int32_t scsi_send_command(SCSIRequest *req, uint8_t *cmd)
     }
 }
 
-static int get_stream_blocksize(BlockDriverState *bdrv)
+static int get_stream_blocksize(BlockBackend *blk)
 {
     uint8_t cmd[6];
     uint8_t buf[12];
@@ -372,7 +354,7 @@ static int get_stream_blocksize(BlockDriverState *bdrv)
     io_header.sbp = sensebuf;
     io_header.timeout = 6000; /* XXX */
 
-    ret = bdrv_ioctl(bdrv, SG_IO, &io_header);
+    ret = blk_ioctl(blk, SG_IO, &io_header);
     if (ret < 0 || io_header.driver_status || io_header.host_status) {
         return -1;
     }
@@ -386,61 +368,52 @@ static void scsi_generic_reset(DeviceState *dev)
     scsi_device_purge_requests(s, SENSE_CODE(RESET));
 }
 
-static void scsi_destroy(SCSIDevice *s)
-{
-    scsi_device_purge_requests(s, SENSE_CODE(NO_SENSE));
-    blockdev_mark_auto_del(s->conf.bs);
-}
-
-static int scsi_generic_initfn(SCSIDevice *s)
+static void scsi_generic_realize(SCSIDevice *s, Error **errp)
 {
     int rc;
     int sg_version;
     struct sg_scsi_id scsiid;
 
-    if (!s->conf.bs) {
-        error_report("drive property not set");
-        return -1;
+    if (!s->conf.blk) {
+        error_setg(errp, "drive property not set");
+        return;
     }
 
-    if (bdrv_get_on_error(s->conf.bs, 0) != BLOCKDEV_ON_ERROR_ENOSPC) {
-        error_report("Device doesn't support drive option werror");
-        return -1;
+    if (blk_get_on_error(s->conf.blk, 0) != BLOCKDEV_ON_ERROR_ENOSPC) {
+        error_setg(errp, "Device doesn't support drive option werror");
+        return;
     }
-    if (bdrv_get_on_error(s->conf.bs, 1) != BLOCKDEV_ON_ERROR_REPORT) {
-        error_report("Device doesn't support drive option rerror");
-        return -1;
+    if (blk_get_on_error(s->conf.blk, 1) != BLOCKDEV_ON_ERROR_REPORT) {
+        error_setg(errp, "Device doesn't support drive option rerror");
+        return;
     }
 
     /* check we are using a driver managing SG_IO (version 3 and after */
-    rc = bdrv_ioctl(s->conf.bs, SG_GET_VERSION_NUM, &sg_version);
+    rc = blk_ioctl(s->conf.blk, SG_GET_VERSION_NUM, &sg_version);
     if (rc < 0) {
-        error_report("cannot get SG_IO version number: %s.  "
-                     "Is this a SCSI device?",
-                     strerror(-rc));
-        return -1;
+        error_setg(errp, "cannot get SG_IO version number: %s.  "
+                         "Is this a SCSI device?",
+                         strerror(-rc));
+        return;
     }
     if (sg_version < 30000) {
-        error_report("scsi generic interface too old");
-        return -1;
+        error_setg(errp, "scsi generic interface too old");
+        return;
     }
 
     /* get LUN of the /dev/sg? */
-    if (bdrv_ioctl(s->conf.bs, SG_GET_SCSI_ID, &scsiid)) {
-        error_report("SG_GET_SCSI_ID ioctl failed");
-        return -1;
+    if (blk_ioctl(s->conf.blk, SG_GET_SCSI_ID, &scsiid)) {
+        error_setg(errp, "SG_GET_SCSI_ID ioctl failed");
+        return;
     }
 
     /* define device state */
     s->type = scsiid.scsi_type;
     DPRINTF("device type %d\n", s->type);
-    if (s->type == TYPE_DISK || s->type == TYPE_ROM) {
-        add_boot_device_path(s->conf.bootindex, &s->qdev, NULL);
-    }
 
     switch (s->type) {
     case TYPE_TAPE:
-        s->blocksize = get_stream_blocksize(s->conf.bs);
+        s->blocksize = get_stream_blocksize(s->conf.blk);
         if (s->blocksize == -1) {
             s->blocksize = 0;
         }
@@ -460,7 +433,6 @@ static int scsi_generic_initfn(SCSIDevice *s)
     }
 
     DPRINTF("block size %d\n", s->blocksize);
-    return 0;
 }
 
 const SCSIReqOps scsi_generic_req_ops = {
@@ -469,7 +441,6 @@ const SCSIReqOps scsi_generic_req_ops = {
     .send_command = scsi_send_command,
     .read_data    = scsi_read_data,
     .write_data   = scsi_write_data,
-    .cancel_io    = scsi_cancel_io,
     .get_buf      = scsi_get_buf,
     .load_request = scsi_generic_load_request,
     .save_request = scsi_generic_save_request,
@@ -485,19 +456,24 @@ static SCSIRequest *scsi_new_request(SCSIDevice *d, uint32_t tag, uint32_t lun,
 }
 
 static Property scsi_generic_properties[] = {
-    DEFINE_PROP_DRIVE("drive", SCSIDevice, conf.bs),
-    DEFINE_PROP_INT32("bootindex", SCSIDevice, conf.bootindex, -1),
+    DEFINE_PROP_DRIVE("drive", SCSIDevice, conf.blk),
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static int scsi_generic_parse_cdb(SCSIDevice *dev, SCSICommand *cmd,
+                                  uint8_t *buf, void *hba_private)
+{
+    return scsi_bus_parse_cdb(dev, cmd, buf, hba_private);
+}
+
 static void scsi_generic_class_initfn(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     SCSIDeviceClass *sc = SCSI_DEVICE_CLASS(klass);
 
-    sc->init         = scsi_generic_initfn;
-    sc->destroy      = scsi_destroy;
+    sc->realize      = scsi_generic_realize;
     sc->alloc_req    = scsi_new_request;
+    sc->parse_cdb    = scsi_generic_parse_cdb;
     dc->fw_name = "disk";
     dc->desc = "pass through generic scsi device (/dev/sg*)";
     dc->reset = scsi_generic_reset;
diff --git a/hw/scsi/spapr_vscsi.c b/hw/scsi/spapr_vscsi.c
index 048cfc7b0..20b20f0ba 100644
--- a/hw/scsi/spapr_vscsi.c
+++ b/hw/scsi/spapr_vscsi.c
@@ -77,8 +77,9 @@ typedef struct vscsi_req {
     SCSIRequest             *sreq;
     uint32_t                qtag; /* qemu tag != srp tag */
     bool                    active;
-    uint32_t                data_len;
     bool                    writing;
+    bool                    dma_error;
+    uint32_t                data_len;
     uint32_t                senselen;
     uint8_t                 sense[SCSI_SENSE_BUF_SIZE];
 
@@ -536,8 +537,8 @@ static void vscsi_transfer_data(SCSIRequest *sreq, uint32_t len)
     }
     if (rc < 0) {
         fprintf(stderr, "VSCSI: RDMA error rc=%d!\n", rc);
-        vscsi_makeup_sense(s, req, HARDWARE_ERROR, 0, 0);
-        scsi_req_abort(req->sreq, CHECK_CONDITION);
+        req->dma_error = true;
+        scsi_req_cancel(req->sreq);
         return;
     }
 
@@ -591,6 +592,12 @@ static void vscsi_request_cancelled(SCSIRequest *sreq)
 {
     vscsi_req *req = sreq->hba_private;
 
+    if (req->dma_error) {
+        VSCSIState *s = VIO_SPAPR_VSCSI_DEVICE(sreq->bus->qbus.parent);
+
+        vscsi_makeup_sense(s, req, HARDWARE_ERROR, 0, 0);
+        vscsi_send_rsp(s, req, CHECK_CONDITION, 0, 0);
+    }
     vscsi_put_req(req);
 }
 
diff --git a/hw/scsi/vhost-scsi.c b/hw/scsi/vhost-scsi.c
index ddfe76aed..dcb2bc5a6 100644
--- a/hw/scsi/vhost-scsi.c
+++ b/hw/scsi/vhost-scsi.c
@@ -23,6 +23,7 @@
 #include "hw/virtio/vhost.h"
 #include "hw/virtio/virtio-scsi.h"
 #include "hw/virtio/virtio-bus.h"
+#include "hw/virtio/virtio-access.h"
 
 /* Features supported by host kernel. */
 static const int kernel_feature_bits[] = {
@@ -163,8 +164,8 @@ static void vhost_scsi_set_config(VirtIODevice *vdev,
     VirtIOSCSIConfig *scsiconf = (VirtIOSCSIConfig *)config;
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(vdev);
 
-    if ((uint32_t) ldl_p(&scsiconf->sense_size) != vs->sense_size ||
-        (uint32_t) ldl_p(&scsiconf->cdb_size) != vs->cdb_size) {
+    if ((uint32_t) virtio_ldl_p(vdev, &scsiconf->sense_size) != vs->sense_size ||
+        (uint32_t) virtio_ldl_p(vdev, &scsiconf->cdb_size) != vs->cdb_size) {
         error_report("vhost-scsi does not support changing the sense data and CDB sizes");
         exit(1);
     }
@@ -232,12 +233,14 @@ static void vhost_scsi_realize(DeviceState *dev, Error **errp)
                                vhost_dummy_handle_output);
     if (err != NULL) {
         error_propagate(errp, err);
+        close(vhostfd);
         return;
     }
 
     s->dev.nvqs = VHOST_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
     s->dev.vqs = g_new(struct vhost_virtqueue, s->dev.nvqs);
     s->dev.vq_index = 0;
+    s->dev.backend_features = 0;
 
     ret = vhost_dev_init(&s->dev, (void *)(uintptr_t)vhostfd,
                          VHOST_BACKEND_TYPE_KERNEL, true);
@@ -246,7 +249,6 @@ static void vhost_scsi_realize(DeviceState *dev, Error **errp)
                    strerror(-ret));
         return;
     }
-    s->dev.backend_features = 0;
 
     error_setg(&s->migration_blocker,
             "vhost-scsi does not support migration");
diff --git a/hw/scsi/virtio-scsi-dataplane.c b/hw/scsi/virtio-scsi-dataplane.c
new file mode 100644
index 000000000..03a1e8cfc
--- /dev/null
+++ b/hw/scsi/virtio-scsi-dataplane.c
@@ -0,0 +1,312 @@
+/*
+ * Virtio SCSI dataplane
+ *
+ * Copyright Red Hat, Inc. 2014
+ *
+ * Authors:
+ *   Fam Zheng <famz@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "hw/virtio/virtio-scsi.h"
+#include "qemu/error-report.h"
+#include "sysemu/block-backend.h"
+#include <hw/scsi/scsi.h>
+#include <block/scsi.h>
+#include <hw/virtio/virtio-bus.h>
+#include "hw/virtio/virtio-access.h"
+#include "stdio.h"
+
+/* Context: QEMU global mutex held */
+void virtio_scsi_set_iothread(VirtIOSCSI *s, IOThread *iothread)
+{
+    BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s)));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(s);
+
+    assert(!s->ctx);
+    s->ctx = iothread_get_aio_context(vs->conf.iothread);
+
+    /* Don't try if transport does not support notifiers. */
+    if (!k->set_guest_notifiers || !k->set_host_notifier) {
+        fprintf(stderr, "virtio-scsi: Failed to set iothread "
+                   "(transport does not support notifiers)");
+        exit(1);
+    }
+}
+
+static VirtIOSCSIVring *virtio_scsi_vring_init(VirtIOSCSI *s,
+                                               VirtQueue *vq,
+                                               EventNotifierHandler *handler,
+                                               int n)
+{
+    BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s)));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
+    VirtIOSCSIVring *r = g_slice_new(VirtIOSCSIVring);
+    int rc;
+
+    /* Set up virtqueue notify */
+    rc = k->set_host_notifier(qbus->parent, n, true);
+    if (rc != 0) {
+        fprintf(stderr, "virtio-scsi: Failed to set host notifier (%d)\n",
+                rc);
+        s->dataplane_fenced = true;
+        return NULL;
+    }
+    r->host_notifier = *virtio_queue_get_host_notifier(vq);
+    r->guest_notifier = *virtio_queue_get_guest_notifier(vq);
+    aio_set_event_notifier(s->ctx, &r->host_notifier, handler);
+
+    r->parent = s;
+
+    if (!vring_setup(&r->vring, VIRTIO_DEVICE(s), n)) {
+        fprintf(stderr, "virtio-scsi: VRing setup failed\n");
+        goto fail_vring;
+    }
+    return r;
+
+fail_vring:
+    aio_set_event_notifier(s->ctx, &r->host_notifier, NULL);
+    k->set_host_notifier(qbus->parent, n, false);
+    g_slice_free(VirtIOSCSIVring, r);
+    return NULL;
+}
+
+VirtIOSCSIReq *virtio_scsi_pop_req_vring(VirtIOSCSI *s,
+                                         VirtIOSCSIVring *vring)
+{
+    VirtIOSCSIReq *req = virtio_scsi_init_req(s, NULL);
+    int r;
+
+    req->vring = vring;
+    r = vring_pop((VirtIODevice *)s, &vring->vring, &req->elem);
+    if (r < 0) {
+        virtio_scsi_free_req(req);
+        req = NULL;
+    }
+    return req;
+}
+
+void virtio_scsi_vring_push_notify(VirtIOSCSIReq *req)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(req->vring->parent);
+
+    vring_push(&req->vring->vring, &req->elem,
+               req->qsgl.size + req->resp_iov.size);
+
+    if (vring_should_notify(vdev, &req->vring->vring)) {
+        event_notifier_set(&req->vring->guest_notifier);
+    }
+}
+
+static void virtio_scsi_iothread_handle_ctrl(EventNotifier *notifier)
+{
+    VirtIOSCSIVring *vring = container_of(notifier,
+                                          VirtIOSCSIVring, host_notifier);
+    VirtIOSCSI *s = VIRTIO_SCSI(vring->parent);
+    VirtIOSCSIReq *req;
+
+    event_notifier_test_and_clear(notifier);
+    while ((req = virtio_scsi_pop_req_vring(s, vring))) {
+        virtio_scsi_handle_ctrl_req(s, req);
+    }
+}
+
+static void virtio_scsi_iothread_handle_event(EventNotifier *notifier)
+{
+    VirtIOSCSIVring *vring = container_of(notifier,
+                                          VirtIOSCSIVring, host_notifier);
+    VirtIOSCSI *s = vring->parent;
+    VirtIODevice *vdev = VIRTIO_DEVICE(s);
+
+    event_notifier_test_and_clear(notifier);
+
+    if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
+        return;
+    }
+
+    if (s->events_dropped) {
+        virtio_scsi_push_event(s, NULL, VIRTIO_SCSI_T_NO_EVENT, 0);
+    }
+}
+
+static void virtio_scsi_iothread_handle_cmd(EventNotifier *notifier)
+{
+    VirtIOSCSIVring *vring = container_of(notifier,
+                                          VirtIOSCSIVring, host_notifier);
+    VirtIOSCSI *s = (VirtIOSCSI *)vring->parent;
+    VirtIOSCSIReq *req, *next;
+    QTAILQ_HEAD(, VirtIOSCSIReq) reqs = QTAILQ_HEAD_INITIALIZER(reqs);
+
+    event_notifier_test_and_clear(notifier);
+    while ((req = virtio_scsi_pop_req_vring(s, vring))) {
+        if (virtio_scsi_handle_cmd_req_prepare(s, req)) {
+            QTAILQ_INSERT_TAIL(&reqs, req, next);
+        }
+    }
+
+    QTAILQ_FOREACH_SAFE(req, &reqs, next, next) {
+        virtio_scsi_handle_cmd_req_submit(s, req);
+    }
+}
+
+/* assumes s->ctx held */
+static void virtio_scsi_clear_aio(VirtIOSCSI *s)
+{
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(s);
+    int i;
+
+    if (s->ctrl_vring) {
+        aio_set_event_notifier(s->ctx, &s->ctrl_vring->host_notifier, NULL);
+    }
+    if (s->event_vring) {
+        aio_set_event_notifier(s->ctx, &s->event_vring->host_notifier, NULL);
+    }
+    if (s->cmd_vrings) {
+        for (i = 0; i < vs->conf.num_queues && s->cmd_vrings[i]; i++) {
+            aio_set_event_notifier(s->ctx, &s->cmd_vrings[i]->host_notifier, NULL);
+        }
+    }
+}
+
+static void virtio_scsi_vring_teardown(VirtIOSCSI *s)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(s);
+    int i;
+
+    if (s->ctrl_vring) {
+        vring_teardown(&s->ctrl_vring->vring, vdev, 0);
+    }
+    if (s->event_vring) {
+        vring_teardown(&s->event_vring->vring, vdev, 1);
+    }
+    if (s->cmd_vrings) {
+        for (i = 0; i < vs->conf.num_queues && s->cmd_vrings[i]; i++) {
+            vring_teardown(&s->cmd_vrings[i]->vring, vdev, 2 + i);
+        }
+        free(s->cmd_vrings);
+        s->cmd_vrings = NULL;
+    }
+}
+
+/* Context: QEMU global mutex held */
+void virtio_scsi_dataplane_start(VirtIOSCSI *s)
+{
+    int i;
+    int rc;
+    BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s)));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(s);
+
+    if (s->dataplane_started ||
+        s->dataplane_starting ||
+        s->dataplane_fenced ||
+        s->ctx != iothread_get_aio_context(vs->conf.iothread)) {
+        return;
+    }
+
+    s->dataplane_starting = true;
+
+    assert(!s->blocker);
+    error_setg(&s->blocker, "block device is in use by data plane");
+    /* Set up guest notifier (irq) */
+    rc = k->set_guest_notifiers(qbus->parent, vs->conf.num_queues + 2, true);
+    if (rc != 0) {
+        fprintf(stderr, "virtio-scsi: Failed to set guest notifiers (%d), "
+                "ensure -enable-kvm is set\n", rc);
+        s->dataplane_fenced = true;
+        goto fail_guest_notifiers;
+    }
+
+    aio_context_acquire(s->ctx);
+    s->ctrl_vring = virtio_scsi_vring_init(s, vs->ctrl_vq,
+                                           virtio_scsi_iothread_handle_ctrl,
+                                           0);
+    if (!s->ctrl_vring) {
+        goto fail_vrings;
+    }
+    s->event_vring = virtio_scsi_vring_init(s, vs->event_vq,
+                                            virtio_scsi_iothread_handle_event,
+                                            1);
+    if (!s->event_vring) {
+        goto fail_vrings;
+    }
+    s->cmd_vrings = g_new(VirtIOSCSIVring *, vs->conf.num_queues);
+    for (i = 0; i < vs->conf.num_queues; i++) {
+        s->cmd_vrings[i] =
+            virtio_scsi_vring_init(s, vs->cmd_vqs[i],
+                                   virtio_scsi_iothread_handle_cmd,
+                                   i + 2);
+        if (!s->cmd_vrings[i]) {
+            goto fail_vrings;
+        }
+    }
+
+    s->dataplane_starting = false;
+    s->dataplane_started = true;
+    aio_context_release(s->ctx);
+    return;
+
+fail_vrings:
+    virtio_scsi_clear_aio(s);
+    aio_context_release(s->ctx);
+    virtio_scsi_vring_teardown(s);
+    for (i = 0; i < vs->conf.num_queues + 2; i++) {
+        k->set_host_notifier(qbus->parent, i, false);
+    }
+    k->set_guest_notifiers(qbus->parent, vs->conf.num_queues + 2, false);
+fail_guest_notifiers:
+    s->dataplane_starting = false;
+}
+
+/* Context: QEMU global mutex held */
+void virtio_scsi_dataplane_stop(VirtIOSCSI *s)
+{
+    BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s)));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(s);
+    int i;
+
+    /* Better luck next time. */
+    if (s->dataplane_fenced) {
+        s->dataplane_fenced = false;
+        return;
+    }
+    if (!s->dataplane_started || s->dataplane_stopping) {
+        return;
+    }
+    error_free(s->blocker);
+    s->blocker = NULL;
+    s->dataplane_stopping = true;
+    assert(s->ctx == iothread_get_aio_context(vs->conf.iothread));
+
+    aio_context_acquire(s->ctx);
+
+    aio_set_event_notifier(s->ctx, &s->ctrl_vring->host_notifier, NULL);
+    aio_set_event_notifier(s->ctx, &s->event_vring->host_notifier, NULL);
+    for (i = 0; i < vs->conf.num_queues; i++) {
+        aio_set_event_notifier(s->ctx, &s->cmd_vrings[i]->host_notifier, NULL);
+    }
+
+    blk_drain_all(); /* ensure there are no in-flight requests */
+
+    aio_context_release(s->ctx);
+
+    /* Sync vring state back to virtqueue so that non-dataplane request
+     * processing can continue when we disable the host notifier below.
+     */
+    virtio_scsi_vring_teardown(s);
+
+    for (i = 0; i < vs->conf.num_queues + 2; i++) {
+        k->set_host_notifier(qbus->parent, i, false);
+    }
+
+    /* Clean up guest notifier (irq) */
+    k->set_guest_notifiers(qbus->parent, vs->conf.num_queues + 2, false);
+    s->dataplane_stopping = false;
+    s->dataplane_started = false;
+}
diff --git a/hw/scsi/virtio-scsi.c b/hw/scsi/virtio-scsi.c
index 0eb069ae9..ef485508b 100644
--- a/hw/scsi/virtio-scsi.c
+++ b/hw/scsi/virtio-scsi.c
@@ -16,38 +16,12 @@
 #include "hw/virtio/virtio-scsi.h"
 #include "qemu/error-report.h"
 #include "qemu/iov.h"
+#include "sysemu/block-backend.h"
 #include <hw/scsi/scsi.h>
 #include <block/scsi.h>
 #include <hw/virtio/virtio-bus.h>
 #include "hw/virtio/virtio-access.h"
-
-typedef struct VirtIOSCSIReq {
-    VirtIOSCSI *dev;
-    VirtQueue *vq;
-    VirtQueueElement elem;
-    QEMUSGList qsgl;
-    SCSIRequest *sreq;
-    size_t resp_size;
-    enum SCSIXferMode mode;
-    QEMUIOVector resp_iov;
-    union {
-        VirtIOSCSICmdResp     cmd;
-        VirtIOSCSICtrlTMFResp tmf;
-        VirtIOSCSICtrlANResp  an;
-        VirtIOSCSIEvent       event;
-    } resp;
-    union {
-        struct {
-            VirtIOSCSICmdReq  cmd;
-            uint8_t           cdb[];
-        } QEMU_PACKED;
-        VirtIOSCSICtrlTMFReq  tmf;
-        VirtIOSCSICtrlANReq   an;
-    } req;
-} VirtIOSCSIReq;
-
-QEMU_BUILD_BUG_ON(offsetof(VirtIOSCSIReq, req.cdb) !=
-                  offsetof(VirtIOSCSIReq, req.cmd) + sizeof(VirtIOSCSICmdReq));
+#include "migration/migration.h"
 
 static inline int virtio_scsi_get_lun(uint8_t *lun)
 {
@@ -65,26 +39,29 @@ static inline SCSIDevice *virtio_scsi_device_find(VirtIOSCSI *s, uint8_t *lun)
     return scsi_device_find(&s->bus, 0, lun[1], virtio_scsi_get_lun(lun));
 }
 
-static VirtIOSCSIReq *virtio_scsi_init_req(VirtIOSCSI *s, VirtQueue *vq)
+VirtIOSCSIReq *virtio_scsi_init_req(VirtIOSCSI *s, VirtQueue *vq)
 {
     VirtIOSCSIReq *req;
-    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(s);
-
-    req = g_malloc0(sizeof(*req) + vs->cdb_size);
+    VirtIOSCSICommon *vs = (VirtIOSCSICommon *)s;
+    const size_t zero_skip = offsetof(VirtIOSCSIReq, elem)
+                             + sizeof(VirtQueueElement);
 
+    req = g_slice_alloc(sizeof(*req) + vs->cdb_size);
     req->vq = vq;
     req->dev = s;
-    req->sreq = NULL;
     qemu_sglist_init(&req->qsgl, DEVICE(s), 8, &address_space_memory);
     qemu_iovec_init(&req->resp_iov, 1);
+    memset((uint8_t *)req + zero_skip, 0, sizeof(*req) - zero_skip);
     return req;
 }
 
-static void virtio_scsi_free_req(VirtIOSCSIReq *req)
+void virtio_scsi_free_req(VirtIOSCSIReq *req)
 {
+    VirtIOSCSICommon *vs = (VirtIOSCSICommon *)req->dev;
+
     qemu_iovec_destroy(&req->resp_iov);
     qemu_sglist_destroy(&req->qsgl);
-    g_free(req);
+    g_slice_free1(sizeof(*req) + vs->cdb_size, req);
 }
 
 static void virtio_scsi_complete_req(VirtIOSCSIReq *req)
@@ -94,13 +71,19 @@ static void virtio_scsi_complete_req(VirtIOSCSIReq *req)
     VirtIODevice *vdev = VIRTIO_DEVICE(s);
 
     qemu_iovec_from_buf(&req->resp_iov, 0, &req->resp, req->resp_size);
-    virtqueue_push(vq, &req->elem, req->qsgl.size + req->resp_iov.size);
+    if (req->vring) {
+        assert(req->vq == NULL);
+        virtio_scsi_vring_push_notify(req);
+    } else {
+        virtqueue_push(vq, &req->elem, req->qsgl.size + req->resp_iov.size);
+        virtio_notify(vdev, vq);
+    }
+
     if (req->sreq) {
         req->sreq->hba_private = NULL;
         scsi_req_unref(req->sreq);
     }
     virtio_scsi_free_req(req);
-    virtio_notify(vdev, vq);
 }
 
 static void virtio_scsi_bad_req(void)
@@ -135,6 +118,7 @@ static size_t qemu_sgl_concat(VirtIOSCSIReq *req, struct iovec *iov,
 static int virtio_scsi_parse_req(VirtIOSCSIReq *req,
                                  unsigned req_size, unsigned resp_size)
 {
+    VirtIODevice *vdev = (VirtIODevice *) req->dev;
     size_t in_size, out_size;
 
     if (iov_to_buf(req->elem.out_sg, req->elem.out_num, 0,
@@ -147,8 +131,24 @@ static int virtio_scsi_parse_req(VirtIOSCSIReq *req,
                               resp_size) < resp_size) {
         return -EINVAL;
     }
+
     req->resp_size = resp_size;
 
+    /* Old BIOSes left some padding by mistake after the req_size/resp_size.
+     * As a workaround, always consider the first buffer as the virtio-scsi
+     * request/response, making the payload start at the second element
+     * of the iovec.
+     *
+     * The actual length of the response header, stored in req->resp_size,
+     * does not change.
+     *
+     * TODO: always disable this workaround for virtio 1.0 devices.
+     */
+    if ((vdev->guest_features & VIRTIO_F_ANY_LAYOUT) == 0) {
+        req_size = req->elem.out_sg[0].iov_len;
+        resp_size = req->elem.in_sg[0].iov_len;
+    }
+
     out_size = qemu_sgl_concat(req, req->elem.out_sg,
                                &req->elem.out_addr[0], req->elem.out_num,
                                req_size);
@@ -226,13 +226,39 @@ static void *virtio_scsi_load_request(QEMUFile *f, SCSIRequest *sreq)
     return req;
 }
 
-static void virtio_scsi_do_tmf(VirtIOSCSI *s, VirtIOSCSIReq *req)
+typedef struct {
+    Notifier        notifier;
+    VirtIOSCSIReq  *tmf_req;
+} VirtIOSCSICancelNotifier;
+
+static void virtio_scsi_cancel_notify(Notifier *notifier, void *data)
+{
+    VirtIOSCSICancelNotifier *n = container_of(notifier,
+                                               VirtIOSCSICancelNotifier,
+                                               notifier);
+
+    if (--n->tmf_req->remaining == 0) {
+        virtio_scsi_complete_req(n->tmf_req);
+    }
+    g_slice_free(VirtIOSCSICancelNotifier, n);
+}
+
+/* Return 0 if the request is ready to be completed and return to guest;
+ * -EINPROGRESS if the request is submitted and will be completed later, in the
+ *  case of async cancellation. */
+static int virtio_scsi_do_tmf(VirtIOSCSI *s, VirtIOSCSIReq *req)
 {
     SCSIDevice *d = virtio_scsi_device_find(s, req->req.tmf.lun);
     SCSIRequest *r, *next;
     BusChild *kid;
     int target;
+    int ret = 0;
 
+    if (s->dataplane_started && blk_get_aio_context(d->conf.blk) != s->ctx) {
+        aio_context_acquire(s->ctx);
+        blk_set_aio_context(d->conf.blk, s->ctx);
+        aio_context_release(s->ctx);
+    }
     /* Here VIRTIO_SCSI_S_OK means "FUNCTION COMPLETE".  */
     req->resp.tmf.response = VIRTIO_SCSI_S_OK;
 
@@ -264,7 +290,14 @@ static void virtio_scsi_do_tmf(VirtIOSCSI *s, VirtIOSCSIReq *req)
                  */
                 req->resp.tmf.response = VIRTIO_SCSI_S_FUNCTION_SUCCEEDED;
             } else {
-                scsi_req_cancel(r);
+                VirtIOSCSICancelNotifier *notifier;
+
+                req->remaining = 1;
+                notifier = g_slice_new(VirtIOSCSICancelNotifier);
+                notifier->tmf_req = req;
+                notifier->notifier.notify = virtio_scsi_cancel_notify;
+                scsi_req_cancel_async(r, &notifier->notifier);
+                ret = -EINPROGRESS;
             }
         }
         break;
@@ -290,6 +323,13 @@ static void virtio_scsi_do_tmf(VirtIOSCSI *s, VirtIOSCSIReq *req)
         if (d->lun != virtio_scsi_get_lun(req->req.tmf.lun)) {
             goto incorrect_lun;
         }
+
+        /* Add 1 to "remaining" until virtio_scsi_do_tmf returns.
+         * This way, if the bus starts calling back to the notifiers
+         * even before we finish the loop, virtio_scsi_cancel_notify
+         * will not complete the TMF too early.
+         */
+        req->remaining = 1;
         QTAILQ_FOREACH_SAFE(r, &d->requests, next, next) {
             if (r->hba_private) {
                 if (req->req.tmf.subtype == VIRTIO_SCSI_T_TMF_QUERY_TASK_SET) {
@@ -299,10 +339,19 @@ static void virtio_scsi_do_tmf(VirtIOSCSI *s, VirtIOSCSIReq *req)
                     req->resp.tmf.response = VIRTIO_SCSI_S_FUNCTION_SUCCEEDED;
                     break;
                 } else {
-                    scsi_req_cancel(r);
+                    VirtIOSCSICancelNotifier *notifier;
+
+                    req->remaining++;
+                    notifier = g_slice_new(VirtIOSCSICancelNotifier);
+                    notifier->notifier.notify = virtio_scsi_cancel_notify;
+                    notifier->tmf_req = req;
+                    scsi_req_cancel_async(r, &notifier->notifier);
                 }
             }
         }
+        if (--req->remaining > 0) {
+            ret = -EINPROGRESS;
+        }
         break;
 
     case VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET:
@@ -323,50 +372,66 @@ static void virtio_scsi_do_tmf(VirtIOSCSI *s, VirtIOSCSIReq *req)
         break;
     }
 
-    return;
+    return ret;
 
 incorrect_lun:
     req->resp.tmf.response = VIRTIO_SCSI_S_INCORRECT_LUN;
-    return;
+    return ret;
 
 fail:
     req->resp.tmf.response = VIRTIO_SCSI_S_BAD_TARGET;
+    return ret;
 }
 
-static void virtio_scsi_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
+void virtio_scsi_handle_ctrl_req(VirtIOSCSI *s, VirtIOSCSIReq *req)
 {
-    VirtIOSCSI *s = (VirtIOSCSI *)vdev;
-    VirtIOSCSIReq *req;
+    VirtIODevice *vdev = (VirtIODevice *)s;
+    uint32_t type;
+    int r = 0;
 
-    while ((req = virtio_scsi_pop_req(s, vq))) {
-        int type;
+    if (iov_to_buf(req->elem.out_sg, req->elem.out_num, 0,
+                &type, sizeof(type)) < sizeof(type)) {
+        virtio_scsi_bad_req();
+        return;
+    }
 
-        if (iov_to_buf(req->elem.out_sg, req->elem.out_num, 0,
-                       &type, sizeof(type)) < sizeof(type)) {
+    virtio_tswap32s(vdev, &type);
+    if (type == VIRTIO_SCSI_T_TMF) {
+        if (virtio_scsi_parse_req(req, sizeof(VirtIOSCSICtrlTMFReq),
+                    sizeof(VirtIOSCSICtrlTMFResp)) < 0) {
             virtio_scsi_bad_req();
-            continue;
+        } else {
+            r = virtio_scsi_do_tmf(s, req);
         }
 
-        virtio_tswap32s(vdev, &req->req.tmf.type);
-        if (req->req.tmf.type == VIRTIO_SCSI_T_TMF) {
-            if (virtio_scsi_parse_req(req, sizeof(VirtIOSCSICtrlTMFReq),
-                                      sizeof(VirtIOSCSICtrlTMFResp)) < 0) {
-                virtio_scsi_bad_req();
-            } else {
-                virtio_scsi_do_tmf(s, req);
-            }
-
-        } else if (req->req.tmf.type == VIRTIO_SCSI_T_AN_QUERY ||
-                   req->req.tmf.type == VIRTIO_SCSI_T_AN_SUBSCRIBE) {
-            if (virtio_scsi_parse_req(req, sizeof(VirtIOSCSICtrlANReq),
-                                      sizeof(VirtIOSCSICtrlANResp)) < 0) {
-                virtio_scsi_bad_req();
-            } else {
-                req->resp.an.event_actual = 0;
-                req->resp.an.response = VIRTIO_SCSI_S_OK;
-            }
+    } else if (type == VIRTIO_SCSI_T_AN_QUERY ||
+               type == VIRTIO_SCSI_T_AN_SUBSCRIBE) {
+        if (virtio_scsi_parse_req(req, sizeof(VirtIOSCSICtrlANReq),
+                    sizeof(VirtIOSCSICtrlANResp)) < 0) {
+            virtio_scsi_bad_req();
+        } else {
+            req->resp.an.event_actual = 0;
+            req->resp.an.response = VIRTIO_SCSI_S_OK;
         }
+    }
+    if (r == 0) {
         virtio_scsi_complete_req(req);
+    } else {
+        assert(r == -EINPROGRESS);
+    }
+}
+
+static void virtio_scsi_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
+{
+    VirtIOSCSI *s = (VirtIOSCSI *)vdev;
+    VirtIOSCSIReq *req;
+
+    if (s->ctx && !s->dataplane_disabled) {
+        virtio_scsi_dataplane_start(s);
+        return;
+    }
+    while ((req = virtio_scsi_pop_req(s, vq))) {
+        virtio_scsi_handle_ctrl_req(s, req);
     }
 }
 
@@ -400,12 +465,30 @@ static void virtio_scsi_command_complete(SCSIRequest *r, uint32_t status,
         sense_len = scsi_req_get_sense(r, sense, sizeof(sense));
         sense_len = MIN(sense_len, req->resp_iov.size - sizeof(req->resp.cmd));
         qemu_iovec_from_buf(&req->resp_iov, sizeof(req->resp.cmd),
-                            &req->resp, sense_len);
+                            sense, sense_len);
         req->resp.cmd.sense_len = virtio_tswap32(vdev, sense_len);
     }
     virtio_scsi_complete_cmd_req(req);
 }
 
+static int virtio_scsi_parse_cdb(SCSIDevice *dev, SCSICommand *cmd,
+                                 uint8_t *buf, void *hba_private)
+{
+    VirtIOSCSIReq *req = hba_private;
+
+    if (cmd->len == 0) {
+        cmd->len = MIN(VIRTIO_SCSI_CDB_SIZE, SCSI_CMD_BUF_SIZE);
+        memcpy(cmd->buf, buf, cmd->len);
+    }
+
+    /* Extract the direction and mode directly from the request, for
+     * host device passthrough.
+     */
+    cmd->xfer = req->qsgl.size;
+    cmd->mode = req->mode;
+    return 0;
+}
+
 static QEMUSGList *virtio_scsi_get_sg_list(SCSIRequest *r)
 {
     VirtIOSCSIReq *req = r->hba_private;
@@ -434,52 +517,79 @@ static void virtio_scsi_fail_cmd_req(VirtIOSCSIReq *req)
     virtio_scsi_complete_cmd_req(req);
 }
 
+bool virtio_scsi_handle_cmd_req_prepare(VirtIOSCSI *s, VirtIOSCSIReq *req)
+{
+    VirtIOSCSICommon *vs = &s->parent_obj;
+    SCSIDevice *d;
+    int rc;
+
+    rc = virtio_scsi_parse_req(req, sizeof(VirtIOSCSICmdReq) + vs->cdb_size,
+                               sizeof(VirtIOSCSICmdResp) + vs->sense_size);
+    if (rc < 0) {
+        if (rc == -ENOTSUP) {
+            virtio_scsi_fail_cmd_req(req);
+        } else {
+            virtio_scsi_bad_req();
+        }
+        return false;
+    }
+
+    d = virtio_scsi_device_find(s, req->req.cmd.lun);
+    if (!d) {
+        req->resp.cmd.response = VIRTIO_SCSI_S_BAD_TARGET;
+        virtio_scsi_complete_cmd_req(req);
+        return false;
+    }
+    if (s->dataplane_started && blk_get_aio_context(d->conf.blk) != s->ctx) {
+        aio_context_acquire(s->ctx);
+        blk_set_aio_context(d->conf.blk, s->ctx);
+        aio_context_release(s->ctx);
+    }
+    req->sreq = scsi_req_new(d, req->req.cmd.tag,
+                             virtio_scsi_get_lun(req->req.cmd.lun),
+                             req->req.cdb, req);
+
+    if (req->sreq->cmd.mode != SCSI_XFER_NONE
+        && (req->sreq->cmd.mode != req->mode ||
+            req->sreq->cmd.xfer > req->qsgl.size)) {
+        req->resp.cmd.response = VIRTIO_SCSI_S_OVERRUN;
+        virtio_scsi_complete_cmd_req(req);
+        return false;
+    }
+    scsi_req_ref(req->sreq);
+    blk_io_plug(d->conf.blk);
+    return true;
+}
+
+void virtio_scsi_handle_cmd_req_submit(VirtIOSCSI *s, VirtIOSCSIReq *req)
+{
+    SCSIRequest *sreq = req->sreq;
+    if (scsi_req_enqueue(sreq)) {
+        scsi_req_continue(sreq);
+    }
+    blk_io_unplug(sreq->dev->conf.blk);
+    scsi_req_unref(sreq);
+}
+
 static void virtio_scsi_handle_cmd(VirtIODevice *vdev, VirtQueue *vq)
 {
     /* use non-QOM casts in the data path */
     VirtIOSCSI *s = (VirtIOSCSI *)vdev;
-    VirtIOSCSICommon *vs = &s->parent_obj;
-
-    VirtIOSCSIReq *req;
-    int n;
+    VirtIOSCSIReq *req, *next;
+    QTAILQ_HEAD(, VirtIOSCSIReq) reqs = QTAILQ_HEAD_INITIALIZER(reqs);
 
+    if (s->ctx && !s->dataplane_disabled) {
+        virtio_scsi_dataplane_start(s);
+        return;
+    }
     while ((req = virtio_scsi_pop_req(s, vq))) {
-        SCSIDevice *d;
-        int rc;
-
-        rc = virtio_scsi_parse_req(req, sizeof(VirtIOSCSICmdReq) + vs->cdb_size,
-                                   sizeof(VirtIOSCSICmdResp) + vs->sense_size);
-        if (rc < 0) {
-            if (rc == -ENOTSUP) {
-                virtio_scsi_fail_cmd_req(req);
-            } else {
-                virtio_scsi_bad_req();
-            }
-            continue;
-        }
-
-        d = virtio_scsi_device_find(s, req->req.cmd.lun);
-        if (!d) {
-            req->resp.cmd.response = VIRTIO_SCSI_S_BAD_TARGET;
-            virtio_scsi_complete_cmd_req(req);
-            continue;
-        }
-        req->sreq = scsi_req_new(d, req->req.cmd.tag,
-                                 virtio_scsi_get_lun(req->req.cmd.lun),
-                                 req->req.cdb, req);
-
-        if (req->sreq->cmd.mode != SCSI_XFER_NONE
-            && (req->sreq->cmd.mode != req->mode ||
-                req->sreq->cmd.xfer > req->qsgl.size)) {
-            req->resp.cmd.response = VIRTIO_SCSI_S_OVERRUN;
-            virtio_scsi_complete_cmd_req(req);
-            continue;
+        if (virtio_scsi_handle_cmd_req_prepare(s, req)) {
+            QTAILQ_INSERT_TAIL(&reqs, req, next);
         }
+    }
 
-        n = scsi_req_enqueue(req->sreq);
-        if (n) {
-            scsi_req_continue(req->sreq);
-        }
+    QTAILQ_FOREACH_SAFE(req, &reqs, next, next) {
+        virtio_scsi_handle_cmd_req_submit(s, req);
     }
 }
 
@@ -528,6 +638,9 @@ static void virtio_scsi_reset(VirtIODevice *vdev)
     VirtIOSCSI *s = VIRTIO_SCSI(vdev);
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(vdev);
 
+    if (s->ctx) {
+        virtio_scsi_dataplane_stop(s);
+    }
     s->resetting++;
     qbus_reset_all(&s->bus.qbus);
     s->resetting--;
@@ -558,8 +671,8 @@ static int virtio_scsi_load(QEMUFile *f, void *opaque, int version_id)
     return 0;
 }
 
-static void virtio_scsi_push_event(VirtIOSCSI *s, SCSIDevice *dev,
-                                   uint32_t event, uint32_t reason)
+void virtio_scsi_push_event(VirtIOSCSI *s, SCSIDevice *dev,
+                            uint32_t event, uint32_t reason)
 {
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(s);
     VirtIOSCSIReq *req;
@@ -570,10 +683,19 @@ static void virtio_scsi_push_event(VirtIOSCSI *s, SCSIDevice *dev,
         return;
     }
 
-    req = virtio_scsi_pop_req(s, vs->event_vq);
+    if (s->dataplane_started) {
+        assert(s->ctx);
+        aio_context_acquire(s->ctx);
+    }
+
+    if (s->dataplane_started) {
+        req = virtio_scsi_pop_req_vring(s, s->event_vring);
+    } else {
+        req = virtio_scsi_pop_req(s, vs->event_vq);
+    }
     if (!req) {
         s->events_dropped = true;
-        return;
+        goto out;
     }
 
     if (s->events_dropped) {
@@ -602,12 +724,20 @@ static void virtio_scsi_push_event(VirtIOSCSI *s, SCSIDevice *dev,
         evt->lun[3] = dev->lun & 0xFF;
     }
     virtio_scsi_complete_req(req);
+out:
+    if (s->dataplane_started) {
+        aio_context_release(s->ctx);
+    }
 }
 
 static void virtio_scsi_handle_event(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIOSCSI *s = VIRTIO_SCSI(vdev);
 
+    if (s->ctx && !s->dataplane_disabled) {
+        virtio_scsi_dataplane_start(s);
+        return;
+    }
     if (s->events_dropped) {
         virtio_scsi_push_event(s, NULL, VIRTIO_SCSI_T_NO_EVENT, 0);
     }
@@ -625,26 +755,44 @@ static void virtio_scsi_change(SCSIBus *bus, SCSIDevice *dev, SCSISense sense)
     }
 }
 
-static void virtio_scsi_hotplug(SCSIBus *bus, SCSIDevice *dev)
+static void virtio_scsi_hotplug(HotplugHandler *hotplug_dev, DeviceState *dev,
+                                Error **errp)
 {
-    VirtIOSCSI *s = container_of(bus, VirtIOSCSI, bus);
-    VirtIODevice *vdev = VIRTIO_DEVICE(s);
+    VirtIODevice *vdev = VIRTIO_DEVICE(hotplug_dev);
+    VirtIOSCSI *s = VIRTIO_SCSI(vdev);
+    SCSIDevice *sd = SCSI_DEVICE(dev);
+
+    if (s->ctx && !s->dataplane_disabled) {
+        if (blk_op_is_blocked(sd->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) {
+            return;
+        }
+        blk_op_block_all(sd->conf.blk, s->blocker);
+    }
 
     if ((vdev->guest_features >> VIRTIO_SCSI_F_HOTPLUG) & 1) {
-        virtio_scsi_push_event(s, dev, VIRTIO_SCSI_T_TRANSPORT_RESET,
+        virtio_scsi_push_event(s, sd,
+                               VIRTIO_SCSI_T_TRANSPORT_RESET,
                                VIRTIO_SCSI_EVT_RESET_RESCAN);
     }
 }
 
-static void virtio_scsi_hot_unplug(SCSIBus *bus, SCSIDevice *dev)
+static void virtio_scsi_hotunplug(HotplugHandler *hotplug_dev, DeviceState *dev,
+                                  Error **errp)
 {
-    VirtIOSCSI *s = container_of(bus, VirtIOSCSI, bus);
-    VirtIODevice *vdev = VIRTIO_DEVICE(s);
+    VirtIODevice *vdev = VIRTIO_DEVICE(hotplug_dev);
+    VirtIOSCSI *s = VIRTIO_SCSI(vdev);
+    SCSIDevice *sd = SCSI_DEVICE(dev);
 
     if ((vdev->guest_features >> VIRTIO_SCSI_F_HOTPLUG) & 1) {
-        virtio_scsi_push_event(s, dev, VIRTIO_SCSI_T_TRANSPORT_RESET,
+        virtio_scsi_push_event(s, sd,
+                               VIRTIO_SCSI_T_TRANSPORT_RESET,
                                VIRTIO_SCSI_EVT_RESET_REMOVED);
     }
+
+    if (s->ctx) {
+        blk_op_unblock_all(sd->conf.blk, s->blocker);
+    }
+    qdev_simple_device_unplug_cb(hotplug_dev, dev, errp);
 }
 
 static struct SCSIBusInfo virtio_scsi_scsi_info = {
@@ -656,8 +804,7 @@ static struct SCSIBusInfo virtio_scsi_scsi_info = {
     .complete = virtio_scsi_command_complete,
     .cancel = virtio_scsi_request_cancelled,
     .change = virtio_scsi_change,
-    .hotplug = virtio_scsi_hotplug,
-    .hot_unplug = virtio_scsi_hot_unplug,
+    .parse_cdb = virtio_scsi_parse_cdb,
     .get_sg_list = virtio_scsi_get_sg_list,
     .save_request = virtio_scsi_save_request,
     .load_request = virtio_scsi_load_request,
@@ -674,6 +821,14 @@ void virtio_scsi_common_realize(DeviceState *dev, Error **errp,
     virtio_init(vdev, "virtio-scsi", VIRTIO_ID_SCSI,
                 sizeof(VirtIOSCSIConfig));
 
+    if (s->conf.num_queues == 0 ||
+            s->conf.num_queues > VIRTIO_PCI_QUEUE_MAX - 2) {
+        error_setg(errp, "Invalid number of queues (= %" PRIu32 "), "
+                         "must be a positive integer less than %d.",
+                   s->conf.num_queues, VIRTIO_PCI_QUEUE_MAX - 2);
+        virtio_cleanup(vdev);
+        return;
+    }
     s->cmd_vqs = g_malloc0(s->conf.num_queues * sizeof(VirtQueue *));
     s->sense_size = VIRTIO_SCSI_SENSE_SIZE;
     s->cdb_size = VIRTIO_SCSI_CDB_SIZE;
@@ -686,6 +841,35 @@ void virtio_scsi_common_realize(DeviceState *dev, Error **errp,
         s->cmd_vqs[i] = virtio_add_queue(vdev, VIRTIO_SCSI_VQ_SIZE,
                                          cmd);
     }
+
+    if (s->conf.iothread) {
+        virtio_scsi_set_iothread(VIRTIO_SCSI(s), s->conf.iothread);
+    }
+}
+
+/* Disable dataplane thread during live migration since it does not
+ * update the dirty memory bitmap yet.
+ */
+static void virtio_scsi_migration_state_changed(Notifier *notifier, void *data)
+{
+    VirtIOSCSI *s = container_of(notifier, VirtIOSCSI,
+                                 migration_state_notifier);
+    MigrationState *mig = data;
+
+    if (migration_in_setup(mig)) {
+        if (!s->dataplane_started) {
+            return;
+        }
+        virtio_scsi_dataplane_stop(s);
+        s->dataplane_disabled = true;
+    } else if (migration_has_finished(mig) ||
+               migration_has_failed(mig)) {
+        if (s->dataplane_started) {
+            return;
+        }
+        blk_drain_all(); /* complete in-flight non-dataplane requests */
+        s->dataplane_disabled = false;
+    }
 }
 
 static void virtio_scsi_device_realize(DeviceState *dev, Error **errp)
@@ -705,6 +889,8 @@ static void virtio_scsi_device_realize(DeviceState *dev, Error **errp)
 
     scsi_bus_new(&s->bus, sizeof(s->bus), dev,
                  &virtio_scsi_scsi_info, vdev->bus_name);
+    /* override default SCSI bus hotplug-handler, with virtio-scsi's one */
+    qbus_set_hotplug_handler(BUS(&s->bus), dev, &error_abort);
 
     if (!dev->hotplugged) {
         scsi_bus_legacy_handle_cmdline(&s->bus, &err);
@@ -716,6 +902,18 @@ static void virtio_scsi_device_realize(DeviceState *dev, Error **errp)
 
     register_savevm(dev, "virtio-scsi", virtio_scsi_id++, 1,
                     virtio_scsi_save, virtio_scsi_load, s);
+    s->migration_state_notifier.notify = virtio_scsi_migration_state_changed;
+    add_migration_state_change_notifier(&s->migration_state_notifier);
+}
+
+static void virtio_scsi_instance_init(Object *obj)
+{
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(obj);
+
+    object_property_add_link(obj, "iothread", TYPE_IOTHREAD,
+                             (Object **)&vs->conf.iothread,
+                             qdev_prop_allow_set_link_before_realize,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE, &error_abort);
 }
 
 void virtio_scsi_common_unrealize(DeviceState *dev, Error **errp)
@@ -732,6 +930,7 @@ static void virtio_scsi_device_unrealize(DeviceState *dev, Error **errp)
     VirtIOSCSI *s = VIRTIO_SCSI(dev);
 
     unregister_savevm(dev, "virtio-scsi", s);
+    remove_migration_state_change_notifier(&s->migration_state_notifier);
 
     virtio_scsi_common_unrealize(dev, errp);
 }
@@ -754,6 +953,7 @@ static void virtio_scsi_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
+    HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass);
 
     dc->props = virtio_scsi_properties;
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
@@ -762,6 +962,8 @@ static void virtio_scsi_class_init(ObjectClass *klass, void *data)
     vdc->set_config = virtio_scsi_set_config;
     vdc->get_features = virtio_scsi_get_features;
     vdc->reset = virtio_scsi_reset;
+    hc->plug = virtio_scsi_hotplug;
+    hc->unplug = virtio_scsi_hotunplug;
 }
 
 static const TypeInfo virtio_scsi_common_info = {
@@ -776,7 +978,12 @@ static const TypeInfo virtio_scsi_info = {
     .name = TYPE_VIRTIO_SCSI,
     .parent = TYPE_VIRTIO_SCSI_COMMON,
     .instance_size = sizeof(VirtIOSCSI),
+    .instance_init = virtio_scsi_instance_init,
     .class_init = virtio_scsi_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_HOTPLUG_HANDLER },
+        { }
+    }
 };
 
 static void virtio_register_types(void)
diff --git a/hw/scsi/vmw_pvscsi.c b/hw/scsi/vmw_pvscsi.c
index f9ed926bd..d3a92fbab 100644
--- a/hw/scsi/vmw_pvscsi.c
+++ b/hw/scsi/vmw_pvscsi.c
@@ -524,17 +524,20 @@ pvscsi_send_msg(PVSCSIState *s, SCSIDevice *dev, uint32_t msg_type)
 }
 
 static void
-pvscsi_hotplug(SCSIBus *bus, SCSIDevice *dev)
+pvscsi_hotplug(HotplugHandler *hotplug_dev, DeviceState *dev, Error **errp)
 {
-    PVSCSIState *s = container_of(bus, PVSCSIState, bus);
-    pvscsi_send_msg(s, dev, PVSCSI_MSG_DEV_ADDED);
+    PVSCSIState *s = PVSCSI(hotplug_dev);
+
+    pvscsi_send_msg(s, SCSI_DEVICE(dev), PVSCSI_MSG_DEV_ADDED);
 }
 
 static void
-pvscsi_hot_unplug(SCSIBus *bus, SCSIDevice *dev)
+pvscsi_hot_unplug(HotplugHandler *hotplug_dev, DeviceState *dev, Error **errp)
 {
-    PVSCSIState *s = container_of(bus, PVSCSIState, bus);
-    pvscsi_send_msg(s, dev, PVSCSI_MSG_DEV_REMOVED);
+    PVSCSIState *s = PVSCSI(hotplug_dev);
+
+    pvscsi_send_msg(s, SCSI_DEVICE(dev), PVSCSI_MSG_DEV_REMOVED);
+    qdev_simple_device_unplug_cb(hotplug_dev, dev, errp);
 }
 
 static void
@@ -1057,8 +1060,6 @@ static const struct SCSIBusInfo pvscsi_scsi_info = {
         .get_sg_list = pvscsi_get_sg_list,
         .complete = pvscsi_command_complete,
         .cancel = pvscsi_request_cancelled,
-        .hotplug = pvscsi_hotplug,
-        .hot_unplug = pvscsi_hot_unplug,
 };
 
 static int
@@ -1087,12 +1088,13 @@ pvscsi_init(PCIDevice *pci_dev)
     s->completion_worker = qemu_bh_new(pvscsi_process_completion_queue, s);
     if (!s->completion_worker) {
         pvscsi_cleanup_msi(s);
-        memory_region_destroy(&s->io_space);
         return -ENOMEM;
     }
 
     scsi_bus_new(&s->bus, sizeof(s->bus), DEVICE(pci_dev),
                  &pvscsi_scsi_info, NULL);
+    /* override default SCSI bus hotplug-handler, with pvscsi's one */
+    qbus_set_hotplug_handler(BUS(&s->bus), DEVICE(s), &error_abort);
     pvscsi_reset_state(s);
 
     return 0;
@@ -1107,8 +1109,6 @@ pvscsi_uninit(PCIDevice *pci_dev)
     qemu_bh_delete(s->completion_worker);
 
     pvscsi_cleanup_msi(s);
-
-    memory_region_destroy(&s->io_space);
 }
 
 static void
@@ -1190,6 +1190,7 @@ static void pvscsi_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+    HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass);
 
     k->init = pvscsi_init;
     k->exit = pvscsi_uninit;
@@ -1202,6 +1203,8 @@ static void pvscsi_class_init(ObjectClass *klass, void *data)
     dc->props = pvscsi_properties;
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
     k->config_write = pvscsi_write_config;
+    hc->unplug = pvscsi_hot_unplug;
+    hc->plug = pvscsi_hotplug;
 }
 
 static const TypeInfo pvscsi_info = {
@@ -1209,6 +1212,10 @@ static const TypeInfo pvscsi_info = {
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PVSCSIState),
     .class_init    = pvscsi_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_HOTPLUG_HANDLER },
+        { }
+    }
 };
 
 static void
diff --git a/hw/sd/milkymist-memcard.c b/hw/sd/milkymist-memcard.c
index 2a40f9273..9661eafac 100644
--- a/hw/sd/milkymist-memcard.c
+++ b/hw/sd/milkymist-memcard.c
@@ -26,6 +26,7 @@
 #include "sysemu/sysemu.h"
 #include "trace.h"
 #include "qemu/error-report.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "hw/sd.h"
 
@@ -252,14 +253,16 @@ static int milkymist_memcard_init(SysBusDevice *dev)
 {
     MilkymistMemcardState *s = MILKYMIST_MEMCARD(dev);
     DriveInfo *dinfo;
+    BlockBackend *blk;
 
     dinfo = drive_get_next(IF_SD);
-    s->card = sd_init(dinfo ? dinfo->bdrv : NULL, false);
+    blk = dinfo ? blk_by_legacy_dinfo(dinfo) : NULL;
+    s->card = sd_init(blk, false);
     if (s->card == NULL) {
         return -1;
     }
 
-    s->enabled = dinfo ? bdrv_is_inserted(dinfo->bdrv) : 0;
+    s->enabled = blk && blk_is_inserted(blk);
 
     memory_region_init_io(&s->regs_region, OBJECT(s), &memcard_mmio_ops, s,
             "milkymist-memcard", R_MAX * 4);
diff --git a/hw/sd/omap_mmc.c b/hw/sd/omap_mmc.c
index 6c92149c0..86c477d72 100644
--- a/hw/sd/omap_mmc.c
+++ b/hw/sd/omap_mmc.c
@@ -574,7 +574,7 @@ static void omap_mmc_cover_cb(void *opaque, int line, int level)
 
 struct omap_mmc_s *omap_mmc_init(hwaddr base,
                 MemoryRegion *sysmem,
-                BlockDriverState *bd,
+                BlockBackend *blk,
                 qemu_irq irq, qemu_irq dma[], omap_clk clk)
 {
     struct omap_mmc_s *s = (struct omap_mmc_s *)
@@ -592,7 +592,7 @@ struct omap_mmc_s *omap_mmc_init(hwaddr base,
     memory_region_add_subregion(sysmem, base, &s->iomem);
 
     /* Instantiate the storage */
-    s->card = sd_init(bd, false);
+    s->card = sd_init(blk, false);
     if (s->card == NULL) {
         exit(1);
     }
@@ -601,7 +601,7 @@ struct omap_mmc_s *omap_mmc_init(hwaddr base,
 }
 
 struct omap_mmc_s *omap2_mmc_init(struct omap_target_agent_s *ta,
-                BlockDriverState *bd, qemu_irq irq, qemu_irq dma[],
+                BlockBackend *blk, qemu_irq irq, qemu_irq dma[],
                 omap_clk fclk, omap_clk iclk)
 {
     struct omap_mmc_s *s = (struct omap_mmc_s *)
@@ -620,7 +620,7 @@ struct omap_mmc_s *omap2_mmc_init(struct omap_target_agent_s *ta,
     omap_l4_attach(ta, 0, &s->iomem);
 
     /* Instantiate the storage */
-    s->card = sd_init(bd, false);
+    s->card = sd_init(blk, false);
     if (s->card == NULL) {
         exit(1);
     }
diff --git a/hw/sd/pl181.c b/hw/sd/pl181.c
index 462558b76..e704b6e97 100644
--- a/hw/sd/pl181.c
+++ b/hw/sd/pl181.c
@@ -7,6 +7,7 @@
  * This code is licensed under the GPL.
  */
 
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "hw/sysbus.h"
 #include "hw/sd.h"
@@ -490,7 +491,7 @@ static int pl181_init(SysBusDevice *sbd)
     sysbus_init_irq(sbd, &s->irq[1]);
     qdev_init_gpio_out(dev, s->cardstatus, 2);
     dinfo = drive_get_next(IF_SD);
-    s->card = sd_init(dinfo ? dinfo->bdrv : NULL, false);
+    s->card = sd_init(dinfo ? blk_by_legacy_dinfo(dinfo) : NULL, false);
     if (s->card == NULL) {
         return -1;
     }
diff --git a/hw/sd/pxa2xx_mmci.c b/hw/sd/pxa2xx_mmci.c
index b9d8b1a3e..ac3ab39be 100644
--- a/hw/sd/pxa2xx_mmci.c
+++ b/hw/sd/pxa2xx_mmci.c
@@ -523,7 +523,7 @@ static int pxa2xx_mmci_load(QEMUFile *f, void *opaque, int version_id)
 
 PXA2xxMMCIState *pxa2xx_mmci_init(MemoryRegion *sysmem,
                 hwaddr base,
-                BlockDriverState *bd, qemu_irq irq,
+                BlockBackend *blk, qemu_irq irq,
                 qemu_irq rx_dma, qemu_irq tx_dma)
 {
     PXA2xxMMCIState *s;
@@ -538,7 +538,7 @@ PXA2xxMMCIState *pxa2xx_mmci_init(MemoryRegion *sysmem,
     memory_region_add_subregion(sysmem, base, &s->iomem);
 
     /* Instantiate the actual storage */
-    s->card = sd_init(bd, false);
+    s->card = sd_init(blk, false);
     if (s->card == NULL) {
         exit(1);
     }
diff --git a/hw/sd/sd.c b/hw/sd/sd.c
index 5efe8c1af..f955265f7 100644
--- a/hw/sd/sd.c
+++ b/hw/sd/sd.c
@@ -30,7 +30,7 @@
  */
 
 #include "hw/hw.h"
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "hw/sd.h"
 #include "qemu/bitmap.h"
 
@@ -110,7 +110,7 @@ struct SDState {
     uint8_t data[512];
     qemu_irq readonly_cb;
     qemu_irq inserted_cb;
-    BlockDriverState *bdrv;
+    BlockBackend *blk;
     uint8_t *buf;
 
     bool enable;
@@ -389,13 +389,13 @@ static inline uint64_t sd_addr_to_wpnum(uint64_t addr)
     return addr >> (HWBLOCK_SHIFT + SECTOR_SHIFT + WPGROUP_SHIFT);
 }
 
-static void sd_reset(SDState *sd, BlockDriverState *bdrv)
+static void sd_reset(SDState *sd, BlockBackend *blk)
 {
     uint64_t size;
     uint64_t sect;
 
-    if (bdrv) {
-        bdrv_get_geometry(bdrv, &sect);
+    if (blk) {
+        blk_get_geometry(blk, &sect);
     } else {
         sect = 0;
     }
@@ -412,11 +412,11 @@ static void sd_reset(SDState *sd, BlockDriverState *bdrv)
     sd_set_cardstatus(sd);
     sd_set_sdstatus(sd);
 
-    sd->bdrv = bdrv;
+    sd->blk = blk;
 
     if (sd->wp_groups)
         g_free(sd->wp_groups);
-    sd->wp_switch = bdrv ? bdrv_is_read_only(bdrv) : false;
+    sd->wp_switch = blk ? blk_is_read_only(blk) : false;
     sd->wpgrps_size = sect;
     sd->wp_groups = bitmap_new(sd->wpgrps_size);
     memset(sd->function_group, 0, sizeof(sd->function_group));
@@ -432,9 +432,9 @@ static void sd_cardchange(void *opaque, bool load)
 {
     SDState *sd = opaque;
 
-    qemu_set_irq(sd->inserted_cb, bdrv_is_inserted(sd->bdrv));
-    if (bdrv_is_inserted(sd->bdrv)) {
-        sd_reset(sd, sd->bdrv);
+    qemu_set_irq(sd->inserted_cb, blk_is_inserted(sd->blk));
+    if (blk_is_inserted(sd->blk)) {
+        sd_reset(sd, sd->blk);
         qemu_set_irq(sd->readonly_cb, sd->wp_switch);
     }
 }
@@ -479,23 +479,23 @@ static const VMStateDescription sd_vmstate = {
    whether card should be in SSI or MMC/SD mode.  It is also up to the
    board to ensure that ssi transfers only occur when the chip select
    is asserted.  */
-SDState *sd_init(BlockDriverState *bs, bool is_spi)
+SDState *sd_init(BlockBackend *blk, bool is_spi)
 {
     SDState *sd;
 
-    if (bs && bdrv_is_read_only(bs)) {
+    if (blk && blk_is_read_only(blk)) {
         fprintf(stderr, "sd_init: Cannot use read-only drive\n");
         return NULL;
     }
 
     sd = (SDState *) g_malloc0(sizeof(SDState));
-    sd->buf = qemu_blockalign(bs, 512);
+    sd->buf = blk_blockalign(blk, 512);
     sd->spi = is_spi;
     sd->enable = true;
-    sd_reset(sd, bs);
-    if (sd->bdrv) {
-        bdrv_attach_dev_nofail(sd->bdrv, sd);
-        bdrv_set_dev_ops(sd->bdrv, &sd_block_ops, sd);
+    sd_reset(sd, blk);
+    if (sd->blk) {
+        blk_attach_dev_nofail(sd->blk, sd);
+        blk_set_dev_ops(sd->blk, &sd_block_ops, sd);
     }
     vmstate_register(NULL, -1, &sd_vmstate, sd);
     return sd;
@@ -505,8 +505,8 @@ void sd_set_cb(SDState *sd, qemu_irq readonly, qemu_irq insert)
 {
     sd->readonly_cb = readonly;
     sd->inserted_cb = insert;
-    qemu_set_irq(readonly, sd->bdrv ? bdrv_is_read_only(sd->bdrv) : 0);
-    qemu_set_irq(insert, sd->bdrv ? bdrv_is_inserted(sd->bdrv) : 0);
+    qemu_set_irq(readonly, sd->blk ? blk_is_read_only(sd->blk) : 0);
+    qemu_set_irq(insert, sd->blk ? blk_is_inserted(sd->blk) : 0);
 }
 
 static void sd_erase(SDState *sd)
@@ -680,7 +680,7 @@ static sd_rsp_type_t sd_normal_command(SDState *sd,
 
         default:
             sd->state = sd_idle_state;
-            sd_reset(sd, sd->bdrv);
+            sd_reset(sd, sd->blk);
             return sd->spi ? sd_r1 : sd_r0;
         }
         break;
@@ -1347,7 +1347,7 @@ int sd_do_command(SDState *sd, SDRequest *req,
     sd_rsp_type_t rtype;
     int rsplen;
 
-    if (!sd->bdrv || !bdrv_is_inserted(sd->bdrv) || !sd->enable) {
+    if (!sd->blk || !blk_is_inserted(sd->blk) || !sd->enable) {
         return 0;
     }
 
@@ -1456,7 +1456,7 @@ static void sd_blk_read(SDState *sd, uint64_t addr, uint32_t len)
 
     DPRINTF("sd_blk_read: addr = 0x%08llx, len = %d\n",
             (unsigned long long) addr, len);
-    if (!sd->bdrv || bdrv_read(sd->bdrv, addr >> 9, sd->buf, 1) < 0) {
+    if (!sd->blk || blk_read(sd->blk, addr >> 9, sd->buf, 1) < 0) {
         fprintf(stderr, "sd_blk_read: read error on host side\n");
         return;
     }
@@ -1464,7 +1464,7 @@ static void sd_blk_read(SDState *sd, uint64_t addr, uint32_t len)
     if (end > (addr & ~511) + 512) {
         memcpy(sd->data, sd->buf + (addr & 511), 512 - (addr & 511));
 
-        if (bdrv_read(sd->bdrv, end >> 9, sd->buf, 1) < 0) {
+        if (blk_read(sd->blk, end >> 9, sd->buf, 1) < 0) {
             fprintf(stderr, "sd_blk_read: read error on host side\n");
             return;
         }
@@ -1478,29 +1478,29 @@ static void sd_blk_write(SDState *sd, uint64_t addr, uint32_t len)
     uint64_t end = addr + len;
 
     if ((addr & 511) || len < 512)
-        if (!sd->bdrv || bdrv_read(sd->bdrv, addr >> 9, sd->buf, 1) < 0) {
+        if (!sd->blk || blk_read(sd->blk, addr >> 9, sd->buf, 1) < 0) {
             fprintf(stderr, "sd_blk_write: read error on host side\n");
             return;
         }
 
     if (end > (addr & ~511) + 512) {
         memcpy(sd->buf + (addr & 511), sd->data, 512 - (addr & 511));
-        if (bdrv_write(sd->bdrv, addr >> 9, sd->buf, 1) < 0) {
+        if (blk_write(sd->blk, addr >> 9, sd->buf, 1) < 0) {
             fprintf(stderr, "sd_blk_write: write error on host side\n");
             return;
         }
 
-        if (bdrv_read(sd->bdrv, end >> 9, sd->buf, 1) < 0) {
+        if (blk_read(sd->blk, end >> 9, sd->buf, 1) < 0) {
             fprintf(stderr, "sd_blk_write: read error on host side\n");
             return;
         }
         memcpy(sd->buf, sd->data + 512 - (addr & 511), end & 511);
-        if (bdrv_write(sd->bdrv, end >> 9, sd->buf, 1) < 0) {
+        if (blk_write(sd->blk, end >> 9, sd->buf, 1) < 0) {
             fprintf(stderr, "sd_blk_write: write error on host side\n");
         }
     } else {
         memcpy(sd->buf + (addr & 511), sd->data, len);
-        if (!sd->bdrv || bdrv_write(sd->bdrv, addr >> 9, sd->buf, 1) < 0) {
+        if (!sd->blk || blk_write(sd->blk, addr >> 9, sd->buf, 1) < 0) {
             fprintf(stderr, "sd_blk_write: write error on host side\n");
         }
     }
@@ -1515,7 +1515,7 @@ void sd_write_data(SDState *sd, uint8_t value)
 {
     int i;
 
-    if (!sd->bdrv || !bdrv_is_inserted(sd->bdrv) || !sd->enable)
+    if (!sd->blk || !blk_is_inserted(sd->blk) || !sd->enable)
         return;
 
     if (sd->state != sd_receivingdata_state) {
@@ -1641,7 +1641,7 @@ uint8_t sd_read_data(SDState *sd)
     uint8_t ret;
     int io_len;
 
-    if (!sd->bdrv || !bdrv_is_inserted(sd->bdrv) || !sd->enable)
+    if (!sd->blk || !blk_is_inserted(sd->blk) || !sd->enable)
         return 0x00;
 
     if (sd->state != sd_sendingdata_state) {
diff --git a/hw/sd/sdhci.c b/hw/sd/sdhci.c
index b5a9eee3e..b38005003 100644
--- a/hw/sd/sdhci.c
+++ b/hw/sd/sdhci.c
@@ -23,6 +23,7 @@
  */
 
 #include "hw/hw.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/dma.h"
 #include "qemu/timer.h"
@@ -702,7 +703,8 @@ static void sdhci_do_adma(SDHCIState *s)
                         length -= block_size - begin;
                     }
                     dma_memory_read(&address_space_memory, dscr.addr,
-                                    &s->fifo_buffer[begin], s->data_count);
+                                    &s->fifo_buffer[begin],
+                                    s->data_count - begin);
                     dscr.addr += s->data_count - begin;
                     if (s->data_count == block_size) {
                         for (n = 0; n < block_size; n++) {
@@ -1164,7 +1166,7 @@ static void sdhci_initfn(Object *obj)
     DriveInfo *di;
 
     di = drive_get_next(IF_SD);
-    s->card = sd_init(di ? di->bdrv : NULL, false);
+    s->card = sd_init(di ? blk_by_legacy_dinfo(di) : NULL, false);
     if (s->card == NULL) {
         exit(1);
     }
diff --git a/hw/sd/ssi-sd.c b/hw/sd/ssi-sd.c
index b012e57f6..a71fbca71 100644
--- a/hw/sd/ssi-sd.c
+++ b/hw/sd/ssi-sd.c
@@ -10,6 +10,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "hw/ssi.h"
 #include "hw/sd.h"
@@ -255,7 +256,7 @@ static int ssi_sd_init(SSISlave *d)
 
     s->mode = SSI_SD_CMD;
     dinfo = drive_get_next(IF_SD);
-    s->sd = sd_init(dinfo ? dinfo->bdrv : NULL, true);
+    s->sd = sd_init(dinfo ? blk_by_legacy_dinfo(dinfo) : NULL, true);
     if (s->sd == NULL) {
         return -1;
     }
diff --git a/hw/sh4/r2d.c b/hw/sh4/r2d.c
index 95c0246d4..12f44d28f 100644
--- a/hw/sh4/r2d.c
+++ b/hw/sh4/r2d.c
@@ -36,7 +36,7 @@
 #include "hw/loader.h"
 #include "hw/usb.h"
 #include "hw/block/flash.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 
 #define FLASH_BASE 0x00000000
@@ -255,7 +255,7 @@ static void r2d_init(MachineState *machine)
     qemu_register_reset(main_cpu_reset, reset_info);
 
     /* Allocate memory space */
-    memory_region_init_ram(sdram, NULL, "r2d.sdram", SDRAM_SIZE);
+    memory_region_init_ram(sdram, NULL, "r2d.sdram", SDRAM_SIZE, &error_abort);
     vmstate_register_ram_global(sdram);
     memory_region_add_subregion(address_space_mem, SDRAM_BASE, sdram);
     /* Register peripherals */
@@ -290,8 +290,8 @@ static void r2d_init(MachineState *machine)
     /* onboard flash memory */
     dinfo = drive_get(IF_PFLASH, 0, 0);
     pflash_cfi02_register(0x0, NULL, "r2d.flash", FLASH_SIZE,
-                          dinfo ? dinfo->bdrv : NULL, (16 * 1024),
-                          FLASH_SIZE >> 16,
+                          dinfo ? blk_by_legacy_dinfo(dinfo) : NULL,
+                          (16 * 1024), FLASH_SIZE >> 16,
                           1, 4, 0x0000, 0x0000, 0x0000, 0x0000,
                           0x555, 0x2aa, 0);
 
diff --git a/hw/sh4/shix.c b/hw/sh4/shix.c
index 7c152b4a3..f93f98e56 100644
--- a/hw/sh4/shix.c
+++ b/hw/sh4/shix.c
@@ -59,14 +59,16 @@ static void shix_init(MachineState *machine)
     }
 
     /* Allocate memory space */
-    memory_region_init_ram(rom, NULL, "shix.rom", 0x4000);
+    memory_region_init_ram(rom, NULL, "shix.rom", 0x4000, &error_abort);
     vmstate_register_ram_global(rom);
     memory_region_set_readonly(rom, true);
     memory_region_add_subregion(sysmem, 0x00000000, rom);
-    memory_region_init_ram(&sdram[0], NULL, "shix.sdram1", 0x01000000);
+    memory_region_init_ram(&sdram[0], NULL, "shix.sdram1", 0x01000000,
+                           &error_abort);
     vmstate_register_ram_global(&sdram[0]);
     memory_region_add_subregion(sysmem, 0x08000000, &sdram[0]);
-    memory_region_init_ram(&sdram[1], NULL, "shix.sdram2", 0x01000000);
+    memory_region_init_ram(&sdram[1], NULL, "shix.sdram2", 0x01000000,
+                           &error_abort);
     vmstate_register_ram_global(&sdram[1]);
     memory_region_add_subregion(sysmem, 0x0c000000, &sdram[1]);
 
diff --git a/hw/sparc/leon3.c b/hw/sparc/leon3.c
index 827383b02..751392e13 100644
--- a/hw/sparc/leon3.c
+++ b/hw/sparc/leon3.c
@@ -151,13 +151,13 @@ static void leon3_generic_hw_init(MachineState *machine)
         exit(1);
     }
 
-    memory_region_init_ram(ram, NULL, "leon3.ram", ram_size);
+    memory_region_init_ram(ram, NULL, "leon3.ram", ram_size, &error_abort);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(address_space_mem, 0x40000000, ram);
 
     /* Allocate BIOS */
     prom_size = 8 * 1024 * 1024; /* 8Mb */
-    memory_region_init_ram(prom, NULL, "Leon3.bios", prom_size);
+    memory_region_init_ram(prom, NULL, "Leon3.bios", prom_size, &error_abort);
     vmstate_register_ram_global(prom);
     memory_region_set_readonly(prom, true);
     memory_region_add_subregion(address_space_mem, 0x00000000, prom);
diff --git a/hw/sparc/sun4m.c b/hw/sparc/sun4m.c
index 67e3663bf..8273199f1 100644
--- a/hw/sparc/sun4m.c
+++ b/hw/sparc/sun4m.c
@@ -40,7 +40,7 @@
 #include "hw/empty_slot.h"
 #include "hw/loader.h"
 #include "elf.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "trace.h"
 
 /*
@@ -527,7 +527,7 @@ static void apc_init(hwaddr power_base, qemu_irq cpu_halt)
     sysbus_connect_irq(s, 0, cpu_halt);
 }
 
-static void tcx_init(hwaddr addr, int vram_size, int width,
+static void tcx_init(hwaddr addr, qemu_irq irq, int vram_size, int width,
                      int height, int depth)
 {
     DeviceState *dev;
@@ -541,25 +541,43 @@ static void tcx_init(hwaddr addr, int vram_size, int width,
     qdev_prop_set_uint64(dev, "prom_addr", addr);
     qdev_init_nofail(dev);
     s = SYS_BUS_DEVICE(dev);
-    /* FCode ROM */
+
+    /* 10/ROM : FCode ROM */
     sysbus_mmio_map(s, 0, addr);
-    /* DAC */
-    sysbus_mmio_map(s, 1, addr + 0x00200000ULL);
-    /* TEC (dummy) */
-    sysbus_mmio_map(s, 2, addr + 0x00700000ULL);
-    /* THC 24 bit: NetBSD writes here even with 8-bit display: dummy */
-    sysbus_mmio_map(s, 3, addr + 0x00301000ULL);
-    /* 8-bit plane */
-    sysbus_mmio_map(s, 4, addr + 0x00800000ULL);
-    if (depth == 24) {
-        /* 24-bit plane */
-        sysbus_mmio_map(s, 5, addr + 0x02000000ULL);
-        /* Control plane */
-        sysbus_mmio_map(s, 6, addr + 0x0a000000ULL);
+    /* 2/STIP : Stipple */
+    sysbus_mmio_map(s, 1, addr + 0x04000000ULL);
+    /* 3/BLIT : Blitter */
+    sysbus_mmio_map(s, 2, addr + 0x06000000ULL);
+    /* 5/RSTIP : Raw Stipple */
+    sysbus_mmio_map(s, 3, addr + 0x0c000000ULL);
+    /* 6/RBLIT : Raw Blitter */
+    sysbus_mmio_map(s, 4, addr + 0x0e000000ULL);
+    /* 7/TEC : Transform Engine */
+    sysbus_mmio_map(s, 5, addr + 0x00700000ULL);
+    /* 8/CMAP  : DAC */
+    sysbus_mmio_map(s, 6, addr + 0x00200000ULL);
+    /* 9/THC : */
+    if (depth == 8) {
+        sysbus_mmio_map(s, 7, addr + 0x00300000ULL);
     } else {
-        /* THC 8 bit (dummy) */
-        sysbus_mmio_map(s, 5, addr + 0x00300000ULL);
+        sysbus_mmio_map(s, 7, addr + 0x00301000ULL);
     }
+    /* 11/DHC : */
+    sysbus_mmio_map(s, 8, addr + 0x00240000ULL);
+    /* 12/ALT : */
+    sysbus_mmio_map(s, 9, addr + 0x00280000ULL);
+    /* 0/DFB8 : 8-bit plane */
+    sysbus_mmio_map(s, 10, addr + 0x00800000ULL);
+    /* 1/DFB24 : 24bit plane */
+    sysbus_mmio_map(s, 11, addr + 0x02000000ULL);
+    /* 4/RDFB32: Raw framebuffer. Control plane */
+    sysbus_mmio_map(s, 12, addr + 0x0a000000ULL);
+    /* 9/THC24bits : NetBSD writes here even with 8-bit display: dummy */
+    if (depth == 8) {
+        sysbus_mmio_map(s, 13, addr + 0x00301000ULL);
+    }
+
+    sysbus_connect_irq(s, 0, irq);
 }
 
 static void cg3_init(hwaddr addr, qemu_irq irq, int vram_size, int width,
@@ -621,7 +639,7 @@ static int idreg_init1(SysBusDevice *dev)
     IDRegState *s = MACIO_ID_REGISTER(dev);
 
     memory_region_init_ram(&s->mem, OBJECT(s),
-                           "sun4m.idreg", sizeof(idreg_data));
+                           "sun4m.idreg", sizeof(idreg_data), &error_abort);
     vmstate_register_ram_global(&s->mem);
     memory_region_set_readonly(&s->mem, true);
     sysbus_init_mmio(dev, &s->mem);
@@ -668,7 +686,7 @@ static int afx_init1(SysBusDevice *dev)
 {
     AFXState *s = TCX_AFX(dev);
 
-    memory_region_init_ram(&s->mem, OBJECT(s), "sun4m.afx", 4);
+    memory_region_init_ram(&s->mem, OBJECT(s), "sun4m.afx", 4, &error_abort);
     vmstate_register_ram_global(&s->mem);
     sysbus_init_mmio(dev, &s->mem);
     return 0;
@@ -742,7 +760,8 @@ static int prom_init1(SysBusDevice *dev)
 {
     PROMState *s = OPENPROM(dev);
 
-    memory_region_init_ram(&s->prom, OBJECT(s), "sun4m.prom", PROM_SIZE_MAX);
+    memory_region_init_ram(&s->prom, OBJECT(s), "sun4m.prom", PROM_SIZE_MAX,
+                           &error_abort);
     vmstate_register_ram_global(&s->prom);
     memory_region_set_readonly(&s->prom, true);
     sysbus_init_mmio(dev, &s->prom);
@@ -784,7 +803,8 @@ static int ram_init1(SysBusDevice *dev)
 {
     RamDevice *d = SUN4M_RAM(dev);
 
-    memory_region_init_ram(&d->ram, OBJECT(d), "sun4m.ram", d->size);
+    memory_region_init_ram(&d->ram, OBJECT(d), "sun4m.ram", d->size,
+                           &error_abort);
     vmstate_register_ram_global(&d->ram);
     sysbus_init_mmio(dev, &d->ram);
     return 0;
@@ -974,8 +994,8 @@ static void sun4m_hw_init(const struct sun4m_hwdef *hwdef,
                 exit(1);
             }
 
-            tcx_init(hwdef->tcx_base, 0x00100000, graphic_width, graphic_height,
-                     graphic_depth);
+            tcx_init(hwdef->tcx_base, slavio_irq[11], 0x00100000,
+                     graphic_width, graphic_height, graphic_depth);
         }
     }
 
diff --git a/hw/sparc64/sun4u.c b/hw/sparc64/sun4u.c
index 33c311bf2..f42112c1b 100644
--- a/hw/sparc64/sun4u.c
+++ b/hw/sparc64/sun4u.c
@@ -38,7 +38,7 @@
 #include "hw/ide.h"
 #include "hw/loader.h"
 #include "elf.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 
 //#define DEBUG_IRQ
@@ -609,8 +609,8 @@ pci_ebus_init1(PCIDevice *pci_dev)
                              0, 0x1000000);
     pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->bar0);
     memory_region_init_alias(&s->bar1, OBJECT(s), "bar1", get_system_io(),
-                             0, 0x800000);
-    pci_register_bar(pci_dev, 1, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->bar1);
+                             0, 0x1000);
+    pci_register_bar(pci_dev, 1, PCI_BASE_ADDRESS_SPACE_IO, &s->bar1);
     return 0;
 }
 
@@ -686,7 +686,8 @@ static int prom_init1(SysBusDevice *dev)
 {
     PROMState *s = OPENPROM(dev);
 
-    memory_region_init_ram(&s->prom, OBJECT(s), "sun4u.prom", PROM_SIZE_MAX);
+    memory_region_init_ram(&s->prom, OBJECT(s), "sun4u.prom", PROM_SIZE_MAX,
+                           &error_abort);
     vmstate_register_ram_global(&s->prom);
     memory_region_set_readonly(&s->prom, true);
     sysbus_init_mmio(dev, &s->prom);
@@ -729,7 +730,8 @@ static int ram_init1(SysBusDevice *dev)
 {
     RamDevice *d = SUN4U_RAM(dev);
 
-    memory_region_init_ram(&d->ram, OBJECT(d), "sun4u.ram", d->size);
+    memory_region_init_ram(&d->ram, OBJECT(d), "sun4u.ram", d->size,
+                           &error_abort);
     vmstate_register_ram_global(&d->ram);
     sysbus_init_mmio(dev, &d->ram);
     return 0;
@@ -862,7 +864,7 @@ static void sun4uv_init(MemoryRegion *address_space_mem,
     for(i = 0; i < nb_nics; i++)
         pci_nic_init_nofail(&nd_table[i], pci_bus, "ne2k_pci", NULL);
 
-    ide_drive_get(hd, MAX_IDE_BUS);
+    ide_drive_get(hd, ARRAY_SIZE(hd));
 
     pci_cmd646_ide_init(pci_bus, hd, 1);
 
diff --git a/hw/ssi/xilinx_spi.c b/hw/ssi/xilinx_spi.c
index 207f47a1c..620573cac 100644
--- a/hw/ssi/xilinx_spi.c
+++ b/hw/ssi/xilinx_spi.c
@@ -329,7 +329,7 @@ static int xilinx_spi_init(SysBusDevice *sbd)
     s->spi = ssi_create_bus(dev, "spi");
 
     sysbus_init_irq(sbd, &s->irq);
-    s->cs_lines = g_new(qemu_irq, s->num_cs);
+    s->cs_lines = g_new0(qemu_irq, s->num_cs);
     ssi_auto_connect_slaves(dev, s->cs_lines, s->spi);
     for (i = 0; i < s->num_cs; ++i) {
         sysbus_init_irq(sbd, &s->cs_lines[i]);
diff --git a/hw/timer/imx_epit.c b/hw/timer/imx_epit.c
index c855eba43..ffefc22f4 100644
--- a/hw/timer/imx_epit.c
+++ b/hw/timer/imx_epit.c
@@ -83,7 +83,7 @@ static char const *imx_epit_reg_name(uint32_t reg)
 #define CR_CLKSRC_SHIFT (24)
 #define CR_CLKSRC_MASK  (0x3 << CR_CLKSRC_SHIFT)
 
-#define TIMER_MAX  0XFFFFFFFFUL
+#define EPIT_TIMER_MAX  0XFFFFFFFFUL
 
 /*
  * Exact clock frequencies vary from board to board.
@@ -155,7 +155,7 @@ static void imx_epit_reset(DeviceState *dev)
      */
     s->cr &= (CR_EN|CR_ENMOD|CR_STOPEN|CR_DOZEN|CR_WAITEN|CR_DBGEN);
     s->sr = 0;
-    s->lr = TIMER_MAX;
+    s->lr = EPIT_TIMER_MAX;
     s->cmp = 0;
     s->cnt = 0;
     /* stop both timers */
@@ -163,9 +163,9 @@ static void imx_epit_reset(DeviceState *dev)
     ptimer_stop(s->timer_reload);
     /* compute new frequency */
     imx_epit_set_freq(s);
-    /* init both timers to TIMER_MAX */
-    ptimer_set_limit(s->timer_cmp, TIMER_MAX, 1);
-    ptimer_set_limit(s->timer_reload, TIMER_MAX, 1);
+    /* init both timers to EPIT_TIMER_MAX */
+    ptimer_set_limit(s->timer_cmp, EPIT_TIMER_MAX, 1);
+    ptimer_set_limit(s->timer_reload, EPIT_TIMER_MAX, 1);
     if (s->freq && (s->cr & CR_EN)) {
         /* if the timer is still enabled, restart it */
         ptimer_run(s->timer_reload, 0);
@@ -227,7 +227,7 @@ static void imx_epit_reload_compare_timer(IMXEPITState *s)
             /* It'll fire in this round of the timer */
             next = tmp - s->cmp;
         } else { /* catch it next time around */
-            next = tmp - s->cmp + ((s->cr & CR_RLD) ? TIMER_MAX : s->lr);
+            next = tmp - s->cmp + ((s->cr & CR_RLD) ? EPIT_TIMER_MAX : s->lr);
         }
         ptimer_set_count(s->timer_cmp, next);
     }
@@ -260,8 +260,8 @@ static void imx_epit_write(void *opaque, hwaddr offset, uint64_t value,
                     ptimer_set_limit(s->timer_reload, s->lr, 1);
                     ptimer_set_limit(s->timer_cmp, s->lr, 1);
                 } else {
-                    ptimer_set_limit(s->timer_reload, TIMER_MAX, 1);
-                    ptimer_set_limit(s->timer_cmp, TIMER_MAX, 1);
+                    ptimer_set_limit(s->timer_reload, EPIT_TIMER_MAX, 1);
+                    ptimer_set_limit(s->timer_cmp, EPIT_TIMER_MAX, 1);
                 }
             }
 
diff --git a/hw/timer/imx_gpt.c b/hw/timer/imx_gpt.c
index 56ee4db99..3b3101084 100644
--- a/hw/timer/imx_gpt.c
+++ b/hw/timer/imx_gpt.c
@@ -80,11 +80,11 @@ static char const *imx_gpt_reg_name(uint32_t reg)
  * GPT : General purpose timer
  *
  * This timer counts up continuously while it is enabled, resetting itself
- * to 0 when it reaches TIMER_MAX (in freerun mode) or when it
+ * to 0 when it reaches GPT_TIMER_MAX (in freerun mode) or when it
  * reaches the value of one of the ocrX (in periodic mode).
  */
 
-#define TIMER_MAX  0XFFFFFFFFUL
+#define GPT_TIMER_MAX  0XFFFFFFFFUL
 
 /* Control register.  Not all of these bits have any effect (yet) */
 #define GPT_CR_EN     (1 << 0)  /* GPT Enable */
@@ -218,7 +218,7 @@ static inline uint32_t imx_gpt_find_limit(uint32_t count, uint32_t reg,
 
 static void imx_gpt_compute_next_timeout(IMXGPTState *s, bool event)
 {
-    uint32_t timeout = TIMER_MAX;
+    uint32_t timeout = GPT_TIMER_MAX;
     uint32_t count = 0;
     long long limit;
 
@@ -230,10 +230,10 @@ static void imx_gpt_compute_next_timeout(IMXGPTState *s, bool event)
     if (event) {
         /* This is a timer event  */
 
-        if ((s->cr & GPT_CR_FRR)  && (s->next_timeout != TIMER_MAX)) {
+        if ((s->cr & GPT_CR_FRR)  && (s->next_timeout != GPT_TIMER_MAX)) {
             /*
              * if we are in free running mode and we have not reached
-             * the TIMER_MAX limit, then update the count
+             * the GPT_TIMER_MAX limit, then update the count
              */
             count = imx_gpt_update_count(s);
         }
@@ -267,7 +267,7 @@ static void imx_gpt_compute_next_timeout(IMXGPTState *s, bool event)
     if ((s->ir & GPT_IR_OF3IE) && (timeout == s->ocr3)) {
         s->next_int |= GPT_SR_OF3;
     }
-    if ((s->ir & GPT_IR_ROVIE) && (timeout == TIMER_MAX)) {
+    if ((s->ir & GPT_IR_ROVIE) && (timeout == GPT_TIMER_MAX)) {
         s->next_int |= GPT_SR_ROV;
     }
 
@@ -370,20 +370,20 @@ static void imx_gpt_reset(DeviceState *dev)
     s->pr = 0;
     s->ir = 0;
     s->cnt = 0;
-    s->ocr1 = TIMER_MAX;
-    s->ocr2 = TIMER_MAX;
-    s->ocr3 = TIMER_MAX;
+    s->ocr1 = GPT_TIMER_MAX;
+    s->ocr2 = GPT_TIMER_MAX;
+    s->ocr3 = GPT_TIMER_MAX;
     s->icr1 = 0;
     s->icr2 = 0;
 
-    s->next_timeout = TIMER_MAX;
+    s->next_timeout = GPT_TIMER_MAX;
     s->next_int = 0;
 
     /* compute new freq */
     imx_gpt_set_freq(s);
 
-    /* reset the limit to TIMER_MAX */
-    ptimer_set_limit(s->timer, TIMER_MAX, 1);
+    /* reset the limit to GPT_TIMER_MAX */
+    ptimer_set_limit(s->timer, GPT_TIMER_MAX, 1);
 
     /* if the timer is still enabled, restart it */
     if (s->freq && (s->cr & GPT_CR_EN)) {
@@ -415,8 +415,8 @@ static void imx_gpt_write(void *opaque, hwaddr offset, uint64_t value,
             if ((oldreg ^ s->cr) & GPT_CR_EN) {
                 if (s->cr & GPT_CR_EN) {
                     if (s->cr & GPT_CR_ENMOD) {
-                        s->next_timeout = TIMER_MAX;
-                        ptimer_set_count(s->timer, TIMER_MAX);
+                        s->next_timeout = GPT_TIMER_MAX;
+                        ptimer_set_count(s->timer, GPT_TIMER_MAX);
                         imx_gpt_compute_next_timeout(s, false);
                     }
                     ptimer_run(s->timer, 1);
@@ -451,8 +451,8 @@ static void imx_gpt_write(void *opaque, hwaddr offset, uint64_t value,
 
         /* In non-freerun mode, reset count when this register is written */
         if (!(s->cr & GPT_CR_FRR)) {
-            s->next_timeout = TIMER_MAX;
-            ptimer_set_limit(s->timer, TIMER_MAX, 1);
+            s->next_timeout = GPT_TIMER_MAX;
+            ptimer_set_limit(s->timer, GPT_TIMER_MAX, 1);
         }
 
         /* compute the new timeout */
diff --git a/hw/timer/mc146818rtc.c b/hw/timer/mc146818rtc.c
index 233fc70d6..f18d1281c 100644
--- a/hw/timer/mc146818rtc.c
+++ b/hw/timer/mc146818rtc.c
@@ -391,7 +391,7 @@ static void cmos_ioport_write(void *opaque, hwaddr addr,
     if ((addr & 1) == 0) {
         s->cmos_index = data & 0x7f;
     } else {
-        CMOS_DPRINTF("cmos: write index=0x%02x val=0x%02x\n",
+        CMOS_DPRINTF("cmos: write index=0x%02x val=0x%02" PRIx64 "\n",
                      s->cmos_index, data);
         switch(s->cmos_index) {
         case RTC_SECONDS_ALARM:
@@ -733,6 +733,22 @@ static int rtc_post_load(void *opaque, int version_id)
     return 0;
 }
 
+static const VMStateDescription vmstate_rtc_irq_reinject_on_ack_count = {
+    .name = "irq_reinject_on_ack_count",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT16(irq_reinject_on_ack_count, RTCState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool rtc_irq_reinject_on_ack_count_needed(void *opaque)
+{
+    RTCState *s = (RTCState *)opaque;
+    return s->irq_reinject_on_ack_count != 0;
+}
+
 static const VMStateDescription vmstate_rtc = {
     .name = "mc146818rtc",
     .version_id = 3,
@@ -753,6 +769,14 @@ static const VMStateDescription vmstate_rtc = {
         VMSTATE_TIMER_V(update_timer, RTCState, 3),
         VMSTATE_UINT64_V(next_alarm_time, RTCState, 3),
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (VMStateSubsection[]) {
+        {
+            .vmsd = &vmstate_rtc_irq_reinject_on_ack_count,
+            .needed = rtc_irq_reinject_on_ack_count_needed,
+        }, {
+            /* empty */
+        }
     }
 };
 
@@ -792,6 +816,7 @@ static void rtc_reset(void *opaque)
 #ifdef TARGET_I386
     if (s->lost_tick_policy == LOST_TICK_POLICY_SLEW) {
         s->irq_coalesced = 0;
+        s->irq_reinject_on_ack_count = 0;		
     }
 #endif
 }
diff --git a/hw/timer/tusb6010.c b/hw/timer/tusb6010.c
index bd2a89e02..459c748e1 100644
--- a/hw/timer/tusb6010.c
+++ b/hw/timer/tusb6010.c
@@ -282,9 +282,6 @@ static void tusb_gpio_intr_update(TUSBState *s)
     /* TODO: How is this signalled?  */
 }
 
-extern CPUReadMemoryFunc * const musb_read[];
-extern CPUWriteMemoryFunc * const musb_write[];
-
 static uint32_t tusb_async_readb(void *opaque, hwaddr addr)
 {
     TUSBState *s = (TUSBState *) opaque;
diff --git a/hw/tpm/tpm_tis.c b/hw/tpm/tpm_tis.c
index 6f0a4d281..c0e7cd738 100644
--- a/hw/tpm/tpm_tis.c
+++ b/hw/tpm/tpm_tis.c
@@ -21,7 +21,7 @@
 
 #include "sysemu/tpm_backend.h"
 #include "tpm_int.h"
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
@@ -896,14 +896,6 @@ static void tpm_tis_initfn(Object *obj)
                                 &s->mmio);
 }
 
-static void tpm_tis_uninitfn(Object *obj)
-{
-    TPMState *s = TPM(obj);
-
-    memory_region_del_subregion(get_system_memory(), &s->mmio);
-    memory_region_destroy(&s->mmio);
-}
-
 static void tpm_tis_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -919,7 +911,6 @@ static const TypeInfo tpm_tis_info = {
     .parent = TYPE_ISA_DEVICE,
     .instance_size = sizeof(TPMState),
     .instance_init = tpm_tis_initfn,
-    .instance_finalize = tpm_tis_uninitfn,
     .class_init  = tpm_tis_class_init,
 };
 
diff --git a/hw/tpm/tpm_tis.h b/hw/tpm/tpm_tis.h
index 916152ae3..1a0db2336 100644
--- a/hw/tpm/tpm_tis.h
+++ b/hw/tpm/tpm_tis.h
@@ -18,23 +18,17 @@
 #define TPM_TPM_TIS_H
 
 #include "hw/isa/isa.h"
+#include "hw/acpi/tpm.h"
 #include "qemu-common.h"
 
-#define TPM_TIS_ADDR_BASE           0xFED40000
-
 #define TPM_TIS_NUM_LOCALITIES      5     /* per spec */
 #define TPM_TIS_LOCALITY_SHIFT      12
 #define TPM_TIS_NO_LOCALITY         0xff
 
 #define TPM_TIS_IS_VALID_LOCTY(x)   ((x) < TPM_TIS_NUM_LOCALITIES)
 
-#define TPM_TIS_IRQ                 5
-
 #define TPM_TIS_BUFFER_MAX          4096
 
-#define TYPE_TPM_TIS                "tpm-tis"
-
-
 typedef enum {
     TPM_TIS_STATE_IDLE = 0,
     TPM_TIS_STATE_READY,
diff --git a/hw/tricore/Makefile.objs b/hw/tricore/Makefile.objs
new file mode 100644
index 000000000..435e095cf
--- /dev/null
+++ b/hw/tricore/Makefile.objs
@@ -0,0 +1 @@
+obj-y += tricore_testboard.o
diff --git a/hw/tricore/tricore_testboard.c b/hw/tricore/tricore_testboard.c
new file mode 100644
index 000000000..a059a20a3
--- /dev/null
+++ b/hw/tricore/tricore_testboard.c
@@ -0,0 +1,124 @@
+/*
+ * TriCore Baseboard System emulation.
+ *
+ * Copyright (c) 2013-2014 Bastian Koppelmann C-Lab/University Paderborn
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "hw/hw.h"
+#include "hw/devices.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
+#include "hw/boards.h"
+#include "hw/loader.h"
+#include "sysemu/block-backend.h"
+#include "exec/address-spaces.h"
+#include "hw/block/flash.h"
+#include "elf.h"
+#include "hw/tricore/tricore.h"
+#include "qemu/error-report.h"
+
+
+/* Board init.  */
+
+static struct tricore_boot_info tricoretb_binfo;
+
+static void tricore_load_kernel(CPUTriCoreState *env)
+{
+    uint64_t entry;
+    long kernel_size;
+
+    kernel_size = load_elf(tricoretb_binfo.kernel_filename, NULL,
+                           NULL, (uint64_t *)&entry, NULL,
+                           NULL, 0,
+                           ELF_MACHINE, 1);
+    if (kernel_size <= 0) {
+        error_report("qemu: no kernel file '%s'",
+                tricoretb_binfo.kernel_filename);
+        exit(1);
+    }
+    env->PC = entry;
+
+}
+
+static void tricore_testboard_init(MachineState *machine, int board_id)
+{
+    TriCoreCPU *cpu;
+    CPUTriCoreState *env;
+
+    MemoryRegion *sysmem = get_system_memory();
+    MemoryRegion *ext_cram = g_new(MemoryRegion, 1);
+    MemoryRegion *ext_dram = g_new(MemoryRegion, 1);
+    MemoryRegion *int_cram = g_new(MemoryRegion, 1);
+    MemoryRegion *int_dram = g_new(MemoryRegion, 1);
+    MemoryRegion *pcp_data = g_new(MemoryRegion, 1);
+    MemoryRegion *pcp_text = g_new(MemoryRegion, 1);
+
+    if (!machine->cpu_model) {
+        machine->cpu_model = "tc1796";
+    }
+    cpu = cpu_tricore_init(machine->cpu_model);
+    if (!cpu) {
+        error_report("Unable to find CPU definition");
+        exit(1);
+    }
+    env = &cpu->env;
+    memory_region_init_ram(ext_cram, NULL, "powerlink_ext_c.ram", 2*1024*1024, &error_abort);
+    vmstate_register_ram_global(ext_cram);
+    memory_region_init_ram(ext_dram, NULL, "powerlink_ext_d.ram", 4*1024*1024, &error_abort);
+    vmstate_register_ram_global(ext_dram);
+    memory_region_init_ram(int_cram, NULL, "powerlink_int_c.ram", 48*1024, &error_abort);
+    vmstate_register_ram_global(int_cram);
+    memory_region_init_ram(int_dram, NULL, "powerlink_int_d.ram", 48*1024, &error_abort);
+    vmstate_register_ram_global(int_dram);
+    memory_region_init_ram(pcp_data, NULL, "powerlink_pcp_data.ram", 16*1024, &error_abort);
+    vmstate_register_ram_global(pcp_data);
+    memory_region_init_ram(pcp_text, NULL, "powerlink_pcp_text.ram", 32*1024, &error_abort);
+    vmstate_register_ram_global(pcp_text);
+
+    memory_region_add_subregion(sysmem, 0x80000000, ext_cram);
+    memory_region_add_subregion(sysmem, 0xa1000000, ext_dram);
+    memory_region_add_subregion(sysmem, 0xd4000000, int_cram);
+    memory_region_add_subregion(sysmem, 0xd0000000, int_dram);
+    memory_region_add_subregion(sysmem, 0xf0050000, pcp_data);
+    memory_region_add_subregion(sysmem, 0xf0060000, pcp_text);
+
+    tricoretb_binfo.ram_size = machine->ram_size;
+    tricoretb_binfo.kernel_filename = machine->kernel_filename;
+
+    if (machine->kernel_filename) {
+        tricore_load_kernel(env);
+    }
+}
+
+static void tricoreboard_init(MachineState *machine)
+{
+    tricore_testboard_init(machine, 0x183);
+}
+
+static QEMUMachine ttb_machine = {
+    .name = "tricore_testboard",
+    .desc = "a minimal TriCore board",
+    .init = tricoreboard_init,
+    .is_default = 0,
+};
+
+static void tricore_testboard_machine_init(void)
+{
+    qemu_register_machine(&ttb_machine);
+}
+
+machine_init(tricore_testboard_machine_init);
diff --git a/hw/unicore32/puv3.c b/hw/unicore32/puv3.c
index 08dd4d04c..c41499e38 100644
--- a/hw/unicore32/puv3.c
+++ b/hw/unicore32/puv3.c
@@ -74,7 +74,8 @@ static void puv3_board_init(CPUUniCore32State *env, ram_addr_t ram_size)
     MemoryRegion *ram_memory = g_new(MemoryRegion, 1);
 
     /* SDRAM at address zero.  */
-    memory_region_init_ram(ram_memory, NULL, "puv3.ram", ram_size);
+    memory_region_init_ram(ram_memory, NULL, "puv3.ram", ram_size,
+                           &error_abort);
     vmstate_register_ram_global(ram_memory);
     memory_region_add_subregion(get_system_memory(), 0, ram_memory);
 }
diff --git a/hw/usb/bus.c b/hw/usb/bus.c
index 927a47bbf..986b2d8da 100644
--- a/hw/usb/bus.c
+++ b/hw/usb/bus.c
@@ -9,7 +9,7 @@ static void usb_bus_dev_print(Monitor *mon, DeviceState *qdev, int indent);
 
 static char *usb_get_dev_path(DeviceState *dev);
 static char *usb_get_fw_dev_path(DeviceState *qdev);
-static int usb_qdev_exit(DeviceState *qdev);
+static void usb_qdev_unrealize(DeviceState *qdev, Error **errp);
 
 static Property usb_props[] = {
     DEFINE_PROP_STRING("port", USBDevice, port_path),
@@ -24,10 +24,12 @@ static Property usb_props[] = {
 static void usb_bus_class_init(ObjectClass *klass, void *data)
 {
     BusClass *k = BUS_CLASS(klass);
+    HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass);
 
     k->print_dev = usb_bus_dev_print;
     k->get_dev_path = usb_get_dev_path;
     k->get_fw_dev_path = usb_get_fw_dev_path;
+    hc->unplug = qdev_simple_device_unplug_cb;
 }
 
 static const TypeInfo usb_bus_info = {
@@ -35,6 +37,10 @@ static const TypeInfo usb_bus_info = {
     .parent = TYPE_BUS,
     .instance_size = sizeof(USBBus),
     .class_init = usb_bus_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_HOTPLUG_HANDLER },
+        { }
+    }
 };
 
 static int next_usb_bus = 0;
@@ -79,14 +85,21 @@ void usb_bus_new(USBBus *bus, size_t bus_size,
                  USBBusOps *ops, DeviceState *host)
 {
     qbus_create_inplace(bus, bus_size, TYPE_USB_BUS, host, NULL);
+    qbus_set_bus_hotplug_handler(BUS(bus), &error_abort);
     bus->ops = ops;
     bus->busnr = next_usb_bus++;
-    bus->qbus.allow_hotplug = 1; /* Yes, we can */
     QTAILQ_INIT(&bus->free);
     QTAILQ_INIT(&bus->used);
     QTAILQ_INSERT_TAIL(&busses, bus, next);
 }
 
+void usb_bus_release(USBBus *bus)
+{
+    assert(next_usb_bus > 0);
+
+    QTAILQ_REMOVE(&busses, bus, next);
+}
+
 USBBus *usb_bus_find(int busnr)
 {
     USBBus *bus;
@@ -100,13 +113,13 @@ USBBus *usb_bus_find(int busnr)
     return NULL;
 }
 
-static int usb_device_init(USBDevice *dev)
+static void usb_device_realize(USBDevice *dev, Error **errp)
 {
     USBDeviceClass *klass = USB_DEVICE_GET_CLASS(dev);
-    if (klass->init) {
-        return klass->init(dev);
+
+    if (klass->realize) {
+        klass->realize(dev, errp);
     }
-    return 0;
 }
 
 USBDevice *usb_device_find_device(USBDevice *dev, uint8_t addr)
@@ -225,36 +238,41 @@ void usb_device_free_streams(USBDevice *dev, USBEndpoint **eps, int nr_eps)
     }
 }
 
-static int usb_qdev_init(DeviceState *qdev)
+static void usb_qdev_realize(DeviceState *qdev, Error **errp)
 {
     USBDevice *dev = USB_DEVICE(qdev);
-    int rc;
+    Error *local_err = NULL;
 
     pstrcpy(dev->product_desc, sizeof(dev->product_desc),
             usb_device_get_product_desc(dev));
     dev->auto_attach = 1;
     QLIST_INIT(&dev->strings);
     usb_ep_init(dev);
-    rc = usb_claim_port(dev);
-    if (rc != 0) {
-        return rc;
+
+    usb_claim_port(dev, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
     }
-    rc = usb_device_init(dev);
-    if (rc != 0) {
+
+    usb_device_realize(dev, &local_err);
+    if (local_err) {
         usb_release_port(dev);
-        return rc;
+        error_propagate(errp, local_err);
+        return;
     }
+
     if (dev->auto_attach) {
-        rc = usb_device_attach(dev);
-        if (rc != 0) {
-            usb_qdev_exit(qdev);
-            return rc;
+        usb_device_attach(dev, &local_err);
+        if (local_err) {
+            usb_qdev_unrealize(qdev, NULL);
+            error_propagate(errp, local_err);
+            return;
         }
     }
-    return 0;
 }
 
-static int usb_qdev_exit(DeviceState *qdev)
+static void usb_qdev_unrealize(DeviceState *qdev, Error **errp)
 {
     USBDevice *dev = USB_DEVICE(qdev);
 
@@ -265,7 +283,6 @@ static int usb_qdev_exit(DeviceState *qdev)
     if (dev->port) {
         usb_release_port(dev);
     }
-    return 0;
 }
 
 typedef struct LegacyUSBFactory
@@ -385,7 +402,7 @@ void usb_unregister_port(USBBus *bus, USBPort *port)
     bus->nfree--;
 }
 
-int usb_claim_port(USBDevice *dev)
+void usb_claim_port(USBDevice *dev, Error **errp)
 {
     USBBus *bus = usb_bus_from_device(dev);
     USBPort *port;
@@ -399,9 +416,9 @@ int usb_claim_port(USBDevice *dev)
             }
         }
         if (port == NULL) {
-            error_report("Error: usb port %s (bus %s) not found (in use?)",
-                         dev->port_path, bus->qbus.name);
-            return -1;
+            error_setg(errp, "Error: usb port %s (bus %s) not found (in use?)",
+                       dev->port_path, bus->qbus.name);
+            return;
         }
     } else {
         if (bus->nfree == 1 && strcmp(object_get_typename(OBJECT(dev)), "usb-hub") != 0) {
@@ -409,9 +426,9 @@ int usb_claim_port(USBDevice *dev)
             usb_create_simple(bus, "usb-hub");
         }
         if (bus->nfree == 0) {
-            error_report("Error: tried to attach usb device %s to a bus "
-                         "with no free ports", dev->product_desc);
-            return -1;
+            error_setg(errp, "Error: tried to attach usb device %s to a bus "
+                       "with no free ports", dev->product_desc);
+            return;
         }
         port = QTAILQ_FIRST(&bus->free);
     }
@@ -425,7 +442,6 @@ int usb_claim_port(USBDevice *dev)
 
     QTAILQ_INSERT_TAIL(&bus->used, port, next);
     bus->nused++;
-    return 0;
 }
 
 void usb_release_port(USBDevice *dev)
@@ -468,7 +484,7 @@ static void usb_mask_to_str(char *dest, size_t size,
     }
 }
 
-int usb_device_attach(USBDevice *dev)
+void usb_check_attach(USBDevice *dev, Error **errp)
 {
     USBBus *bus = usb_bus_from_device(dev);
     USBPort *port = dev->port;
@@ -482,18 +498,28 @@ int usb_device_attach(USBDevice *dev)
                           devspeed, portspeed);
 
     if (!(port->speedmask & dev->speedmask)) {
-        error_report("Warning: speed mismatch trying to attach"
-                     " usb device \"%s\" (%s speed)"
-                     " to bus \"%s\", port \"%s\" (%s speed)",
-                     dev->product_desc, devspeed,
-                     bus->qbus.name, port->path, portspeed);
-        return -1;
+        error_setg(errp, "Warning: speed mismatch trying to attach"
+                   " usb device \"%s\" (%s speed)"
+                   " to bus \"%s\", port \"%s\" (%s speed)",
+                   dev->product_desc, devspeed,
+                   bus->qbus.name, port->path, portspeed);
+        return;
+    }
+}
+
+void usb_device_attach(USBDevice *dev, Error **errp)
+{
+    USBPort *port = dev->port;
+    Error *local_err = NULL;
+
+    usb_check_attach(dev, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
     }
 
     dev->attached++;
     usb_attach(port);
-
-    return 0;
 }
 
 int usb_device_detach(USBDevice *dev)
@@ -589,11 +615,11 @@ static char *usb_get_fw_dev_path(DeviceState *qdev)
         nr = strtol(in, &in, 10);
         if (in[0] == '.') {
             /* some hub between root port and device */
-            pos += snprintf(fw_path + pos, fw_len - pos, "hub@%ld/", nr);
+            pos += snprintf(fw_path + pos, fw_len - pos, "hub@%lx/", nr);
             in++;
         } else {
             /* the device itself */
-            pos += snprintf(fw_path + pos, fw_len - pos, "%s@%ld",
+            pos += snprintf(fw_path + pos, fw_len - pos, "%s@%lx",
                             qdev_fw_name(qdev), nr);
             break;
         }
@@ -681,9 +707,8 @@ static void usb_device_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *k = DEVICE_CLASS(klass);
     k->bus_type = TYPE_USB_BUS;
-    k->init     = usb_qdev_init;
-    k->unplug   = qdev_simple_unplug_cb;
-    k->exit     = usb_qdev_exit;
+    k->realize  = usb_qdev_realize;
+    k->unrealize = usb_qdev_unrealize;
     k->props    = usb_props;
 }
 
diff --git a/hw/usb/dev-audio.c b/hw/usb/dev-audio.c
index bfebfe90f..67deffebc 100644
--- a/hw/usb/dev-audio.c
+++ b/hw/usb/dev-audio.c
@@ -371,7 +371,7 @@ static void output_callback(void *opaque, int avail)
             return;
         }
         data = streambuf_get(&s->out.buf);
-        if (NULL == data) {
+        if (!data) {
             return;
         }
         AUD_write(s->out.voice, data, USBAUDIO_PACKET_SIZE);
@@ -628,7 +628,7 @@ static void usb_audio_handle_destroy(USBDevice *dev)
     streambuf_fini(&s->out.buf);
 }
 
-static int usb_audio_initfn(USBDevice *dev)
+static void usb_audio_realize(USBDevice *dev, Error **errp)
 {
     USBAudioState *s = DO_UPCAST(USBAudioState, dev, dev);
 
@@ -651,7 +651,6 @@ static int usb_audio_initfn(USBDevice *dev)
                                 s, output_callback, &s->out.as);
     AUD_set_volume_out(s->out.voice, s->out.mute, s->out.vol[0], s->out.vol[1]);
     AUD_set_active_out(s->out.voice, 0);
-    return 0;
 }
 
 static const VMStateDescription vmstate_usb_audio = {
@@ -676,7 +675,7 @@ static void usb_audio_class_init(ObjectClass *klass, void *data)
     set_bit(DEVICE_CATEGORY_SOUND, dc->categories);
     k->product_desc   = "QEMU USB Audio Interface";
     k->usb_desc       = &desc_audio;
-    k->init           = usb_audio_initfn;
+    k->realize        = usb_audio_realize;
     k->handle_reset   = usb_audio_handle_reset;
     k->handle_control = usb_audio_handle_control;
     k->handle_data    = usb_audio_handle_data;
diff --git a/hw/usb/dev-bluetooth.c b/hw/usb/dev-bluetooth.c
index a76e58191..390d475c1 100644
--- a/hw/usb/dev-bluetooth.c
+++ b/hw/usb/dev-bluetooth.c
@@ -501,7 +501,7 @@ static void usb_bt_handle_destroy(USBDevice *dev)
     s->hci->acl_recv = NULL;
 }
 
-static int usb_bt_initfn(USBDevice *dev)
+static void usb_bt_realize(USBDevice *dev, Error **errp)
 {
     struct USBBtState *s = DO_UPCAST(struct USBBtState, dev, dev);
 
@@ -516,8 +516,6 @@ static int usb_bt_initfn(USBDevice *dev)
     s->hci->acl_recv = usb_bt_out_hci_packet_acl;
     usb_bt_handle_reset(&s->dev);
     s->intr = usb_ep_get(dev, USB_TOKEN_IN, USB_EVT_EP);
-
-    return 0;
 }
 
 static USBDevice *usb_bt_init(USBBus *bus, const char *cmdline)
@@ -560,7 +558,7 @@ static void usb_bt_class_initfn(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
-    uc->init           = usb_bt_initfn;
+    uc->realize        = usb_bt_realize;
     uc->product_desc   = "QEMU BT dongle";
     uc->usb_desc       = &desc_bluetooth;
     uc->handle_reset   = usb_bt_handle_reset;
diff --git a/hw/usb/dev-hid.c b/hw/usb/dev-hid.c
index 67a57f1dc..507c9663c 100644
--- a/hw/usb/dev-hid.c
+++ b/hw/usb/dev-hid.c
@@ -104,6 +104,37 @@ static const USBDescIface desc_iface_mouse = {
     },
 };
 
+static const USBDescIface desc_iface_mouse2 = {
+    .bInterfaceNumber              = 0,
+    .bNumEndpoints                 = 1,
+    .bInterfaceClass               = USB_CLASS_HID,
+    .bInterfaceSubClass            = 0x01, /* boot */
+    .bInterfaceProtocol            = 0x02,
+    .ndesc                         = 1,
+    .descs = (USBDescOther[]) {
+        {
+            /* HID descriptor */
+            .data = (uint8_t[]) {
+                0x09,          /*  u8  bLength */
+                USB_DT_HID,    /*  u8  bDescriptorType */
+                0x01, 0x00,    /*  u16 HID_class */
+                0x00,          /*  u8  country_code */
+                0x01,          /*  u8  num_descriptors */
+                USB_DT_REPORT, /*  u8  type: Report */
+                52, 0,         /*  u16 len */
+            },
+        },
+    },
+    .eps = (USBDescEndpoint[]) {
+        {
+            .bEndpointAddress      = USB_DIR_IN | 0x01,
+            .bmAttributes          = USB_ENDPOINT_XFER_INT,
+            .wMaxPacketSize        = 4,
+            .bInterval             = 7, /* 2 ^ (8-1) * 125 usecs = 8 ms */
+        },
+    },
+};
+
 static const USBDescIface desc_iface_tablet = {
     .bInterfaceNumber              = 0,
     .bNumEndpoints                 = 1,
@@ -195,6 +226,37 @@ static const USBDescIface desc_iface_keyboard = {
     },
 };
 
+static const USBDescIface desc_iface_keyboard2 = {
+    .bInterfaceNumber              = 0,
+    .bNumEndpoints                 = 1,
+    .bInterfaceClass               = USB_CLASS_HID,
+    .bInterfaceSubClass            = 0x01, /* boot */
+    .bInterfaceProtocol            = 0x01, /* keyboard */
+    .ndesc                         = 1,
+    .descs = (USBDescOther[]) {
+        {
+            /* HID descriptor */
+            .data = (uint8_t[]) {
+                0x09,          /*  u8  bLength */
+                USB_DT_HID,    /*  u8  bDescriptorType */
+                0x11, 0x01,    /*  u16 HID_class */
+                0x00,          /*  u8  country_code */
+                0x01,          /*  u8  num_descriptors */
+                USB_DT_REPORT, /*  u8  type: Report */
+                0x3f, 0,       /*  u16 len */
+            },
+        },
+    },
+    .eps = (USBDescEndpoint[]) {
+        {
+            .bEndpointAddress      = USB_DIR_IN | 0x01,
+            .bmAttributes          = USB_ENDPOINT_XFER_INT,
+            .wMaxPacketSize        = 8,
+            .bInterval             = 7, /* 2 ^ (8-1) * 125 usecs = 8 ms */
+        },
+    },
+};
+
 static const USBDescDevice desc_device_mouse = {
     .bcdUSB                        = 0x0100,
     .bMaxPacketSize0               = 8,
@@ -212,6 +274,23 @@ static const USBDescDevice desc_device_mouse = {
     },
 };
 
+static const USBDescDevice desc_device_mouse2 = {
+    .bcdUSB                        = 0x0200,
+    .bMaxPacketSize0               = 64,
+    .bNumConfigurations            = 1,
+    .confs = (USBDescConfig[]) {
+        {
+            .bNumInterfaces        = 1,
+            .bConfigurationValue   = 1,
+            .iConfiguration        = STR_CONFIG_MOUSE,
+            .bmAttributes          = USB_CFG_ATT_ONE | USB_CFG_ATT_WAKEUP,
+            .bMaxPower             = 50,
+            .nif = 1,
+            .ifs = &desc_iface_mouse2,
+        },
+    },
+};
+
 static const USBDescDevice desc_device_tablet = {
     .bcdUSB                        = 0x0100,
     .bMaxPacketSize0               = 8,
@@ -263,6 +342,23 @@ static const USBDescDevice desc_device_keyboard = {
     },
 };
 
+static const USBDescDevice desc_device_keyboard2 = {
+    .bcdUSB                        = 0x0200,
+    .bMaxPacketSize0               = 64,
+    .bNumConfigurations            = 1,
+    .confs = (USBDescConfig[]) {
+        {
+            .bNumInterfaces        = 1,
+            .bConfigurationValue   = 1,
+            .iConfiguration        = STR_CONFIG_KEYBOARD,
+            .bmAttributes          = USB_CFG_ATT_ONE | USB_CFG_ATT_WAKEUP,
+            .bMaxPower             = 50,
+            .nif = 1,
+            .ifs = &desc_iface_keyboard2,
+        },
+    },
+};
+
 static const USBDescMSOS desc_msos_suspend = {
     .SelectiveSuspendEnabled = true,
 };
@@ -281,6 +377,21 @@ static const USBDesc desc_mouse = {
     .msos = &desc_msos_suspend,
 };
 
+static const USBDesc desc_mouse2 = {
+    .id = {
+        .idVendor          = 0x0627,
+        .idProduct         = 0x0001,
+        .bcdDevice         = 0,
+        .iManufacturer     = STR_MANUFACTURER,
+        .iProduct          = STR_PRODUCT_MOUSE,
+        .iSerialNumber     = STR_SERIALNUMBER,
+    },
+    .full = &desc_device_mouse,
+    .high = &desc_device_mouse2,
+    .str  = desc_strings,
+    .msos = &desc_msos_suspend,
+};
+
 static const USBDesc desc_tablet = {
     .id = {
         .idVendor          = 0x0627,
@@ -324,6 +435,21 @@ static const USBDesc desc_keyboard = {
     .msos = &desc_msos_suspend,
 };
 
+static const USBDesc desc_keyboard2 = {
+    .id = {
+        .idVendor          = 0x0627,
+        .idProduct         = 0x0001,
+        .bcdDevice         = 0,
+        .iManufacturer     = STR_MANUFACTURER,
+        .iProduct          = STR_PRODUCT_KEYBOARD,
+        .iSerialNumber     = STR_SERIALNUMBER,
+    },
+    .full = &desc_device_keyboard,
+    .high = &desc_device_keyboard2,
+    .str  = desc_strings,
+    .msos = &desc_msos_suspend,
+};
+
 static const uint8_t qemu_mouse_hid_report_descriptor[] = {
     0x05, 0x01,		/* Usage Page (Generic Desktop) */
     0x09, 0x02,		/* Usage (Mouse) */
@@ -566,9 +692,26 @@ static void usb_hid_handle_destroy(USBDevice *dev)
     hid_free(&us->hid);
 }
 
-static int usb_hid_initfn(USBDevice *dev, int kind)
+static void usb_hid_initfn(USBDevice *dev, int kind,
+                           const USBDesc *usb1, const USBDesc *usb2,
+                           Error **errp)
 {
     USBHIDState *us = DO_UPCAST(USBHIDState, dev, dev);
+    switch (us->usb_version) {
+    case 1:
+        dev->usb_desc = usb1;
+        break;
+    case 2:
+        dev->usb_desc = usb2;
+        break;
+    default:
+        dev->usb_desc = NULL;
+    }
+    if (!dev->usb_desc) {
+        error_setg(errp, "Invalid usb version %d for usb hid device",
+                   us->usb_version);
+        return;
+    }
 
     if (dev->serial) {
         usb_desc_set_string(dev, STR_SERIALNUMBER, dev->serial);
@@ -579,37 +722,22 @@ static int usb_hid_initfn(USBDevice *dev, int kind)
     if (us->display && us->hid.s) {
         qemu_input_handler_bind(us->hid.s, us->display, us->head, NULL);
     }
-    return 0;
 }
 
-static int usb_tablet_initfn(USBDevice *dev)
+static void usb_tablet_realize(USBDevice *dev, Error **errp)
 {
-    USBHIDState *us = DO_UPCAST(USBHIDState, dev, dev);
 
-    switch (us->usb_version) {
-    case 1:
-        dev->usb_desc = &desc_tablet;
-        break;
-    case 2:
-        dev->usb_desc = &desc_tablet2;
-        break;
-    default:
-        error_report("Invalid usb version %d for usb-tabler (must be 1 or 2)",
-                     us->usb_version);
-        return -1;
-    }
-
-    return usb_hid_initfn(dev, HID_TABLET);
+    usb_hid_initfn(dev, HID_TABLET, &desc_tablet, &desc_tablet2, errp);
 }
 
-static int usb_mouse_initfn(USBDevice *dev)
+static void usb_mouse_realize(USBDevice *dev, Error **errp)
 {
-    return usb_hid_initfn(dev, HID_MOUSE);
+    usb_hid_initfn(dev, HID_MOUSE, &desc_mouse, &desc_mouse2, errp);
 }
 
-static int usb_keyboard_initfn(USBDevice *dev)
+static void usb_keyboard_realize(USBDevice *dev, Error **errp)
 {
-    return usb_hid_initfn(dev, HID_KEYBOARD);
+    usb_hid_initfn(dev, HID_KEYBOARD, &desc_keyboard, &desc_keyboard2, errp);
 }
 
 static int usb_ptr_post_load(void *opaque, int version_id)
@@ -669,7 +797,7 @@ static void usb_tablet_class_initfn(ObjectClass *klass, void *data)
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
     usb_hid_class_initfn(klass, data);
-    uc->init           = usb_tablet_initfn;
+    uc->realize        = usb_tablet_realize;
     uc->product_desc   = "QEMU USB Tablet";
     dc->vmsd = &vmstate_usb_ptr;
     dc->props = usb_tablet_properties;
@@ -683,16 +811,21 @@ static const TypeInfo usb_tablet_info = {
     .class_init    = usb_tablet_class_initfn,
 };
 
+static Property usb_mouse_properties[] = {
+        DEFINE_PROP_UINT32("usb_version", USBHIDState, usb_version, 2),
+        DEFINE_PROP_END_OF_LIST(),
+};
+
 static void usb_mouse_class_initfn(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
     usb_hid_class_initfn(klass, data);
-    uc->init           = usb_mouse_initfn;
+    uc->realize        = usb_mouse_realize;
     uc->product_desc   = "QEMU USB Mouse";
-    uc->usb_desc       = &desc_mouse;
     dc->vmsd = &vmstate_usb_ptr;
+    dc->props = usb_mouse_properties;
     set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
 }
 
@@ -704,6 +837,7 @@ static const TypeInfo usb_mouse_info = {
 };
 
 static Property usb_keyboard_properties[] = {
+        DEFINE_PROP_UINT32("usb_version", USBHIDState, usb_version, 2),
         DEFINE_PROP_STRING("display", USBHIDState, display),
         DEFINE_PROP_END_OF_LIST(),
 };
@@ -714,9 +848,8 @@ static void usb_keyboard_class_initfn(ObjectClass *klass, void *data)
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
     usb_hid_class_initfn(klass, data);
-    uc->init           = usb_keyboard_initfn;
+    uc->realize        = usb_keyboard_realize;
     uc->product_desc   = "QEMU USB Keyboard";
-    uc->usb_desc       = &desc_keyboard;
     dc->vmsd = &vmstate_usb_kbd;
     dc->props = usb_keyboard_properties;
     set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
diff --git a/hw/usb/dev-hub.c b/hw/usb/dev-hub.c
index 749217497..0482f5871 100644
--- a/hw/usb/dev-hub.c
+++ b/hw/usb/dev-hub.c
@@ -511,15 +511,15 @@ static USBPortOps usb_hub_port_ops = {
     .complete = usb_hub_complete,
 };
 
-static int usb_hub_initfn(USBDevice *dev)
+static void usb_hub_realize(USBDevice *dev, Error **errp)
 {
     USBHubState *s = DO_UPCAST(USBHubState, dev, dev);
     USBHubPort *port;
     int i;
 
     if (dev->port->hubcount == 5) {
-        error_report("usb hub chain too deep");
-        return -1;
+        error_setg(errp, "usb hub chain too deep");
+        return;
     }
 
     usb_desc_create_serial(dev);
@@ -533,7 +533,6 @@ static int usb_hub_initfn(USBDevice *dev)
         usb_port_location(&port->port, dev->port, i+1);
     }
     usb_hub_handle_reset(dev);
-    return 0;
 }
 
 static const VMStateDescription vmstate_usb_hub_port = {
@@ -564,7 +563,7 @@ static void usb_hub_class_initfn(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
-    uc->init           = usb_hub_initfn;
+    uc->realize        = usb_hub_realize;
     uc->product_desc   = "QEMU USB Hub";
     uc->usb_desc       = &desc_hub;
     uc->find_device    = usb_hub_find_device;
diff --git a/hw/usb/dev-mtp.c b/hw/usb/dev-mtp.c
index 384d4a5ac..108ece819 100644
--- a/hw/usb/dev-mtp.c
+++ b/hw/usb/dev-mtp.c
@@ -832,7 +832,7 @@ static void usb_mtp_command(MTPState *s, MTPControl *c)
             return;
         }
         data_in = usb_mtp_get_object(s, c, o);
-        if (NULL == data_in) {
+        if (data_in == NULL) {
             usb_mtp_queue_result(s, RES_GENERAL_ERROR,
                                  c->trans, 0, 0, 0);
             return;
@@ -851,7 +851,7 @@ static void usb_mtp_command(MTPState *s, MTPControl *c)
             return;
         }
         data_in = usb_mtp_get_partial_object(s, c, o);
-        if (NULL == data_in) {
+        if (data_in == NULL) {
             usb_mtp_queue_result(s, RES_GENERAL_ERROR,
                                  c->trans, 0, 0, 0);
             return;
@@ -1060,7 +1060,7 @@ static void usb_mtp_handle_data(USBDevice *dev, USBPacket *p)
     }
 }
 
-static int usb_mtp_initfn(USBDevice *dev)
+static void usb_mtp_realize(USBDevice *dev, Error **errp)
 {
     MTPState *s = DO_UPCAST(MTPState, dev, dev);
 
@@ -1075,7 +1075,6 @@ static int usb_mtp_initfn(USBDevice *dev)
             s->desc = g_strdup("none");
         }
     }
-    return 0;
 }
 
 static const VMStateDescription vmstate_usb_mtp = {
@@ -1100,7 +1099,7 @@ static void usb_mtp_class_initfn(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
-    uc->init           = usb_mtp_initfn;
+    uc->realize        = usb_mtp_realize;
     uc->product_desc   = "QEMU USB MTP";
     uc->usb_desc       = &desc;
     uc->cancel_packet  = usb_mtp_cancel_packet;
diff --git a/hw/usb/dev-network.c b/hw/usb/dev-network.c
index 518d5366d..5b95d5c38 100644
--- a/hw/usb/dev-network.c
+++ b/hw/usb/dev-network.c
@@ -27,7 +27,7 @@
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
 #include "net/net.h"
-#include "qapi/qmp/qerror.h"
+#include "qemu/error-report.h"
 #include "qemu/queue.h"
 #include "qemu/config-file.h"
 #include "sysemu/sysemu.h"
@@ -1341,7 +1341,7 @@ static NetClientInfo net_usbnet_info = {
     .cleanup = usbnet_cleanup,
 };
 
-static int usb_net_initfn(USBDevice *dev)
+static void usb_net_realize(USBDevice *dev, Error **errrp)
 {
     USBNetState *s = DO_UPCAST(USBNetState, dev, dev);
 
@@ -1371,9 +1371,16 @@ static int usb_net_initfn(USBDevice *dev)
              s->conf.macaddr.a[4],
              s->conf.macaddr.a[5]);
     usb_desc_set_string(dev, STRING_ETHADDR, s->usbstring_mac);
+}
 
-    add_boot_device_path(s->conf.bootindex, &dev->qdev, "/ethernet@0");
-    return 0;
+static void usb_net_instance_init(Object *obj)
+{
+    USBDevice *dev = USB_DEVICE(obj);
+    USBNetState *s = DO_UPCAST(USBNetState, dev, dev);
+
+    device_add_bootindex_property(obj, &s->conf.bootindex,
+                                  "bootindex", "/ethernet-phy@0",
+                                  &dev->qdev, NULL);
 }
 
 static USBDevice *usb_net_init(USBBus *bus, const char *cmdline)
@@ -1392,7 +1399,7 @@ static USBDevice *usb_net_init(USBBus *bus, const char *cmdline)
 
     idx = net_client_init(opts, 0, &local_err);
     if (local_err) {
-        qerror_report_err(local_err);
+        error_report("%s", error_get_pretty(local_err));
         error_free(local_err);
         return NULL;
     }
@@ -1421,7 +1428,7 @@ static void usb_net_class_initfn(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
-    uc->init           = usb_net_initfn;
+    uc->realize        = usb_net_realize;
     uc->product_desc   = "QEMU USB Network Interface";
     uc->usb_desc       = &desc_net;
     uc->handle_reset   = usb_net_handle_reset;
@@ -1439,6 +1446,7 @@ static const TypeInfo net_info = {
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBNetState),
     .class_init    = usb_net_class_initfn,
+    .instance_init = usb_net_instance_init,
 };
 
 static void usb_net_register_types(void)
diff --git a/hw/usb/dev-serial.c b/hw/usb/dev-serial.c
index d3606142c..1cee45025 100644
--- a/hw/usb/dev-serial.c
+++ b/hw/usb/dev-serial.c
@@ -9,7 +9,7 @@
  */
 
 #include "qemu-common.h"
-#include "qemu/error-report.h"
+#include "monitor/monitor.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
 #include "sysemu/char.h"
@@ -460,7 +460,7 @@ static void usb_serial_event(void *opaque, int event)
             break;
         case CHR_EVENT_OPENED:
             if (!s->dev.attached) {
-                usb_device_attach(&s->dev);
+                usb_device_attach(&s->dev, &error_abort);
             }
             break;
         case CHR_EVENT_CLOSED:
@@ -471,17 +471,24 @@ static void usb_serial_event(void *opaque, int event)
     }
 }
 
-static int usb_serial_initfn(USBDevice *dev)
+static void usb_serial_realize(USBDevice *dev, Error **errp)
 {
     USBSerialState *s = DO_UPCAST(USBSerialState, dev, dev);
+    Error *local_err = NULL;
 
     usb_desc_create_serial(dev);
     usb_desc_init(dev);
     dev->auto_attach = 0;
 
     if (!s->cs) {
-        error_report("Property chardev is required");
-        return -1;
+        error_setg(errp, "Property chardev is required");
+        return;
+    }
+
+    usb_check_attach(dev, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
     }
 
     qemu_chr_add_handlers(s->cs, usb_serial_can_read, usb_serial_read,
@@ -489,9 +496,8 @@ static int usb_serial_initfn(USBDevice *dev)
     usb_serial_handle_reset(dev);
 
     if (s->cs->be_open && !dev->attached) {
-        usb_device_attach(dev);
+        usb_device_attach(dev, &error_abort);
     }
-    return 0;
 }
 
 static USBDevice *usb_serial_init(USBBus *bus, const char *filename)
@@ -582,7 +588,7 @@ static void usb_serial_class_initfn(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
-    uc->init = usb_serial_initfn;
+    uc->realize = usb_serial_realize;
     uc->product_desc   = "QEMU USB Serial";
     uc->usb_desc       = &desc_serial;
     uc->handle_reset   = usb_serial_handle_reset;
@@ -610,7 +616,7 @@ static void usb_braille_class_initfn(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
-    uc->init           = usb_serial_initfn;
+    uc->realize        = usb_serial_realize;
     uc->product_desc   = "QEMU USB Braille";
     uc->usb_desc       = &desc_braille;
     uc->handle_reset   = usb_serial_handle_reset;
diff --git a/hw/usb/dev-smartcard-reader.c b/hw/usb/dev-smartcard-reader.c
index 470e69ffc..78ce68167 100644
--- a/hw/usb/dev-smartcard-reader.c
+++ b/hw/usb/dev-smartcard-reader.c
@@ -1304,7 +1304,7 @@ static int ccid_card_init(DeviceState *qdev)
     return ret;
 }
 
-static int ccid_initfn(USBDevice *dev)
+static void ccid_realize(USBDevice *dev, Error **errp)
 {
     USBCCIDState *s = DO_UPCAST(USBCCIDState, dev, dev);
 
@@ -1312,8 +1312,8 @@ static int ccid_initfn(USBDevice *dev)
     usb_desc_init(dev);
     qbus_create_inplace(&s->bus, sizeof(s->bus), TYPE_CCID_BUS, DEVICE(dev),
                         NULL);
+    qbus_set_hotplug_handler(BUS(&s->bus), DEVICE(dev), &error_abort);
     s->intr = usb_ep_get(dev, USB_TOKEN_IN, CCID_INT_IN_EP);
-    s->bus.qbus.allow_hotplug = 1;
     s->card = NULL;
     s->migration_state = MIGRATION_NONE;
     s->migration_target_ip = 0;
@@ -1332,7 +1332,6 @@ static int ccid_initfn(USBDevice *dev)
     ccid_reset_parameters(s);
     ccid_reset(s);
     s->debug = parse_debug_env("QEMU_CCID_DEBUG", D_VERBOSE, s->debug);
-    return 0;
 }
 
 static int ccid_post_load(void *opaque, int version_id)
@@ -1440,8 +1439,9 @@ static void ccid_class_initfn(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
+    HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass);
 
-    uc->init           = ccid_initfn;
+    uc->realize        = ccid_realize;
     uc->product_desc   = "QEMU USB CCID";
     uc->usb_desc       = &desc_ccid;
     uc->handle_reset   = ccid_handle_reset;
@@ -1452,6 +1452,7 @@ static void ccid_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd = &ccid_vmstate;
     dc->props = ccid_properties;
     set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
+    hc->unplug = qdev_simple_device_unplug_cb;
 }
 
 static const TypeInfo ccid_info = {
@@ -1459,6 +1460,10 @@ static const TypeInfo ccid_info = {
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBCCIDState),
     .class_init    = ccid_class_initfn,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_HOTPLUG_HANDLER },
+        { }
+    }
 };
 
 static void ccid_card_class_init(ObjectClass *klass, void *data)
diff --git a/hw/usb/dev-storage.c b/hw/usb/dev-storage.c
index ae4efcbd2..4539733e4 100644
--- a/hw/usb/dev-storage.c
+++ b/hw/usb/dev-storage.c
@@ -16,7 +16,9 @@
 #include "ui/console.h"
 #include "monitor/monitor.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
+#include "qapi/visitor.h"
 
 //#define DEBUG_MSD
 
@@ -59,6 +61,7 @@ typedef struct {
     /* usb-storage only */
     BlockConf conf;
     uint32_t removable;
+    SCSIDevice *scsi_dev;
 } MSDState;
 
 struct usb_msd_cbw {
@@ -409,19 +412,19 @@ static void usb_msd_handle_data(USBDevice *dev, USBPacket *p)
         switch (s->mode) {
         case USB_MSDM_CBW:
             if (p->iov.size != 31) {
-                fprintf(stderr, "usb-msd: Bad CBW size");
+                error_report("usb-msd: Bad CBW size");
                 goto fail;
             }
             usb_packet_copy(p, &cbw, 31);
             if (le32_to_cpu(cbw.sig) != 0x43425355) {
-                fprintf(stderr, "usb-msd: Bad signature %08x\n",
-                        le32_to_cpu(cbw.sig));
+                error_report("usb-msd: Bad signature %08x",
+                             le32_to_cpu(cbw.sig));
                 goto fail;
             }
             DPRINTF("Command on LUN %d\n", cbw.lun);
             scsi_dev = scsi_device_find(&s->bus, 0, 0, cbw.lun);
             if (scsi_dev == NULL) {
-                fprintf(stderr, "usb-msd: Bad LUN %d\n", cbw.lun);
+                error_report("usb-msd: Bad LUN %d", cbw.lun);
                 goto fail;
             }
             tag = le32_to_cpu(cbw.tag);
@@ -549,12 +552,17 @@ static void usb_msd_handle_data(USBDevice *dev, USBPacket *p)
 static void usb_msd_password_cb(void *opaque, int err)
 {
     MSDState *s = opaque;
+    Error *local_err = NULL;
 
-    if (!err)
-        err = usb_device_attach(&s->dev);
+    if (!err) {
+        usb_device_attach(&s->dev, &local_err);
+    }
 
-    if (err)
+    if (local_err) {
+        qerror_report_err(local_err);
+        error_free(local_err);
         qdev_unplug(&s->dev.qdev, NULL);
+    }
 }
 
 static void *usb_msd_load_request(QEMUFile *f, SCSIRequest *req)
@@ -590,16 +598,16 @@ static const struct SCSIBusInfo usb_msd_scsi_info_bot = {
     .load_request = usb_msd_load_request,
 };
 
-static int usb_msd_initfn_storage(USBDevice *dev)
+static void usb_msd_realize_storage(USBDevice *dev, Error **errp)
 {
     MSDState *s = DO_UPCAST(MSDState, dev, dev);
-    BlockDriverState *bs = s->conf.bs;
+    BlockBackend *blk = s->conf.blk;
     SCSIDevice *scsi_dev;
     Error *err = NULL;
 
-    if (!bs) {
-        error_report("drive property not set");
-        return -1;
+    if (!blk) {
+        error_setg(errp, "drive property not set");
+        return;
     }
 
     blkconf_serial(&s->conf, &dev->serial);
@@ -613,35 +621,35 @@ static int usb_msd_initfn_storage(USBDevice *dev)
      *
      * The hack is probably a bad idea.
      */
-    bdrv_detach_dev(bs, &s->dev.qdev);
-    s->conf.bs = NULL;
+    blk_detach_dev(blk, &s->dev.qdev);
+    s->conf.blk = NULL;
 
     usb_desc_create_serial(dev);
     usb_desc_init(dev);
     scsi_bus_new(&s->bus, sizeof(s->bus), DEVICE(dev),
                  &usb_msd_scsi_info_storage, NULL);
-    scsi_dev = scsi_bus_legacy_add_drive(&s->bus, bs, 0, !!s->removable,
+    scsi_dev = scsi_bus_legacy_add_drive(&s->bus, blk, 0, !!s->removable,
                                          s->conf.bootindex, dev->serial,
                                          &err);
     if (!scsi_dev) {
-        return -1;
+        error_propagate(errp, err);
+        return;
     }
-    s->bus.qbus.allow_hotplug = 0;
     usb_msd_handle_reset(dev);
+    s->scsi_dev = scsi_dev;
 
-    if (bdrv_key_required(bs)) {
+    if (bdrv_key_required(blk_bs(blk))) {
         if (cur_mon) {
-            monitor_read_bdrv_key_start(cur_mon, bs, usb_msd_password_cb, s);
+            monitor_read_bdrv_key_start(cur_mon, blk_bs(blk),
+                                        usb_msd_password_cb, s);
             s->dev.auto_attach = 0;
         } else {
             autostart = 0;
         }
     }
-
-    return 0;
 }
 
-static int usb_msd_initfn_bot(USBDevice *dev)
+static void usb_msd_realize_bot(USBDevice *dev, Error **errp)
 {
     MSDState *s = DO_UPCAST(MSDState, dev, dev);
 
@@ -649,10 +657,7 @@ static int usb_msd_initfn_bot(USBDevice *dev)
     usb_desc_init(dev);
     scsi_bus_new(&s->bus, sizeof(s->bus), DEVICE(dev),
                  &usb_msd_scsi_info_bot, NULL);
-    s->bus.qbus.allow_hotplug = 0;
     usb_msd_handle_reset(dev);
-
-    return 0;
 }
 
 static USBDevice *usb_msd_init(USBBus *bus, const char *filename)
@@ -666,8 +671,10 @@ static USBDevice *usb_msd_init(USBBus *bus, const char *filename)
     char fmt[32];
 
     /* parse -usbdevice disk: syntax into drive opts */
-    snprintf(id, sizeof(id), "usb%d", nr++);
-    opts = qemu_opts_create(qemu_find_opts("drive"), id, 0, NULL);
+    do {
+        snprintf(id, sizeof(id), "usb%d", nr++);
+        opts = qemu_opts_create(qemu_find_opts("drive"), id, 1, NULL);
+    } while (!opts);
 
     p1 = strchr(filename, ':');
     if (p1++) {
@@ -678,13 +685,13 @@ static USBDevice *usb_msd_init(USBBus *bus, const char *filename)
             pstrcpy(fmt, len, p2);
             qemu_opt_set(opts, "format", fmt);
         } else if (*filename != ':') {
-            printf("unrecognized USB mass-storage option %s\n", filename);
+            error_report("unrecognized USB mass-storage option %s", filename);
             return NULL;
         }
         filename = p1;
     }
     if (!*filename) {
-        printf("block device specification needed\n");
+        error_report("block device specification needed");
         return NULL;
     }
     qemu_opt_set(opts, "file", filename);
@@ -702,7 +709,8 @@ static USBDevice *usb_msd_init(USBBus *bus, const char *filename)
     if (!dev) {
         return NULL;
     }
-    if (qdev_prop_set_drive(&dev->qdev, "drive", dinfo->bdrv) < 0) {
+    if (qdev_prop_set_drive(&dev->qdev, "drive",
+                            blk_by_legacy_dinfo(dinfo)) < 0) {
         object_unparent(OBJECT(dev));
         return NULL;
     }
@@ -758,17 +766,67 @@ static void usb_msd_class_initfn_storage(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
-    uc->init = usb_msd_initfn_storage;
+    uc->realize = usb_msd_realize_storage;
     dc->props = msd_properties;
     usb_msd_class_initfn_common(klass);
 }
 
+static void usb_msd_get_bootindex(Object *obj, Visitor *v, void *opaque,
+                                  const char *name, Error **errp)
+{
+    USBDevice *dev = USB_DEVICE(obj);
+    MSDState *s = DO_UPCAST(MSDState, dev, dev);
+
+    visit_type_int32(v, &s->conf.bootindex, name, errp);
+}
+
+static void usb_msd_set_bootindex(Object *obj, Visitor *v, void *opaque,
+                                  const char *name, Error **errp)
+{
+    USBDevice *dev = USB_DEVICE(obj);
+    MSDState *s = DO_UPCAST(MSDState, dev, dev);
+    int32_t boot_index;
+    Error *local_err = NULL;
+
+    visit_type_int32(v, &boot_index, name, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    /* check whether bootindex is present in fw_boot_order list  */
+    check_boot_index(boot_index, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    /* change bootindex to a new one */
+    s->conf.bootindex = boot_index;
+
+    if (s->scsi_dev) {
+        object_property_set_int(OBJECT(s->scsi_dev), boot_index, "bootindex",
+                                &error_abort);
+    }
+
+out:
+    if (local_err) {
+        error_propagate(errp, local_err);
+    }
+}
+
+static void usb_msd_instance_init(Object *obj)
+{
+    object_property_add(obj, "bootindex", "int32",
+                        usb_msd_get_bootindex,
+                        usb_msd_set_bootindex, NULL, NULL, NULL);
+    object_property_set_int(obj, -1, "bootindex", NULL);
+}
+
 static void usb_msd_class_initfn_bot(ObjectClass *klass, void *data)
 {
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
-    uc->init = usb_msd_initfn_bot;
+    uc->realize = usb_msd_realize_bot;
     usb_msd_class_initfn_common(klass);
+    dc->hotpluggable = false;
 }
 
 static const TypeInfo msd_info = {
@@ -776,6 +834,7 @@ static const TypeInfo msd_info = {
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(MSDState),
     .class_init    = usb_msd_class_initfn_storage,
+    .instance_init = usb_msd_instance_init,
 };
 
 static const TypeInfo bot_info = {
diff --git a/hw/usb/dev-uas.c b/hw/usb/dev-uas.c
index 983238511..04fc515db 100644
--- a/hw/usb/dev-uas.c
+++ b/hw/usb/dev-uas.c
@@ -13,6 +13,7 @@
 #include "qemu/option.h"
 #include "qemu/config-file.h"
 #include "trace.h"
+#include "qemu/error-report.h"
 
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
@@ -648,7 +649,7 @@ static void usb_uas_handle_control(USBDevice *dev, USBPacket *p,
     if (ret >= 0) {
         return;
     }
-    fprintf(stderr, "%s: unhandled control request\n", __func__);
+    error_report("%s: unhandled control request", __func__);
     p->status = USB_RET_STALL;
 }
 
@@ -814,8 +815,8 @@ static void usb_uas_handle_data(USBDevice *dev, USBPacket *p)
             usb_uas_task(uas, &iu);
             break;
         default:
-            fprintf(stderr, "%s: unknown command iu: id 0x%x\n",
-                    __func__, iu.hdr.id);
+            error_report("%s: unknown command iu: id 0x%x",
+                         __func__, iu.hdr.id);
             p->status = USB_RET_STALL;
             break;
         }
@@ -861,7 +862,7 @@ static void usb_uas_handle_data(USBDevice *dev, USBPacket *p)
                 p->status = USB_RET_ASYNC;
                 break;
             } else {
-                fprintf(stderr, "%s: no inflight request\n", __func__);
+                error_report("%s: no inflight request", __func__);
                 p->status = USB_RET_STALL;
                 break;
             }
@@ -879,7 +880,7 @@ static void usb_uas_handle_data(USBDevice *dev, USBPacket *p)
         usb_uas_start_next_transfer(uas);
         break;
     default:
-        fprintf(stderr, "%s: invalid endpoint %d\n", __func__, p->ep->nr);
+        error_report("%s: invalid endpoint %d", __func__, p->ep->nr);
         p->status = USB_RET_STALL;
         break;
     }
@@ -892,7 +893,7 @@ static void usb_uas_handle_destroy(USBDevice *dev)
     qemu_bh_delete(uas->status_bh);
 }
 
-static int usb_uas_init(USBDevice *dev)
+static void usb_uas_realize(USBDevice *dev, Error **errp)
 {
     UASDevice *uas = DO_UPCAST(UASDevice, dev, dev);
 
@@ -905,8 +906,6 @@ static int usb_uas_init(USBDevice *dev)
 
     scsi_bus_new(&uas->bus, sizeof(uas->bus), DEVICE(dev),
                  &usb_uas_scsi_info, NULL);
-
-    return 0;
 }
 
 static const VMStateDescription vmstate_usb_uas = {
@@ -928,7 +927,7 @@ static void usb_uas_class_initfn(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
-    uc->init           = usb_uas_init;
+    uc->realize        = usb_uas_realize;
     uc->product_desc   = desc_strings[STR_PRODUCT];
     uc->usb_desc       = &desc;
     uc->cancel_packet  = usb_uas_cancel_io;
diff --git a/hw/usb/dev-wacom.c b/hw/usb/dev-wacom.c
index 1b73fd0aa..844eafadf 100644
--- a/hw/usb/dev-wacom.c
+++ b/hw/usb/dev-wacom.c
@@ -335,14 +335,13 @@ static void usb_wacom_handle_destroy(USBDevice *dev)
     }
 }
 
-static int usb_wacom_initfn(USBDevice *dev)
+static void usb_wacom_realize(USBDevice *dev, Error **errp)
 {
     USBWacomState *s = DO_UPCAST(USBWacomState, dev, dev);
     usb_desc_create_serial(dev);
     usb_desc_init(dev);
     s->intr = usb_ep_get(dev, USB_TOKEN_IN, 1);
     s->changed = 1;
-    return 0;
 }
 
 static const VMStateDescription vmstate_usb_wacom = {
@@ -357,7 +356,7 @@ static void usb_wacom_class_init(ObjectClass *klass, void *data)
 
     uc->product_desc   = "QEMU PenPartner Tablet";
     uc->usb_desc       = &desc_wacom;
-    uc->init           = usb_wacom_initfn;
+    uc->realize        = usb_wacom_realize;
     uc->handle_reset   = usb_wacom_handle_reset;
     uc->handle_control = usb_wacom_handle_control;
     uc->handle_data    = usb_wacom_handle_data;
diff --git a/hw/usb/hcd-ehci-pci.c b/hw/usb/hcd-ehci-pci.c
index 505741a78..490f2b6af 100644
--- a/hw/usb/hcd-ehci-pci.c
+++ b/hw/usb/hcd-ehci-pci.c
@@ -23,6 +23,7 @@ typedef struct EHCIPCIInfo {
     uint16_t vendor_id;
     uint16_t device_id;
     uint8_t  revision;
+    bool companion;
 } EHCIPCIInfo;
 
 static int usb_ehci_pci_initfn(PCIDevice *dev)
@@ -71,6 +72,7 @@ static int usb_ehci_pci_initfn(PCIDevice *dev)
 
 static void usb_ehci_pci_init(Object *obj)
 {
+    DeviceClass *dc = OBJECT_GET_CLASS(DeviceClass, obj, TYPE_DEVICE);
     EHCIPCIState *i = PCI_EHCI(obj);
     EHCIState *s = &i->ehci;
 
@@ -81,9 +83,26 @@ static void usb_ehci_pci_init(Object *obj)
     s->portscbase = 0x44;
     s->portnr = NB_PORTS;
 
+    if (!dc->hotpluggable) {
+        s->companion_enable = true;
+    }
+
     usb_ehci_init(s, DEVICE(obj));
 }
 
+static void usb_ehci_pci_exit(PCIDevice *dev)
+{
+    EHCIPCIState *i = PCI_EHCI(dev);
+    EHCIState *s = &i->ehci;
+
+    usb_ehci_unrealize(s, DEVICE(dev), NULL);
+
+    if (s->irq) {
+        g_free(s->irq);
+        s->irq = NULL;
+    }
+}
+
 static void usb_ehci_pci_write_config(PCIDevice *dev, uint32_t addr,
                                       uint32_t val, int l)
 {
@@ -121,9 +140,9 @@ static void ehci_class_init(ObjectClass *klass, void *data)
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
     k->init = usb_ehci_pci_initfn;
+    k->exit = usb_ehci_pci_exit;
     k->class_id = PCI_CLASS_SERIAL_USB;
     k->config_write = usb_ehci_pci_write_config;
-    dc->hotpluggable = false;
     dc->vmsd = &vmstate_ehci_pci;
     dc->props = ehci_pci_properties;
 }
@@ -147,6 +166,9 @@ static void ehci_data_class_init(ObjectClass *klass, void *data)
     k->device_id = i->device_id;
     k->revision = i->revision;
     set_bit(DEVICE_CATEGORY_USB, dc->categories);
+    if (i->companion) {
+        dc->hotpluggable = false;
+    }
 }
 
 static struct EHCIPCIInfo ehci_pci_info[] = {
@@ -160,11 +182,13 @@ static struct EHCIPCIInfo ehci_pci_info[] = {
         .vendor_id = PCI_VENDOR_ID_INTEL,
         .device_id = PCI_DEVICE_ID_INTEL_82801I_EHCI1,
         .revision  = 0x03,
+        .companion = true,
     },{
         .name      = "ich9-usb-ehci2", /* 00:1a.7 */
         .vendor_id = PCI_VENDOR_ID_INTEL,
         .device_id = PCI_DEVICE_ID_INTEL_82801I_EHCI2,
         .revision  = 0x03,
+        .companion = true,
     }
 };
 
diff --git a/hw/usb/hcd-ehci.c b/hw/usb/hcd-ehci.c
index a00a93c3e..1cc0fc116 100644
--- a/hw/usb/hcd-ehci.c
+++ b/hw/usb/hcd-ehci.c
@@ -1596,7 +1596,7 @@ static EHCIQueue *ehci_state_fetchqh(EHCIState *ehci, int async)
 
     entry = ehci_get_fetch_addr(ehci, async);
     q = ehci_find_queue_by_qh(ehci, entry, async);
-    if (NULL == q) {
+    if (q == NULL) {
         q = ehci_alloc_queue(ehci, entry, async);
     }
 
@@ -2347,10 +2347,13 @@ static USBPortOps ehci_port_ops = {
     .complete = ehci_async_complete_packet,
 };
 
-static USBBusOps ehci_bus_ops = {
+static USBBusOps ehci_bus_ops_companion = {
     .register_companion = ehci_register_companion,
     .wakeup_endpoint = ehci_wakeup_endpoint,
 };
+static USBBusOps ehci_bus_ops_standalone = {
+    .wakeup_endpoint = ehci_wakeup_endpoint,
+};
 
 static void usb_ehci_pre_save(void *opaque)
 {
@@ -2456,7 +2459,8 @@ void usb_ehci_realize(EHCIState *s, DeviceState *dev, Error **errp)
         return;
     }
 
-    usb_bus_new(&s->bus, sizeof(s->bus), &ehci_bus_ops, dev);
+    usb_bus_new(&s->bus, sizeof(s->bus), s->companion_enable ?
+                &ehci_bus_ops_companion : &ehci_bus_ops_standalone, dev);
     for (i = 0; i < s->portnr; i++) {
         usb_register_port(&s->bus, &s->ports[i], s, i, &ehci_port_ops,
                           USB_SPEED_MASK_HIGH);
@@ -2468,7 +2472,34 @@ void usb_ehci_realize(EHCIState *s, DeviceState *dev, Error **errp)
     s->device = dev;
 
     qemu_register_reset(ehci_reset, s);
-    qemu_add_vm_change_state_handler(usb_ehci_vm_state_change, s);
+    s->vmstate = qemu_add_vm_change_state_handler(usb_ehci_vm_state_change, s);
+}
+
+void usb_ehci_unrealize(EHCIState *s, DeviceState *dev, Error **errp)
+{
+    trace_usb_ehci_unrealize();
+
+    if (s->frame_timer) {
+        timer_del(s->frame_timer);
+        timer_free(s->frame_timer);
+        s->frame_timer = NULL;
+    }
+    if (s->async_bh) {
+        qemu_bh_delete(s->async_bh);
+    }
+
+    ehci_queues_rip_all(s, 0);
+    ehci_queues_rip_all(s, 1);
+
+    memory_region_del_subregion(&s->mem, &s->mem_caps);
+    memory_region_del_subregion(&s->mem, &s->mem_opreg);
+    memory_region_del_subregion(&s->mem, &s->mem_ports);
+
+    usb_bus_release(&s->bus);
+
+    if (s->vmstate) {
+        qemu_del_vm_change_state_handler(s->vmstate);
+    }
 }
 
 void usb_ehci_init(EHCIState *s, DeviceState *dev)
diff --git a/hw/usb/hcd-ehci.h b/hw/usb/hcd-ehci.h
index 1ad4b96cc..2bc259c9b 100644
--- a/hw/usb/hcd-ehci.h
+++ b/hw/usb/hcd-ehci.h
@@ -262,6 +262,7 @@ struct EHCIState {
     MemoryRegion mem_opreg;
     MemoryRegion mem_ports;
     int companion_count;
+    bool companion_enable;
     uint16_t capsbase;
     uint16_t opregbase;
     uint16_t portscbase;
@@ -316,12 +317,14 @@ struct EHCIState {
     uint32_t async_stepdown;
     uint32_t periodic_sched_active;
     bool int_req_by_async;
+    VMChangeStateEntry *vmstate;
 };
 
 extern const VMStateDescription vmstate_ehci;
 
 void usb_ehci_init(EHCIState *s, DeviceState *dev);
 void usb_ehci_realize(EHCIState *s, DeviceState *dev, Error **errp);
+void usb_ehci_unrealize(EHCIState *s, DeviceState *dev, Error **errp);
 
 #define TYPE_PCI_EHCI "pci-ehci-usb"
 #define PCI_EHCI(obj) OBJECT_CHECK(EHCIPCIState, (obj), TYPE_PCI_EHCI)
diff --git a/hw/usb/hcd-musb.c b/hw/usb/hcd-musb.c
index 66bc61ae1..40809f610 100644
--- a/hw/usb/hcd-musb.c
+++ b/hw/usb/hcd-musb.c
@@ -608,6 +608,7 @@ static void musb_packet(MUSBState *s, MUSBEndPoint *ep,
     USBDevice *dev;
     USBEndpoint *uep;
     int idx = epnum && dir;
+    int id;
     int ttype;
 
     /* ep->type[0,1] contains:
@@ -625,8 +626,11 @@ static void musb_packet(MUSBState *s, MUSBEndPoint *ep,
     /* A wild guess on the FADDR semantics... */
     dev = usb_find_device(&s->port, ep->faddr[idx]);
     uep = usb_ep_get(dev, pid, ep->type[idx] & 0xf);
-    usb_packet_setup(&ep->packey[dir].p, pid, uep, 0,
-                     (dev->addr << 16) | (uep->nr << 8) | pid, false, true);
+    id = pid;
+    if (uep) {
+        id |= (dev->addr << 16) | (uep->nr << 8);
+    }
+    usb_packet_setup(&ep->packey[dir].p, pid, uep, 0, id, false, true);
     usb_packet_addbuf(&ep->packey[dir].p, ep->buf[idx], len);
     ep->packey[dir].ep = ep;
     ep->packey[dir].dir = dir;
diff --git a/hw/usb/hcd-ohci.c b/hw/usb/hcd-ohci.c
index 13afdf591..9a84eb695 100644
--- a/hw/usb/hcd-ohci.c
+++ b/hw/usb/hcd-ohci.c
@@ -31,20 +31,11 @@
 #include "hw/pci/pci.h"
 #include "hw/sysbus.h"
 #include "hw/qdev-dma.h"
+#include "trace.h"
 
-//#define DEBUG_OHCI
-/* Dump packet contents.  */
-//#define DEBUG_PACKET
-//#define DEBUG_ISOCH
 /* This causes frames to occur 1000x slower */
 //#define OHCI_TIME_WARP 1
 
-#ifdef DEBUG_OHCI
-#define DPRINTF printf
-#else
-#define DPRINTF(...)
-#endif
-
 /* Number of Downstream Ports on the root hub.  */
 
 #define OHCI_MAX_PORTS 15
@@ -350,7 +341,7 @@ static void ohci_attach(USBPort *port1)
         ohci_set_interrupt(s, OHCI_INTR_RD);
     }
 
-    DPRINTF("usb-ohci: Attached port %d\n", port1->index);
+    trace_usb_ohci_port_attach(port1->index);
 
     if (old_state != port->ctrl) {
         ohci_set_interrupt(s, OHCI_INTR_RHSC);
@@ -375,7 +366,7 @@ static void ohci_detach(USBPort *port1)
         port->ctrl &= ~OHCI_PORT_PES;
         port->ctrl |= OHCI_PORT_PESC;
     }
-    DPRINTF("usb-ohci: Detached port %d\n", port1->index);
+    trace_usb_ohci_port_detach(port1->index);
 
     if (old_state != port->ctrl) {
         ohci_set_interrupt(s, OHCI_INTR_RHSC);
@@ -388,14 +379,14 @@ static void ohci_wakeup(USBPort *port1)
     OHCIPort *port = &s->rhport[port1->index];
     uint32_t intr = 0;
     if (port->ctrl & OHCI_PORT_PSS) {
-        DPRINTF("usb-ohci: port %d: wakeup\n", port1->index);
+        trace_usb_ohci_port_wakeup(port1->index);
         port->ctrl |= OHCI_PORT_PSSC;
         port->ctrl &= ~OHCI_PORT_PSS;
         intr = OHCI_INTR_RHSC;
     }
     /* Note that the controller can be suspended even if this port is not */
     if ((s->ctl & OHCI_CTL_HCFS) == OHCI_USB_SUSPEND) {
-        DPRINTF("usb-ohci: remote-wakeup: SUSPEND->RESUME\n");
+        trace_usb_ohci_remote_wakeup(s->name);
         /* This is the one state transition the controller can do by itself */
         s->ctl &= ~OHCI_CTL_HCFS;
         s->ctl |= OHCI_USB_RESUME;
@@ -497,7 +488,7 @@ static void ohci_reset(void *opaque)
         ohci->async_td = 0;
     }
     ohci_stop_endpoints(ohci);
-    DPRINTF("usb-ohci: Reset %s\n", ohci->name);
+    trace_usb_ohci_reset(ohci->name);
 }
 
 /* Get an array of dwords from main memory */
@@ -619,8 +610,8 @@ static inline int ohci_put_td(OHCIState *ohci,
 static inline int ohci_put_iso_td(OHCIState *ohci,
                                   dma_addr_t addr, struct ohci_iso_td *td)
 {
-    return put_dwords(ohci, addr, (uint32_t *)td, 4 ||
-           put_words(ohci, addr + 16, td->offset, 8));
+    return put_dwords(ohci, addr, (uint32_t *)td, 4) ||
+           put_words(ohci, addr + 16, td->offset, 8);
 }
 
 static inline int ohci_put_hcca(OHCIState *ohci,
@@ -690,9 +681,8 @@ static void ohci_process_lists(OHCIState *ohci, int completion);
 static void ohci_async_complete_packet(USBPort *port, USBPacket *packet)
 {
     OHCIState *ohci = container_of(packet, OHCIState, usb_packet);
-#ifdef DEBUG_PACKET
-    DPRINTF("Async packet complete\n");
-#endif
+
+    trace_usb_ohci_async_complete();
     ohci->async_complete = true;
     ohci_process_lists(ohci, 1);
 }
@@ -704,9 +694,7 @@ static int ohci_service_iso_td(OHCIState *ohci, struct ohci_ed *ed,
 {
     int dir;
     size_t len = 0;
-#ifdef DEBUG_ISOCH
     const char *str = NULL;
-#endif
     int pid;
     int ret;
     int i;
@@ -723,7 +711,7 @@ static int ohci_service_iso_td(OHCIState *ohci, struct ohci_ed *ed,
     addr = ed->head & OHCI_DPTR_MASK;
 
     if (ohci_read_iso_td(ohci, addr, &iso_td)) {
-        printf("usb-ohci: ISO_TD read error at %x\n", addr);
+        trace_usb_ohci_iso_td_read_failed(addr);
         ohci_die(ohci);
         return 0;
     }
@@ -732,31 +720,25 @@ static int ohci_service_iso_td(OHCIState *ohci, struct ohci_ed *ed,
     frame_count = OHCI_BM(iso_td.flags, TD_FC);
     relative_frame_number = USUB(ohci->frame_number, starting_frame); 
 
-#ifdef DEBUG_ISOCH
-    printf("--- ISO_TD ED head 0x%.8x tailp 0x%.8x\n"
-           "0x%.8x 0x%.8x 0x%.8x 0x%.8x\n"
-           "0x%.8x 0x%.8x 0x%.8x 0x%.8x\n"
-           "0x%.8x 0x%.8x 0x%.8x 0x%.8x\n"
-           "frame_number 0x%.8x starting_frame 0x%.8x\n"
-           "frame_count  0x%.8x relative %d\n"
-           "di 0x%.8x cc 0x%.8x\n",
+    trace_usb_ohci_iso_td_head(
            ed->head & OHCI_DPTR_MASK, ed->tail & OHCI_DPTR_MASK,
            iso_td.flags, iso_td.bp, iso_td.next, iso_td.be,
-           iso_td.offset[0], iso_td.offset[1], iso_td.offset[2], iso_td.offset[3],
-           iso_td.offset[4], iso_td.offset[5], iso_td.offset[6], iso_td.offset[7],
-           ohci->frame_number, starting_frame, 
-           frame_count, relative_frame_number,         
-           OHCI_BM(iso_td.flags, TD_DI), OHCI_BM(iso_td.flags, TD_CC));
-#endif
+           ohci->frame_number, starting_frame,
+           frame_count, relative_frame_number);
+    trace_usb_ohci_iso_td_head_offset(
+           iso_td.offset[0], iso_td.offset[1],
+           iso_td.offset[2], iso_td.offset[3],
+           iso_td.offset[4], iso_td.offset[5],
+           iso_td.offset[6], iso_td.offset[7]);
 
     if (relative_frame_number < 0) {
-        DPRINTF("usb-ohci: ISO_TD R=%d < 0\n", relative_frame_number);
+        trace_usb_ohci_iso_td_relative_frame_number_neg(relative_frame_number);
         return 1;
     } else if (relative_frame_number > frame_count) {
         /* ISO TD expired - retire the TD to the Done Queue and continue with
            the next ISO TD of the same ED */
-        DPRINTF("usb-ohci: ISO_TD R=%d > FC=%d\n", relative_frame_number, 
-               frame_count);
+        trace_usb_ohci_iso_td_relative_frame_number_big(relative_frame_number,
+                                                        frame_count);
         OHCI_SET_BM(iso_td.flags, TD_CC, OHCI_CC_DATAOVERRUN);
         ed->head &= ~OHCI_DPTR_MASK;
         ed->head |= (iso_td.next & OHCI_DPTR_MASK);
@@ -775,30 +757,24 @@ static int ohci_service_iso_td(OHCIState *ohci, struct ohci_ed *ed,
     dir = OHCI_BM(ed->flags, ED_D);
     switch (dir) {
     case OHCI_TD_DIR_IN:
-#ifdef DEBUG_ISOCH
         str = "in";
-#endif
         pid = USB_TOKEN_IN;
         break;
     case OHCI_TD_DIR_OUT:
-#ifdef DEBUG_ISOCH
         str = "out";
-#endif
         pid = USB_TOKEN_OUT;
         break;
     case OHCI_TD_DIR_SETUP:
-#ifdef DEBUG_ISOCH
         str = "setup";
-#endif
         pid = USB_TOKEN_SETUP;
         break;
     default:
-        printf("usb-ohci: Bad direction %d\n", dir);
+        trace_usb_ohci_iso_td_bad_direction(dir);
         return 1;
     }
 
     if (!iso_td.bp || !iso_td.be) {
-        printf("usb-ohci: ISO_TD bp 0x%.8x be 0x%.8x\n", iso_td.bp, iso_td.be);
+        trace_usb_ohci_iso_td_bad_bp_be(iso_td.bp, iso_td.be);
         return 1;
     }
 
@@ -808,14 +784,12 @@ static int ohci_service_iso_td(OHCIState *ohci, struct ohci_ed *ed,
     if (!(OHCI_BM(start_offset, TD_PSW_CC) & 0xe) || 
         ((relative_frame_number < frame_count) && 
          !(OHCI_BM(next_offset, TD_PSW_CC) & 0xe))) {
-        printf("usb-ohci: ISO_TD cc != not accessed 0x%.8x 0x%.8x\n",
-               start_offset, next_offset);
+        trace_usb_ohci_iso_td_bad_cc_not_accessed(start_offset, next_offset);
         return 1;
     }
 
     if ((relative_frame_number < frame_count) && (start_offset > next_offset)) {
-        printf("usb-ohci: ISO_TD start_offset=0x%.8x > next_offset=0x%.8x\n",
-                start_offset, next_offset);
+        trace_usb_ohci_iso_td_bad_cc_overrun(start_offset, next_offset);
         return 1;
     }
 
@@ -875,10 +849,8 @@ static int ohci_service_iso_td(OHCIState *ohci, struct ohci_ed *ed,
         ret = ohci->usb_packet.status;
     }
 
-#ifdef DEBUG_ISOCH
-    printf("so 0x%.8x eo 0x%.8x\nsa 0x%.8x ea 0x%.8x\ndir %s len %zu ret %d\n",
-           start_offset, end_offset, start_addr, end_addr, str, len, ret);
-#endif
+    trace_usb_ohci_iso_td_so(start_offset, end_offset, start_addr, end_addr,
+                             str, len, ret);
 
     /* Writeback */
     if (dir == OHCI_TD_DIR_IN && ret >= 0 && ret <= len) {
@@ -898,13 +870,13 @@ static int ohci_service_iso_td(OHCIState *ohci, struct ohci_ed *ed,
         OHCI_SET_BM(iso_td.offset[relative_frame_number], TD_PSW_SIZE, 0);
     } else {
         if (ret > (ssize_t) len) {
-            printf("usb-ohci: DataOverrun %d > %zu\n", ret, len);
+            trace_usb_ohci_iso_td_data_overrun(ret, len);
             OHCI_SET_BM(iso_td.offset[relative_frame_number], TD_PSW_CC,
                         OHCI_CC_DATAOVERRUN);
             OHCI_SET_BM(iso_td.offset[relative_frame_number], TD_PSW_SIZE,
                         len);
         } else if (ret >= 0) {
-            printf("usb-ohci: DataUnderrun %d\n", ret);
+            trace_usb_ohci_iso_td_data_underrun(ret);
             OHCI_SET_BM(iso_td.offset[relative_frame_number], TD_PSW_CC,
                         OHCI_CC_DATAUNDERRUN);
         } else {
@@ -918,14 +890,14 @@ static int ohci_service_iso_td(OHCIState *ohci, struct ohci_ed *ed,
                 break;
             case USB_RET_NAK:
             case USB_RET_STALL:
-                printf("usb-ohci: got NAK/STALL %d\n", ret);
+                trace_usb_ohci_iso_td_nak(ret);
                 OHCI_SET_BM(iso_td.offset[relative_frame_number], TD_PSW_CC,
                             OHCI_CC_STALL);
                 OHCI_SET_BM(iso_td.offset[relative_frame_number], TD_PSW_SIZE,
                             0);
                 break;
             default:
-                printf("usb-ohci: Bad device response %d\n", ret);
+                trace_usb_ohci_iso_td_bad_response(ret);
                 OHCI_SET_BM(iso_td.offset[relative_frame_number], TD_PSW_CC,
                             OHCI_CC_UNDEXPETEDPID);
                 break;
@@ -950,6 +922,43 @@ static int ohci_service_iso_td(OHCIState *ohci, struct ohci_ed *ed,
     return 1;
 }
 
+#ifdef trace_event_get_state
+static void ohci_td_pkt(const char *msg, const uint8_t *buf, size_t len)
+{
+    bool print16 = !!trace_event_get_state(TRACE_USB_OHCI_TD_PKT_SHORT);
+    bool printall = !!trace_event_get_state(TRACE_USB_OHCI_TD_PKT_FULL);
+    const int width = 16;
+    int i;
+    char tmp[3 * width + 1];
+    char *p = tmp;
+
+    if (!printall && !print16) {
+        return;
+    }
+
+    for (i = 0; ; i++) {
+        if (i && (!(i % width) || (i == len))) {
+            if (!printall) {
+                trace_usb_ohci_td_pkt_short(msg, tmp);
+                break;
+            }
+            trace_usb_ohci_td_pkt_full(msg, tmp);
+            p = tmp;
+            *p = 0;
+        }
+        if (i == len) {
+            break;
+        }
+
+        p += sprintf(p, " %.2x", buf[i]);
+    }
+}
+#else
+static void ohci_td_pkt(const char *msg, const uint8_t *buf, size_t len)
+{
+}
+#endif
+
 /* Service a transport descriptor.
    Returns nonzero to terminate processing of this endpoint.  */
 
@@ -957,9 +966,7 @@ static int ohci_service_td(OHCIState *ohci, struct ohci_ed *ed)
 {
     int dir;
     size_t len = 0, pktlen = 0;
-#ifdef DEBUG_PACKET
     const char *str = NULL;
-#endif
     int pid;
     int ret;
     int i;
@@ -974,13 +981,11 @@ static int ohci_service_td(OHCIState *ohci, struct ohci_ed *ed)
     /* See if this TD has already been submitted to the device.  */
     completion = (addr == ohci->async_td);
     if (completion && !ohci->async_complete) {
-#ifdef DEBUG_PACKET
-        DPRINTF("Skipping async TD\n");
-#endif
+        trace_usb_ohci_td_skip_async();
         return 1;
     }
     if (ohci_read_td(ohci, addr, &td)) {
-        fprintf(stderr, "usb-ohci: TD read error at %x\n", addr);
+        trace_usb_ohci_td_read_error(addr);
         ohci_die(ohci);
         return 0;
     }
@@ -998,25 +1003,19 @@ static int ohci_service_td(OHCIState *ohci, struct ohci_ed *ed)
 
     switch (dir) {
     case OHCI_TD_DIR_IN:
-#ifdef DEBUG_PACKET
         str = "in";
-#endif
         pid = USB_TOKEN_IN;
         break;
     case OHCI_TD_DIR_OUT:
-#ifdef DEBUG_PACKET
         str = "out";
-#endif
         pid = USB_TOKEN_OUT;
         break;
     case OHCI_TD_DIR_SETUP:
-#ifdef DEBUG_PACKET
         str = "setup";
-#endif
         pid = USB_TOKEN_SETUP;
         break;
     default:
-        fprintf(stderr, "usb-ohci: Bad direction\n");
+        trace_usb_ohci_td_bad_direction(dir);
         return 1;
     }
     if (td.cbp && td.be) {
@@ -1043,19 +1042,10 @@ static int ohci_service_td(OHCIState *ohci, struct ohci_ed *ed)
     }
 
     flag_r = (td.flags & OHCI_TD_R) != 0;
-#ifdef DEBUG_PACKET
-    DPRINTF(" TD @ 0x%.8x %" PRId64 " of %" PRId64
-            " bytes %s r=%d cbp=0x%.8x be=0x%.8x\n",
-            addr, (int64_t)pktlen, (int64_t)len, str, flag_r, td.cbp, td.be);
-
-    if (pktlen > 0 && dir != OHCI_TD_DIR_IN) {
-        DPRINTF("  data:");
-        for (i = 0; i < pktlen; i++) {
-            printf(" %.2x", ohci->usb_buf[i]);
-        }
-        DPRINTF("\n");
-    }
-#endif
+    trace_usb_ohci_td_pkt_hdr(addr, (int64_t)pktlen, (int64_t)len, str,
+                              flag_r, td.cbp, td.be);
+    ohci_td_pkt("OUT", ohci->usb_buf, pktlen);
+
     if (completion) {
         ohci->async_td = 0;
         ohci->async_complete = false;
@@ -1066,9 +1056,7 @@ static int ohci_service_td(OHCIState *ohci, struct ohci_ed *ed)
                This should be sufficient as long as devices respond in a
                timely manner.
             */
-#ifdef DEBUG_PACKET
-            DPRINTF("Too many pending packets\n");
-#endif
+            trace_usb_ohci_td_too_many_pending();
             return 1;
         }
         dev = ohci_find_device(ohci, OHCI_BM(ed->flags, ED_FA));
@@ -1077,9 +1065,8 @@ static int ohci_service_td(OHCIState *ohci, struct ohci_ed *ed)
                          OHCI_BM(td.flags, TD_DI) == 0);
         usb_packet_addbuf(&ohci->usb_packet, ohci->usb_buf, pktlen);
         usb_handle_packet(dev, &ohci->usb_packet);
-#ifdef DEBUG_PACKET
-        DPRINTF("status=%d\n", ohci->usb_packet.status);
-#endif
+        trace_usb_ohci_td_packet_status(ohci->usb_packet.status);
+
         if (ohci->usb_packet.status == USB_RET_ASYNC) {
             usb_device_flush_ep_queue(dev, ep);
             ohci->async_td = addr;
@@ -1098,12 +1085,7 @@ static int ohci_service_td(OHCIState *ohci, struct ohci_ed *ed)
                              DMA_DIRECTION_FROM_DEVICE)) {
                 ohci_die(ohci);
             }
-#ifdef DEBUG_PACKET
-            DPRINTF("  data:");
-            for (i = 0; i < ret; i++)
-                printf(" %.2x", ohci->usb_buf[i]);
-            DPRINTF("\n");
-#endif
+            ohci_td_pkt("IN", ohci->usb_buf, pktlen);
         } else {
             ret = pktlen;
         }
@@ -1137,28 +1119,28 @@ static int ohci_service_td(OHCIState *ohci, struct ohci_ed *ed)
             ed->head |= OHCI_ED_C;
     } else {
         if (ret >= 0) {
-            DPRINTF("usb-ohci: Underrun\n");
+            trace_usb_ohci_td_underrun();
             OHCI_SET_BM(td.flags, TD_CC, OHCI_CC_DATAUNDERRUN);
         } else {
             switch (ret) {
             case USB_RET_IOERROR:
             case USB_RET_NODEV:
-                DPRINTF("usb-ohci: got DEV ERROR\n");
+                trace_usb_ohci_td_dev_error();
                 OHCI_SET_BM(td.flags, TD_CC, OHCI_CC_DEVICENOTRESPONDING);
                 break;
             case USB_RET_NAK:
-                DPRINTF("usb-ohci: got NAK\n");
+                trace_usb_ohci_td_nak();
                 return 1;
             case USB_RET_STALL:
-                DPRINTF("usb-ohci: got STALL\n");
+                trace_usb_ohci_td_stall();
                 OHCI_SET_BM(td.flags, TD_CC, OHCI_CC_STALL);
                 break;
             case USB_RET_BABBLE:
-                DPRINTF("usb-ohci: got BABBLE\n");
+                trace_usb_ohci_td_babble();
                 OHCI_SET_BM(td.flags, TD_CC, OHCI_CC_DATAOVERRUN);
                 break;
             default:
-                fprintf(stderr, "usb-ohci: Bad device response %d\n", ret);
+                trace_usb_ohci_td_bad_device_response(ret);
                 OHCI_SET_BM(td.flags, TD_CC, OHCI_CC_UNDEXPETEDPID);
                 OHCI_SET_BM(td.flags, TD_EC, 3);
                 break;
@@ -1198,7 +1180,7 @@ static int ohci_service_ed_list(OHCIState *ohci, uint32_t head, int completion)
 
     for (cur = head; cur; cur = next_ed) {
         if (ohci_read_ed(ohci, cur, &ed)) {
-            fprintf(stderr, "usb-ohci: ED read error at %x\n", cur);
+            trace_usb_ohci_ed_read_error(cur);
             ohci_die(ohci);
             return 0;
         }
@@ -1219,16 +1201,15 @@ static int ohci_service_ed_list(OHCIState *ohci, uint32_t head, int completion)
         }
 
         while ((ed.head & OHCI_DPTR_MASK) != ed.tail) {
-#ifdef DEBUG_PACKET
-            DPRINTF("ED @ 0x%.8x fa=%u en=%u d=%u s=%u k=%u f=%u mps=%u "
-                    "h=%u c=%u\n  head=0x%.8x tailp=0x%.8x next=0x%.8x\n", cur,
+            trace_usb_ohci_ed_pkt(cur, (ed.head & OHCI_ED_H) != 0,
+                    (ed.head & OHCI_ED_C) != 0, ed.head & OHCI_DPTR_MASK,
+                    ed.tail & OHCI_DPTR_MASK, ed.next & OHCI_DPTR_MASK);
+            trace_usb_ohci_ed_pkt_flags(
                     OHCI_BM(ed.flags, ED_FA), OHCI_BM(ed.flags, ED_EN),
                     OHCI_BM(ed.flags, ED_D), (ed.flags & OHCI_ED_S)!= 0,
                     (ed.flags & OHCI_ED_K) != 0, (ed.flags & OHCI_ED_F) != 0,
-                    OHCI_BM(ed.flags, ED_MPS), (ed.head & OHCI_ED_H) != 0,
-                    (ed.head & OHCI_ED_C) != 0, ed.head & OHCI_DPTR_MASK,
-                    ed.tail & OHCI_DPTR_MASK, ed.next & OHCI_DPTR_MASK);
-#endif
+                    OHCI_BM(ed.flags, ED_MPS));
+
             active = 1;
 
             if ((ed.flags & OHCI_ED_F) == 0) {
@@ -1263,8 +1244,7 @@ static void ohci_process_lists(OHCIState *ohci, int completion)
 {
     if ((ohci->ctl & OHCI_CTL_CLE) && (ohci->status & OHCI_STATUS_CLF)) {
         if (ohci->ctrl_cur && ohci->ctrl_cur != ohci->ctrl_head) {
-            DPRINTF("usb-ohci: head %x, cur %x\n",
-                    ohci->ctrl_head, ohci->ctrl_cur);
+            trace_usb_ohci_process_lists(ohci->ctrl_head, ohci->ctrl_cur);
         }
         if (!ohci_service_ed_list(ohci, ohci->ctrl_head, completion)) {
             ohci->ctrl_cur = 0;
@@ -1287,7 +1267,7 @@ static void ohci_frame_boundary(void *opaque)
     struct ohci_hcca hcca;
 
     if (ohci_read_hcca(ohci, ohci->hcca, &hcca)) {
-        fprintf(stderr, "usb-ohci: HCCA read error at %x\n", ohci->hcca);
+        trace_usb_ohci_hcca_read_error(ohci->hcca);
         ohci_die(ohci);
         return;
     }
@@ -1356,12 +1336,12 @@ static int ohci_bus_start(OHCIState *ohci)
                     ohci);
 
     if (ohci->eof_timer == NULL) {
-        fprintf(stderr, "usb-ohci: %s: timer_new_ns failed\n", ohci->name);
+        trace_usb_ohci_bus_eof_timer_failed(ohci->name);
         ohci_die(ohci);
         return 0;
     }
 
-    DPRINTF("usb-ohci: %s: USB Operational\n", ohci->name);
+    trace_usb_ohci_start(ohci->name);
 
     ohci_sof(ohci);
 
@@ -1371,8 +1351,11 @@ static int ohci_bus_start(OHCIState *ohci)
 /* Stop sending SOF tokens on the bus */
 static void ohci_bus_stop(OHCIState *ohci)
 {
-    if (ohci->eof_timer)
+    trace_usb_ohci_stop(ohci->name);
+    if (ohci->eof_timer) {
         timer_del(ohci->eof_timer);
+        timer_free(ohci->eof_timer);
+    }
     ohci->eof_timer = NULL;
 }
 
@@ -1414,8 +1397,7 @@ static void ohci_set_frame_interval(OHCIState *ohci, uint16_t val)
     val &= OHCI_FMI_FI;
 
     if (val != ohci->fi) {
-        DPRINTF("usb-ohci: %s: FrameInterval = 0x%x (%u)\n",
-            ohci->name, ohci->fi, ohci->fi);
+        trace_usb_ohci_set_frame_interval(ohci->name, ohci->fi, ohci->fi);
     }
 
     ohci->fi = val;
@@ -1447,20 +1429,19 @@ static void ohci_set_ctl(OHCIState *ohci, uint32_t val)
     if (old_state == new_state)
         return;
 
+    trace_usb_ohci_set_ctl(ohci->name, new_state);
     switch (new_state) {
     case OHCI_USB_OPERATIONAL:
         ohci_bus_start(ohci);
         break;
     case OHCI_USB_SUSPEND:
         ohci_bus_stop(ohci);
-        DPRINTF("usb-ohci: %s: USB Suspended\n", ohci->name);
         break;
     case OHCI_USB_RESUME:
-        DPRINTF("usb-ohci: %s: USB Resume\n", ohci->name);
+        trace_usb_ohci_resume(ohci->name);
         break;
     case OHCI_USB_RESET:
         ohci_reset(ohci);
-        DPRINTF("usb-ohci: %s: USB Reset\n", ohci->name);
         break;
     }
 }
@@ -1505,7 +1486,7 @@ static void ohci_set_hub_status(OHCIState *ohci, uint32_t val)
 
         for (i = 0; i < ohci->num_ports; i++)
             ohci_port_power(ohci, i, 0);
-        DPRINTF("usb-ohci: powered down all ports\n");
+        trace_usb_ohci_hub_power_down();
     }
 
     if (val & OHCI_RHS_LPSC) {
@@ -1513,7 +1494,7 @@ static void ohci_set_hub_status(OHCIState *ohci, uint32_t val)
 
         for (i = 0; i < ohci->num_ports; i++)
             ohci_port_power(ohci, i, 1);
-        DPRINTF("usb-ohci: powered up all ports\n");
+        trace_usb_ohci_hub_power_up();
     }
 
     if (val & OHCI_RHS_DRWE)
@@ -1545,11 +1526,11 @@ static void ohci_port_set_status(OHCIState *ohci, int portnum, uint32_t val)
     ohci_port_set_if_connected(ohci, portnum, val & OHCI_PORT_PES);
 
     if (ohci_port_set_if_connected(ohci, portnum, val & OHCI_PORT_PSS)) {
-        DPRINTF("usb-ohci: port %d: SUSPEND\n", portnum);
+        trace_usb_ohci_port_suspend(portnum);
     }
 
     if (ohci_port_set_if_connected(ohci, portnum, val & OHCI_PORT_PRS)) {
-        DPRINTF("usb-ohci: port %d: RESET\n", portnum);
+        trace_usb_ohci_port_reset(portnum);
         usb_device_reset(port->port.dev);
         port->ctrl &= ~OHCI_PORT_PRS;
         /* ??? Should this also set OHCI_PORT_PESC.  */
@@ -1577,7 +1558,7 @@ static uint64_t ohci_mem_read(void *opaque,
 
     /* Only aligned reads are allowed on OHCI */
     if (addr & 3) {
-        fprintf(stderr, "usb-ohci: Mis-aligned read\n");
+        trace_usb_ohci_mem_read_unaligned(addr);
         return 0xffffffff;
     } else if (addr >= 0x54 && addr < 0x54 + ohci->num_ports * 4) {
         /* HcRhPortStatus */
@@ -1683,7 +1664,7 @@ static uint64_t ohci_mem_read(void *opaque,
             break;
 
         default:
-            fprintf(stderr, "ohci_read: Bad offset %x\n", (int)addr);
+            trace_usb_ohci_mem_read_bad_offset(addr);
             retval = 0xffffffff;
         }
     }
@@ -1700,7 +1681,7 @@ static void ohci_mem_write(void *opaque,
 
     /* Only aligned reads are allowed on OHCI */
     if (addr & 3) {
-        fprintf(stderr, "usb-ohci: Mis-aligned write\n");
+        trace_usb_ohci_mem_write_unaligned(addr);
         return;
     }
 
@@ -1814,7 +1795,7 @@ static void ohci_mem_write(void *opaque,
         break;
 
     default:
-        fprintf(stderr, "ohci_write: Bad offset %x\n", (int)addr);
+        trace_usb_ohci_mem_write_bad_offset(addr);
         break;
     }
 }
@@ -1867,8 +1848,7 @@ static int usb_ohci_init(OHCIState *ohci, DeviceState *dev,
             usb_bit_time = 1;
         }
 #endif
-        DPRINTF("usb-ohci: usb_bit_time=%" PRId64 " usb_frame_time=%" PRId64 "\n",
-                usb_frame_time, usb_bit_time);
+        trace_usb_ohci_init_time(usb_frame_time, usb_bit_time);
     }
 
     ohci->num_ports = num_ports;
@@ -1926,7 +1906,7 @@ static void ohci_die(OHCIState *ohci)
 {
     OHCIPCIState *dev = container_of(ohci, OHCIPCIState, state);
 
-    fprintf(stderr, "%s: DMA error\n", __func__);
+    trace_usb_ohci_die();
 
     ohci_set_interrupt(ohci, OHCI_INTR_UE);
     ohci_bus_stop(ohci);
@@ -1952,6 +1932,25 @@ static int usb_ohci_initfn_pci(PCIDevice *dev)
     return 0;
 }
 
+static void usb_ohci_exit(PCIDevice *dev)
+{
+    OHCIPCIState *ohci = PCI_OHCI(dev);
+    OHCIState *s = &ohci->state;
+
+    trace_usb_ohci_exit(s->name);
+    ohci_bus_stop(s);
+
+    if (s->async_td) {
+        usb_cancel_packet(&s->usb_packet);
+        s->async_td = 0;
+    }
+    ohci_stop_endpoints(s);
+
+    if (!ohci->masterbus) {
+        usb_bus_release(&s->bus);
+    }
+}
+
 #define TYPE_SYSBUS_OHCI "sysbus-ohci"
 #define SYSBUS_OHCI(obj) OBJECT_CHECK(OHCISysBusState, (obj), TYPE_SYSBUS_OHCI)
 
@@ -2089,6 +2088,7 @@ static void ohci_pci_class_init(ObjectClass *klass, void *data)
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
     k->init = usb_ohci_initfn_pci;
+    k->exit = usb_ohci_exit;
     k->vendor_id = PCI_VENDOR_ID_APPLE;
     k->device_id = PCI_DEVICE_ID_APPLE_IPID_USB;
     k->class_id = PCI_CLASS_SERIAL_USB;
diff --git a/hw/usb/hcd-uhci.c b/hw/usb/hcd-uhci.c
index c3bf72cc1..4a4215d33 100644
--- a/hw/usb/hcd-uhci.c
+++ b/hw/usb/hcd-uhci.c
@@ -35,9 +35,6 @@
 #include "trace.h"
 #include "qemu/main-loop.h"
 
-//#define DEBUG
-//#define DEBUG_DUMP_DATA
-
 #define FRAME_TIMER_FREQ 1000
 
 #define FRAME_MAX_LOOPS  256
@@ -1260,16 +1257,37 @@ static void usb_uhci_exit(PCIDevice *dev)
 {
     UHCIState *s = DO_UPCAST(UHCIState, dev, dev);
 
-    memory_region_destroy(&s->io_bar);
+    trace_usb_uhci_exit();
+
+    if (s->frame_timer) {
+        timer_del(s->frame_timer);
+        timer_free(s->frame_timer);
+        s->frame_timer = NULL;
+    }
+
+    if (s->bh) {
+        qemu_bh_delete(s->bh);
+    }
+
+    uhci_async_cancel_all(s);
+
+    if (!s->masterbus) {
+        usb_bus_release(&s->bus);
+    }
 }
 
-static Property uhci_properties[] = {
+static Property uhci_properties_companion[] = {
     DEFINE_PROP_STRING("masterbus", UHCIState, masterbus),
     DEFINE_PROP_UINT32("firstport", UHCIState, firstport, 0),
     DEFINE_PROP_UINT32("bandwidth", UHCIState, frame_bandwidth, 1280),
     DEFINE_PROP_UINT32("maxframes", UHCIState, maxframes, 128),
     DEFINE_PROP_END_OF_LIST(),
 };
+static Property uhci_properties_standalone[] = {
+    DEFINE_PROP_UINT32("bandwidth", UHCIState, frame_bandwidth, 1280),
+    DEFINE_PROP_UINT32("maxframes", UHCIState, maxframes, 128),
+    DEFINE_PROP_END_OF_LIST(),
+};
 
 static void uhci_class_init(ObjectClass *klass, void *data)
 {
@@ -1284,9 +1302,14 @@ static void uhci_class_init(ObjectClass *klass, void *data)
     k->device_id = info->device_id;
     k->revision  = info->revision;
     k->class_id  = PCI_CLASS_SERIAL_USB;
-    dc->hotpluggable = false;
     dc->vmsd = &vmstate_uhci;
-    dc->props = uhci_properties;
+    if (!info->unplug) {
+        /* uhci controllers in companion setups can't be hotplugged */
+        dc->hotpluggable = false;
+        dc->props = uhci_properties_companion;
+    } else {
+        dc->props = uhci_properties_standalone;
+    }
     set_bit(DEVICE_CATEGORY_USB, dc->categories);
     u->info = *info;
 }
diff --git a/hw/usb/hcd-xhci.c b/hw/usb/hcd-xhci.c
index 58c4b1152..9a942cfad 100644
--- a/hw/usb/hcd-xhci.c
+++ b/hw/usb/hcd-xhci.c
@@ -459,6 +459,7 @@ struct XHCIState {
     uint32_t numintrs;
     uint32_t numslots;
     uint32_t flags;
+    uint32_t max_pstreams_mask;
 
     /* Operational Registers */
     uint32_t usbcmd;
@@ -499,6 +500,8 @@ enum xhci_flags {
     XHCI_FLAG_USE_MSI = 1,
     XHCI_FLAG_USE_MSI_X,
     XHCI_FLAG_SS_FIRST,
+    XHCI_FLAG_FORCE_PCIE_ENDCAP,
+    XHCI_FLAG_ENABLE_STREAMS,
 };
 
 static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid,
@@ -1151,7 +1154,7 @@ static void xhci_reset_streams(XHCIEPContext *epctx)
 static void xhci_alloc_streams(XHCIEPContext *epctx, dma_addr_t base)
 {
     assert(epctx->pstreams == NULL);
-    epctx->nr_pstreams = 2 << (epctx->max_pstreams + 1);
+    epctx->nr_pstreams = 2 << epctx->max_pstreams;
     epctx->pstreams = xhci_alloc_stream_contexts(epctx->nr_pstreams, base);
 }
 
@@ -1180,7 +1183,7 @@ static int xhci_epmask_to_eps_with_streams(XHCIState *xhci,
     slot = &xhci->slots[slotid - 1];
 
     for (i = 2, j = 0; i <= 31; i++) {
-        if (!(epmask & (1 << i))) {
+        if (!(epmask & (1u << i))) {
             continue;
         }
 
@@ -1380,14 +1383,11 @@ static void xhci_init_epctx(XHCIEPContext *epctx,
     dequeue = xhci_addr64(ctx[2] & ~0xf, ctx[3]);
 
     epctx->type = (ctx[1] >> EP_TYPE_SHIFT) & EP_TYPE_MASK;
-    DPRINTF("xhci: endpoint %d.%d type is %d\n", epid/2, epid%2, epctx->type);
     epctx->pctx = pctx;
     epctx->max_psize = ctx[1]>>16;
     epctx->max_psize *= 1+((ctx[1]>>8)&0xff);
-    epctx->max_pstreams = (ctx[0] >> 10) & 0xf;
+    epctx->max_pstreams = (ctx[0] >> 10) & epctx->xhci->max_pstreams_mask;
     epctx->lsa = (ctx[0] >> 15) & 1;
-    DPRINTF("xhci: endpoint %d.%d max transaction (burst) size is %d\n",
-            epid/2, epid%2, epctx->max_psize);
     if (epctx->max_pstreams) {
         xhci_alloc_streams(epctx, dequeue);
     } else {
@@ -1418,6 +1418,9 @@ static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,
     slot->eps[epid-1] = epctx;
     xhci_init_epctx(epctx, pctx, ctx);
 
+    DPRINTF("xhci: endpoint %d.%d type is %d, max transaction (burst) "
+            "size is %d\n", epid/2, epid%2, epctx->type, epctx->max_psize);
+
     epctx->mfindex_last = 0;
 
     epctx->state = EP_RUNNING;
@@ -1607,12 +1610,6 @@ static TRBCCode xhci_reset_ep(XHCIState *xhci, unsigned int slotid,
                 "data might be lost\n");
     }
 
-    uint8_t ep = epid>>1;
-
-    if (epid & 1) {
-        ep |= 0x80;
-    }
-
     if (!xhci->slots[slotid-1].uport ||
         !xhci->slots[slotid-1].uport->dev ||
         !xhci->slots[slotid-1].uport->dev->attached) {
@@ -2265,6 +2262,9 @@ static USBPort *xhci_lookup_uport(XHCIState *xhci, uint32_t *slot_ctx)
     int i, pos, port;
 
     port = (slot_ctx[1]>>16) & 0xFF;
+    if (port < 1 || port > xhci->numports) {
+        return NULL;
+    }
     port = xhci->ports[port-1].uport->index+1;
     pos = snprintf(path, sizeof(path), "%d", port);
     for (i = 0; i < 5; i++) {
@@ -2465,7 +2465,7 @@ static TRBCCode xhci_configure_slot(XHCIState *xhci, unsigned int slotid,
     res = xhci_alloc_device_streams(xhci, slotid, ictl_ctx[1]);
     if (res != CC_SUCCESS) {
         for (i = 2; i <= 31; i++) {
-            if (ictl_ctx[1] & (1 << i)) {
+            if (ictl_ctx[1] & (1u << i)) {
                 xhci_disable_ep(xhci, slotid, i);
             }
         }
@@ -2961,9 +2961,9 @@ static uint64_t xhci_cap_read(void *ptr, hwaddr reg, unsigned size)
         break;
     case 0x10: /* HCCPARAMS */
         if (sizeof(dma_addr_t) == 4) {
-            ret = 0x00087000;
+            ret = 0x00080000 | (xhci->max_pstreams_mask << 12);
         } else {
-            ret = 0x00087001;
+            ret = 0x00080001 | (xhci->max_pstreams_mask << 12);
         }
         break;
     case 0x14: /* DBOFF */
@@ -3595,6 +3595,11 @@ static int usb_xhci_initfn(struct PCIDevice *dev)
     if (xhci->numslots < 1) {
         xhci->numslots = 1;
     }
+    if (xhci_get_flag(xhci, XHCI_FLAG_ENABLE_STREAMS)) {
+        xhci->max_pstreams_mask = 7; /* == 256 primary streams */
+    } else {
+        xhci->max_pstreams_mask = 0;
+    }
 
     xhci->mfwrap_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, xhci_mfwrap_timer, xhci);
 
@@ -3626,7 +3631,8 @@ static int usb_xhci_initfn(struct PCIDevice *dev)
                      PCI_BASE_ADDRESS_SPACE_MEMORY|PCI_BASE_ADDRESS_MEM_TYPE_64,
                      &xhci->mem);
 
-    if (pci_bus_is_express(dev->bus)) {
+    if (pci_bus_is_express(dev->bus) ||
+        xhci_get_flag(xhci, XHCI_FLAG_FORCE_PCIE_ENDCAP)) {
         ret = pcie_endpoint_cap_init(dev, 0xa0);
         assert(ret >= 0);
     }
@@ -3644,6 +3650,43 @@ static int usb_xhci_initfn(struct PCIDevice *dev)
     return 0;
 }
 
+static void usb_xhci_exit(PCIDevice *dev)
+{
+    int i;
+    XHCIState *xhci = XHCI(dev);
+
+    trace_usb_xhci_exit();
+
+    for (i = 0; i < xhci->numslots; i++) {
+        xhci_disable_slot(xhci, i + 1);
+    }
+
+    if (xhci->mfwrap_timer) {
+        timer_del(xhci->mfwrap_timer);
+        timer_free(xhci->mfwrap_timer);
+        xhci->mfwrap_timer = NULL;
+    }
+
+    memory_region_del_subregion(&xhci->mem, &xhci->mem_cap);
+    memory_region_del_subregion(&xhci->mem, &xhci->mem_oper);
+    memory_region_del_subregion(&xhci->mem, &xhci->mem_runtime);
+    memory_region_del_subregion(&xhci->mem, &xhci->mem_doorbell);
+
+    for (i = 0; i < xhci->numports; i++) {
+        XHCIPort *port = &xhci->ports[i];
+        memory_region_del_subregion(&xhci->mem, &port->mem);
+    }
+
+    /* destroy msix memory region */
+    if (dev->msix_table && dev->msix_pba
+        && dev->msix_entry_used) {
+        memory_region_del_subregion(&xhci->mem, &dev->msix_table_mmio);
+        memory_region_del_subregion(&xhci->mem, &dev->msix_pba_mmio);
+    }
+
+    usb_bus_release(&xhci->bus);
+}
+
 static int usb_xhci_post_load(void *opaque, int version_id)
 {
     XHCIState *xhci = opaque;
@@ -3666,6 +3709,12 @@ static int usb_xhci_post_load(void *opaque, int version_id)
             xhci_mask64(ldq_le_pci_dma(pci_dev, dcbaap + 8 * slotid));
         xhci_dma_read_u32s(xhci, slot->ctx, slot_ctx, sizeof(slot_ctx));
         slot->uport = xhci_lookup_uport(xhci, slot_ctx);
+        if (!slot->uport) {
+            /* should not happen, but may trigger on guest bugs */
+            slot->enabled = 0;
+            slot->addressed = 0;
+            continue;
+        }
         assert(slot->uport && slot->uport->dev);
 
         for (epid = 1; epid <= 31; epid++) {
@@ -3818,6 +3867,10 @@ static Property xhci_properties[] = {
     DEFINE_PROP_BIT("msix",     XHCIState, flags, XHCI_FLAG_USE_MSI_X, true),
     DEFINE_PROP_BIT("superspeed-ports-first",
                     XHCIState, flags, XHCI_FLAG_SS_FIRST, true),
+    DEFINE_PROP_BIT("force-pcie-endcap", XHCIState, flags,
+                    XHCI_FLAG_FORCE_PCIE_ENDCAP, false),
+    DEFINE_PROP_BIT("streams", XHCIState, flags,
+                    XHCI_FLAG_ENABLE_STREAMS, true),
     DEFINE_PROP_UINT32("intrs", XHCIState, numintrs, MAXINTRS),
     DEFINE_PROP_UINT32("slots", XHCIState, numslots, MAXSLOTS),
     DEFINE_PROP_UINT32("p2",    XHCIState, numports_2, 4),
@@ -3833,9 +3886,9 @@ static void xhci_class_init(ObjectClass *klass, void *data)
     dc->vmsd    = &vmstate_xhci;
     dc->props   = xhci_properties;
     dc->reset   = xhci_reset;
-    dc->hotpluggable   = false;
     set_bit(DEVICE_CATEGORY_USB, dc->categories);
     k->init         = usb_xhci_initfn;
+    k->exit         = usb_xhci_exit;
     k->vendor_id    = PCI_VENDOR_ID_NEC;
     k->device_id    = PCI_DEVICE_ID_NEC_UPD720200;
     k->class_id     = PCI_CLASS_SERIAL_USB;
diff --git a/hw/usb/host-libusb.c b/hw/usb/host-libusb.c
index c189147f9..a5f9dab0c 100644
--- a/hw/usb/host-libusb.c
+++ b/hw/usb/host-libusb.c
@@ -143,6 +143,12 @@ static void usb_host_attach_kernel(USBHostDevice *s);
 
 /* ------------------------------------------------------------------------ */
 
+#ifndef LIBUSB_LOG_LEVEL_WARNING /* older libusb didn't define these */
+#define LIBUSB_LOG_LEVEL_WARNING 2
+#endif
+
+/* ------------------------------------------------------------------------ */
+
 #define CONTROL_TIMEOUT  10000        /* 10 sec    */
 #define BULK_TIMEOUT         0        /* unlimited */
 #define INTR_TIMEOUT         0        /* unlimited */
@@ -275,7 +281,7 @@ static void usb_host_libusb_error(const char *func, int rc)
     } else {
         errname = "?";
     }
-    fprintf(stderr, "%s: %d [%s]\n", func, rc, errname);
+    error_report("%s: %d [%s]", func, rc, errname);
 }
 
 /* ------------------------------------------------------------------------ */
@@ -743,13 +749,13 @@ static void usb_host_speed_compat(USBHostDevice *s)
 
     udev->speedmask = (1 << udev->speed);
     if (udev->speed == USB_SPEED_SUPER && compat_high) {
-        udev->speedmask |= USB_SPEED_HIGH;
+        udev->speedmask |= USB_SPEED_MASK_HIGH;
     }
     if (udev->speed == USB_SPEED_SUPER && compat_full) {
-        udev->speedmask |= USB_SPEED_FULL;
+        udev->speedmask |= USB_SPEED_MASK_FULL;
     }
     if (udev->speed == USB_SPEED_HIGH && compat_full) {
-        udev->speedmask |= USB_SPEED_FULL;
+        udev->speedmask |= USB_SPEED_MASK_FULL;
     }
 }
 
@@ -834,6 +840,7 @@ static int usb_host_open(USBHostDevice *s, libusb_device *dev)
     int bus_num = libusb_get_bus_number(dev);
     int addr    = libusb_get_device_address(dev);
     int rc;
+    Error *local_err = NULL;
 
     trace_usb_host_open_started(bus_num, addr);
 
@@ -869,8 +876,10 @@ static int usb_host_open(USBHostDevice *s, libusb_device *dev)
                  "host:%d.%d", bus_num, addr);
     }
 
-    rc = usb_device_attach(udev);
-    if (rc) {
+    usb_device_attach(udev, &local_err);
+    if (local_err) {
+        error_report("%s", error_get_pretty(local_err));
+        error_free(local_err);
         goto fail;
     }
 
@@ -948,21 +957,21 @@ static void usb_host_exit_notifier(struct Notifier *n, void *data)
     }
 }
 
-static int usb_host_initfn(USBDevice *udev)
+static void usb_host_realize(USBDevice *udev, Error **errp)
 {
     USBHostDevice *s = USB_HOST_DEVICE(udev);
 
     if (s->match.vendor_id > 0xffff) {
-        error_report("vendorid out of range");
-        return -1;
+        error_setg(errp, "vendorid out of range");
+        return;
     }
     if (s->match.product_id > 0xffff) {
-        error_report("productid out of range");
-        return -1;
+        error_setg(errp, "productid out of range");
+        return;
     }
     if (s->match.addr > 127) {
-        error_report("hostaddr out of range");
-        return -1;
+        error_setg(errp, "hostaddr out of range");
+        return;
     }
 
     loglevel = s->loglevel;
@@ -975,9 +984,17 @@ static int usb_host_initfn(USBDevice *udev)
     qemu_add_exit_notifier(&s->exit);
 
     QTAILQ_INSERT_TAIL(&hostdevs, s, next);
-    add_boot_device_path(s->bootindex, &udev->qdev, NULL);
     usb_host_auto_check(NULL);
-    return 0;
+}
+
+static void usb_host_instance_init(Object *obj)
+{
+    USBDevice *udev = USB_DEVICE(obj);
+    USBHostDevice *s = USB_HOST_DEVICE(udev);
+
+    device_add_bootindex_property(obj, &s->bootindex,
+                                  "bootindex", NULL,
+                                  &udev->qdev, NULL);
 }
 
 static void usb_host_handle_destroy(USBDevice *udev)
@@ -1374,14 +1391,13 @@ static int usb_host_alloc_streams(USBDevice *udev, USBEndpoint **eps,
     if (rc < 0) {
         usb_host_libusb_error("libusb_alloc_streams", rc);
     } else if (rc != streams) {
-        fprintf(stderr,
-            "libusb_alloc_streams: got less streams then requested %d < %d\n",
-            rc, streams);
+        error_report("libusb_alloc_streams: got less streams "
+                     "then requested %d < %d", rc, streams);
     }
 
     return (rc == streams) ? 0 : -1;
 #else
-    fprintf(stderr, "libusb_alloc_streams: error not implemented\n");
+    error_report("libusb_alloc_streams: error not implemented");
     return -1;
 #endif
 }
@@ -1464,7 +1480,6 @@ static Property usb_host_dev_properties[] = {
     DEFINE_PROP_UINT32("productid", USBHostDevice, match.product_id, 0),
     DEFINE_PROP_UINT32("isobufs",  USBHostDevice, iso_urb_count,    4),
     DEFINE_PROP_UINT32("isobsize", USBHostDevice, iso_urb_frames,   32),
-    DEFINE_PROP_INT32("bootindex", USBHostDevice, bootindex,        -1),
     DEFINE_PROP_UINT32("loglevel",  USBHostDevice, loglevel,
                        LIBUSB_LOG_LEVEL_WARNING),
     DEFINE_PROP_BIT("pipeline",    USBHostDevice, options,
@@ -1477,7 +1492,7 @@ static void usb_host_class_initfn(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
-    uc->init           = usb_host_initfn;
+    uc->realize        = usb_host_realize;
     uc->product_desc   = "USB Host Device";
     uc->cancel_packet  = usb_host_cancel_packet;
     uc->handle_data    = usb_host_handle_data;
@@ -1497,6 +1512,7 @@ static TypeInfo usb_host_dev_info = {
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBHostDevice),
     .class_init    = usb_host_class_initfn,
+    .instance_init = usb_host_instance_init,
 };
 
 static void usb_host_register_types(void)
diff --git a/hw/usb/redirect.c b/hw/usb/redirect.c
index 44522d900..9fbd59e5e 100644
--- a/hw/usb/redirect.c
+++ b/hw/usb/redirect.c
@@ -1256,6 +1256,7 @@ static void usbredir_device_reject_bh(void *opaque)
 static void usbredir_do_attach(void *opaque)
 {
     USBRedirDevice *dev = opaque;
+    Error *local_err = NULL;
 
     /* In order to work properly with XHCI controllers we need these caps */
     if ((dev->dev.port->speedmask & USB_SPEED_MASK_SUPER) && !(
@@ -1270,7 +1271,10 @@ static void usbredir_do_attach(void *opaque)
         return;
     }
 
-    if (usb_device_attach(&dev->dev) != 0) {
+    usb_device_attach(&dev->dev, &local_err);
+    if (local_err) {
+        error_report("%s", error_get_pretty(local_err));
+        error_free(local_err);
         WARNING("rejecting device due to speed mismatch\n");
         usbredir_reject_device(dev);
     }
@@ -1357,14 +1361,14 @@ static void usbredir_init_endpoints(USBRedirDevice *dev)
     }
 }
 
-static int usbredir_initfn(USBDevice *udev)
+static void usbredir_realize(USBDevice *udev, Error **errp)
 {
     USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
     int i;
 
     if (dev->cs == NULL) {
-        qerror_report(QERR_MISSING_PARAMETER, "chardev");
-        return -1;
+        error_set(errp, QERR_MISSING_PARAMETER, "chardev");
+        return;
     }
 
     if (dev->filter_str) {
@@ -1372,9 +1376,9 @@ static int usbredir_initfn(USBDevice *udev)
                                            &dev->filter_rules,
                                            &dev->filter_rules_count);
         if (i) {
-            qerror_report(QERR_INVALID_PARAMETER_VALUE, "filter",
-                          "a usb device filter string");
-            return -1;
+            error_set(errp, QERR_INVALID_PARAMETER_VALUE, "filter",
+                      "a usb device filter string");
+            return;
         }
     }
 
@@ -1397,8 +1401,6 @@ static int usbredir_initfn(USBDevice *udev)
                           usbredir_chardev_read, usbredir_chardev_event, dev);
 
     qemu_add_vm_change_state_handler(usbredir_vm_state_change, dev);
-    add_boot_device_path(dev->bootindex, &udev->qdev, NULL);
-    return 0;
 }
 
 static void usbredir_cleanup_device_queues(USBRedirDevice *dev)
@@ -2468,7 +2470,6 @@ static Property usbredir_properties[] = {
     DEFINE_PROP_CHR("chardev", USBRedirDevice, cs),
     DEFINE_PROP_UINT8("debug", USBRedirDevice, debug, usbredirparser_warning),
     DEFINE_PROP_STRING("filter", USBRedirDevice, filter_str),
-    DEFINE_PROP_INT32("bootindex", USBRedirDevice, bootindex, -1),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -2477,7 +2478,7 @@ static void usbredir_class_initfn(ObjectClass *klass, void *data)
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    uc->init           = usbredir_initfn;
+    uc->realize        = usbredir_realize;
     uc->product_desc   = "USB Redirection Device";
     uc->handle_destroy = usbredir_handle_destroy;
     uc->cancel_packet  = usbredir_cancel_packet;
@@ -2493,11 +2494,22 @@ static void usbredir_class_initfn(ObjectClass *klass, void *data)
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 }
 
+static void usbredir_instance_init(Object *obj)
+{
+    USBDevice *udev = USB_DEVICE(obj);
+    USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
+
+    device_add_bootindex_property(obj, &dev->bootindex,
+                                  "bootindex", NULL,
+                                  &udev->qdev, NULL);
+}
+
 static const TypeInfo usbredir_dev_info = {
     .name          = "usb-redir",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBRedirDevice),
     .class_init    = usbredir_class_initfn,
+    .instance_init = usbredir_instance_init,
 };
 
 static void usbredir_register_types(void)
diff --git a/hw/virtio/Makefile.objs b/hw/virtio/Makefile.objs
index ec9e855bc..d21c39775 100644
--- a/hw/virtio/Makefile.objs
+++ b/hw/virtio/Makefile.objs
@@ -2,7 +2,7 @@ common-obj-y += virtio-rng.o
 common-obj-$(CONFIG_VIRTIO_PCI) += virtio-pci.o
 common-obj-y += virtio-bus.o
 common-obj-y += virtio-mmio.o
-common-obj-$(CONFIG_VIRTIO_BLK_DATA_PLANE) += dataplane/
+common-obj-$(CONFIG_VIRTIO) += dataplane/
 
 obj-y += virtio.o virtio-balloon.o 
 obj-$(CONFIG_LINUX) += vhost.o vhost-backend.o vhost-user.o
diff --git a/hw/virtio/dataplane/vring.c b/hw/virtio/dataplane/vring.c
index 67cb2b823..61f6d8368 100644
--- a/hw/virtio/dataplane/vring.c
+++ b/hw/virtio/dataplane/vring.c
@@ -181,7 +181,8 @@ static int get_desc(Vring *vring, VirtQueueElement *elem,
 
     /* Stop for now if there are not enough iovecs available. */
     if (*num >= VIRTQUEUE_MAX_SIZE) {
-        return -ENOBUFS;
+        error_report("Invalid SG num: %u", *num);
+        return -EFAULT;
     }
 
     /* TODO handle non-contiguous memory across region boundaries */
@@ -351,10 +352,6 @@ int vring_pop(VirtIODevice *vdev, Vring *vring,
         goto out;
     }
 
-    if (vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX)) {
-        vring_avail_event(&vring->vr) = vring->vr.avail->idx;
-    }
-
     i = head;
     do {
         if (unlikely(i >= num)) {
@@ -391,6 +388,10 @@ int vring_pop(VirtIODevice *vdev, Vring *vring,
 
     /* On success, increment avail index. */
     vring->last_avail_idx++;
+    if (vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX)) {
+        vring_avail_event(&vring->vr) = vring->last_avail_idx;
+    }
+
     return head;
 
 out:
diff --git a/hw/virtio/vhost-backend.c b/hw/virtio/vhost-backend.c
index 35316c40d..ff4f2001b 100644
--- a/hw/virtio/vhost-backend.c
+++ b/hw/virtio/vhost-backend.c
@@ -14,8 +14,6 @@
 
 #include <sys/ioctl.h>
 
-extern const VhostOps user_ops;
-
 static int vhost_kernel_call(struct vhost_dev *dev, unsigned long int request,
                              void *arg)
 {
diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
index 4e88d9c5e..aefe0bbaa 100644
--- a/hw/virtio/vhost-user.c
+++ b/hw/virtio/vhost-user.c
@@ -226,7 +226,7 @@ static int vhost_user_call(struct vhost_dev *dev, unsigned long int request,
                 msg.memory.regions[fd_num].memory_size  = reg->memory_size;
                 msg.memory.regions[fd_num].guest_phys_addr = reg->guest_phys_addr;
                 msg.memory.regions[fd_num].mmap_offset = reg->userspace_addr -
-                    (uintptr_t) qemu_get_ram_block_host_ptr(reg->guest_phys_addr);
+                    (uintptr_t) qemu_get_ram_block_host_ptr(ram_addr);
                 assert(fd_num < VHOST_MEMORY_MAX_NREGIONS);
                 fds[fd_num++] = fd;
             }
diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c
index e55fe1cc7..5a128613b 100644
--- a/hw/virtio/vhost.c
+++ b/hw/virtio/vhost.c
@@ -817,10 +817,12 @@ int vhost_dev_init(struct vhost_dev *hdev, void *opaque,
     int i, r;
 
     if (vhost_set_backend_type(hdev, backend_type) < 0) {
+        close((uintptr_t)opaque);
         return -1;
     }
 
     if (hdev->vhost_ops->vhost_backend_init(hdev, opaque) < 0) {
+        close((uintptr_t)opaque);
         return -errno;
     }
 
@@ -976,7 +978,6 @@ void vhost_dev_disable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
 bool vhost_virtqueue_pending(struct vhost_dev *hdev, int n)
 {
     struct vhost_virtqueue *vq = hdev->vqs + n - hdev->vq_index;
-    assert(hdev->started);
     assert(n >= hdev->vq_index && n < hdev->vq_index + hdev->nvqs);
     return event_notifier_test_and_clear(&vq->masked_notifier);
 }
@@ -988,7 +989,6 @@ void vhost_virtqueue_mask(struct vhost_dev *hdev, VirtIODevice *vdev, int n,
     struct VirtQueue *vvq = virtio_get_queue(vdev, n);
     int r, index = n - hdev->vq_index;
 
-    assert(hdev->started);
     assert(n >= hdev->vq_index && n < hdev->vq_index + hdev->nvqs);
 
     struct vhost_vring_file file = {
diff --git a/hw/virtio/virtio-balloon.c b/hw/virtio/virtio-balloon.c
index 2c30b3d8b..7bfbb75ad 100644
--- a/hw/virtio/virtio-balloon.c
+++ b/hw/virtio/virtio-balloon.c
@@ -87,7 +87,7 @@ static void balloon_stats_destroy_timer(VirtIOBalloon *s)
     }
 }
 
-static void balloon_stats_change_timer(VirtIOBalloon *s, int secs)
+static void balloon_stats_change_timer(VirtIOBalloon *s, int64_t secs)
 {
     timer_mod(s->stats_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + secs * 1000);
 }
@@ -170,6 +170,11 @@ static void balloon_stats_set_poll_interval(Object *obj, struct Visitor *v,
         return;
     }
 
+    if (value > UINT32_MAX) {
+        error_setg(errp, "timer value is too big");
+        return;
+    }
+
     if (value == s->stats_poll_interval) {
         return;
     }
diff --git a/hw/virtio/virtio-mmio.c b/hw/virtio/virtio-mmio.c
index 18c6e5b55..2450c1374 100644
--- a/hw/virtio/virtio-mmio.c
+++ b/hw/virtio/virtio-mmio.c
@@ -89,9 +89,6 @@ typedef struct {
     VirtioBusState bus;
 } VirtIOMMIOProxy;
 
-static void virtio_mmio_bus_new(VirtioBusState *bus, size_t bus_size,
-                                VirtIOMMIOProxy *dev);
-
 static uint64_t virtio_mmio_read(void *opaque, hwaddr offset, unsigned size)
 {
     VirtIOMMIOProxy *proxy = (VirtIOMMIOProxy *)opaque;
@@ -362,7 +359,8 @@ static void virtio_mmio_realizefn(DeviceState *d, Error **errp)
     VirtIOMMIOProxy *proxy = VIRTIO_MMIO(d);
     SysBusDevice *sbd = SYS_BUS_DEVICE(d);
 
-    virtio_mmio_bus_new(&proxy->bus, sizeof(proxy->bus), proxy);
+    qbus_create_inplace(&proxy->bus, sizeof(proxy->bus), TYPE_VIRTIO_MMIO_BUS,
+                        d, NULL);
     sysbus_init_irq(sbd, &proxy->irq);
     memory_region_init_io(&proxy->iomem, OBJECT(d), &virtio_mem_ops, proxy,
                           TYPE_VIRTIO_MMIO, 0x200);
@@ -393,17 +391,6 @@ static const TypeInfo virtio_mmio_info = {
 
 /* virtio-mmio-bus. */
 
-static void virtio_mmio_bus_new(VirtioBusState *bus, size_t bus_size,
-                                VirtIOMMIOProxy *dev)
-{
-    DeviceState *qdev = DEVICE(dev);
-    BusState *qbus;
-
-    qbus_create_inplace(bus, bus_size, TYPE_VIRTIO_MMIO_BUS, qdev, NULL);
-    qbus = BUS(bus);
-    qbus->allow_hotplug = 0;
-}
-
 static void virtio_mmio_bus_class_init(ObjectClass *klass, void *data)
 {
     BusClass *bus_class = BUS_CLASS(klass);
diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
index 300731974..dde1d73b5 100644
--- a/hw/virtio/virtio-pci.c
+++ b/hw/virtio/virtio-pci.c
@@ -29,7 +29,7 @@
 #include "hw/pci/msix.h"
 #include "hw/loader.h"
 #include "sysemu/kvm.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "virtio-pci.h"
 #include "qemu/range.h"
 #include "hw/virtio/virtio-bus.h"
@@ -86,9 +86,6 @@
  * 12 is historical, and due to x86 page size. */
 #define VIRTIO_PCI_QUEUE_ADDR_SHIFT    12
 
-/* Flags track per-device state like workarounds for quirks in older guests. */
-#define VIRTIO_PCI_FLAG_BUS_MASTER_BUG  (1 << 0)
-
 static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size,
                                VirtIOPCIProxy *dev);
 
@@ -314,12 +311,14 @@ static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
             msix_unuse_all_vectors(&proxy->pci_dev);
         }
 
-        /* Linux before 2.6.34 sets the device as OK without enabling
-           the PCI device bus master bit. In this case we need to disable
-           some safety checks. */
-        if ((val & VIRTIO_CONFIG_S_DRIVER_OK) &&
-            !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
-            proxy->flags |= VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
+        /* Linux before 2.6.34 drives the device without enabling
+           the PCI device bus master bit. Enable it automatically
+           for the guest. This is a PCI spec violation but so is
+           initiating DMA with bus master bit clear. */
+        if (val == (VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER)) {
+            pci_default_write_config(&proxy->pci_dev, PCI_COMMAND,
+                                     proxy->pci_dev.config[PCI_COMMAND] |
+                                     PCI_COMMAND_MASTER, 1);
         }
         break;
     case VIRTIO_MSI_CONFIG_VECTOR:
@@ -473,8 +472,7 @@ static void virtio_write_config(PCIDevice *pci_dev, uint32_t address,
     pci_default_write_config(pci_dev, address, val, len);
 
     if (range_covers_byte(address, len, PCI_COMMAND) &&
-        !(pci_dev->config[PCI_COMMAND] & PCI_COMMAND_MASTER) &&
-        !(proxy->flags & VIRTIO_PCI_FLAG_BUS_MASTER_BUG)) {
+        !(pci_dev->config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
         virtio_pci_stop_ioeventfd(proxy);
         virtio_set_status(vdev, vdev->status & ~VIRTIO_CONFIG_S_DRIVER_OK);
     }
@@ -885,11 +883,15 @@ static void virtio_pci_vmstate_change(DeviceState *d, bool running)
     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 
     if (running) {
-        /* Try to find out if the guest has bus master disabled, but is
-           in ready state. Then we have a buggy guest OS. */
-        if ((vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) &&
+        /* Old QEMU versions did not set bus master enable on status write.
+         * Detect DRIVER set and enable it.
+         */
+        if ((proxy->flags & VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION) &&
+            (vdev->status & VIRTIO_CONFIG_S_DRIVER) &&
             !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
-            proxy->flags |= VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
+            pci_default_write_config(&proxy->pci_dev, PCI_COMMAND,
+                                     proxy->pci_dev.config[PCI_COMMAND] |
+                                     PCI_COMMAND_MASTER, 1);
         }
         virtio_pci_start_ioeventfd(proxy);
     } else {
@@ -914,7 +916,6 @@ static Property virtio_9p_pci_properties[] = {
     DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags,
                     VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 2),
-    DEFINE_VIRTIO_9P_PROPERTIES(V9fsPCIState, vdev.fsconf),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -936,8 +937,9 @@ static void virtio_9p_pci_class_init(ObjectClass *klass, void *data)
 static void virtio_9p_pci_instance_init(Object *obj)
 {
     V9fsPCIState *dev = VIRTIO_9P_PCI(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_9P);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_9P);
 }
 
 static const TypeInfo virtio_9p_pci_info = {
@@ -1020,10 +1022,7 @@ static int virtio_pci_init(PCIDevice *pci_dev)
 
 static void virtio_pci_exit(PCIDevice *pci_dev)
 {
-    VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
-
     msix_uninit_exclusive_bar(pci_dev);
-    memory_region_destroy(&proxy->bar);
 }
 
 static void virtio_pci_reset(DeviceState *qdev)
@@ -1033,10 +1032,11 @@ static void virtio_pci_reset(DeviceState *qdev)
     virtio_pci_stop_ioeventfd(proxy);
     virtio_bus_reset(bus);
     msix_unuse_all_vectors(&proxy->pci_dev);
-    proxy->flags &= ~VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
 }
 
 static Property virtio_pci_properties[] = {
+    DEFINE_PROP_BIT("virtio-pci-bus-master-bug-migration", VirtIOPCIProxy, flags,
+                    VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT, false),
     DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
     DEFINE_PROP_END_OF_LIST(),
 };
@@ -1103,12 +1103,13 @@ static void virtio_blk_pci_class_init(ObjectClass *klass, void *data)
 static void virtio_blk_pci_instance_init(Object *obj)
 {
     VirtIOBlkPCI *dev = VIRTIO_BLK_PCI(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_BLK);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
-    object_unref(OBJECT(&dev->vdev));
-    qdev_alias_all_properties(DEVICE(&dev->vdev), obj);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_BLK);
     object_property_add_alias(obj, "iothread", OBJECT(&dev->vdev),"iothread",
                               &error_abort);
+    object_property_add_alias(obj, "bootindex", OBJECT(&dev->vdev),
+                              "bootindex", &error_abort);
 }
 
 static const TypeInfo virtio_blk_pci_info = {
@@ -1127,7 +1128,6 @@ static Property virtio_scsi_pci_properties[] = {
     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors,
                        DEV_NVECTORS_UNSPECIFIED),
     DEFINE_VIRTIO_SCSI_FEATURES(VirtIOPCIProxy, host_features),
-    DEFINE_VIRTIO_SCSI_PROPERTIES(VirtIOSCSIPCI, vdev.parent_obj.conf),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1177,8 +1177,11 @@ static void virtio_scsi_pci_class_init(ObjectClass *klass, void *data)
 static void virtio_scsi_pci_instance_init(Object *obj)
 {
     VirtIOSCSIPCI *dev = VIRTIO_SCSI_PCI(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_SCSI);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_SCSI);
+    object_property_add_alias(obj, "iothread", OBJECT(&dev->vdev), "iothread",
+                              &error_abort);
 }
 
 static const TypeInfo virtio_scsi_pci_info = {
@@ -1195,7 +1198,6 @@ static const TypeInfo virtio_scsi_pci_info = {
 static Property vhost_scsi_pci_properties[] = {
     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors,
                        DEV_NVECTORS_UNSPECIFIED),
-    DEFINE_VHOST_SCSI_PROPERTIES(VHostSCSIPCI, vdev.parent_obj.conf),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1233,8 +1235,9 @@ static void vhost_scsi_pci_class_init(ObjectClass *klass, void *data)
 static void vhost_scsi_pci_instance_init(Object *obj)
 {
     VHostSCSIPCI *dev = VHOST_SCSI_PCI(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VHOST_SCSI);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VHOST_SCSI);
 }
 
 static const TypeInfo vhost_scsi_pci_info = {
@@ -1315,7 +1318,7 @@ static void virtio_balloon_pci_instance_init(Object *obj)
     VirtIOBalloonPCI *dev = VIRTIO_BALLOON_PCI(obj);
     object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_BALLOON);
     object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
-
+    object_unref(OBJECT(&dev->vdev));
     object_property_add(obj, "guest-stats", "guest statistics",
                         balloon_pci_stats_get_all, NULL, NULL, dev,
                         NULL);
@@ -1377,7 +1380,6 @@ static Property virtio_serial_pci_properties[] = {
                     VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 2),
     DEFINE_PROP_UINT32("class", VirtIOPCIProxy, class_code, 0),
-    DEFINE_VIRTIO_SERIAL_PROPERTIES(VirtIOSerialPCI, vdev.serial),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1398,8 +1400,9 @@ static void virtio_serial_pci_class_init(ObjectClass *klass, void *data)
 static void virtio_serial_pci_instance_init(Object *obj)
 {
     VirtIOSerialPCI *dev = VIRTIO_SERIAL_PCI(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_SERIAL);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_SERIAL);
 }
 
 static const TypeInfo virtio_serial_pci_info = {
@@ -1417,8 +1420,6 @@ static Property virtio_net_properties[] = {
                     VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, false),
     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 3),
     DEFINE_VIRTIO_NET_FEATURES(VirtIOPCIProxy, host_features),
-    DEFINE_NIC_PROPERTIES(VirtIONetPCI, vdev.nic_conf),
-    DEFINE_VIRTIO_NET_PROPERTIES(VirtIONetPCI, vdev.net_conf),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1457,8 +1458,11 @@ static void virtio_net_pci_class_init(ObjectClass *klass, void *data)
 static void virtio_net_pci_instance_init(Object *obj)
 {
     VirtIONetPCI *dev = VIRTIO_NET_PCI(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_NET);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_NET);
+    object_property_add_alias(obj, "bootindex", OBJECT(&dev->vdev),
+                              "bootindex", &error_abort);
 }
 
 static const TypeInfo virtio_net_pci_info = {
@@ -1472,7 +1476,6 @@ static const TypeInfo virtio_net_pci_info = {
 /* virtio-rng-pci */
 
 static Property virtio_rng_pci_properties[] = {
-    DEFINE_VIRTIO_RNG_PROPERTIES(VirtIORngPCI, vdev.conf),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1512,13 +1515,11 @@ static void virtio_rng_pci_class_init(ObjectClass *klass, void *data)
 static void virtio_rng_initfn(Object *obj)
 {
     VirtIORngPCI *dev = VIRTIO_RNG_PCI(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_RNG);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
-    object_property_add_link(obj, "rng", TYPE_RNG_BACKEND,
-                             (Object **)&dev->vdev.conf.rng,
-                             qdev_prop_allow_set_link_before_realize,
-                             OBJ_PROP_LINK_UNREF_ON_RELEASE, NULL);
 
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_RNG);
+    object_property_add_alias(obj, "rng", OBJECT(&dev->vdev), "rng",
+                              &error_abort);
 }
 
 static const TypeInfo virtio_rng_pci_info = {
@@ -1535,13 +1536,10 @@ static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size,
                                VirtIOPCIProxy *dev)
 {
     DeviceState *qdev = DEVICE(dev);
-    BusState *qbus;
     char virtio_bus_name[] = "virtio-bus";
 
     qbus_create_inplace(bus, bus_size, TYPE_VIRTIO_PCI_BUS, qdev,
                         virtio_bus_name);
-    qbus = BUS(bus);
-    qbus->allow_hotplug = 1;
 }
 
 static void virtio_pci_bus_class_init(ObjectClass *klass, void *data)
diff --git a/hw/virtio/virtio-pci.h b/hw/virtio/virtio-pci.h
index 1cea157a4..8873b6d13 100644
--- a/hw/virtio/virtio-pci.h
+++ b/hw/virtio/virtio-pci.h
@@ -53,6 +53,11 @@ typedef struct VirtioBusClass VirtioPCIBusClass;
 #define VIRTIO_PCI_BUS_CLASS(klass) \
         OBJECT_CLASS_CHECK(VirtioPCIBusClass, klass, TYPE_VIRTIO_PCI_BUS)
 
+/* Need to activate work-arounds for buggy guests at vmstate load. */
+#define VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT  0
+#define VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION \
+    (1 << VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT)
+
 /* Performance improves when virtqueue kick processing is decoupled from the
  * vcpu thread using ioeventfd for some devices. */
 #define VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT 1
diff --git a/hw/virtio/virtio-rng.c b/hw/virtio/virtio-rng.c
index 7c5a67567..473c04410 100644
--- a/hw/virtio/virtio-rng.c
+++ b/hw/virtio/virtio-rng.c
@@ -16,6 +16,7 @@
 #include "hw/virtio/virtio-rng.h"
 #include "sysemu/rng.h"
 #include "qom/object_interfaces.h"
+#include "trace.h"
 
 static bool is_guest_ready(VirtIORNG *vrng)
 {
@@ -24,6 +25,7 @@ static bool is_guest_ready(VirtIORNG *vrng)
         && (vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
         return true;
     }
+    trace_virtio_rng_guest_not_ready(vrng);
     return false;
 }
 
@@ -62,6 +64,7 @@ static void chr_read(void *opaque, const void *buf, size_t size)
         offset += len;
 
         virtqueue_push(vrng->vq, &elem, len);
+        trace_virtio_rng_pushed(vrng, len);
     }
     virtio_notify(vdev, vrng->vq);
 }
@@ -81,6 +84,9 @@ static void virtio_rng_process(VirtIORNG *vrng)
         quota = MIN((uint64_t)vrng->quota_remaining, (uint64_t)UINT32_MAX);
     }
     size = get_request_size(vrng->vq, quota);
+
+    trace_virtio_rng_request(vrng, size, quota);
+
     size = MIN(vrng->quota_remaining, size);
     if (size) {
         rng_backend_request_entropy(vrng->rng, size, chr_read, vrng);
@@ -107,20 +113,22 @@ static void virtio_rng_save(QEMUFile *f, void *opaque)
 
 static int virtio_rng_load(QEMUFile *f, void *opaque, int version_id)
 {
+    VirtIORNG *vrng = opaque;
+    int ret;
+
     if (version_id != 1) {
         return -EINVAL;
     }
-    return virtio_load(VIRTIO_DEVICE(opaque), f, version_id);
-}
+    ret = virtio_load(VIRTIO_DEVICE(vrng), f, version_id);
+    if (ret != 0) {
+        return ret;
+    }
 
-static int virtio_rng_load_device(VirtIODevice *vdev, QEMUFile *f,
-                                  int version_id)
-{
     /* We may have an element ready but couldn't process it due to a quota
      * limit.  Make sure to try again after live migration when the quota may
      * have been reset.
      */
-    virtio_rng_process(VIRTIO_RNG(vdev));
+    virtio_rng_process(vrng);
 
     return 0;
 }
@@ -142,8 +150,15 @@ static void virtio_rng_device_realize(DeviceState *dev, Error **errp)
     Error *local_err = NULL;
 
     if (!vrng->conf.period_ms > 0) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "period",
-                  "a positive number");
+        error_setg(errp, "'period' parameter expects a positive integer");
+        return;
+    }
+
+    /* Workaround: Property parsing does not enforce unsigned integers,
+     * So this is a hack to reject such numbers. */
+    if (vrng->conf.max_bytes > INT64_MAX) {
+        error_setg(errp, "'max-bytes' parameter must be non-negative, "
+                   "and less than 2^63");
         return;
     }
 
@@ -171,23 +186,15 @@ static void virtio_rng_device_realize(DeviceState *dev, Error **errp)
                                  "rng", NULL);
     }
 
-    virtio_init(vdev, "virtio-rng", VIRTIO_ID_RNG, 0);
-
     vrng->rng = vrng->conf.rng;
     if (vrng->rng == NULL) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "rng", "a valid object");
+        error_setg(errp, "'rng' parameter expects a valid object");
         return;
     }
 
-    vrng->vq = virtio_add_queue(vdev, 8, handle_input);
+    virtio_init(vdev, "virtio-rng", VIRTIO_ID_RNG, 0);
 
-    /* Workaround: Property parsing does not enforce unsigned integers,
-     * So this is a hack to reject such numbers. */
-    if (vrng->conf.max_bytes > INT64_MAX) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "max-bytes",
-                  "a non-negative integer below 2^63");
-        return;
-    }
+    vrng->vq = virtio_add_queue(vdev, 8, handle_input);
     vrng->quota_remaining = vrng->conf.max_bytes;
 
     vrng->rate_limit_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
@@ -226,7 +233,6 @@ static void virtio_rng_class_init(ObjectClass *klass, void *data)
     vdc->realize = virtio_rng_device_realize;
     vdc->unrealize = virtio_rng_device_unrealize;
     vdc->get_features = get_features;
-    vdc->load = virtio_rng_load_device;
 }
 
 static void virtio_rng_initfn(Object *obj)
diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c
index 5c981801f..013979a6b 100644
--- a/hw/virtio/virtio.c
+++ b/hw/virtio/virtio.c
@@ -469,7 +469,7 @@ int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem)
 
     i = head = virtqueue_get_head(vq, vq->last_avail_idx++);
     if (vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX)) {
-        vring_avail_event(vq, vring_avail_idx(vq));
+        vring_avail_event(vq, vq->last_avail_idx);
     }
 
     if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_INDIRECT) {
@@ -1123,6 +1123,17 @@ static void virtio_vmstate_change(void *opaque, int running, RunState state)
     }
 }
 
+void virtio_instance_init_common(Object *proxy_obj, void *data,
+                                 size_t vdev_size, const char *vdev_name)
+{
+    DeviceState *vdev = data;
+
+    object_initialize(vdev, vdev_size, vdev_name);
+    object_property_add_child(proxy_obj, "virtio-backend", OBJECT(vdev), NULL);
+    object_unref(OBJECT(vdev));
+    qdev_alias_all_properties(vdev, proxy_obj);
+}
+
 void virtio_init(VirtIODevice *vdev, const char *name,
                  uint16_t device_id, size_t config_size)
 {
diff --git a/hw/watchdog/wdt_i6300esb.c b/hw/watchdog/wdt_i6300esb.c
index b3d6f39dd..687c8b1d4 100644
--- a/hw/watchdog/wdt_i6300esb.c
+++ b/hw/watchdog/wdt_i6300esb.c
@@ -425,13 +425,6 @@ static int i6300esb_init(PCIDevice *dev)
     return 0;
 }
 
-static void i6300esb_exit(PCIDevice *dev)
-{
-    I6300State *d = DO_UPCAST(I6300State, dev, dev);
-
-    memory_region_destroy(&d->io_mem);
-}
-
 static WatchdogTimerModel model = {
     .wdt_name = "i6300esb",
     .wdt_description = "Intel 6300ESB",
@@ -445,7 +438,6 @@ static void i6300esb_class_init(ObjectClass *klass, void *data)
     k->config_read = i6300esb_config_read;
     k->config_write = i6300esb_config_write;
     k->init = i6300esb_init;
-    k->exit = i6300esb_exit;
     k->vendor_id = PCI_VENDOR_ID_INTEL;
     k->device_id = PCI_DEVICE_ID_INTEL_ESB_9;
     k->class_id = PCI_CLASS_SYSTEM_OTHER;
diff --git a/hw/xen/xen_devconfig.c b/hw/xen/xen_devconfig.c
index fa998eff0..e138dbbec 100644
--- a/hw/xen/xen_devconfig.c
+++ b/hw/xen/xen_devconfig.c
@@ -1,4 +1,5 @@
 #include "hw/xen/xen_backend.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 
 /* ------------------------------------------------------------- */
diff --git a/hw/xen/xen_pt.c b/hw/xen/xen_pt.c
index be4220b41..c1bf35715 100644
--- a/hw/xen/xen_pt.c
+++ b/hw/xen/xen_pt.c
@@ -453,25 +453,6 @@ static int xen_pt_register_regions(XenPCIPassthroughState *s)
     return 0;
 }
 
-static void xen_pt_unregister_regions(XenPCIPassthroughState *s)
-{
-    XenHostPCIDevice *d = &s->real_device;
-    int i;
-
-    for (i = 0; i < PCI_NUM_REGIONS - 1; i++) {
-        XenHostPCIIORegion *r = &d->io_regions[i];
-
-        if (r->base_addr == 0 || r->size == 0) {
-            continue;
-        }
-
-        memory_region_destroy(&s->bar[i]);
-    }
-    if (d->rom.base_addr && d->rom.size) {
-        memory_region_destroy(&s->rom);
-    }
-}
-
 /* region mapping */
 
 static int xen_pt_bar_from_region(XenPCIPassthroughState *s, MemoryRegion *mr)
@@ -810,7 +791,6 @@ static void xen_pt_unregister_device(PCIDevice *d)
     /* delete all emulated config registers */
     xen_pt_config_delete(s);
 
-    xen_pt_unregister_regions(s);
     memory_listener_unregister(&s->memory_listener);
     memory_listener_unregister(&s->io_listener);
 
diff --git a/hw/xen/xen_pt_msi.c b/hw/xen/xen_pt_msi.c
index 12b4c4560..9ed9321c9 100644
--- a/hw/xen/xen_pt_msi.c
+++ b/hw/xen/xen_pt_msi.c
@@ -593,7 +593,6 @@ int xen_pt_msix_init(XenPCIPassthroughState *s, uint32_t base)
     return 0;
 
 error_out:
-    memory_region_destroy(&msix->mmio);
     g_free(s->msix);
     s->msix = NULL;
     return rc;
@@ -616,7 +615,6 @@ void xen_pt_msix_delete(XenPCIPassthroughState *s)
     }
 
     memory_region_del_subregion(&s->bar[msix->bar_index], &msix->mmio);
-    memory_region_destroy(&msix->mmio);
 
     g_free(s->msix);
     s->msix = NULL;
diff --git a/hw/xenpv/xen_machine_pv.c b/hw/xenpv/xen_machine_pv.c
index 31500643a..2e545d241 100644
--- a/hw/xenpv/xen_machine_pv.c
+++ b/hw/xenpv/xen_machine_pv.c
@@ -26,7 +26,7 @@
 #include "hw/boards.h"
 #include "hw/xen/xen_backend.h"
 #include "xen_domainbuild.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 
 static void xen_init_pv(MachineState *machine)
 {
diff --git a/hw/xtensa/pic_cpu.c b/hw/xtensa/pic_cpu.c
index e2005bd98..18825d19f 100644
--- a/hw/xtensa/pic_cpu.c
+++ b/hw/xtensa/pic_cpu.c
@@ -31,14 +31,14 @@
 
 void xtensa_advance_ccount(CPUXtensaState *env, uint32_t d)
 {
-    uint32_t old_ccount = env->sregs[CCOUNT];
+    uint32_t old_ccount = env->sregs[CCOUNT] + 1;
 
     env->sregs[CCOUNT] += d;
 
     if (xtensa_option_enabled(env->config, XTENSA_OPTION_TIMER_INTERRUPT)) {
         int i;
         for (i = 0; i < env->config->nccompare; ++i) {
-            if (env->sregs[CCOMPARE + i] - old_ccount <= d) {
+            if (env->sregs[CCOMPARE + i] - old_ccount < d) {
                 xtensa_timer_irq(env, i, 1);
             }
         }
diff --git a/hw/xtensa/sim.c b/hw/xtensa/sim.c
index 9642bf54c..37ea9ae9c 100644
--- a/hw/xtensa/sim.c
+++ b/hw/xtensa/sim.c
@@ -79,12 +79,12 @@ static void xtensa_sim_init(MachineState *machine)
     }
 
     ram = g_malloc(sizeof(*ram));
-    memory_region_init_ram(ram, NULL, "xtensa.sram", ram_size);
+    memory_region_init_ram(ram, NULL, "xtensa.sram", ram_size, &error_abort);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(get_system_memory(), 0, ram);
 
     rom = g_malloc(sizeof(*rom));
-    memory_region_init_ram(rom, NULL, "xtensa.rom", 0x1000);
+    memory_region_init_ram(rom, NULL, "xtensa.rom", 0x1000, &error_abort);
     vmstate_register_ram_global(rom);
     memory_region_add_subregion(get_system_memory(), 0xfe000000, rom);
 
diff --git a/hw/xtensa/xtfpga.c b/hw/xtensa/xtfpga.c
index a2dff5a13..e5a6bba78 100644
--- a/hw/xtensa/xtfpga.c
+++ b/hw/xtensa/xtfpga.c
@@ -35,7 +35,7 @@
 #include "net/net.h"
 #include "hw/sysbus.h"
 #include "hw/block/flash.h"
-#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "sysemu/char.h"
 #include "sysemu/device_tree.h"
 #include "qemu/error-report.h"
@@ -143,7 +143,7 @@ static void lx60_net_init(MemoryRegion *address_space,
             sysbus_mmio_get_region(s, 1));
 
     ram = g_malloc(sizeof(*ram));
-    memory_region_init_ram(ram, OBJECT(s), "open_eth.ram", 16384);
+    memory_region_init_ram(ram, OBJECT(s), "open_eth.ram", 16384, &error_abort);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(address_space, buffers, ram);
 }
@@ -205,7 +205,8 @@ static void lx_init(const LxBoardDesc *board, MachineState *machine)
     }
 
     ram = g_malloc(sizeof(*ram));
-    memory_region_init_ram(ram, NULL, "lx60.dram", machine->ram_size);
+    memory_region_init_ram(ram, NULL, "lx60.dram", machine->ram_size,
+                           &error_abort);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(system_memory, 0, ram);
 
@@ -229,7 +230,8 @@ static void lx_init(const LxBoardDesc *board, MachineState *machine)
     if (dinfo) {
         flash = pflash_cfi01_register(board->flash_base,
                 NULL, "lx60.io.flash", board->flash_size,
-                dinfo->bdrv, board->flash_sector_size,
+                blk_by_legacy_dinfo(dinfo),
+                board->flash_sector_size,
                 board->flash_size / board->flash_sector_size,
                 4, 0x0000, 0x0000, 0x0000, 0x0000, be);
         if (flash == NULL) {
@@ -254,7 +256,8 @@ static void lx_init(const LxBoardDesc *board, MachineState *machine)
         uint32_t cur_lowmem = QEMU_ALIGN_UP(lowmem_end / 2, 4096);
 
         rom = g_malloc(sizeof(*rom));
-        memory_region_init_ram(rom, NULL, "lx60.sram", board->sram_size);
+        memory_region_init_ram(rom, NULL, "lx60.sram", board->sram_size,
+                               &error_abort);
         vmstate_register_ram_global(rom);
         memory_region_add_subregion(system_memory, 0xfe000000, rom);
 
@@ -325,7 +328,8 @@ static void lx_init(const LxBoardDesc *board, MachineState *machine)
         } else {
             hwaddr ep;
             int is_linux;
-            success = load_uimage(kernel_filename, &ep, NULL, &is_linux);
+            success = load_uimage(kernel_filename, &ep, NULL, &is_linux,
+                                  translate_phys_addr, cpu);
             if (success > 0 && is_linux) {
                 entry_point = ep;
             } else {
diff --git a/include/block/accounting.h b/include/block/accounting.h
new file mode 100644
index 000000000..50b42b380
--- /dev/null
+++ b/include/block/accounting.h
@@ -0,0 +1,57 @@
+/*
+ * QEMU System Emulator block accounting
+ *
+ * Copyright (c) 2011 Christoph Hellwig
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef BLOCK_ACCOUNTING_H
+#define BLOCK_ACCOUNTING_H
+
+#include <stdint.h>
+
+#include "qemu/typedefs.h"
+
+enum BlockAcctType {
+    BLOCK_ACCT_READ,
+    BLOCK_ACCT_WRITE,
+    BLOCK_ACCT_FLUSH,
+    BLOCK_MAX_IOTYPE,
+};
+
+typedef struct BlockAcctStats {
+    uint64_t nr_bytes[BLOCK_MAX_IOTYPE];
+    uint64_t nr_ops[BLOCK_MAX_IOTYPE];
+    uint64_t total_time_ns[BLOCK_MAX_IOTYPE];
+    uint64_t wr_highest_sector;
+} BlockAcctStats;
+
+typedef struct BlockAcctCookie {
+    int64_t bytes;
+    int64_t start_time_ns;
+    enum BlockAcctType type;
+} BlockAcctCookie;
+
+void block_acct_start(BlockAcctStats *stats, BlockAcctCookie *cookie,
+                      int64_t bytes, enum BlockAcctType type);
+void block_acct_done(BlockAcctStats *stats, BlockAcctCookie *cookie);
+void block_acct_highest_sector(BlockAcctStats *stats, int64_t sector_num,
+                               unsigned int nb_sectors);
+
+#endif
diff --git a/include/block/aio.h b/include/block/aio.h
index c23de3cd1..6bf0e0456 100644
--- a/include/block/aio.h
+++ b/include/block/aio.h
@@ -22,24 +22,27 @@
 #include "qemu/rfifolock.h"
 #include "qemu/timer.h"
 
-typedef struct BlockDriverAIOCB BlockDriverAIOCB;
-typedef void BlockDriverCompletionFunc(void *opaque, int ret);
+typedef struct BlockAIOCB BlockAIOCB;
+typedef void BlockCompletionFunc(void *opaque, int ret);
 
 typedef struct AIOCBInfo {
-    void (*cancel)(BlockDriverAIOCB *acb);
+    void (*cancel_async)(BlockAIOCB *acb);
+    AioContext *(*get_aio_context)(BlockAIOCB *acb);
     size_t aiocb_size;
 } AIOCBInfo;
 
-struct BlockDriverAIOCB {
+struct BlockAIOCB {
     const AIOCBInfo *aiocb_info;
     BlockDriverState *bs;
-    BlockDriverCompletionFunc *cb;
+    BlockCompletionFunc *cb;
     void *opaque;
+    int refcnt;
 };
 
 void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
-                   BlockDriverCompletionFunc *cb, void *opaque);
-void qemu_aio_release(void *p);
+                   BlockCompletionFunc *cb, void *opaque);
+void qemu_aio_unref(void *p);
+void qemu_aio_ref(void *p);
 
 typedef struct AioHandler AioHandler;
 typedef void QEMUBHFunc(void *opaque);
@@ -99,7 +102,7 @@ void aio_set_dispatching(AioContext *ctx, bool dispatching);
  * They also provide bottom halves, a service to execute a piece of code
  * as soon as possible.
  */
-AioContext *aio_context_new(void);
+AioContext *aio_context_new(Error **errp);
 
 /**
  * aio_context_ref:
@@ -205,12 +208,25 @@ void qemu_bh_cancel(QEMUBH *bh);
 void qemu_bh_delete(QEMUBH *bh);
 
 /* Return whether there are any pending callbacks from the GSource
- * attached to the AioContext.
+ * attached to the AioContext, before g_poll is invoked.
+ *
+ * This is used internally in the implementation of the GSource.
+ */
+bool aio_prepare(AioContext *ctx);
+
+/* Return whether there are any pending callbacks from the GSource
+ * attached to the AioContext, after g_poll is invoked.
  *
  * This is used internally in the implementation of the GSource.
  */
 bool aio_pending(AioContext *ctx);
 
+/* Dispatch any pending callbacks from the GSource attached to the AioContext.
+ *
+ * This is used internally in the implementation of the GSource.
+ */
+bool aio_dispatch(AioContext *ctx);
+
 /* Progress in completing AIO work to occur.  This can issue new pending
  * aio as a result of executing I/O completion or bh callbacks.
  *
@@ -226,7 +242,6 @@ bool aio_pending(AioContext *ctx);
  */
 bool aio_poll(AioContext *ctx, bool blocking);
 
-#ifdef CONFIG_POSIX
 /* Register a file descriptor and associated callbacks.  Behaves very similarly
  * to qemu_set_fd_handler2.  Unlike qemu_set_fd_handler2, these callbacks will
  * be invoked when using aio_poll().
@@ -239,7 +254,6 @@ void aio_set_fd_handler(AioContext *ctx,
                         IOHandler *io_read,
                         IOHandler *io_write,
                         void *opaque);
-#endif
 
 /* Register an event notifier and associated callbacks.  Behaves very similarly
  * to event_notifier_set_handler.  Unlike event_notifier_set_handler, these callbacks
@@ -303,4 +317,12 @@ static inline void aio_timer_init(AioContext *ctx,
     timer_init(ts, ctx->tlg.tl[type], scale, cb, opaque);
 }
 
+/**
+ * aio_compute_timeout:
+ * @ctx: the aio context
+ *
+ * Compute the timeout that a blocking aio_poll should use.
+ */
+int64_t aio_compute_timeout(AioContext *ctx);
+
 #endif
diff --git a/include/block/block.h b/include/block/block.h
index f08471d7e..5450610bc 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -5,6 +5,7 @@
 #include "qemu-common.h"
 #include "qemu/option.h"
 #include "block/coroutine.h"
+#include "block/accounting.h"
 #include "qapi/qmp/qobject.h"
 #include "qapi-types.h"
 
@@ -47,41 +48,6 @@ typedef struct BlockFragInfo {
     uint64_t compressed_clusters;
 } BlockFragInfo;
 
-/* Callbacks for block device models */
-typedef struct BlockDevOps {
-    /*
-     * Runs when virtual media changed (monitor commands eject, change)
-     * Argument load is true on load and false on eject.
-     * Beware: doesn't run when a host device's physical media
-     * changes.  Sure would be useful if it did.
-     * Device models with removable media must implement this callback.
-     */
-    void (*change_media_cb)(void *opaque, bool load);
-    /*
-     * Runs when an eject request is issued from the monitor, the tray
-     * is closed, and the medium is locked.
-     * Device models that do not implement is_medium_locked will not need
-     * this callback.  Device models that can lock the medium or tray might
-     * want to implement the callback and unlock the tray when "force" is
-     * true, even if they do not support eject requests.
-     */
-    void (*eject_request_cb)(void *opaque, bool force);
-    /*
-     * Is the virtual tray open?
-     * Device models implement this only when the device has a tray.
-     */
-    bool (*is_tray_open)(void *opaque);
-    /*
-     * Is the virtual medium locked into the device?
-     * Device models implement this only when device has such a lock.
-     */
-    bool (*is_medium_locked)(void *opaque);
-    /*
-     * Runs when the size changed (e.g. monitor command block_resize)
-     */
-    void (*resize_cb)(void *opaque);
-} BlockDevOps;
-
 typedef enum {
     BDRV_REQ_COPY_ON_READ = 0x1,
     BDRV_REQ_ZERO_WRITE   = 0x2,
@@ -117,7 +83,9 @@ typedef enum {
 #define BDRV_SECTOR_SIZE   (1ULL << BDRV_SECTOR_BITS)
 #define BDRV_SECTOR_MASK   ~(BDRV_SECTOR_SIZE - 1)
 
-/* BDRV_BLOCK_DATA: data is read from bs->file or another file
+/*
+ * Allocation status flags
+ * BDRV_BLOCK_DATA: data is read from bs->file or another file
  * BDRV_BLOCK_ZERO: sectors read as zero
  * BDRV_BLOCK_OFFSET_VALID: sector stored in bs->file as raw data
  * BDRV_BLOCK_ALLOCATED: the content of the block is determined by this
@@ -203,7 +171,8 @@ BlockDriver *bdrv_find_whitelisted_format(const char *format_name,
 int bdrv_create(BlockDriver *drv, const char* filename,
                 QemuOpts *opts, Error **errp);
 int bdrv_create_file(const char *filename, QemuOpts *opts, Error **errp);
-BlockDriverState *bdrv_new(const char *device_name, Error **errp);
+BlockDriverState *bdrv_new_root(void);
+BlockDriverState *bdrv_new(void);
 void bdrv_make_anon(BlockDriverState *bs);
 void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old);
 void bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top);
@@ -228,16 +197,6 @@ void bdrv_reopen_commit(BDRVReopenState *reopen_state);
 void bdrv_reopen_abort(BDRVReopenState *reopen_state);
 void bdrv_close(BlockDriverState *bs);
 void bdrv_add_close_notifier(BlockDriverState *bs, Notifier *notify);
-int bdrv_attach_dev(BlockDriverState *bs, void *dev);
-void bdrv_attach_dev_nofail(BlockDriverState *bs, void *dev);
-void bdrv_detach_dev(BlockDriverState *bs, void *dev);
-void *bdrv_get_attached_dev(BlockDriverState *bs);
-void bdrv_set_dev_ops(BlockDriverState *bs, const BlockDevOps *ops,
-                      void *opaque);
-void bdrv_dev_eject_request(BlockDriverState *bs, bool force);
-bool bdrv_dev_has_removable_media(BlockDriverState *bs);
-bool bdrv_dev_is_tray_open(BlockDriverState *bs);
-bool bdrv_dev_is_medium_locked(BlockDriverState *bs);
 int bdrv_read(BlockDriverState *bs, int64_t sector_num,
               uint8_t *buf, int nb_sectors);
 int bdrv_read_unthrottled(BlockDriverState *bs, int64_t sector_num,
@@ -246,9 +205,9 @@ int bdrv_write(BlockDriverState *bs, int64_t sector_num,
                const uint8_t *buf, int nb_sectors);
 int bdrv_write_zeroes(BlockDriverState *bs, int64_t sector_num,
                int nb_sectors, BdrvRequestFlags flags);
-BlockDriverAIOCB *bdrv_aio_write_zeroes(BlockDriverState *bs, int64_t sector_num,
-                                        int nb_sectors, BdrvRequestFlags flags,
-                                        BlockDriverCompletionFunc *cb, void *opaque);
+BlockAIOCB *bdrv_aio_write_zeroes(BlockDriverState *bs, int64_t sector_num,
+                                  int nb_sectors, BdrvRequestFlags flags,
+                                  BlockCompletionFunc *cb, void *opaque);
 int bdrv_make_zero(BlockDriverState *bs, BdrvRequestFlags flags);
 int bdrv_pread(BlockDriverState *bs, int64_t offset,
                void *buf, int count);
@@ -274,7 +233,9 @@ int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs, int64_t sector_num,
 BlockDriverState *bdrv_find_backing_image(BlockDriverState *bs,
     const char *backing_file);
 int bdrv_get_backing_file_depth(BlockDriverState *bs);
+void bdrv_refresh_filename(BlockDriverState *bs);
 int bdrv_truncate(BlockDriverState *bs, int64_t offset);
+int64_t bdrv_nb_sectors(BlockDriverState *bs);
 int64_t bdrv_getlength(BlockDriverState *bs);
 int64_t bdrv_get_allocated_file_size(BlockDriverState *bs);
 void bdrv_get_geometry(BlockDriverState *bs, uint64_t *nb_sectors_ptr);
@@ -309,7 +270,13 @@ typedef enum {
 
 int bdrv_check(BlockDriverState *bs, BdrvCheckResult *res, BdrvCheckMode fix);
 
-int bdrv_amend_options(BlockDriverState *bs_new, QemuOpts *opts);
+/* The units of offset and total_work_size may be chosen arbitrarily by the
+ * block driver; total_work_size may change during the course of the amendment
+ * operation */
+typedef void BlockDriverAmendStatusCB(BlockDriverState *bs, int64_t offset,
+                                      int64_t total_work_size);
+int bdrv_amend_options(BlockDriverState *bs_new, QemuOpts *opts,
+                       BlockDriverAmendStatusCB *status_cb);
 
 /* external snapshots */
 bool bdrv_recurse_is_first_non_filter(BlockDriverState *bs,
@@ -322,18 +289,19 @@ BlockDriverState *check_to_replace_node(const char *node_name, Error **errp);
 /* async block I/O */
 typedef void BlockDriverDirtyHandler(BlockDriverState *bs, int64_t sector,
                                      int sector_num);
-BlockDriverAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
-                                 QEMUIOVector *iov, int nb_sectors,
-                                 BlockDriverCompletionFunc *cb, void *opaque);
-BlockDriverAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num,
-                                  QEMUIOVector *iov, int nb_sectors,
-                                  BlockDriverCompletionFunc *cb, void *opaque);
-BlockDriverAIOCB *bdrv_aio_flush(BlockDriverState *bs,
-                                 BlockDriverCompletionFunc *cb, void *opaque);
-BlockDriverAIOCB *bdrv_aio_discard(BlockDriverState *bs,
-                                   int64_t sector_num, int nb_sectors,
-                                   BlockDriverCompletionFunc *cb, void *opaque);
-void bdrv_aio_cancel(BlockDriverAIOCB *acb);
+BlockAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
+                           QEMUIOVector *iov, int nb_sectors,
+                           BlockCompletionFunc *cb, void *opaque);
+BlockAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num,
+                            QEMUIOVector *iov, int nb_sectors,
+                            BlockCompletionFunc *cb, void *opaque);
+BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs,
+                           BlockCompletionFunc *cb, void *opaque);
+BlockAIOCB *bdrv_aio_discard(BlockDriverState *bs,
+                             int64_t sector_num, int nb_sectors,
+                             BlockCompletionFunc *cb, void *opaque);
+void bdrv_aio_cancel(BlockAIOCB *acb);
+void bdrv_aio_cancel_async(BlockAIOCB *acb);
 
 typedef struct BlockRequest {
     /* Fields to be filled by multiwrite caller */
@@ -341,7 +309,7 @@ typedef struct BlockRequest {
     int nb_sectors;
     int flags;
     QEMUIOVector *qiov;
-    BlockDriverCompletionFunc *cb;
+    BlockCompletionFunc *cb;
     void *opaque;
 
     /* Filled by multiwrite implementation */
@@ -353,9 +321,9 @@ int bdrv_aio_multiwrite(BlockDriverState *bs, BlockRequest *reqs,
 
 /* sg packet commands */
 int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf);
-BlockDriverAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
+BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
         unsigned long int req, void *buf,
-        BlockDriverCompletionFunc *cb, void *opaque);
+        BlockCompletionFunc *cb, void *opaque);
 
 /* Invalidate any cached metadata used by image formats */
 void bdrv_invalidate_cache(BlockDriverState *bs, Error **errp);
@@ -368,6 +336,7 @@ int bdrv_flush(BlockDriverState *bs);
 int coroutine_fn bdrv_co_flush(BlockDriverState *bs);
 int bdrv_flush_all(void);
 void bdrv_close_all(void);
+void bdrv_drain(BlockDriverState *bs);
 void bdrv_drain_all(void);
 
 int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors);
@@ -406,15 +375,13 @@ BlockDriverState *bdrv_lookup_bs(const char *device,
                                  Error **errp);
 bool bdrv_chain_contains(BlockDriverState *top, BlockDriverState *base);
 BlockDriverState *bdrv_next(BlockDriverState *bs);
-void bdrv_iterate(void (*it)(void *opaque, BlockDriverState *bs),
-                  void *opaque);
 int bdrv_is_encrypted(BlockDriverState *bs);
 int bdrv_key_required(BlockDriverState *bs);
 int bdrv_set_key(BlockDriverState *bs, const char *key);
 int bdrv_query_missing_keys(void);
 void bdrv_iterate_format(void (*it)(void *opaque, const char *name),
                          void *opaque);
-const char *bdrv_get_device_name(BlockDriverState *bs);
+const char *bdrv_get_device_name(const BlockDriverState *bs);
 int bdrv_get_flags(BlockDriverState *bs);
 int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
                           const uint8_t *buf, int nb_sectors);
@@ -454,6 +421,9 @@ void bdrv_img_create(const char *filename, const char *fmt,
 size_t bdrv_opt_mem_align(BlockDriverState *bs);
 void bdrv_set_guest_block_size(BlockDriverState *bs, int align);
 void *qemu_blockalign(BlockDriverState *bs, size_t size);
+void *qemu_blockalign0(BlockDriverState *bs, size_t size);
+void *qemu_try_blockalign(BlockDriverState *bs, size_t size);
+void *qemu_try_blockalign0(BlockDriverState *bs, size_t size);
 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov);
 
 struct HBitmapIter;
@@ -482,23 +452,6 @@ void bdrv_op_block_all(BlockDriverState *bs, Error *reason);
 void bdrv_op_unblock_all(BlockDriverState *bs, Error *reason);
 bool bdrv_op_blocker_is_empty(BlockDriverState *bs);
 
-enum BlockAcctType {
-    BDRV_ACCT_READ,
-    BDRV_ACCT_WRITE,
-    BDRV_ACCT_FLUSH,
-    BDRV_MAX_IOTYPE,
-};
-
-typedef struct BlockAcctCookie {
-    int64_t bytes;
-    int64_t start_time_ns;
-    enum BlockAcctType type;
-} BlockAcctCookie;
-
-void bdrv_acct_start(BlockDriverState *bs, BlockAcctCookie *cookie,
-        int64_t bytes, enum BlockAcctType type);
-void bdrv_acct_done(BlockDriverState *bs, BlockAcctCookie *cookie);
-
 typedef enum {
     BLKDBG_L1_UPDATE,
 
@@ -554,6 +507,8 @@ typedef enum {
     BLKDBG_PWRITEV_ZERO,
     BLKDBG_PWRITEV_DONE,
 
+    BLKDBG_EMPTY_IMAGE_PREPARE,
+
     BLKDBG_EVENT_MAX,
 } BlkDebugEvent;
 
@@ -588,4 +543,6 @@ void bdrv_io_plug(BlockDriverState *bs);
 void bdrv_io_unplug(BlockDriverState *bs);
 void bdrv_flush_io_queue(BlockDriverState *bs);
 
+BlockAcctStats *bdrv_get_stats(BlockDriverState *bs);
+
 #endif
diff --git a/include/block/block_int.h b/include/block/block_int.h
index 7b541a069..a1c17b957 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -24,6 +24,7 @@
 #ifndef BLOCK_INT_H
 #define BLOCK_INT_H
 
+#include "block/accounting.h"
 #include "block/block.h"
 #include "qemu/option.h"
 #include "qemu/queue.h"
@@ -123,18 +124,21 @@ struct BlockDriver {
     int (*bdrv_create)(const char *filename, QemuOpts *opts, Error **errp);
     int (*bdrv_set_key)(BlockDriverState *bs, const char *key);
     int (*bdrv_make_empty)(BlockDriverState *bs);
+
+    void (*bdrv_refresh_filename)(BlockDriverState *bs);
+
     /* aio */
-    BlockDriverAIOCB *(*bdrv_aio_readv)(BlockDriverState *bs,
+    BlockAIOCB *(*bdrv_aio_readv)(BlockDriverState *bs,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque);
-    BlockDriverAIOCB *(*bdrv_aio_writev)(BlockDriverState *bs,
+        BlockCompletionFunc *cb, void *opaque);
+    BlockAIOCB *(*bdrv_aio_writev)(BlockDriverState *bs,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque);
-    BlockDriverAIOCB *(*bdrv_aio_flush)(BlockDriverState *bs,
-        BlockDriverCompletionFunc *cb, void *opaque);
-    BlockDriverAIOCB *(*bdrv_aio_discard)(BlockDriverState *bs,
+        BlockCompletionFunc *cb, void *opaque);
+    BlockAIOCB *(*bdrv_aio_flush)(BlockDriverState *bs,
+        BlockCompletionFunc *cb, void *opaque);
+    BlockAIOCB *(*bdrv_aio_discard)(BlockDriverState *bs,
         int64_t sector_num, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque);
+        BlockCompletionFunc *cb, void *opaque);
 
     int coroutine_fn (*bdrv_co_readv)(BlockDriverState *bs,
         int64_t sector_num, int nb_sectors, QEMUIOVector *qiov);
@@ -214,9 +218,9 @@ struct BlockDriver {
 
     /* to control generic scsi devices */
     int (*bdrv_ioctl)(BlockDriverState *bs, unsigned long int req, void *buf);
-    BlockDriverAIOCB *(*bdrv_aio_ioctl)(BlockDriverState *bs,
+    BlockAIOCB *(*bdrv_aio_ioctl)(BlockDriverState *bs,
         unsigned long int req, void *buf,
-        BlockDriverCompletionFunc *cb, void *opaque);
+        BlockCompletionFunc *cb, void *opaque);
 
     /* List of options for creating images, terminated by name == NULL */
     QemuOptsList *create_opts;
@@ -228,7 +232,8 @@ struct BlockDriver {
     int (*bdrv_check)(BlockDriverState* bs, BdrvCheckResult *result,
         BdrvCheckMode fix);
 
-    int (*bdrv_amend_options)(BlockDriverState *bs, QemuOpts *opts);
+    int (*bdrv_amend_options)(BlockDriverState *bs, QemuOpts *opts,
+                              BlockDriverAmendStatusCB *status_cb);
 
     void (*bdrv_debug_event)(BlockDriverState *bs, BlkDebugEvent event);
 
@@ -285,12 +290,24 @@ typedef struct BlockLimits {
     /* optimal transfer length in sectors */
     int opt_transfer_length;
 
+    /* maximal transfer length in sectors */
+    int max_transfer_length;
+
     /* memory alignment so that no bounce buffer is needed */
     size_t opt_mem_alignment;
 } BlockLimits;
 
 typedef struct BdrvOpBlocker BdrvOpBlocker;
 
+typedef struct BdrvAioNotifier {
+    void (*attached_aio_context)(AioContext *new_context, void *opaque);
+    void (*detach_aio_context)(void *opaque);
+
+    void *opaque;
+
+    QLIST_ENTRY(BdrvAioNotifier) list;
+} BdrvAioNotifier;
+
 /*
  * Note: the function bdrv_append() copies and swaps contents of
  * BlockDriverStates, so if you add new fields to this struct, please
@@ -311,18 +328,22 @@ struct BlockDriverState {
     BlockDriver *drv; /* NULL means no media */
     void *opaque;
 
-    void *dev;                  /* attached device model, if any */
-    /* TODO change to DeviceState when all users are qdevified */
-    const BlockDevOps *dev_ops;
-    void *dev_opaque;
+    BlockBackend *blk;          /* owning backend, if any */
 
     AioContext *aio_context; /* event loop used for fd handlers, timers, etc */
+    /* long-running tasks intended to always use the same AioContext as this
+     * BDS may register themselves in this list to be notified of changes
+     * regarding this BDS's context */
+    QLIST_HEAD(, BdrvAioNotifier) aio_notifiers;
 
     char filename[1024];
     char backing_file[1024]; /* if non zero, the image is a diff of
                                 this file image */
     char backing_format[16]; /* if non-zero and backing_file exists */
 
+    QDict *full_open_options;
+    char exact_filename[1024];
+
     BlockDriverState *backing_hd;
     BlockDriverState *file;
 
@@ -340,10 +361,7 @@ struct BlockDriverState {
     bool         io_limits_enabled;
 
     /* I/O stats (display with "info blockstats"). */
-    uint64_t nr_bytes[BDRV_MAX_IOTYPE];
-    uint64_t nr_ops[BDRV_MAX_IOTYPE];
-    uint64_t total_time_ns[BDRV_MAX_IOTYPE];
-    uint64_t wr_highest_sector;
+    BlockAcctStats stats;
 
     /* I/O Limits */
     BlockLimits bl;
@@ -373,8 +391,6 @@ struct BlockDriverState {
     char node_name[32];
     /* element of the list of named nodes building the graph */
     QTAILQ_ENTRY(BlockDriverState) node_list;
-    /* Device name is the name associated with the "drive" the guest sees */
-    char device_name[32];
     /* element of the list of "drives" the guest sees */
     QTAILQ_ENTRY(BlockDriverState) device_list;
     QLIST_HEAD(, BdrvDirtyBitmap) dirty_bitmaps;
@@ -431,6 +447,34 @@ void bdrv_detach_aio_context(BlockDriverState *bs);
 void bdrv_attach_aio_context(BlockDriverState *bs,
                              AioContext *new_context);
 
+/**
+ * bdrv_add_aio_context_notifier:
+ *
+ * If a long-running job intends to be always run in the same AioContext as a
+ * certain BDS, it may use this function to be notified of changes regarding the
+ * association of the BDS to an AioContext.
+ *
+ * attached_aio_context() is called after the target BDS has been attached to a
+ * new AioContext; detach_aio_context() is called before the target BDS is being
+ * detached from its old AioContext.
+ */
+void bdrv_add_aio_context_notifier(BlockDriverState *bs,
+        void (*attached_aio_context)(AioContext *new_context, void *opaque),
+        void (*detach_aio_context)(void *opaque), void *opaque);
+
+/**
+ * bdrv_remove_aio_context_notifier:
+ *
+ * Unsubscribe of change notifications regarding the BDS's AioContext. The
+ * parameters given here have to be the same as those given to
+ * bdrv_add_aio_context_notifier().
+ */
+void bdrv_remove_aio_context_notifier(BlockDriverState *bs,
+                                      void (*aio_context_attached)(AioContext *,
+                                                                   void *),
+                                      void (*aio_context_detached)(void *),
+                                      void *opaque);
+
 #ifdef _WIN32
 int is_windows_drive(const char *filename);
 #endif
@@ -456,7 +500,7 @@ int is_windows_drive(const char *filename);
  */
 void stream_start(BlockDriverState *bs, BlockDriverState *base,
                   const char *base_id, int64_t speed, BlockdevOnError on_error,
-                  BlockDriverCompletionFunc *cb,
+                  BlockCompletionFunc *cb,
                   void *opaque, Error **errp);
 
 /**
@@ -474,7 +518,7 @@ void stream_start(BlockDriverState *bs, BlockDriverState *base,
  */
 void commit_start(BlockDriverState *bs, BlockDriverState *base,
                  BlockDriverState *top, int64_t speed,
-                 BlockdevOnError on_error, BlockDriverCompletionFunc *cb,
+                 BlockdevOnError on_error, BlockCompletionFunc *cb,
                  void *opaque, const char *backing_file_str, Error **errp);
 /**
  * commit_active_start:
@@ -490,7 +534,7 @@ void commit_start(BlockDriverState *bs, BlockDriverState *base,
 void commit_active_start(BlockDriverState *bs, BlockDriverState *base,
                          int64_t speed,
                          BlockdevOnError on_error,
-                         BlockDriverCompletionFunc *cb,
+                         BlockCompletionFunc *cb,
                          void *opaque, Error **errp);
 /*
  * mirror_start:
@@ -518,7 +562,7 @@ void mirror_start(BlockDriverState *bs, BlockDriverState *target,
                   int64_t speed, int64_t granularity, int64_t buf_size,
                   MirrorSyncMode mode, BlockdevOnError on_source_error,
                   BlockdevOnError on_target_error,
-                  BlockDriverCompletionFunc *cb,
+                  BlockCompletionFunc *cb,
                   void *opaque, Error **errp);
 
 /*
@@ -539,7 +583,14 @@ void backup_start(BlockDriverState *bs, BlockDriverState *target,
                   int64_t speed, MirrorSyncMode sync_mode,
                   BlockdevOnError on_source_error,
                   BlockdevOnError on_target_error,
-                  BlockDriverCompletionFunc *cb, void *opaque,
+                  BlockCompletionFunc *cb, void *opaque,
                   Error **errp);
 
+void blk_dev_change_media_cb(BlockBackend *blk, bool load);
+bool blk_dev_has_removable_media(BlockBackend *blk);
+void blk_dev_eject_request(BlockBackend *blk, bool force);
+bool blk_dev_is_tray_open(BlockBackend *blk);
+bool blk_dev_is_medium_locked(BlockBackend *blk);
+void blk_dev_resize_cb(BlockBackend *blk);
+
 #endif /* BLOCK_INT_H */
diff --git a/include/block/blockjob.h b/include/block/blockjob.h
index 60aa835ec..b6d4ebbe0 100644
--- a/include/block/blockjob.h
+++ b/include/block/blockjob.h
@@ -91,6 +91,11 @@ struct BlockJob {
      */
     bool busy;
 
+    /**
+     * Set to true when the job is ready to be completed.
+     */
+    bool ready;
+
     /** Status that is published by the query-block-jobs QMP API */
     BlockDeviceIoStatus iostatus;
 
@@ -104,7 +109,7 @@ struct BlockJob {
     int64_t speed;
 
     /** The completion function that will be called when the job completes.  */
-    BlockDriverCompletionFunc *cb;
+    BlockCompletionFunc *cb;
 
     /** Block other operations when block job is running */
     Error *blocker;
@@ -132,7 +137,7 @@ struct BlockJob {
  * called from a wrapper that is specific to the job type.
  */
 void *block_job_create(const BlockJobDriver *driver, BlockDriverState *bs,
-                       int64_t speed, BlockDriverCompletionFunc *cb,
+                       int64_t speed, BlockCompletionFunc *cb,
                        void *opaque, Error **errp);
 
 /**
@@ -273,6 +278,21 @@ bool block_job_is_paused(BlockJob *job);
 int block_job_cancel_sync(BlockJob *job);
 
 /**
+ * block_job_complete_sync:
+ * @job: The job to be completed.
+ * @errp: Error object which may be set by block_job_complete(); this is not
+ *        necessarily set on every error, the job return value has to be
+ *        checked as well.
+ *
+ * Synchronously complete the job.  The completion callback is called before the
+ * function returns, unless it is NULL (which is permissible when using this
+ * function).
+ *
+ * Returns the return value from the job.
+ */
+int block_job_complete_sync(BlockJob *job, Error **errp);
+
+/**
  * block_job_iostatus_reset:
  * @job: The job whose I/O status should be reset.
  *
@@ -295,4 +315,23 @@ void block_job_iostatus_reset(BlockJob *job);
 BlockErrorAction block_job_error_action(BlockJob *job, BlockDriverState *bs,
                                         BlockdevOnError on_err,
                                         int is_read, int error);
+
+typedef void BlockJobDeferToMainLoopFn(BlockJob *job, void *opaque);
+
+/**
+ * block_job_defer_to_main_loop:
+ * @job: The job
+ * @fn: The function to run in the main loop
+ * @opaque: The opaque value that is passed to @fn
+ *
+ * Execute a given function in the main loop with the BlockDriverState
+ * AioContext acquired.  Block jobs must call bdrv_unref(), bdrv_close(), and
+ * anything that uses bdrv_drain_all() in the main loop.
+ *
+ * The @job AioContext is held while @fn executes.
+ */
+void block_job_defer_to_main_loop(BlockJob *job,
+                                  BlockJobDeferToMainLoopFn *fn,
+                                  void *opaque);
+
 #endif
diff --git a/include/block/coroutine.h b/include/block/coroutine.h
index b408f96b8..793df0ef8 100644
--- a/include/block/coroutine.h
+++ b/include/block/coroutine.h
@@ -203,14 +203,6 @@ void qemu_co_rwlock_unlock(CoRwlock *lock);
 /**
  * Yield the coroutine for a given duration
  *
- * Note this function uses timers and hence only works when a main loop is in
- * use.  See main-loop.h and do not use from qemu-tool programs.
- */
-void coroutine_fn co_sleep_ns(QEMUClockType type, int64_t ns);
-
-/**
- * Yield the coroutine for a given duration
- *
  * Behaves similarly to co_sleep_ns(), but the sleeping coroutine will be
  * resumed when using aio_poll().
  */
@@ -223,4 +215,15 @@ void coroutine_fn co_aio_sleep_ns(AioContext *ctx, QEMUClockType type,
  * Note that this function clobbers the handlers for the file descriptor.
  */
 void coroutine_fn yield_until_fd_readable(int fd);
+
+/**
+ * Add or subtract from the coroutine pool size
+ *
+ * The coroutine implementation keeps a pool of coroutines to be reused by
+ * qemu_coroutine_create().  This makes coroutine creation cheap.  Heavy
+ * coroutine users should call this to reserve pool space.  Call it again with
+ * a negative number to release pool space.
+ */
+void qemu_coroutine_adjust_pool_size(int n);
+
 #endif /* QEMU_COROUTINE_H */
diff --git a/include/block/qapi.h b/include/block/qapi.h
index 03745464d..168d78852 100644
--- a/include/block/qapi.h
+++ b/include/block/qapi.h
@@ -36,9 +36,6 @@ int bdrv_query_snapshot_info_list(BlockDriverState *bs,
 void bdrv_query_image_info(BlockDriverState *bs,
                            ImageInfo **p_info,
                            Error **errp);
-void bdrv_query_info(BlockDriverState *bs,
-                     BlockInfo **p_info,
-                     Error **errp);
 
 void bdrv_snapshot_dump(fprintf_function func_fprintf, void *f,
                         QEMUSnapshotInfo *sn);
diff --git a/include/block/thread-pool.h b/include/block/thread-pool.h
index 32afcdd1d..42eb5e842 100644
--- a/include/block/thread-pool.h
+++ b/include/block/thread-pool.h
@@ -18,11 +18,7 @@
 #ifndef QEMU_THREAD_POOL_H
 #define QEMU_THREAD_POOL_H 1
 
-#include "qemu-common.h"
-#include "qemu/queue.h"
-#include "qemu/thread.h"
-#include "block/coroutine.h"
-#include "block/block_int.h"
+#include "block/block.h"
 
 typedef int ThreadPoolFunc(void *opaque);
 
@@ -31,9 +27,9 @@ typedef struct ThreadPool ThreadPool;
 ThreadPool *thread_pool_new(struct AioContext *ctx);
 void thread_pool_free(ThreadPool *pool);
 
-BlockDriverAIOCB *thread_pool_submit_aio(ThreadPool *pool,
+BlockAIOCB *thread_pool_submit_aio(ThreadPool *pool,
         ThreadPoolFunc *func, void *arg,
-        BlockDriverCompletionFunc *cb, void *opaque);
+        BlockCompletionFunc *cb, void *opaque);
 int coroutine_fn thread_pool_submit_co(ThreadPool *pool,
         ThreadPoolFunc *func, void *arg);
 void thread_pool_submit(ThreadPool *pool, ThreadPoolFunc *func, void *arg);
diff --git a/include/elf.h b/include/elf.h
index e88d52fd7..a51658448 100644
--- a/include/elf.h
+++ b/include/elf.h
@@ -92,6 +92,8 @@ typedef int64_t  Elf64_Sxword;
 
 #define EM_SPARCV9     43	/* SPARC v9 64-bit */
 
+#define EM_TRICORE      44      /* Infineon TriCore */
+
 #define EM_IA_64	50	/* HP/Intel IA-64 */
 
 #define EM_X86_64	62	/* AMD x86-64 */
@@ -471,14 +473,35 @@ typedef struct {
 #define PPC_FEATURE_TRUE_LE             0x00000002
 #define PPC_FEATURE_PPC_LE              0x00000001
 
-/* Bits present in AT_HWCAP, primarily for Sparc32.  */
-
-#define HWCAP_SPARC_FLUSH       1    /* CPU supports flush instruction. */
-#define HWCAP_SPARC_STBAR       2
-#define HWCAP_SPARC_SWAP        4
-#define HWCAP_SPARC_MULDIV      8
-#define HWCAP_SPARC_V9		16
-#define HWCAP_SPARC_ULTRA3	32
+/* Bits present in AT_HWCAP for Sparc.  */
+
+#define HWCAP_SPARC_FLUSH               0x00000001
+#define HWCAP_SPARC_STBAR               0x00000002
+#define HWCAP_SPARC_SWAP                0x00000004
+#define HWCAP_SPARC_MULDIV              0x00000008
+#define HWCAP_SPARC_V9                  0x00000010
+#define HWCAP_SPARC_ULTRA3              0x00000020
+#define HWCAP_SPARC_BLKINIT             0x00000040
+#define HWCAP_SPARC_N2                  0x00000080
+#define HWCAP_SPARC_MUL32               0x00000100
+#define HWCAP_SPARC_DIV32               0x00000200
+#define HWCAP_SPARC_FSMULD              0x00000400
+#define HWCAP_SPARC_V8PLUS              0x00000800
+#define HWCAP_SPARC_POPC                0x00001000
+#define HWCAP_SPARC_VIS                 0x00002000
+#define HWCAP_SPARC_VIS2                0x00004000
+#define HWCAP_SPARC_ASI_BLK_INIT        0x00008000
+#define HWCAP_SPARC_FMAF                0x00010000
+#define HWCAP_SPARC_VIS3                0x00020000
+#define HWCAP_SPARC_HPC                 0x00040000
+#define HWCAP_SPARC_RANDOM              0x00080000
+#define HWCAP_SPARC_TRANS               0x00100000
+#define HWCAP_SPARC_FJFMAU              0x00200000
+#define HWCAP_SPARC_IMA                 0x00400000
+#define HWCAP_SPARC_ASI_CACHE_SPARING   0x00800000
+#define HWCAP_SPARC_PAUSE               0x01000000
+#define HWCAP_SPARC_CBCOND              0x02000000
+#define HWCAP_SPARC_CRYPTO              0x04000000
 
 /* Bits present in AT_HWCAP for s390.  */
 
diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h
index f91581fc6..c085804ae 100644
--- a/include/exec/cpu-all.h
+++ b/include/exec/cpu-all.h
@@ -198,6 +198,8 @@ extern unsigned long reserved_va;
 #define RESERVED_VA 0ul
 #endif
 
+#define GUEST_ADDR_MAX (RESERVED_VA ? RESERVED_VA : \
+                                    (1ul << TARGET_VIRT_ADDR_SPACE_BITS) - 1)
 #endif
 
 /* page related stuff */
@@ -230,8 +232,8 @@ extern uintptr_t qemu_host_page_mask;
 #if defined(CONFIG_USER_ONLY)
 void page_dump(FILE *f);
 
-typedef int (*walk_memory_regions_fn)(void *, abi_ulong,
-                                      abi_ulong, unsigned long);
+typedef int (*walk_memory_regions_fn)(void *, target_ulong,
+                                      target_ulong, unsigned long);
 int walk_memory_regions(void *, walk_memory_regions_fn);
 
 int page_get_flags(target_ulong address);
diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h
index e3ec4c8e0..427b8515a 100644
--- a/include/exec/cpu-common.h
+++ b/include/exec/cpu-common.h
@@ -26,6 +26,12 @@ typedef struct CPUListState {
     FILE *file;
 } CPUListState;
 
+typedef enum MMUAccessType {
+    MMU_DATA_LOAD  = 0,
+    MMU_DATA_STORE = 1,
+    MMU_INST_FETCH = 2
+} MMUAccessType;
+
 #if !defined(CONFIG_USER_ONLY)
 
 enum device_endian {
diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h
index 2dd6206d4..0ca6f0b95 100644
--- a/include/exec/cpu-defs.h
+++ b/include/exec/cpu-defs.h
@@ -71,6 +71,8 @@ typedef uint64_t target_ulong;
 #if !defined(CONFIG_USER_ONLY)
 #define CPU_TLB_BITS 8
 #define CPU_TLB_SIZE (1 << CPU_TLB_BITS)
+/* use a fully associative victim tlb of 8 entries */
+#define CPU_VTLB_SIZE 8
 
 #if HOST_LONG_BITS == 32 && TARGET_LONG_BITS == 32
 #define CPU_TLB_ENTRY_BITS 4
@@ -103,9 +105,12 @@ QEMU_BUILD_BUG_ON(sizeof(CPUTLBEntry) != (1 << CPU_TLB_ENTRY_BITS));
 #define CPU_COMMON_TLB \
     /* The meaning of the MMU modes is defined in the target code. */   \
     CPUTLBEntry tlb_table[NB_MMU_MODES][CPU_TLB_SIZE];                  \
-    hwaddr iotlb[NB_MMU_MODES][CPU_TLB_SIZE];               \
+    CPUTLBEntry tlb_v_table[NB_MMU_MODES][CPU_VTLB_SIZE];               \
+    hwaddr iotlb[NB_MMU_MODES][CPU_TLB_SIZE];                           \
+    hwaddr iotlb_v[NB_MMU_MODES][CPU_VTLB_SIZE];                        \
     target_ulong tlb_flush_addr;                                        \
-    target_ulong tlb_flush_mask;
+    target_ulong tlb_flush_mask;                                        \
+    target_ulong vtlb_index;                                            \
 
 #else
 
diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
index 5e5d86ec4..0844885ed 100644
--- a/include/exec/exec-all.h
+++ b/include/exec/exec-all.h
@@ -333,7 +333,7 @@ extern uintptr_t tci_tb_ptr;
 
 #if !defined(CONFIG_USER_ONLY)
 
-void phys_mem_set_alloc(void *(*alloc)(size_t));
+void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align));
 
 struct MemoryRegion *iotlb_to_region(AddressSpace *as, hwaddr index);
 bool io_mem_read(struct MemoryRegion *mr, hwaddr addr,
@@ -356,10 +356,6 @@ static inline tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong
 tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr);
 #endif
 
-typedef void (CPUDebugExcpHandler)(CPUArchState *env);
-
-void cpu_set_debug_excp_handler(CPUDebugExcpHandler *handler);
-
 /* vl.c */
 extern int singlestep;
 
diff --git a/include/exec/helper-gen.h b/include/exec/helper-gen.h
index a04a0341e..0d0da3aeb 100644
--- a/include/exec/helper-gen.h
+++ b/include/exec/helper-gen.h
@@ -57,6 +57,8 @@ static inline void glue(gen_helper_, name)(dh_retvar_decl(ret)          \
 }
 
 #include "helper.h"
+#include "trace/generated-helpers.h"
+#include "trace/generated-helpers-wrappers.h"
 #include "tcg-runtime.h"
 
 #undef DEF_HELPER_FLAGS_0
diff --git a/include/exec/helper-proto.h b/include/exec/helper-proto.h
index 828951c60..effdd4383 100644
--- a/include/exec/helper-proto.h
+++ b/include/exec/helper-proto.h
@@ -27,6 +27,7 @@ dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \
                             dh_ctype(t4), dh_ctype(t5));
 
 #include "helper.h"
+#include "trace/generated-helpers.h"
 #include "tcg-runtime.h"
 
 #undef DEF_HELPER_FLAGS_0
diff --git a/include/exec/helper-tcg.h b/include/exec/helper-tcg.h
index d704c8112..79fa3c8c8 100644
--- a/include/exec/helper-tcg.h
+++ b/include/exec/helper-tcg.h
@@ -36,6 +36,7 @@
     | dh_sizemask(t5, 5) },
 
 #include "helper.h"
+#include "trace/generated-helpers.h"
 #include "tcg-runtime.h"
 
 #undef DEF_HELPER_FLAGS_0
diff --git a/include/exec/memory.h b/include/exec/memory.h
index e2c8e3e0a..f64ab5e3e 100644
--- a/include/exec/memory.h
+++ b/include/exec/memory.h
@@ -129,7 +129,7 @@ typedef struct MemoryRegionIOMMUOps MemoryRegionIOMMUOps;
 
 struct MemoryRegionIOMMUOps {
     /* Return a TLB entry that contains a given address. */
-    IOMMUTLBEntry (*translate)(MemoryRegion *iommu, hwaddr addr);
+    IOMMUTLBEntry (*translate)(MemoryRegion *iommu, hwaddr addr, bool is_write);
 };
 
 typedef struct CoalescedMemoryRange CoalescedMemoryRange;
@@ -146,10 +146,12 @@ struct MemoryRegion {
     hwaddr addr;
     void (*destructor)(MemoryRegion *mr);
     ram_addr_t ram_addr;
+    uint64_t align;
     bool subpage;
     bool terminates;
     bool romd_mode;
     bool ram;
+    bool skip_dump;
     bool readonly; /* For RAM regions */
     bool enabled;
     bool rom_device;
@@ -311,11 +313,13 @@ void memory_region_init_io(MemoryRegion *mr,
  * @owner: the object that tracks the region's reference count
  * @name: the name of the region.
  * @size: size of the region.
+ * @errp: pointer to Error*, to store an error if it happens.
  */
 void memory_region_init_ram(MemoryRegion *mr,
                             struct Object *owner,
                             const char *name,
-                            uint64_t size);
+                            uint64_t size,
+                            Error **errp);
 
 #ifdef __linux__
 /**
@@ -384,13 +388,15 @@ void memory_region_init_alias(MemoryRegion *mr,
  * @ops: callbacks for write access handling.
  * @name: the name of the region.
  * @size: size of the region.
+ * @errp: pointer to Error*, to store an error if it happens.
  */
 void memory_region_init_rom_device(MemoryRegion *mr,
                                    struct Object *owner,
                                    const MemoryRegionOps *ops,
                                    void *opaque,
                                    const char *name,
-                                   uint64_t size);
+                                   uint64_t size,
+                                   Error **errp);
 
 /**
  * memory_region_init_reservation: Initialize a memory region that reserves
@@ -430,15 +436,6 @@ void memory_region_init_iommu(MemoryRegion *mr,
                               uint64_t size);
 
 /**
- * memory_region_destroy: Destroy a memory region and reclaim all resources.
- *
- * @mr: the region to be destroyed.  May not currently be a subregion
- *      (see memory_region_add_subregion()) or referenced in an alias
- *      (see memory_region_init_alias()).
- */
-void memory_region_destroy(MemoryRegion *mr);
-
-/**
  * memory_region_owner: get a memory region's owner.
  *
  * @mr: the memory region being queried.
@@ -462,6 +459,24 @@ uint64_t memory_region_size(MemoryRegion *mr);
 bool memory_region_is_ram(MemoryRegion *mr);
 
 /**
+ * memory_region_is_skip_dump: check whether a memory region should not be
+ *                             dumped
+ *
+ * Returns %true is a memory region should not be dumped(e.g. VFIO BAR MMAP).
+ *
+ * @mr: the memory region being queried
+ */
+bool memory_region_is_skip_dump(MemoryRegion *mr);
+
+/**
+ * memory_region_set_skip_dump: Set skip_dump flag, dump will ignore this memory
+ *                              region
+ *
+ * @mr: the memory region being queried
+ */
+void memory_region_set_skip_dump(MemoryRegion *mr);
+
+/**
  * memory_region_is_romd: check whether a memory region is in ROMD mode
  *
  * Returns %true if a memory region is a ROM device and currently set to allow
@@ -520,7 +535,7 @@ void memory_region_unregister_iommu_notifier(Notifier *n);
  *
  * @mr: the memory region being queried
  */
-const char *memory_region_name(MemoryRegion *mr);
+const char *memory_region_name(const MemoryRegion *mr);
 
 /**
  * memory_region_is_logging: return whether a memory region is logging writes
@@ -824,6 +839,7 @@ void memory_region_add_subregion_overlap(MemoryRegion *mr,
  */
 ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr);
 
+uint64_t memory_region_get_alignment(const MemoryRegion *mr);
 /**
  * memory_region_del_subregion: Remove a subregion.
  *
diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h
index 6593be131..8fc75cdd2 100644
--- a/include/exec/ram_addr.h
+++ b/include/exec/ram_addr.h
@@ -26,8 +26,8 @@ ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
                                     bool share, const char *mem_path,
                                     Error **errp);
 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
-                                   MemoryRegion *mr);
-ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr);
+                                   MemoryRegion *mr, Error **errp);
+ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp);
 int qemu_get_ram_fd(ram_addr_t addr);
 void *qemu_get_ram_block_host_ptr(ram_addr_t addr);
 void *qemu_get_ram_ptr(ram_addr_t addr);
@@ -49,6 +49,21 @@ static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
     return next < end;
 }
 
+static inline bool cpu_physical_memory_get_clean(ram_addr_t start,
+                                                 ram_addr_t length,
+                                                 unsigned client)
+{
+    unsigned long end, page, next;
+
+    assert(client < DIRTY_MEMORY_NUM);
+
+    end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
+    page = start >> TARGET_PAGE_BITS;
+    next = find_next_zero_bit(ram_list.dirty_memory[client], end, page);
+
+    return next < end;
+}
+
 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
                                                       unsigned client)
 {
@@ -64,6 +79,16 @@ static inline bool cpu_physical_memory_is_clean(ram_addr_t addr)
     return !(vga && code && migration);
 }
 
+static inline bool cpu_physical_memory_range_includes_clean(ram_addr_t start,
+                                                            ram_addr_t length)
+{
+    bool vga = cpu_physical_memory_get_clean(start, length, DIRTY_MEMORY_VGA);
+    bool code = cpu_physical_memory_get_clean(start, length, DIRTY_MEMORY_CODE);
+    bool migration =
+        cpu_physical_memory_get_clean(start, length, DIRTY_MEMORY_MIGRATION);
+    return vga || code || migration;
+}
+
 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
                                                       unsigned client)
 {
diff --git a/include/fpu/softfloat.h b/include/fpu/softfloat.h
index 77177c531..e32e25d54 100644
--- a/include/fpu/softfloat.h
+++ b/include/fpu/softfloat.h
@@ -374,6 +374,8 @@ float32 float32_min(float32, float32 STATUS_PARAM);
 float32 float32_max(float32, float32 STATUS_PARAM);
 float32 float32_minnum(float32, float32 STATUS_PARAM);
 float32 float32_maxnum(float32, float32 STATUS_PARAM);
+float32 float32_minnummag(float32, float32 STATUS_PARAM);
+float32 float32_maxnummag(float32, float32 STATUS_PARAM);
 int float32_is_quiet_nan( float32 );
 int float32_is_signaling_nan( float32 );
 float32 float32_maybe_silence_nan( float32 );
@@ -484,6 +486,8 @@ float64 float64_min(float64, float64 STATUS_PARAM);
 float64 float64_max(float64, float64 STATUS_PARAM);
 float64 float64_minnum(float64, float64 STATUS_PARAM);
 float64 float64_maxnum(float64, float64 STATUS_PARAM);
+float64 float64_minnummag(float64, float64 STATUS_PARAM);
+float64 float64_maxnummag(float64, float64 STATUS_PARAM);
 int float64_is_quiet_nan( float64 a );
 int float64_is_signaling_nan( float64 );
 float64 float64_maybe_silence_nan( float64 );
diff --git a/include/glib-compat.h b/include/glib-compat.h
index 4ae0671a8..f0615c99c 100644
--- a/include/glib-compat.h
+++ b/include/glib-compat.h
@@ -18,6 +18,11 @@
 
 #include <glib.h>
 
+/* GLIB version compatibility flags */
+#if !GLIB_CHECK_VERSION(2, 26, 0)
+#define G_TIME_SPAN_SECOND              (G_GINT64_CONSTANT(1000000))
+#endif
+
 #if !GLIB_CHECK_VERSION(2, 14, 0)
 static inline guint g_timeout_add_seconds(guint interval, GSourceFunc function,
                                           gpointer data)
@@ -26,6 +31,37 @@ static inline guint g_timeout_add_seconds(guint interval, GSourceFunc function,
 }
 #endif
 
+#if !GLIB_CHECK_VERSION(2, 28, 0)
+static inline gint64 g_get_monotonic_time(void)
+{
+    /* g_get_monotonic_time() is best-effort so we can use the wall clock as a
+     * fallback.
+     */
+
+    GTimeVal time;
+    g_get_current_time(&time);
+
+    return time.tv_sec * G_TIME_SPAN_SECOND + time.tv_usec;
+}
+#endif
+
+#if !GLIB_CHECK_VERSION(2, 16, 0)
+static inline int g_strcmp0(const char *str1, const char *str2)
+{
+    int result;
+
+    if (!str1) {
+        result = -(str1 != str2);
+    } else if (!str2) {
+        result = (str1 != str2);
+    } else {
+        result = strcmp(str1, str2);
+    }
+
+    return result;
+}
+#endif
+
 #ifdef _WIN32
 /*
  * g_poll has a problem on Windows when using
diff --git a/include/hw/acpi/cpu_hotplug.h b/include/hw/acpi/cpu_hotplug.h
index 9e5d30c9d..f6d358def 100644
--- a/include/hw/acpi/cpu_hotplug.h
+++ b/include/hw/acpi/cpu_hotplug.h
@@ -20,8 +20,9 @@ typedef struct AcpiCpuHotplug {
     uint8_t sts[ACPI_GPE_PROC_LEN];
 } AcpiCpuHotplug;
 
-void AcpiCpuHotplug_add(ACPIGPE *gpe, AcpiCpuHotplug *g, CPUState *cpu);
+void acpi_cpu_plug_cb(ACPIREGS *ar, qemu_irq irq,
+                      AcpiCpuHotplug *g, DeviceState *dev, Error **errp);
 
-void AcpiCpuHotplug_init(MemoryRegion *parent, Object *owner,
-                         AcpiCpuHotplug *gpe_cpu, uint16_t base);
+void acpi_cpu_hotplug_init(MemoryRegion *parent, Object *owner,
+                           AcpiCpuHotplug *gpe_cpu, uint16_t base);
 #endif
diff --git a/include/hw/acpi/ich9.h b/include/hw/acpi/ich9.h
index 7e42448ef..fe975e662 100644
--- a/include/hw/acpi/ich9.h
+++ b/include/hw/acpi/ich9.h
@@ -47,7 +47,6 @@ typedef struct ICH9LPCPMRegs {
     Notifier powerdown_notifier;
 
     AcpiCpuHotplug gpe_cpu;
-    Notifier cpu_added_notifier;
 
     MemHotplugState acpi_memory_hotplug;
 } ICH9LPCPMRegs;
diff --git a/include/hw/acpi/pc-hotplug.h b/include/hw/acpi/pc-hotplug.h
index bf5157d7c..b9db29576 100644
--- a/include/hw/acpi/pc-hotplug.h
+++ b/include/hw/acpi/pc-hotplug.h
@@ -32,7 +32,7 @@
 #define ACPI_MEMORY_HOTPLUG_IO_LEN 24
 #define ACPI_MEMORY_HOTPLUG_BASE 0x0a00
 
-#define MEMORY_HOPTLUG_DEVICE        MHPD
+#define MEMORY_HOTPLUG_DEVICE        MHPD
 #define MEMORY_SLOTS_NUMBER          MDNR
 #define MEMORY_HOTPLUG_IO_REGION     HPMR
 #define MEMORY_SLOT_ADDR_LOW         MRBL
diff --git a/include/hw/acpi/tpm.h b/include/hw/acpi/tpm.h
new file mode 100644
index 000000000..792fcbf5b
--- /dev/null
+++ b/include/hw/acpi/tpm.h
@@ -0,0 +1,29 @@
+/*
+ * tpm.h - TPM ACPI definitions
+ *
+ * Copyright (C) 2014 IBM Corporation
+ *
+ * Authors:
+ *  Stefan Berger <stefanb@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ * Implementation of the TIS interface according to specs found at
+ * http://www.trustedcomputinggroup.org
+ *
+ */
+#ifndef HW_ACPI_TPM_H
+#define HW_ACPI_TPM_H
+
+#define TPM_TIS_ADDR_BASE           0xFED40000
+#define TPM_TIS_ADDR_SIZE           0x5000
+
+#define TPM_TIS_IRQ                 5
+
+#define TPM_LOG_AREA_MINIMUM_SIZE   (64 * 1024)
+
+#define TPM_TCPA_ACPI_CLASS_CLIENT  0
+#define TPM_TCPA_ACPI_CLASS_SERVER  1
+
+#endif /* HW_ACPI_TPM_H */
diff --git a/include/hw/arm/arm.h b/include/hw/arm/arm.h
index cbbf4ca4c..cefc9e698 100644
--- a/include/hw/arm/arm.h
+++ b/include/hw/arm/arm.h
@@ -15,7 +15,7 @@
 #include "hw/irq.h"
 
 /* armv7m.c */
-qemu_irq *armv7m_init(MemoryRegion *address_space_mem,
+qemu_irq *armv7m_init(MemoryRegion *system_memory,
                       int flash_size, int sram_size,
                       const char *kernel_filename, const char *cpu_model);
 
diff --git a/include/hw/arm/omap.h b/include/hw/arm/omap.h
index b9655ee39..0ad5fb883 100644
--- a/include/hw/arm/omap.h
+++ b/include/hw/arm/omap.h
@@ -755,10 +755,10 @@ void omap_rfbi_attach(struct omap_dss_s *s, int cs, struct rfbi_chip_s *chip);
 struct omap_mmc_s;
 struct omap_mmc_s *omap_mmc_init(hwaddr base,
                 MemoryRegion *sysmem,
-                BlockDriverState *bd,
+                BlockBackend *blk,
                 qemu_irq irq, qemu_irq dma[], omap_clk clk);
 struct omap_mmc_s *omap2_mmc_init(struct omap_target_agent_s *ta,
-                BlockDriverState *bd, qemu_irq irq, qemu_irq dma[],
+                BlockBackend *blk, qemu_irq irq, qemu_irq dma[],
                 omap_clk fclk, omap_clk iclk);
 void omap_mmc_reset(struct omap_mmc_s *s);
 void omap_mmc_handlers(struct omap_mmc_s *s, qemu_irq ro, qemu_irq cover);
diff --git a/include/hw/arm/pxa.h b/include/hw/arm/pxa.h
index c5079067b..259b85249 100644
--- a/include/hw/arm/pxa.h
+++ b/include/hw/arm/pxa.h
@@ -87,7 +87,7 @@ void pxa2xx_lcdc_oritentation(void *opaque, int angle);
 typedef struct PXA2xxMMCIState PXA2xxMMCIState;
 PXA2xxMMCIState *pxa2xx_mmci_init(MemoryRegion *sysmem,
                 hwaddr base,
-                BlockDriverState *bd, qemu_irq irq,
+                BlockBackend *blk, qemu_irq irq,
                 qemu_irq rx_dma, qemu_irq tx_dma);
 void pxa2xx_mmci_handlers(PXA2xxMMCIState *s, qemu_irq readonly,
                 qemu_irq coverswitch);
diff --git a/include/hw/block/block.h b/include/hw/block/block.h
index 7c3d6c817..0d0ce9a46 100644
--- a/include/hw/block/block.h
+++ b/include/hw/block/block.h
@@ -12,11 +12,12 @@
 #define HW_BLOCK_COMMON_H
 
 #include "qemu-common.h"
+#include "qapi/error.h"
 
 /* Configuration */
 
 typedef struct BlockConf {
-    BlockDriverState *bs;
+    BlockBackend *blk;
     uint16_t physical_block_size;
     uint16_t logical_block_size;
     uint16_t min_io_size;
@@ -41,14 +42,13 @@ static inline unsigned int get_physical_block_exp(BlockConf *conf)
 }
 
 #define DEFINE_BLOCK_PROPERTIES(_state, _conf)                          \
-    DEFINE_PROP_DRIVE("drive", _state, _conf.bs),                       \
+    DEFINE_PROP_DRIVE("drive", _state, _conf.blk),                      \
     DEFINE_PROP_BLOCKSIZE("logical_block_size", _state,                 \
                           _conf.logical_block_size, 512),               \
     DEFINE_PROP_BLOCKSIZE("physical_block_size", _state,                \
                           _conf.physical_block_size, 512),              \
     DEFINE_PROP_UINT16("min_io_size", _state, _conf.min_io_size, 0),  \
     DEFINE_PROP_UINT32("opt_io_size", _state, _conf.opt_io_size, 0),    \
-    DEFINE_PROP_INT32("bootindex", _state, _conf.bootindex, -1),        \
     DEFINE_PROP_UINT32("discard_granularity", _state, \
                        _conf.discard_granularity, -1)
 
@@ -60,12 +60,13 @@ static inline unsigned int get_physical_block_exp(BlockConf *conf)
 /* Configuration helpers */
 
 void blkconf_serial(BlockConf *conf, char **serial);
-int blkconf_geometry(BlockConf *conf, int *trans,
-                     unsigned cyls_max, unsigned heads_max, unsigned secs_max);
+void blkconf_geometry(BlockConf *conf, int *trans,
+                      unsigned cyls_max, unsigned heads_max, unsigned secs_max,
+                      Error **errp);
 
 /* Hard disk geometry */
 
-void hd_geometry_guess(BlockDriverState *bs,
+void hd_geometry_guess(BlockBackend *blk,
                        uint32_t *pcyls, uint32_t *pheads, uint32_t *psecs,
                        int *ptrans);
 int hd_bios_chs_auto_trans(uint32_t cyls, uint32_t heads, uint32_t secs);
diff --git a/include/hw/block/flash.h b/include/hw/block/flash.h
index 920d7596e..50ccbbcf1 100644
--- a/include/hw/block/flash.h
+++ b/include/hw/block/flash.h
@@ -11,7 +11,7 @@ typedef struct pflash_t pflash_t;
 pflash_t *pflash_cfi01_register(hwaddr base,
                                 DeviceState *qdev, const char *name,
                                 hwaddr size,
-                                BlockDriverState *bs,
+                                BlockBackend *blk,
                                 uint32_t sector_len, int nb_blocs, int width,
                                 uint16_t id0, uint16_t id1,
                                 uint16_t id2, uint16_t id3, int be);
@@ -20,7 +20,7 @@ pflash_t *pflash_cfi01_register(hwaddr base,
 pflash_t *pflash_cfi02_register(hwaddr base,
                                 DeviceState *qdev, const char *name,
                                 hwaddr size,
-                                BlockDriverState *bs, uint32_t sector_len,
+                                BlockBackend *blk, uint32_t sector_len,
                                 int nb_blocs, int nb_mappings, int width,
                                 uint16_t id0, uint16_t id1,
                                 uint16_t id2, uint16_t id3,
@@ -30,7 +30,7 @@ pflash_t *pflash_cfi02_register(hwaddr base,
 MemoryRegion *pflash_cfi01_get_memory(pflash_t *fl);
 
 /* nand.c */
-DeviceState *nand_init(BlockDriverState *bdrv, int manf_id, int chip_id);
+DeviceState *nand_init(BlockBackend *blk, int manf_id, int chip_id);
 void nand_setpins(DeviceState *dev, uint8_t cle, uint8_t ale,
                   uint8_t ce, uint8_t wp, uint8_t gnd);
 void nand_getpins(DeviceState *dev, int *rb);
diff --git a/include/hw/boards.h b/include/hw/boards.h
index 605a97093..e0a67903e 100644
--- a/include/hw/boards.h
+++ b/include/hw/boards.h
@@ -5,12 +5,11 @@
 
 #include "qemu/typedefs.h"
 #include "sysemu/blockdev.h"
+#include "sysemu/accel.h"
 #include "hw/qdev.h"
 #include "qom/object.h"
 
 
-typedef struct MachineState MachineState;
-
 typedef void QEMUMachineInitFunc(MachineState *ms);
 
 typedef void QEMUMachineResetFunc(void);
@@ -20,6 +19,7 @@ typedef void QEMUMachineHotAddCPUFunc(const int64_t id, Error **errp);
 typedef int QEMUMachineGetKvmtypeFunc(const char *arg);
 
 struct QEMUMachine {
+    const char *family; /* NULL iff @name identifies a standalone machtype */
     const char *name;
     const char *alias;
     const char *desc;
@@ -28,6 +28,7 @@ struct QEMUMachine {
     QEMUMachineHotAddCPUFunc *hot_add_cpu;
     QEMUMachineGetKvmtypeFunc *kvm_type;
     BlockInterfaceType block_default_type;
+    int units_per_default_bus;
     int max_cpus;
     unsigned int no_serial:1,
         no_parallel:1,
@@ -35,10 +36,12 @@ struct QEMUMachine {
         use_sclp:1,
         no_floppy:1,
         no_cdrom:1,
-        no_sdcard:1;
+        no_sdcard:1,
+        has_dynamic_sysbus:1;
     int is_default;
     const char *default_machine_opts;
     const char *default_boot_order;
+    const char *default_display;
     GlobalProperty *compat_props;
     const char *hw_version;
 };
@@ -76,6 +79,7 @@ struct MachineClass {
     ObjectClass parent_class;
     /*< public >*/
 
+    const char *family; /* NULL iff @name identifies a standalone machtype */
     const char *name;
     const char *alias;
     const char *desc;
@@ -86,6 +90,7 @@ struct MachineClass {
     int (*kvm_type)(const char *arg);
 
     BlockInterfaceType block_default_type;
+    int units_per_default_bus;
     int max_cpus;
     unsigned int no_serial:1,
         no_parallel:1,
@@ -93,10 +98,12 @@ struct MachineClass {
         use_sclp:1,
         no_floppy:1,
         no_cdrom:1,
-        no_sdcard:1;
+        no_sdcard:1,
+        has_dynamic_sysbus:1;
     int is_default;
     const char *default_machine_opts;
     const char *default_boot_order;
+    const char *default_display;
     GlobalProperty *compat_props;
     const char *hw_version;
 
@@ -110,6 +117,8 @@ struct MachineClass {
 struct MachineState {
     /*< private >*/
     Object parent_obj;
+    Notifier sysbus_notifier;
+
     /*< public >*/
 
     char *accel;
@@ -123,6 +132,7 @@ struct MachineState {
     bool mem_merge;
     bool usb;
     char *firmware;
+    bool iommu;
 
     ram_addr_t ram_size;
     ram_addr_t maxram_size;
@@ -132,6 +142,7 @@ struct MachineState {
     char *kernel_cmdline;
     char *initrd_filename;
     const char *cpu_model;
+    AccelState *accelerator;
 };
 
 #endif
diff --git a/include/hw/compat.h b/include/hw/compat.h
new file mode 100644
index 000000000..313682a70
--- /dev/null
+++ b/include/hw/compat.h
@@ -0,0 +1,35 @@
+#ifndef HW_COMPAT_H
+#define HW_COMPAT_H
+
+#define HW_COMPAT_2_1 \
+        {\
+            .driver   = "intel-hda",\
+            .property = "old_msi_addr",\
+            .value    = "on",\
+        },{\
+            .driver   = "VGA",\
+            .property = "qemu-extended-regs",\
+            .value    = "off",\
+        },{\
+            .driver   = "secondary-vga",\
+            .property = "qemu-extended-regs",\
+            .value    = "off",\
+        },{\
+            .driver   = "virtio-scsi-pci",\
+            .property = "any_layout",\
+            .value    = "off",\
+        },{\
+            .driver   = "usb-mouse",\
+            .property = "usb_version",\
+            .value    = stringify(1),\
+        },{\
+            .driver   = "usb-kbd",\
+            .property = "usb_version",\
+            .value    = stringify(1),\
+        },{\
+            .driver   = "virtio-pci",\
+            .property = "virtio-pci-bus-master-bug-migration",\
+            .value    = "on",\
+        }
+
+#endif /* HW_COMPAT_H */
diff --git a/include/hw/elf_ops.h b/include/hw/elf_ops.h
index c6b5129ba..a517753a6 100644
--- a/include/hw/elf_ops.h
+++ b/include/hw/elf_ops.h
@@ -147,18 +147,13 @@ static int glue(load_symbols, SZ)(struct elfhdr *ehdr, int fd, int must_swab,
         }
         i++;
     }
-    if (nsyms) {
-        syms = g_realloc(syms, nsyms * sizeof(*syms));
+    syms = g_realloc(syms, nsyms * sizeof(*syms));
 
-        qsort(syms, nsyms, sizeof(*syms), glue(symcmp, SZ));
-        for (i = 0; i < nsyms - 1; i++) {
-            if (syms[i].st_size == 0) {
-                syms[i].st_size = syms[i + 1].st_value - syms[i].st_value;
-            }
+    qsort(syms, nsyms, sizeof(*syms), glue(symcmp, SZ));
+    for (i = 0; i < nsyms - 1; i++) {
+        if (syms[i].st_size == 0) {
+            syms[i].st_size = syms[i + 1].st_value - syms[i].st_value;
         }
-    } else {
-        g_free(syms);
-        syms = NULL;
     }
 
     /* String table */
diff --git a/include/hw/hotplug.h b/include/hw/hotplug.h
index a6533cb0b..050d2f053 100644
--- a/include/hw/hotplug.h
+++ b/include/hw/hotplug.h
@@ -47,7 +47,12 @@ typedef void (*hotplug_fn)(HotplugHandler *plug_handler,
  *
  * @parent: Opaque parent interface.
  * @plug: plug callback.
+ * @unplug_request: unplug request callback.
+ *                  Used as a means to initiate device unplug for devices that
+ *                  require asynchronous unplug handling.
  * @unplug: unplug callback.
+ *          Used for device removal with devices that implement
+ *          asynchronous and synchronous (suprise) removal.
  */
 typedef struct HotplugHandlerClass {
     /* <private> */
@@ -55,6 +60,7 @@ typedef struct HotplugHandlerClass {
 
     /* <public> */
     hotplug_fn plug;
+    hotplug_fn unplug_request;
     hotplug_fn unplug;
 } HotplugHandlerClass;
 
@@ -68,9 +74,17 @@ void hotplug_handler_plug(HotplugHandler *plug_handler,
                           Error **errp);
 
 /**
+ * hotplug_handler_unplug_request:
+ *
+ * Calls #HotplugHandlerClass.unplug_request callback of @plug_handler.
+ */
+void hotplug_handler_unplug_request(HotplugHandler *plug_handler,
+                                    DeviceState *plugged_dev,
+                                    Error **errp);
+/**
  * hotplug_handler_unplug:
  *
- * Call #HotplugHandlerClass.unplug callback of @plug_handler.
+ * Calls #HotplugHandlerClass.unplug callback of @plug_handler.
  */
 void hotplug_handler_unplug(HotplugHandler *plug_handler,
                             DeviceState *plugged_dev,
diff --git a/include/hw/i386/intel_iommu.h b/include/hw/i386/intel_iommu.h
new file mode 100644
index 000000000..e321ee4fb
--- /dev/null
+++ b/include/hw/i386/intel_iommu.h
@@ -0,0 +1,120 @@
+/*
+ * QEMU emulation of an Intel IOMMU (VT-d)
+ *   (DMA Remapping device)
+ *
+ * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com>
+ * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef INTEL_IOMMU_H
+#define INTEL_IOMMU_H
+#include "hw/qdev.h"
+#include "sysemu/dma.h"
+
+#define TYPE_INTEL_IOMMU_DEVICE "intel-iommu"
+#define INTEL_IOMMU_DEVICE(obj) \
+     OBJECT_CHECK(IntelIOMMUState, (obj), TYPE_INTEL_IOMMU_DEVICE)
+
+/* DMAR Hardware Unit Definition address (IOMMU unit) */
+#define Q35_HOST_BRIDGE_IOMMU_ADDR  0xfed90000ULL
+
+#define VTD_PCI_BUS_MAX             256
+#define VTD_PCI_SLOT_MAX            32
+#define VTD_PCI_FUNC_MAX            8
+#define VTD_PCI_DEVFN_MAX           256
+#define VTD_PCI_SLOT(devfn)         (((devfn) >> 3) & 0x1f)
+#define VTD_PCI_FUNC(devfn)         ((devfn) & 0x07)
+#define VTD_SID_TO_BUS(sid)         (((sid) >> 8) & 0xff)
+#define VTD_SID_TO_DEVFN(sid)       ((sid) & 0xff)
+
+#define DMAR_REG_SIZE               0x230
+#define VTD_HOST_ADDRESS_WIDTH      39
+#define VTD_HAW_MASK                ((1ULL << VTD_HOST_ADDRESS_WIDTH) - 1)
+
+typedef struct VTDContextEntry VTDContextEntry;
+typedef struct VTDContextCacheEntry VTDContextCacheEntry;
+typedef struct IntelIOMMUState IntelIOMMUState;
+typedef struct VTDAddressSpace VTDAddressSpace;
+typedef struct VTDIOTLBEntry VTDIOTLBEntry;
+
+/* Context-Entry */
+struct VTDContextEntry {
+    uint64_t lo;
+    uint64_t hi;
+};
+
+struct VTDContextCacheEntry {
+    /* The cache entry is obsolete if
+     * context_cache_gen!=IntelIOMMUState.context_cache_gen
+     */
+    uint32_t context_cache_gen;
+    struct VTDContextEntry context_entry;
+};
+
+struct VTDAddressSpace {
+    uint8_t bus_num;
+    uint8_t devfn;
+    AddressSpace as;
+    MemoryRegion iommu;
+    IntelIOMMUState *iommu_state;
+    VTDContextCacheEntry context_cache_entry;
+};
+
+struct VTDIOTLBEntry {
+    uint64_t gfn;
+    uint16_t domain_id;
+    uint64_t slpte;
+    bool read_flags;
+    bool write_flags;
+};
+
+/* The iommu (DMAR) device state struct */
+struct IntelIOMMUState {
+    SysBusDevice busdev;
+    MemoryRegion csrmem;
+    uint8_t csr[DMAR_REG_SIZE];     /* register values */
+    uint8_t wmask[DMAR_REG_SIZE];   /* R/W bytes */
+    uint8_t w1cmask[DMAR_REG_SIZE]; /* RW1C(Write 1 to Clear) bytes */
+    uint8_t womask[DMAR_REG_SIZE];  /* WO (write only - read returns 0) */
+    uint32_t version;
+
+    dma_addr_t root;                /* Current root table pointer */
+    bool root_extended;             /* Type of root table (extended or not) */
+    bool dmar_enabled;              /* Set if DMA remapping is enabled */
+
+    uint16_t iq_head;               /* Current invalidation queue head */
+    uint16_t iq_tail;               /* Current invalidation queue tail */
+    dma_addr_t iq;                  /* Current invalidation queue pointer */
+    uint16_t iq_size;               /* IQ Size in number of entries */
+    bool qi_enabled;                /* Set if the QI is enabled */
+    uint8_t iq_last_desc_type;      /* The type of last completed descriptor */
+
+    /* The index of the Fault Recording Register to be used next.
+     * Wraps around from N-1 to 0, where N is the number of FRCD_REG.
+     */
+    uint16_t next_frcd_reg;
+
+    uint64_t cap;                   /* The value of capability reg */
+    uint64_t ecap;                  /* The value of extended capability reg */
+
+    uint32_t context_cache_gen;     /* Should be in [1,MAX] */
+    GHashTable *iotlb;              /* IOTLB */
+
+    MemoryRegionIOMMUOps iommu_ops;
+    VTDAddressSpace **address_spaces[VTD_PCI_BUS_MAX];
+};
+
+#endif
diff --git a/include/hw/i386/pc.h b/include/hw/i386/pc.h
index f4b9b2bb7..69d9cf8e6 100644
--- a/include/hw/i386/pc.h
+++ b/include/hw/i386/pc.h
@@ -14,6 +14,7 @@
 #include "sysemu/sysemu.h"
 #include "hw/pci/pci.h"
 #include "hw/boards.h"
+#include "hw/compat.h"
 
 #define HPET_INTCAP "hpet-intcap"
 
@@ -23,6 +24,8 @@
  * address space begins.
  * @hotplug_memory: hotplug memory addess space container
  * @acpi_dev: link to ACPI PM device that performs ACPI hotplug handling
+ * @enforce_aligned_dimm: check that DIMM's address/size is aligned by
+ *                        backend's alignment value if provided
  */
 struct PCMachineState {
     /*< private >*/
@@ -33,13 +36,18 @@ struct PCMachineState {
     MemoryRegion hotplug_memory;
 
     HotplugHandler *acpi_dev;
+    ISADevice *rtc;
 
     uint64_t max_ram_below_4g;
+    OnOffAuto vmport;
+    bool enforce_aligned_dimm;
 };
 
 #define PC_MACHINE_ACPI_DEVICE_PROP "acpi-device"
 #define PC_MACHINE_MEMHP_REGION_SIZE "hotplug-memory-region-size"
 #define PC_MACHINE_MAX_RAM_BELOW_4G "max-ram-below-4g"
+#define PC_MACHINE_VMPORT           "vmport"
+#define PC_MACHINE_ENFORCE_ALIGNED_DIMM "enforce-aligned-dimm"
 
 /**
  * PCMachineClass:
@@ -85,7 +93,6 @@ typedef struct PcPciInfo {
 #define ACPI_PM_PROP_GPE0_BLK_LEN "gpe0_blk_len"
 
 struct PcGuestInfo {
-    bool has_pci_info;
     bool isapc_ram_fw;
     hwaddr ram_size, ram_size_below_4g;
     unsigned apic_id_limit;
@@ -177,6 +184,8 @@ void pc_acpi_init(const char *default_dsdt);
 PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
                                 ram_addr_t above_4g_mem_size);
 
+void pc_set_legacy_acpi_data_size(void);
+
 #define PCI_HOST_PROP_PCI_HOLE_START   "pci-hole-start"
 #define PCI_HOST_PROP_PCI_HOLE_END     "pci-hole-end"
 #define PCI_HOST_PROP_PCI_HOLE64_START "pci-hole64-start"
@@ -188,6 +197,11 @@ PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
 void pc_pci_as_mapping_init(Object *owner, MemoryRegion *system_memory,
                             MemoryRegion *pci_address_space);
 
+FWCfgState *xen_load_linux(const char *kernel_filename,
+                           const char *kernel_cmdline,
+                           const char *initrd_filename,
+                           ram_addr_t below_4g_mem_size,
+                           PcGuestInfo *guest_info);
 FWCfgState *pc_memory_init(MachineState *machine,
                            MemoryRegion *system_memory,
                            ram_addr_t below_4g_mem_size,
@@ -204,7 +218,7 @@ void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
                           uint32 hpet_irqs);
 void pc_init_ne2k_isa(ISABus *bus, NICInfo *nd);
 void pc_cmos_init(ram_addr_t ram_size, ram_addr_t above_4g_mem_size,
-                  const char *boot_device,
+                  const char *boot_device, MachineState *machine,
                   ISADevice *floppy, BusState *ide0, BusState *ide1,
                   ISADevice *s);
 void pc_nic_init(ISABus *isa_bus, PCIBus *pci_bus);
@@ -296,6 +310,7 @@ int e820_get_num_entries(void);
 bool e820_get_entry(int, uint32_t, uint64_t *, uint64_t *);
 
 #define PC_COMPAT_2_0 \
+        HW_COMPAT_2_1, \
         {\
             .driver   = "virtio-scsi-pci",\
             .property = "any_layout",\
@@ -316,6 +331,11 @@ bool e820_get_entry(int, uint32_t, uint64_t *, uint64_t *);
             .value    = "off",\
         },\
         {\
+            .driver   = "nec-usb-xhci",\
+            .property = "force-pcie-endcap",\
+            .value    = "on",\
+        },\
+        {\
             .driver   = "pci-serial",\
             .property = "prog_if",\
             .value    = stringify(0),\
diff --git a/include/hw/i386/smbios.h b/include/hw/i386/smbios.h
index a3f4d88bf..d2850bed2 100644
--- a/include/hw/i386/smbios.h
+++ b/include/hw/i386/smbios.h
@@ -20,7 +20,8 @@
 void smbios_entry_add(QemuOpts *opts);
 void smbios_set_cpuid(uint32_t version, uint32_t features);
 void smbios_set_defaults(const char *manufacturer, const char *product,
-                         const char *version, bool legacy_mode);
+                         const char *version, bool legacy_mode,
+                         bool uuid_encoded);
 uint8_t *smbios_get_table_legacy(size_t *length);
 void smbios_get_tables(uint8_t **tables, size_t *tables_len,
                        uint8_t **anchor, size_t *anchor_len);
@@ -72,6 +73,18 @@ struct smbios_type_0 {
     uint8_t embedded_controller_minor_release;
 } QEMU_PACKED;
 
+/* UUID encoding. The time_* fields are little-endian, as specified by SMBIOS
+ * version 2.6.
+ */
+struct smbios_uuid {
+    uint32_t time_low;
+    uint16_t time_mid;
+    uint16_t time_hi_and_version;
+    uint8_t clock_seq_hi_and_reserved;
+    uint8_t clock_seq_low;
+    uint8_t node[6];
+} QEMU_PACKED;
+
 /* SMBIOS type 1 - System Information */
 struct smbios_type_1 {
     struct smbios_structure_header header;
@@ -79,7 +92,7 @@ struct smbios_type_1 {
     uint8_t product_name_str;
     uint8_t version_str;
     uint8_t serial_number_str;
-    uint8_t uuid[16];
+    struct smbios_uuid uuid;
     uint8_t wake_up_type;
     uint8_t sku_number_str;
     uint8_t family_str;
diff --git a/include/hw/irq.h b/include/hw/irq.h
index 6f874f5ac..4c4c2eaf9 100644
--- a/include/hw/irq.h
+++ b/include/hw/irq.h
@@ -61,6 +61,5 @@ qemu_irq *qemu_irq_proxy(qemu_irq **target, int n);
 /* For internal use in qtest.  Similar to qemu_irq_split, but operating
    on an existing vector of qemu_irq.  */
 void qemu_irq_intercept_in(qemu_irq *gpio_in, qemu_irq_handler handler, int n);
-void qemu_irq_intercept_out(qemu_irq **gpio_out, qemu_irq_handler handler, int n);
 
 #endif
diff --git a/include/hw/isa/pc87312.h b/include/hw/isa/pc87312.h
index befc8bdc6..bf74470d4 100644
--- a/include/hw/isa/pc87312.h
+++ b/include/hw/isa/pc87312.h
@@ -47,13 +47,10 @@ typedef struct PC87312State {
 
     struct {
         ISADevice *dev;
-        BlockDriverState *drive[2];
-        uint32_t base;
     } fdc;
 
     struct {
         ISADevice *dev;
-        uint32_t base;
     } ide;
 
     MemoryRegion io;
diff --git a/include/hw/loader.h b/include/hw/loader.h
index 796cbf9b3..64816395d 100644
--- a/include/hw/loader.h
+++ b/include/hw/loader.h
@@ -13,8 +13,10 @@
  */
 int get_image_size(const char *filename);
 int load_image(const char *filename, uint8_t *addr); /* deprecated */
+ssize_t load_image_size(const char *filename, void *addr, size_t size);
 int load_image_targphys(const char *filename, hwaddr,
                         uint64_t max_sz);
+int load_image_gzipped(const char *filename, hwaddr addr, uint64_t max_sz);
 
 #define ELF_LOAD_FAILED       -1
 #define ELF_LOAD_NOT_ELF      -2
@@ -28,7 +30,9 @@ int load_elf(const char *filename, uint64_t (*translate_fn)(void *, uint64_t),
 int load_aout(const char *filename, hwaddr addr, int max_sz,
               int bswap_needed, hwaddr target_page_size);
 int load_uimage(const char *filename, hwaddr *ep,
-                hwaddr *loadaddr, int *is_linux);
+                hwaddr *loadaddr, int *is_linux,
+                uint64_t (*translate_fn)(void *, uint64_t),
+                void *translate_opaque);
 
 /**
  * load_ramdisk:
@@ -55,7 +59,7 @@ extern bool rom_file_has_mr;
 int rom_add_file(const char *file, const char *fw_dir,
                  hwaddr addr, int32_t bootindex,
                  bool option_rom);
-void *rom_add_blob(const char *name, const void *blob, size_t len,
+ram_addr_t rom_add_blob(const char *name, const void *blob, size_t len,
                    hwaddr addr, const char *fw_file_name,
                    FWCfgReadCallback fw_callback, void *callback_opaque);
 int rom_add_elf_program(const char *name, void *data, size_t datasize,
diff --git a/include/hw/mem/pc-dimm.h b/include/hw/mem/pc-dimm.h
index 761eeef80..e1dcbbcd5 100644
--- a/include/hw/mem/pc-dimm.h
+++ b/include/hw/mem/pc-dimm.h
@@ -72,7 +72,7 @@ typedef struct PCDIMMDeviceClass {
 
 uint64_t pc_dimm_get_free_addr(uint64_t address_space_start,
                                uint64_t address_space_size,
-                               uint64_t *hint, uint64_t size,
+                               uint64_t *hint, uint64_t align, uint64_t size,
                                Error **errp);
 
 int pc_dimm_get_free_slot(const int *hint, int max_slots, Error **errp);
diff --git a/include/hw/nmi.h b/include/hw/nmi.h
new file mode 100644
index 000000000..b541772e1
--- /dev/null
+++ b/include/hw/nmi.h
@@ -0,0 +1,49 @@
+/*
+ *  NMI monitor handler class and helpers definitions.
+ *
+ *  Copyright IBM Corp., 2014
+ *
+ *  Author: Alexey Kardashevskiy <aik@ozlabs.ru>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License,
+ *  or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef NMI_H
+#define NMI_H 1
+
+#include "qemu-common.h"
+#include "qom/object.h"
+
+#define TYPE_NMI "nmi"
+
+#define NMI_CLASS(klass) \
+     OBJECT_CLASS_CHECK(NMIClass, (klass), TYPE_NMI)
+#define NMI_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(NMIClass, (obj), TYPE_NMI)
+#define NMI(obj) \
+     INTERFACE_CHECK(NMI, (obj), TYPE_NMI)
+
+typedef struct NMIState {
+    Object parent_obj;
+} NMIState;
+
+typedef struct NMIClass {
+    InterfaceClass parent_class;
+
+    void (*nmi_monitor_handler)(NMIState *n, int cpu_index, Error **errp);
+} NMIClass;
+
+void nmi_monitor_handle(int cpu_index, Error **errp);
+
+#endif /* NMI_H */
diff --git a/include/hw/nvram/fw_cfg.h b/include/hw/nvram/fw_cfg.h
index 72b1549dc..56e1ed712 100644
--- a/include/hw/nvram/fw_cfg.h
+++ b/include/hw/nvram/fw_cfg.h
@@ -76,6 +76,8 @@ void fw_cfg_add_file(FWCfgState *s, const char *filename, void *data,
 void fw_cfg_add_file_callback(FWCfgState *s, const char *filename,
                               FWCfgReadCallback callback, void *callback_opaque,
                               void *data, size_t len);
+void *fw_cfg_modify_file(FWCfgState *s, const char *filename, void *data,
+                         size_t len);
 FWCfgState *fw_cfg_init(uint32_t ctl_port, uint32_t data_port,
                         hwaddr crl_addr, hwaddr data_addr);
 
diff --git a/include/hw/pci-host/pam.h b/include/hw/pci-host/pam.h
index a8b87b89a..4d03e4bf1 100644
--- a/include/hw/pci-host/pam.h
+++ b/include/hw/pci-host/pam.h
@@ -7,7 +7,7 @@
  *               VA Linux Systems Japan K.K.
  * Copyright (c) 2012 Jason Baron <jbaron@redhat.com>
  *
- * Split out from piix_pci.c
+ * Split out from piix.c
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
diff --git a/include/hw/pci-host/q35.h b/include/hw/pci-host/q35.h
index d9ee97845..025d6e69a 100644
--- a/include/hw/pci-host/q35.h
+++ b/include/hw/pci-host/q35.h
@@ -33,6 +33,7 @@
 #include "hw/acpi/acpi.h"
 #include "hw/acpi/ich9.h"
 #include "hw/pci-host/pam.h"
+#include "hw/i386/intel_iommu.h"
 
 #define TYPE_Q35_HOST_DEVICE "q35-pcihost"
 #define Q35_HOST_DEVICE(obj) \
@@ -60,6 +61,7 @@ typedef struct MCHPCIState {
     uint64_t pci_hole64_size;
     PcGuestInfo *guest_info;
     uint32_t short_root_bus;
+    IntelIOMMUState *iommu;
 } MCHPCIState;
 
 typedef struct Q35PCIHost {
diff --git a/include/hw/pci-host/spapr.h b/include/hw/pci-host/spapr.h
index 32f0aa7d1..4ea2a0d14 100644
--- a/include/hw/pci-host/spapr.h
+++ b/include/hw/pci-host/spapr.h
@@ -70,7 +70,7 @@ struct sPAPRPHBState {
 
     MemoryRegion memspace, iospace;
     hwaddr mem_win_addr, mem_win_size, io_win_addr, io_win_size;
-    MemoryRegion memwindow, iowindow;
+    MemoryRegion memwindow, iowindow, msiwindow;
 
     uint32_t dma_liobn;
     AddressSpace iommu_as;
diff --git a/include/hw/pci/pci_ids.h b/include/hw/pci/pci_ids.h
index e597070ab..321d622b7 100644
--- a/include/hw/pci/pci_ids.h
+++ b/include/hw/pci/pci_ids.h
@@ -56,6 +56,7 @@
 #define PCI_DEVICE_ID_LSI_53C810         0x0001
 #define PCI_DEVICE_ID_LSI_53C895A        0x0012
 #define PCI_DEVICE_ID_LSI_SAS1078        0x0060
+#define PCI_DEVICE_ID_LSI_SAS0079        0x0079
 
 #define PCI_VENDOR_ID_DEC                0x1011
 #define PCI_DEVICE_ID_DEC_21154          0x0026
diff --git a/include/hw/pci/pcie.h b/include/hw/pci/pcie.h
index 7fe81f31e..b48a7a2c5 100644
--- a/include/hw/pci/pcie.h
+++ b/include/hw/pci/pcie.h
@@ -103,9 +103,10 @@ void pcie_cap_flr_init(PCIDevice *dev);
 void pcie_cap_flr_write_config(PCIDevice *dev,
                            uint32_t addr, uint32_t val, int len);
 
-void pcie_cap_ari_init(PCIDevice *dev);
-void pcie_cap_ari_reset(PCIDevice *dev);
-bool pcie_cap_is_ari_enabled(const PCIDevice *dev);
+/* ARI forwarding capability and control */
+void pcie_cap_arifwd_init(PCIDevice *dev);
+void pcie_cap_arifwd_reset(PCIDevice *dev);
+bool pcie_cap_is_arifwd_enabled(const PCIDevice *dev);
 
 /* PCI express extended capability helper functions */
 uint16_t pcie_find_capability(PCIDevice *dev, uint16_t cap_id);
@@ -127,6 +128,6 @@ extern const VMStateDescription vmstate_pcie_device;
 
 void pcie_cap_slot_hotplug_cb(HotplugHandler *hotplug_dev, DeviceState *dev,
                               Error **errp);
-void pcie_cap_slot_hot_unplug_cb(HotplugHandler *hotplug_dev, DeviceState *dev,
-                                 Error **errp);
+void pcie_cap_slot_hot_unplug_request_cb(HotplugHandler *hotplug_dev,
+                                         DeviceState *dev, Error **errp);
 #endif /* QEMU_PCIE_H */
diff --git a/include/hw/pci/shpc.h b/include/hw/pci/shpc.h
index eef1a1ad6..025bc5b26 100644
--- a/include/hw/pci/shpc.h
+++ b/include/hw/pci/shpc.h
@@ -46,8 +46,8 @@ void shpc_cap_write_config(PCIDevice *d, uint32_t addr, uint32_t val, int len);
 
 void shpc_device_hotplug_cb(HotplugHandler *hotplug_dev, DeviceState *dev,
                             Error **errp);
-void shpc_device_hot_unplug_cb(HotplugHandler *hotplug_dev, DeviceState *dev,
-                               Error **errp);
+void shpc_device_hot_unplug_request_cb(HotplugHandler *hotplug_dev,
+                                       DeviceState *dev, Error **errp);
 
 extern VMStateInfo shpc_vmstate_info;
 #define SHPC_VMSTATE(_field, _type) \
diff --git a/include/hw/pcmcia.h b/include/hw/pcmcia.h
index 2695d3cba..98406ffbc 100644
--- a/include/hw/pcmcia.h
+++ b/include/hw/pcmcia.h
@@ -8,14 +8,8 @@
 typedef struct PCMCIASocket {
     qemu_irq irq;
     bool attached;
-    const char *slot_string;
-    const char *card_string;
 } PCMCIASocket;
 
-void pcmcia_socket_register(PCMCIASocket *socket);
-void pcmcia_socket_unregister(PCMCIASocket *socket);
-void pcmcia_info(Monitor *mon, const QDict *qdict);
-
 #define TYPE_PCMCIA_CARD "pcmcia-card"
 #define PCMCIA_CARD(obj) \
     OBJECT_CHECK(PCMCIACardState, (obj), TYPE_PCMCIA_CARD)
diff --git a/include/hw/platform-bus.h b/include/hw/platform-bus.h
new file mode 100644
index 000000000..bd42b8380
--- /dev/null
+++ b/include/hw/platform-bus.h
@@ -0,0 +1,57 @@
+#ifndef HW_PLATFORM_BUS_H
+#define HW_PLATFORM_BUS_H 1
+
+/*
+ *  Platform Bus device to support dynamic Sysbus devices
+ *
+ * Copyright (C) 2014 Freescale Semiconductor, Inc. All rights reserved.
+ *
+ * Author: Alexander Graf, <agraf@suse.de>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/sysbus.h"
+
+typedef struct PlatformBusDevice PlatformBusDevice;
+
+#define TYPE_PLATFORM_BUS_DEVICE "platform-bus-device"
+#define PLATFORM_BUS_DEVICE(obj) \
+     OBJECT_CHECK(PlatformBusDevice, (obj), TYPE_PLATFORM_BUS_DEVICE)
+#define PLATFORM_BUS_DEVICE_CLASS(klass) \
+     OBJECT_CLASS_CHECK(PlatformBusDeviceClass, (klass), TYPE_PLATFORM_BUS_DEVICE)
+#define PLATFORM_BUS_DEVICE_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(PlatformBusDeviceClass, (obj), TYPE_PLATFORM_BUS_DEVICE)
+
+struct PlatformBusDevice {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    Notifier notifier;
+    bool done_gathering;
+
+    /*< public >*/
+    uint32_t mmio_size;
+    MemoryRegion mmio;
+
+    uint32_t num_irqs;
+    qemu_irq *irqs;
+    unsigned long *used_irqs;
+};
+
+int platform_bus_get_irqn(PlatformBusDevice *platform_bus, SysBusDevice *sbdev,
+                          int n);
+hwaddr platform_bus_get_mmio_addr(PlatformBusDevice *pbus, SysBusDevice *sbdev,
+                                  int n);
+
+#endif /* !HW_PLATFORM_BUS_H */
diff --git a/include/hw/ppc/ppc.h b/include/hw/ppc/ppc.h
index 7e16e2e06..14efd0ca3 100644
--- a/include/hw/ppc/ppc.h
+++ b/include/hw/ppc/ppc.h
@@ -92,7 +92,7 @@ enum {
 #define FW_CFG_PPC_IS_KVM       (FW_CFG_ARCH_LOCAL + 0x05)
 #define FW_CFG_PPC_KVM_HC       (FW_CFG_ARCH_LOCAL + 0x06)
 #define FW_CFG_PPC_KVM_PID      (FW_CFG_ARCH_LOCAL + 0x07)
-/* OpenBIOS has FW_CFG_PPC_NVRAM_ADDR as +0x08 */
+#define FW_CFG_PPC_NVRAM_ADDR   (FW_CFG_ARCH_LOCAL + 0x08)
 #define FW_CFG_PPC_BUSFREQ      (FW_CFG_ARCH_LOCAL + 0x09)
 
 #define PPC_SERIAL_MM_BAUDBASE 399193
diff --git a/include/hw/ppc/spapr.h b/include/hw/ppc/spapr.h
index bbba51a70..749daf4dd 100644
--- a/include/hw/ppc/spapr.h
+++ b/include/hw/ppc/spapr.h
@@ -13,8 +13,6 @@ struct sPAPRNVRAM;
 typedef struct sPAPREnvironment {
     struct VIOsPAPRBus *vio_bus;
     QLIST_HEAD(, sPAPRPHBState) phbs;
-    hwaddr msi_win_addr;
-    MemoryRegion msiwindow;
     struct sPAPRNVRAM *nvram;
     XICSState *icp;
 
@@ -24,7 +22,8 @@ typedef struct sPAPREnvironment {
     hwaddr rma_size;
     int vrma_adjust;
     hwaddr fdt_addr, rtas_addr;
-    long rtas_size;
+    ssize_t rtas_size;
+    void *rtas_blob;
     void *fdt_skel;
     target_ulong entry_point;
     uint64_t rtc_offset;
@@ -382,9 +381,8 @@ int spapr_allocate_irq_block(int num, bool lsi, bool msi);
 #define RTAS_GET_SENSOR_STATE                   (RTAS_TOKEN_BASE + 0x1D)
 #define RTAS_IBM_CONFIGURE_CONNECTOR            (RTAS_TOKEN_BASE + 0x1E)
 #define RTAS_IBM_OS_TERM                        (RTAS_TOKEN_BASE + 0x1F)
-#define RTAS_IBM_EXTENDED_OS_TERM               (RTAS_TOKEN_BASE + 0x20)
 
-#define RTAS_TOKEN_MAX                          (RTAS_TOKEN_BASE + 0x21)
+#define RTAS_TOKEN_MAX                          (RTAS_TOKEN_BASE + 0x20)
 
 /* RTAS ibm,get-system-parameter token values */
 #define RTAS_SYSPARM_SPLPAR_CHARACTERISTICS      20
diff --git a/include/hw/qdev-core.h b/include/hw/qdev-core.h
index 0799ff29b..589bbe736 100644
--- a/include/hw/qdev-core.h
+++ b/include/hw/qdev-core.h
@@ -126,7 +126,6 @@ typedef struct DeviceClass {
 
     /* Private to qdev / bus.  */
     qdev_initfn init; /* TODO remove, once users are converted to realize */
-    qdev_event unplug;
     qdev_event exit; /* TODO remove, once users are converted to unrealize */
     const char *bus_type;
 } DeviceClass;
@@ -137,7 +136,6 @@ struct NamedGPIOList {
     char *name;
     qemu_irq *in;
     int num_in;
-    qemu_irq *out;
     int num_out;
     QLIST_ENTRY(NamedGPIOList) node;
 };
@@ -210,7 +208,6 @@ struct BusState {
     Object obj;
     DeviceState *parent;
     const char *name;
-    int allow_hotplug;
     HotplugHandler *hotplug_handler;
     int max_index;
     bool realized;
@@ -232,7 +229,7 @@ struct Property {
 
 struct PropertyInfo {
     const char *name;
-    const char *legacy_name;
+    const char *description;
     const char **enum_table;
     int (*print)(DeviceState *dev, Property *prop, char *dest, size_t len);
     ObjectPropertyAccessor *get;
@@ -242,16 +239,16 @@ struct PropertyInfo {
 
 /**
  * GlobalProperty:
- * @not_used: Track use of a global property.  Defaults to false in all C99
- * struct initializations.
- *
- * This prevents reports of .compat_props when they are not used.
+ * @user_provided: Set to true if property comes from user-provided config
+ * (command-line or config file).
+ * @used: Set to true if property was used when initializing a device.
  */
 typedef struct GlobalProperty {
     const char *driver;
     const char *property;
     const char *value;
-    bool not_used;
+    bool user_provided;
+    bool used;
     QTAILQ_ENTRY(GlobalProperty) next;
 } GlobalProperty;
 
@@ -264,7 +261,8 @@ void qdev_init_nofail(DeviceState *dev);
 void qdev_set_legacy_instance_id(DeviceState *dev, int alias_id,
                                  int required_for_version);
 void qdev_unplug(DeviceState *dev, Error **errp);
-int qdev_simple_unplug_cb(DeviceState *dev);
+void qdev_simple_device_unplug_cb(HotplugHandler *hotplug_dev,
+                                  DeviceState *dev, Error **errp);
 void qdev_machine_creation_done(void);
 bool qdev_machine_modified(void);
 
@@ -274,6 +272,9 @@ qemu_irq qdev_get_gpio_in_named(DeviceState *dev, const char *name, int n);
 void qdev_connect_gpio_out(DeviceState *dev, int n, qemu_irq pin);
 void qdev_connect_gpio_out_named(DeviceState *dev, const char *name, int n,
                                  qemu_irq pin);
+qemu_irq qdev_get_gpio_out_connector(DeviceState *dev, const char *name, int n);
+qemu_irq qdev_intercept_gpio_out(DeviceState *dev, qemu_irq icpt,
+                                 const char *name, int n);
 
 BusState *qdev_get_child_bus(DeviceState *dev, const char *name);
 
@@ -288,6 +289,9 @@ void qdev_init_gpio_in_named(DeviceState *dev, qemu_irq_handler handler,
 void qdev_init_gpio_out_named(DeviceState *dev, qemu_irq *pins,
                               const char *name, int n);
 
+void qdev_pass_gpios(DeviceState *dev, DeviceState *container,
+                     const char *name);
+
 BusState *qdev_get_parent_bus(DeviceState *dev);
 
 /*** BUS API. ***/
@@ -361,11 +365,15 @@ extern int qdev_hotplug;
 
 char *qdev_get_dev_path(DeviceState *dev);
 
-static inline void qbus_set_hotplug_handler(BusState *bus, DeviceState *handler,
-                                            Error **errp)
+GSList *qdev_build_hotpluggable_device_list(Object *peripheral);
+
+void qbus_set_hotplug_handler(BusState *bus, DeviceState *handler,
+                              Error **errp);
+
+void qbus_set_bus_hotplug_handler(BusState *bus, Error **errp);
+
+static inline bool qbus_is_hotpluggable(BusState *bus)
 {
-    object_property_set_link(OBJECT(bus), OBJECT(handler),
-                             QDEV_HOTPLUG_HANDLER_PROPERTY, errp);
-    bus->allow_hotplug = 1;
+   return bus->hotplug_handler;
 }
 #endif
diff --git a/include/hw/qdev-properties.h b/include/hw/qdev-properties.h
index 77fe3a12b..070006c71 100644
--- a/include/hw/qdev-properties.h
+++ b/include/hw/qdev-properties.h
@@ -141,7 +141,7 @@ extern PropertyInfo qdev_prop_arraylen;
 #define DEFINE_PROP_VLAN(_n, _s, _f)             \
     DEFINE_PROP(_n, _s, _f, qdev_prop_vlan, NICPeers)
 #define DEFINE_PROP_DRIVE(_n, _s, _f) \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_drive, BlockDriverState *)
+    DEFINE_PROP(_n, _s, _f, qdev_prop_drive, BlockBackend *)
 #define DEFINE_PROP_MACADDR(_n, _s, _f)         \
     DEFINE_PROP(_n, _s, _f, qdev_prop_macaddr, MACAddr)
 #define DEFINE_PROP_LOSTTICKPOLICY(_n, _s, _f, _d) \
@@ -168,8 +168,10 @@ void qdev_prop_set_uint64(DeviceState *dev, const char *name, uint64_t value);
 void qdev_prop_set_string(DeviceState *dev, const char *name, const char *value);
 void qdev_prop_set_chr(DeviceState *dev, const char *name, CharDriverState *value);
 void qdev_prop_set_netdev(DeviceState *dev, const char *name, NetClientState *value);
-int qdev_prop_set_drive(DeviceState *dev, const char *name, BlockDriverState *value) QEMU_WARN_UNUSED_RESULT;
-void qdev_prop_set_drive_nofail(DeviceState *dev, const char *name, BlockDriverState *value);
+int qdev_prop_set_drive(DeviceState *dev, const char *name,
+                        BlockBackend *value) QEMU_WARN_UNUSED_RESULT;
+void qdev_prop_set_drive_nofail(DeviceState *dev, const char *name,
+                                BlockBackend *value);
 void qdev_prop_set_macaddr(DeviceState *dev, const char *name, uint8_t *value);
 void qdev_prop_set_enum(DeviceState *dev, const char *name, int value);
 /* FIXME: Remove opaque pointer properties.  */
@@ -177,7 +179,7 @@ void qdev_prop_set_ptr(DeviceState *dev, const char *name, void *value);
 
 void qdev_prop_register_global(GlobalProperty *prop);
 void qdev_prop_register_global_list(GlobalProperty *props);
-int qdev_prop_check_global(void);
+int qdev_prop_check_globals(void);
 void qdev_prop_set_globals(DeviceState *dev, Error **errp);
 void qdev_prop_set_globals_for_type(DeviceState *dev, const char *typename,
                                     Error **errp);
diff --git a/include/hw/s390x/sclp.h b/include/hw/s390x/sclp.h
index 7ef16226d..ec07a118f 100644
--- a/include/hw/s390x/sclp.h
+++ b/include/hw/s390x/sclp.h
@@ -28,8 +28,6 @@
 #define SCLP_UNASSIGN_STORAGE                   0x000C0001
 #define SCLP_CMD_READ_EVENT_DATA                0x00770005
 #define SCLP_CMD_WRITE_EVENT_DATA               0x00760005
-#define SCLP_CMD_READ_EVENT_DATA                0x00770005
-#define SCLP_CMD_WRITE_EVENT_DATA               0x00760005
 #define SCLP_CMD_WRITE_EVENT_MASK               0x00780005
 
 /* SCLP Memory hotplug codes */
@@ -37,6 +35,7 @@
 #define SCLP_STARTING_SUBINCREMENT_ID           0x10001
 #define SCLP_INCREMENT_UNIT                     0x10000
 #define MAX_AVAIL_SLOTS                         32
+#define MAX_STORAGE_INCREMENTS                  1020
 
 /* CPU hotplug SCLP codes */
 #define SCLP_HAS_CPU_INFO                       0x0C00000000000000ULL
@@ -156,6 +155,23 @@ typedef struct SCCB {
     char data[SCCB_DATA_LEN];
  } QEMU_PACKED SCCB;
 
+typedef struct sclpMemoryHotplugDev sclpMemoryHotplugDev;
+
+#define TYPE_SCLP_MEMORY_HOTPLUG_DEV "sclp-memory-hotplug-dev"
+#define SCLP_MEMORY_HOTPLUG_DEV(obj) \
+  OBJECT_CHECK(sclpMemoryHotplugDev, (obj), TYPE_SCLP_MEMORY_HOTPLUG_DEV)
+
+struct sclpMemoryHotplugDev {
+    SysBusDevice parent;
+    ram_addr_t standby_mem_size;
+    ram_addr_t padded_ram_size;
+    ram_addr_t pad_size;
+    ram_addr_t standby_subregion_size;
+    ram_addr_t rzm;
+    int increment_size;
+    char *standby_state_map;
+};
+
 static inline int sccb_data_len(SCCB *sccb)
 {
     return be16_to_cpu(sccb->h.length) - sizeof(sccb->h);
@@ -163,6 +179,8 @@ static inline int sccb_data_len(SCCB *sccb)
 
 
 void s390_sclp_init(void);
+sclpMemoryHotplugDev *init_sclp_memory_hotplug_dev(void);
+sclpMemoryHotplugDev *get_sclp_memory_hotplug_dev(void);
 void sclp_service_interrupt(uint32_t sccb);
 void raise_irq_cpu_hotplug(void);
 
diff --git a/include/hw/scsi/esp.h b/include/hw/scsi/esp.h
index e079fb8d1..6c795276c 100644
--- a/include/hw/scsi/esp.h
+++ b/include/hw/scsi/esp.h
@@ -22,6 +22,7 @@ struct ESPState {
     uint8_t wregs[ESP_REGS];
     qemu_irq irq;
     uint8_t chip_id;
+    bool tchi_written;
     int32_t ti_size;
     uint32_t ti_rptr, ti_wptr;
     uint32_t status;
diff --git a/include/hw/scsi/scsi.h b/include/hw/scsi/scsi.h
index 1adb54906..cdaf0f8eb 100644
--- a/include/hw/scsi/scsi.h
+++ b/include/hw/scsi/scsi.h
@@ -2,9 +2,10 @@
 #define QEMU_HW_SCSI_H
 
 #include "hw/qdev.h"
-#include "block/block.h"
+#include "qemu/typedefs.h"
 #include "hw/block/block.h"
 #include "sysemu/sysemu.h"
+#include "qemu/notify.h"
 
 #define MAX_SCSI_DEVS	255
 
@@ -50,17 +51,25 @@ struct SCSIRequest {
     uint32_t          tag;
     uint32_t          lun;
     uint32_t          status;
+    void              *hba_private;
     size_t            resid;
     SCSICommand       cmd;
-    BlockDriverAIOCB  *aiocb;
-    QEMUSGList        *sg;
+    NotifierList      cancel_notifiers;
+
+    /* Note:
+     * - fields before sense are initialized by scsi_req_alloc;
+     * - sense[] is uninitialized;
+     * - fields after sense are memset to 0 by scsi_req_alloc.
+     * */
+
+    uint8_t           sense[SCSI_SENSE_BUF_SIZE];
+    uint32_t          sense_len;
+    bool              enqueued;
+    bool              io_canceled;
+    bool              retry;
     bool              dma_started;
-    uint8_t sense[SCSI_SENSE_BUF_SIZE];
-    uint32_t sense_len;
-    bool enqueued;
-    bool io_canceled;
-    bool retry;
-    void *hba_private;
+    BlockAIOCB        *aiocb;
+    QEMUSGList        *sg;
     QTAILQ_ENTRY(SCSIRequest) next;
 };
 
@@ -74,8 +83,9 @@ struct SCSIRequest {
 
 typedef struct SCSIDeviceClass {
     DeviceClass parent_class;
-    int (*init)(SCSIDevice *dev);
-    void (*destroy)(SCSIDevice *s);
+    void (*realize)(SCSIDevice *dev, Error **errp);
+    int (*parse_cdb)(SCSIDevice *dev, SCSICommand *cmd, uint8_t *buf,
+                     void *hba_private);
     SCSIRequest *(*alloc_req)(SCSIDevice *s, uint32_t tag, uint32_t lun,
                               uint8_t *buf, void *hba_private);
     void (*unit_attention_reported)(SCSIDevice *s);
@@ -121,7 +131,6 @@ struct SCSIReqOps {
     int32_t (*send_command)(SCSIRequest *req, uint8_t *buf);
     void (*read_data)(SCSIRequest *req);
     void (*write_data)(SCSIRequest *req);
-    void (*cancel_io)(SCSIRequest *req);
     uint8_t *(*get_buf)(SCSIRequest *req);
 
     void (*save_request)(QEMUFile *f, SCSIRequest *req);
@@ -131,11 +140,11 @@ struct SCSIReqOps {
 struct SCSIBusInfo {
     int tcq;
     int max_channel, max_target, max_lun;
+    int (*parse_cdb)(SCSIDevice *dev, SCSICommand *cmd, uint8_t *buf,
+                     void *hba_private);
     void (*transfer_data)(SCSIRequest *req, uint32_t arg);
     void (*complete)(SCSIRequest *req, uint32_t arg, size_t resid);
     void (*cancel)(SCSIRequest *req);
-    void (*hotplug)(SCSIBus *bus, SCSIDevice *dev);
-    void (*hot_unplug)(SCSIBus *bus, SCSIDevice *dev);
     void (*change)(SCSIBus *bus, SCSIDevice *dev, SCSISense sense);
     QEMUSGList *(*get_sg_list)(SCSIRequest *req);
 
@@ -163,7 +172,7 @@ static inline SCSIBus *scsi_bus_from_device(SCSIDevice *d)
     return DO_UPCAST(SCSIBus, qbus, d->qdev.parent_bus);
 }
 
-SCSIDevice *scsi_bus_legacy_add_drive(SCSIBus *bus, BlockDriverState *bdrv,
+SCSIDevice *scsi_bus_legacy_add_drive(SCSIBus *bus, BlockBackend *blk,
                                       int unit, bool removable, int bootindex,
                                       const char *serial, Error **errp);
 void scsi_bus_legacy_handle_cmdline(SCSIBus *bus, Error **errp);
@@ -229,8 +238,9 @@ extern const struct SCSISense sense_code_SPACE_ALLOC_FAILED;
 
 #define SENSE_CODE(x) sense_code_ ## x
 
-uint32_t scsi_data_cdb_length(uint8_t *buf);
-uint32_t scsi_cdb_length(uint8_t *buf);
+uint32_t scsi_data_cdb_xfer(uint8_t *buf);
+uint32_t scsi_cdb_xfer(uint8_t *buf);
+int scsi_cdb_length(uint8_t *buf);
 int scsi_sense_valid(SCSISense sense);
 int scsi_build_sense(uint8_t *in_buf, int in_len,
                      uint8_t *buf, int len, bool fixed);
@@ -244,6 +254,9 @@ void scsi_req_free(SCSIRequest *req);
 SCSIRequest *scsi_req_ref(SCSIRequest *req);
 void scsi_req_unref(SCSIRequest *req);
 
+int scsi_bus_parse_cdb(SCSIDevice *dev, SCSICommand *cmd, uint8_t *buf,
+                       void *hba_private);
+int scsi_req_parse_cdb(SCSIDevice *dev, SCSICommand *cmd, uint8_t *buf);
 void scsi_req_build_sense(SCSIRequest *req, SCSISense sense);
 void scsi_req_print(SCSIRequest *req);
 void scsi_req_continue(SCSIRequest *req);
@@ -251,12 +264,14 @@ void scsi_req_data(SCSIRequest *req, int len);
 void scsi_req_complete(SCSIRequest *req, int status);
 uint8_t *scsi_req_get_buf(SCSIRequest *req);
 int scsi_req_get_sense(SCSIRequest *req, uint8_t *buf, int len);
-void scsi_req_abort(SCSIRequest *req, int status);
+void scsi_req_cancel_complete(SCSIRequest *req);
 void scsi_req_cancel(SCSIRequest *req);
+void scsi_req_cancel_async(SCSIRequest *req, Notifier *notifier);
 void scsi_req_retry(SCSIRequest *req);
 void scsi_device_purge_requests(SCSIDevice *sdev, SCSISense sense);
 void scsi_device_set_ua(SCSIDevice *sdev, SCSISense sense);
 void scsi_device_report_change(SCSIDevice *dev, SCSISense sense);
+void scsi_device_unit_attention_reported(SCSIDevice *dev);
 int scsi_device_get_sense(SCSIDevice *dev, uint8_t *buf, int len, bool fixed);
 SCSIDevice *scsi_device_find(SCSIBus *bus, int channel, int target, int lun);
 
diff --git a/include/hw/sd.h b/include/hw/sd.h
index d9b97e446..79adb5bb4 100644
--- a/include/hw/sd.h
+++ b/include/hw/sd.h
@@ -68,7 +68,7 @@ typedef struct {
 
 typedef struct SDState SDState;
 
-SDState *sd_init(BlockDriverState *bs, bool is_spi);
+SDState *sd_init(BlockBackend *bs, bool is_spi);
 int sd_do_command(SDState *sd, SDRequest *req,
                   uint8_t *response);
 void sd_write_data(SDState *sd, uint8_t value);
diff --git a/include/hw/sysbus.h b/include/hw/sysbus.h
index f5aaa05ee..d1f3f000f 100644
--- a/include/hw/sysbus.h
+++ b/include/hw/sysbus.h
@@ -8,10 +8,9 @@
 
 #define QDEV_MAX_MMIO 32
 #define QDEV_MAX_PIO 32
-#define QDEV_MAX_IRQ 512
 
 #define TYPE_SYSTEM_BUS "System"
-#define SYSTEM_BUS(obj) OBJECT_CHECK(IDEBus, (obj), TYPE_IDE_BUS)
+#define SYSTEM_BUS(obj) OBJECT_CHECK(BusState, (obj), TYPE_SYSTEM_BUS)
 
 typedef struct SysBusDevice SysBusDevice;
 
@@ -33,6 +32,9 @@ typedef struct SysBusDevice SysBusDevice;
  * SysBusDeviceClass is not overriding #DeviceClass.realize, so derived
  * classes overriding it are not required to invoke its implementation.
  */
+
+#define SYSBUS_DEVICE_GPIO_IRQ "sysbus-irq"
+
 typedef struct SysBusDeviceClass {
     /*< private >*/
     DeviceClass parent_class;
@@ -46,9 +48,6 @@ struct SysBusDevice {
     DeviceState parent_obj;
     /*< public >*/
 
-    int num_irq;
-    qemu_irq irqs[QDEV_MAX_IRQ];
-    qemu_irq *irqp[QDEV_MAX_IRQ];
     int num_mmio;
     struct {
         hwaddr addr;
@@ -58,6 +57,8 @@ struct SysBusDevice {
     pio_addr_t pio[QDEV_MAX_PIO];
 };
 
+typedef int FindSysbusDeviceFunc(SysBusDevice *sbdev, void *opaque);
+
 void sysbus_init_mmio(SysBusDevice *dev, MemoryRegion *memory);
 MemoryRegion *sysbus_mmio_get_region(SysBusDevice *dev, int n);
 void sysbus_init_irq(SysBusDevice *dev, qemu_irq *p);
@@ -65,15 +66,21 @@ void sysbus_pass_irq(SysBusDevice *dev, SysBusDevice *target);
 void sysbus_init_ioports(SysBusDevice *dev, pio_addr_t ioport, pio_addr_t size);
 
 
+bool sysbus_has_irq(SysBusDevice *dev, int n);
+bool sysbus_has_mmio(SysBusDevice *dev, unsigned int n);
 void sysbus_connect_irq(SysBusDevice *dev, int n, qemu_irq irq);
+bool sysbus_is_irq_connected(SysBusDevice *dev, int n);
+qemu_irq sysbus_get_connected_irq(SysBusDevice *dev, int n);
 void sysbus_mmio_map(SysBusDevice *dev, int n, hwaddr addr);
 void sysbus_mmio_map_overlap(SysBusDevice *dev, int n, hwaddr addr,
                              int priority);
 void sysbus_add_io(SysBusDevice *dev, hwaddr addr,
                    MemoryRegion *mem);
-void sysbus_del_io(SysBusDevice *dev, MemoryRegion *mem);
 MemoryRegion *sysbus_address_space(SysBusDevice *dev);
 
+/* Call func for every dynamically created sysbus device in the system */
+void foreach_dynamic_sysbus_device(FindSysbusDeviceFunc *func, void *opaque);
+
 /* Legacy helper function for creating devices.  */
 DeviceState *sysbus_create_varargs(const char *name,
                                  hwaddr addr, ...);
diff --git a/include/hw/tricore/tricore.h b/include/hw/tricore/tricore.h
new file mode 100644
index 000000000..5f1325278
--- /dev/null
+++ b/include/hw/tricore/tricore.h
@@ -0,0 +1,11 @@
+#ifndef TRICORE_MISC_H
+#define TRICORE_MISC_H 1
+
+#include "exec/memory.h"
+#include "hw/irq.h"
+
+struct tricore_boot_info {
+    uint64_t ram_size;
+    const char *kernel_filename;
+};
+#endif
diff --git a/include/hw/usb.h b/include/hw/usb.h
index 8bcab48d2..b20b95912 100644
--- a/include/hw/usb.h
+++ b/include/hw/usb.h
@@ -267,10 +267,14 @@ struct USBDevice {
 #define USB_DEVICE_GET_CLASS(obj) \
      OBJECT_GET_CLASS(USBDeviceClass, (obj), TYPE_USB_DEVICE)
 
+typedef void (*USBDeviceRealize)(USBDevice *dev, Error **errp);
+typedef void (*USBDeviceUnrealize)(USBDevice *dev, Error **errp);
+
 typedef struct USBDeviceClass {
     DeviceClass parent_class;
 
-    int (*init)(USBDevice *dev);
+    USBDeviceRealize realize;
+    USBDeviceUnrealize unrealize;
 
     /*
      * Walk (enabled) downstream ports, check for a matching device.
@@ -475,7 +479,8 @@ void usb_host_info(Monitor *mon, const QDict *qdict);
 
 #define VM_USB_HUB_SIZE 8
 
-/* usb-musb.c */
+/* hw/usb/hdc-musb.c */
+
 enum musb_irq_source_e {
     musb_irq_suspend = 0,
     musb_irq_resume,
@@ -494,6 +499,10 @@ enum musb_irq_source_e {
 };
 
 typedef struct MUSBState MUSBState;
+
+extern CPUReadMemoryFunc * const musb_read[];
+extern CPUWriteMemoryFunc * const musb_write[];
+
 MUSBState *musb_init(DeviceState *parent_device, int gpio_base);
 void musb_reset(MUSBState *s);
 uint32_t musb_core_intr_get(MUSBState *s);
@@ -524,6 +533,7 @@ struct USBBusOps {
 
 void usb_bus_new(USBBus *bus, size_t bus_size,
                  USBBusOps *ops, DeviceState *host);
+void usb_bus_release(USBBus *bus);
 USBBus *usb_bus_find(int busnr);
 void usb_legacy_register(const char *typename, const char *usbdevice_name,
                          USBDevice *(*usbdevice_init)(USBBus *bus,
@@ -538,11 +548,12 @@ int usb_register_companion(const char *masterbus, USBPort *ports[],
                            void *opaque, USBPortOps *ops, int speedmask);
 void usb_port_location(USBPort *downstream, USBPort *upstream, int portnr);
 void usb_unregister_port(USBBus *bus, USBPort *port);
-int usb_claim_port(USBDevice *dev);
+void usb_claim_port(USBDevice *dev, Error **errp);
 void usb_release_port(USBDevice *dev);
-int usb_device_attach(USBDevice *dev);
+void usb_device_attach(USBDevice *dev, Error **errp);
 int usb_device_detach(USBDevice *dev);
 int usb_device_delete_addr(int busnr, int addr);
+void usb_check_attach(USBDevice *dev, Error **errp);
 
 static inline USBBus *usb_bus_from_device(USBDevice *d)
 {
diff --git a/include/hw/virtio/dataplane/vring.h b/include/hw/virtio/dataplane/vring.h
index af73ee2ae..d3e086aef 100644
--- a/include/hw/virtio/dataplane/vring.h
+++ b/include/hw/virtio/dataplane/vring.h
@@ -17,8 +17,8 @@
 #ifndef VRING_H
 #define VRING_H
 
-#include <linux/virtio_ring.h>
 #include "qemu-common.h"
+#include "hw/virtio/virtio_ring.h"
 #include "hw/virtio/virtio.h"
 
 typedef struct {
diff --git a/include/hw/virtio/vhost-backend.h b/include/hw/virtio/vhost-backend.h
index d31768a1d..e472f2971 100644
--- a/include/hw/virtio/vhost-backend.h
+++ b/include/hw/virtio/vhost-backend.h
@@ -32,6 +32,8 @@ typedef struct VhostOps {
     vhost_backend_cleanup vhost_backend_cleanup;
 } VhostOps;
 
+extern const VhostOps user_ops;
+
 int vhost_set_backend_type(struct vhost_dev *dev,
                            VhostBackendType backend_type);
 
diff --git a/include/hw/virtio/virtio-blk.h b/include/hw/virtio/virtio-blk.h
index afb7b8db3..3979dc41a 100644
--- a/include/hw/virtio/virtio-blk.h
+++ b/include/hw/virtio/virtio-blk.h
@@ -17,7 +17,7 @@
 #include "hw/virtio/virtio.h"
 #include "hw/block/block.h"
 #include "sysemu/iothread.h"
-#include "block/block.h"
+#include "sysemu/block-backend.h"
 
 #define TYPE_VIRTIO_BLK "virtio-blk-device"
 #define VIRTIO_BLK(obj) \
@@ -120,21 +120,18 @@ struct VirtIOBlockDataPlane;
 struct VirtIOBlockReq;
 typedef struct VirtIOBlock {
     VirtIODevice parent_obj;
-    BlockDriverState *bs;
+    BlockBackend *blk;
     VirtQueue *vq;
     void *rq;
     QEMUBH *bh;
-    BlockConf *conf;
-    VirtIOBlkConf blk;
+    VirtIOBlkConf conf;
     unsigned short sector_mask;
     bool original_wce;
     VMChangeStateEntry *change;
     /* Function to push to vq and notify guest */
     void (*complete_request)(struct VirtIOBlockReq *req, unsigned char status);
-#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
     Notifier migration_state_notifier;
     struct VirtIOBlockDataPlane *dataplane;
-#endif
 } VirtIOBlock;
 
 typedef struct MultiReqBuffer {
@@ -161,6 +158,6 @@ int virtio_blk_handle_scsi_req(VirtIOBlock *blk,
 
 void virtio_blk_handle_request(VirtIOBlockReq *req, MultiReqBuffer *mrb);
 
-void virtio_submit_multiwrite(BlockDriverState *bs, MultiReqBuffer *mrb);
+void virtio_submit_multiwrite(BlockBackend *blk, MultiReqBuffer *mrb);
 
 #endif
diff --git a/include/hw/virtio/virtio-scsi.h b/include/hw/virtio/virtio-scsi.h
index 188a2d914..bf17cc9ea 100644
--- a/include/hw/virtio/virtio-scsi.h
+++ b/include/hw/virtio/virtio-scsi.h
@@ -17,6 +17,8 @@
 #include "hw/virtio/virtio.h"
 #include "hw/pci/pci.h"
 #include "hw/scsi/scsi.h"
+#include "sysemu/iothread.h"
+#include "hw/virtio/dataplane/vring.h"
 
 #define TYPE_VIRTIO_SCSI_COMMON "virtio-scsi-common"
 #define VIRTIO_SCSI_COMMON(obj) \
@@ -151,8 +153,18 @@ struct VirtIOSCSIConf {
     uint32_t cmd_per_lun;
     char *vhostfd;
     char *wwpn;
+    IOThread *iothread;
 };
 
+struct VirtIOSCSI;
+
+typedef struct {
+    struct VirtIOSCSI *parent;
+    Vring vring;
+    EventNotifier host_notifier;
+    EventNotifier guest_notifier;
+} VirtIOSCSIVring;
+
 typedef struct VirtIOSCSICommon {
     VirtIODevice parent_obj;
     VirtIOSCSIConf conf;
@@ -164,14 +176,77 @@ typedef struct VirtIOSCSICommon {
     VirtQueue **cmd_vqs;
 } VirtIOSCSICommon;
 
-typedef struct {
+typedef struct VirtIOSCSI {
     VirtIOSCSICommon parent_obj;
 
     SCSIBus bus;
     int resetting;
     bool events_dropped;
+
+    /* Fields for dataplane below */
+    AioContext *ctx; /* one iothread per virtio-scsi-pci for now */
+
+    /* Vring is used instead of vq in dataplane code, because of the underlying
+     * memory layer thread safety */
+    VirtIOSCSIVring *ctrl_vring;
+    VirtIOSCSIVring *event_vring;
+    VirtIOSCSIVring **cmd_vrings;
+    bool dataplane_started;
+    bool dataplane_starting;
+    bool dataplane_stopping;
+    bool dataplane_disabled;
+    bool dataplane_fenced;
+    Error *blocker;
+    Notifier migration_state_notifier;
 } VirtIOSCSI;
 
+typedef struct VirtIOSCSIReq {
+    VirtIOSCSI *dev;
+    VirtQueue *vq;
+    QEMUSGList qsgl;
+    QEMUIOVector resp_iov;
+
+    /* Note:
+     * - fields before elem are initialized by virtio_scsi_init_req;
+     * - elem is uninitialized at the time of allocation.
+     * - fields after elem (except the ending cdb[]) are zeroed by
+     *   virtio_scsi_init_req.
+     * */
+
+    VirtQueueElement elem;
+    /* Set by dataplane code. */
+    VirtIOSCSIVring *vring;
+
+    union {
+        /* Used for two-stage request submission */
+        QTAILQ_ENTRY(VirtIOSCSIReq) next;
+
+        /* Used for cancellation of request during TMFs */
+        int remaining;
+    };
+
+    SCSIRequest *sreq;
+    size_t resp_size;
+    enum SCSIXferMode mode;
+    union {
+        VirtIOSCSICmdResp     cmd;
+        VirtIOSCSICtrlTMFResp tmf;
+        VirtIOSCSICtrlANResp  an;
+        VirtIOSCSIEvent       event;
+    } resp;
+    union {
+        struct {
+            VirtIOSCSICmdReq  cmd;
+            uint8_t           cdb[];
+        } QEMU_PACKED;
+        VirtIOSCSICtrlTMFReq  tmf;
+        VirtIOSCSICtrlANReq   an;
+    } req;
+} VirtIOSCSIReq;
+
+QEMU_BUILD_BUG_ON(offsetof(VirtIOSCSIReq, req.cdb) !=
+                  offsetof(VirtIOSCSIReq, req.cmd) + sizeof(VirtIOSCSICmdReq));
+
 #define DEFINE_VIRTIO_SCSI_PROPERTIES(_state, _conf_field)                     \
     DEFINE_PROP_UINT32("num_queues", _state, _conf_field.num_queues, 1),       \
     DEFINE_PROP_UINT32("max_sectors", _state, _conf_field.max_sectors, 0xFFFF),\
@@ -192,5 +267,19 @@ void virtio_scsi_common_realize(DeviceState *dev, Error **errp,
                                 HandleOutput cmd);
 
 void virtio_scsi_common_unrealize(DeviceState *dev, Error **errp);
+void virtio_scsi_handle_ctrl_req(VirtIOSCSI *s, VirtIOSCSIReq *req);
+bool virtio_scsi_handle_cmd_req_prepare(VirtIOSCSI *s, VirtIOSCSIReq *req);
+void virtio_scsi_handle_cmd_req_submit(VirtIOSCSI *s, VirtIOSCSIReq *req);
+VirtIOSCSIReq *virtio_scsi_init_req(VirtIOSCSI *s, VirtQueue *vq);
+void virtio_scsi_free_req(VirtIOSCSIReq *req);
+void virtio_scsi_push_event(VirtIOSCSI *s, SCSIDevice *dev,
+                            uint32_t event, uint32_t reason);
+
+void virtio_scsi_set_iothread(VirtIOSCSI *s, IOThread *iothread);
+void virtio_scsi_dataplane_start(VirtIOSCSI *s);
+void virtio_scsi_dataplane_stop(VirtIOSCSI *s);
+void virtio_scsi_vring_push_notify(VirtIOSCSIReq *req);
+VirtIOSCSIReq *virtio_scsi_pop_req_vring(VirtIOSCSI *s,
+                                         VirtIOSCSIVring *vring);
 
 #endif /* _QEMU_VIRTIO_SCSI_H */
diff --git a/include/hw/virtio/virtio-serial.h b/include/hw/virtio/virtio-serial.h
index 4746312a8..a679e54aa 100644
--- a/include/hw/virtio/virtio-serial.h
+++ b/include/hw/virtio/virtio-serial.h
@@ -202,6 +202,8 @@ struct VirtIOSerial {
 
     QTAILQ_HEAD(, VirtIOSerialPort) ports;
 
+    QLIST_ENTRY(VirtIOSerial) next;
+
     /* bitmap for identifying active ports */
     uint32_t *ports_map;
 
diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h
index a60104ca2..0726d76f4 100644
--- a/include/hw/virtio/virtio.h
+++ b/include/hw/virtio/virtio.h
@@ -161,6 +161,9 @@ typedef struct VirtioDeviceClass {
     int (*load)(VirtIODevice *vdev, QEMUFile *f, int version_id);
 } VirtioDeviceClass;
 
+void virtio_instance_init_common(Object *proxy_obj, void *data,
+                                 size_t vdev_size, const char *vdev_name);
+
 void virtio_init(VirtIODevice *vdev, const char *name,
                          uint16_t device_id, size_t config_size);
 void virtio_cleanup(VirtIODevice *vdev);
diff --git a/include/hw/virtio/virtio_ring.h b/include/hw/virtio/virtio_ring.h
new file mode 100644
index 000000000..0b42e6eae
--- /dev/null
+++ b/include/hw/virtio/virtio_ring.h
@@ -0,0 +1,167 @@
+#ifndef _LINUX_VIRTIO_RING_H
+#define _LINUX_VIRTIO_RING_H
+/*
+ * This file is copied from /usr/include/linux while converting __uNN types
+ * to uXX_t, __inline__ to inline, and tab to spaces.
+ * */
+
+/* An interface for efficient virtio implementation, currently for use by KVM
+ * and lguest, but hopefully others soon.  Do NOT change this since it will
+ * break existing servers and clients.
+ *
+ * This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of IBM nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Copyright Rusty Russell IBM Corporation 2007. */
+
+/* This marks a buffer as continuing via the next field. */
+#define VRING_DESC_F_NEXT   1
+/* This marks a buffer as write-only (otherwise read-only). */
+#define VRING_DESC_F_WRITE  2
+/* This means the buffer contains a list of buffer descriptors. */
+#define VRING_DESC_F_INDIRECT   4
+
+/* The Host uses this in used->flags to advise the Guest: don't kick me when
+ * you add a buffer.  It's unreliable, so it's simply an optimization.  Guest
+ * will still kick if it's out of buffers. */
+#define VRING_USED_F_NO_NOTIFY  1
+/* The Guest uses this in avail->flags to advise the Host: don't interrupt me
+ * when you consume a buffer.  It's unreliable, so it's simply an
+ * optimization.  */
+#define VRING_AVAIL_F_NO_INTERRUPT  1
+
+/* We support indirect buffer descriptors */
+#define VIRTIO_RING_F_INDIRECT_DESC 28
+
+/* The Guest publishes the used index for which it expects an interrupt
+ * at the end of the avail ring. Host should ignore the avail->flags field. */
+/* The Host publishes the avail index for which it expects a kick
+ * at the end of the used ring. Guest should ignore the used->flags field. */
+#define VIRTIO_RING_F_EVENT_IDX     29
+
+/* Virtio ring descriptors: 16 bytes.  These can chain together via "next". */
+struct vring_desc {
+    /* Address (guest-physical). */
+    uint64_t addr;
+    /* Length. */
+    uint32_t len;
+    /* The flags as indicated above. */
+    uint16_t flags;
+    /* We chain unused descriptors via this, too */
+    uint16_t next;
+};
+
+struct vring_avail {
+    uint16_t flags;
+    uint16_t idx;
+    uint16_t ring[];
+};
+
+/* u32 is used here for ids for padding reasons. */
+struct vring_used_elem {
+    /* Index of start of used descriptor chain. */
+    uint32_t id;
+    /* Total length of the descriptor chain which was used (written to) */
+    uint32_t len;
+};
+
+struct vring_used {
+    uint16_t flags;
+    uint16_t idx;
+    struct vring_used_elem ring[];
+};
+
+struct vring {
+    unsigned int num;
+
+    struct vring_desc *desc;
+
+    struct vring_avail *avail;
+
+    struct vring_used *used;
+};
+
+/* The standard layout for the ring is a continuous chunk of memory which looks
+ * like this.  We assume num is a power of 2.
+ *
+ * struct vring
+ * {
+ *  // The actual descriptors (16 bytes each)
+ *  struct vring_desc desc[num];
+ *
+ *  // A ring of available descriptor heads with free-running index.
+ *  uint16_t avail_flags;
+ *  uint16_t avail_idx;
+ *  uint16_t available[num];
+ *  uint16_t used_event_idx;
+ *
+ *  // Padding to the next align boundary.
+ *  char pad[];
+ *
+ *  // A ring of used descriptor heads with free-running index.
+ *  uint16_t used_flags;
+ *  uint16_t used_idx;
+ *  struct vring_used_elem used[num];
+ *  uint16_t avail_event_idx;
+ * };
+ */
+/* We publish the used event index at the end of the available ring, and vice
+ * versa. They are at the end for backwards compatibility. */
+#define vring_used_event(vr) ((vr)->avail->ring[(vr)->num])
+#define vring_avail_event(vr) (*(uint16_t *)&(vr)->used->ring[(vr)->num])
+
+static inline void vring_init(struct vring *vr, unsigned int num, void *p,
+                  unsigned long align)
+{
+    vr->num = num;
+    vr->desc = p;
+    vr->avail = p + num*sizeof(struct vring_desc);
+    vr->used = (void *)(((uintptr_t)&vr->avail->ring[num] + sizeof(uint16_t)
+        + align - 1) & ~(align - 1));
+}
+
+static inline unsigned vring_size(unsigned int num, unsigned long align)
+{
+    return ((sizeof(struct vring_desc) * num + sizeof(uint16_t) * (3 + num)
+         + align - 1) & ~(align - 1))
+        + sizeof(uint16_t) * 3 + sizeof(struct vring_used_elem) * num;
+}
+
+/* The following is used with USED_EVENT_IDX and AVAIL_EVENT_IDX */
+/* Assuming a given event_idx value from the other size, if
+ * we have just incremented index from old to new_idx,
+ * should we trigger an event? */
+static inline int vring_need_event(uint16_t event_idx, uint16_t new_idx, uint16_t old)
+{
+    /* Note: Xen has similar logic for notification hold-off
+     * in include/xen/interface/io/ring.h with req_event and req_prod
+     * corresponding to event_idx + 1 and new_idx respectively.
+     * Note also that req_event and req_prod in Xen start at 1,
+     * event indexes in virtio start at 0. */
+    return (uint16_t)(new_idx - event_idx - 1) < (uint16_t)(new_idx - old);
+}
+
+#endif /* _LINUX_VIRTIO_RING_H */
diff --git a/include/hw/xen/xen.h b/include/hw/xen/xen.h
index f71f2d896..b0ed04caa 100644
--- a/include/hw/xen/xen.h
+++ b/include/hw/xen/xen.h
@@ -36,7 +36,6 @@ void xen_cmos_set_s3_resume(void *opaque, int irq, int level);
 
 qemu_irq *xen_interrupt_controller_init(void);
 
-int xen_init(MachineClass *mc);
 void xenstore_store_pv_console_info(int i, struct CharDriverState *chr);
 
 #if defined(NEED_CPU_H) && !defined(CONFIG_USER_ONLY)
diff --git a/include/hw/xen/xen_common.h b/include/hw/xen/xen_common.h
index 07731b928..95612a40c 100644
--- a/include/hw/xen/xen_common.h
+++ b/include/hw/xen/xen_common.h
@@ -164,4 +164,19 @@ void destroy_hvm_domain(bool reboot);
 /* shutdown/destroy current domain because of an error */
 void xen_shutdown_fatal_error(const char *fmt, ...) GCC_FMT_ATTR(1, 2);
 
+#ifdef HVM_PARAM_VMPORT_REGS_PFN
+static inline int xen_get_vmport_regs_pfn(XenXC xc, domid_t dom,
+                                          unsigned long *vmport_regs_pfn)
+{
+    return xc_get_hvm_param(xc, dom, HVM_PARAM_VMPORT_REGS_PFN,
+                            vmport_regs_pfn);
+}
+#else
+static inline int xen_get_vmport_regs_pfn(XenXC xc, domid_t dom,
+                                          unsigned long *vmport_regs_pfn)
+{
+    return -ENOSYS;
+}
+#endif
+
 #endif /* QEMU_HW_XEN_COMMON_H */
diff --git a/include/migration/qemu-file.h b/include/migration/qemu-file.h
index c90f5298a..401676bf4 100644
--- a/include/migration/qemu-file.h
+++ b/include/migration/qemu-file.h
@@ -25,6 +25,8 @@
 #define QEMU_FILE_H 1
 #include "exec/cpu-common.h"
 
+#include <stdint.h>
+
 /* This function writes a chunk of data to a file at the given position.
  * The pos argument can be ignored if the file is only being used for
  * streaming.  The handler should try to write all of the data it can.
@@ -94,11 +96,19 @@ typedef struct QEMUFileOps {
     QEMURamSaveFunc *save_page;
 } QEMUFileOps;
 
+struct QEMUSizedBuffer {
+    struct iovec *iov;
+    size_t n_iov;
+    size_t size; /* total allocated size in all iov's */
+    size_t used; /* number of used bytes */
+};
+
 QEMUFile *qemu_fopen_ops(void *opaque, const QEMUFileOps *ops);
 QEMUFile *qemu_fopen(const char *filename, const char *mode);
 QEMUFile *qemu_fdopen(int fd, const char *mode);
 QEMUFile *qemu_fopen_socket(int fd, const char *mode);
 QEMUFile *qemu_popen_cmd(const char *command, const char *mode);
+QEMUFile *qemu_bufopen(const char *mode, QEMUSizedBuffer *input);
 int qemu_get_fd(QEMUFile *f);
 int qemu_fclose(QEMUFile *f);
 int64_t qemu_ftell(QEMUFile *f);
@@ -110,6 +120,23 @@ void qemu_put_byte(QEMUFile *f, int v);
  */
 void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, int size);
 bool qemu_file_mode_is_not_valid(const char *mode);
+bool qemu_file_is_writable(QEMUFile *f);
+
+QEMUSizedBuffer *qsb_create(const uint8_t *buffer, size_t len);
+QEMUSizedBuffer *qsb_clone(const QEMUSizedBuffer *);
+void qsb_free(QEMUSizedBuffer *);
+size_t qsb_set_length(QEMUSizedBuffer *qsb, size_t length);
+size_t qsb_get_length(const QEMUSizedBuffer *qsb);
+ssize_t qsb_get_buffer(const QEMUSizedBuffer *, off_t start, size_t count,
+                       uint8_t *buf);
+ssize_t qsb_write_at(QEMUSizedBuffer *qsb, const uint8_t *buf,
+                     off_t pos, size_t count);
+
+
+/*
+ * For use on files opened with qemu_bufopen
+ */
+const QEMUSizedBuffer *qemu_buf_get(QEMUFile *f);
 
 static inline void qemu_put_ubyte(QEMUFile *f, unsigned int v)
 {
diff --git a/include/migration/vmstate.h b/include/migration/vmstate.h
index 9a001bd28..e45fc49cb 100644
--- a/include/migration/vmstate.h
+++ b/include/migration/vmstate.h
@@ -484,6 +484,17 @@ extern const VMStateInfo vmstate_info_bitmap;
     .start        = (_start),                                        \
 }
 
+#define VMSTATE_VBUFFER_ALLOC_UINT32(_field, _state, _version, _test, _start, _field_size) { \
+    .name         = (stringify(_field)),                             \
+    .version_id   = (_version),                                      \
+    .field_exists = (_test),                                         \
+    .size_offset  = vmstate_offset_value(_state, _field_size, uint32_t),\
+    .info         = &vmstate_info_buffer,                            \
+    .flags        = VMS_VBUFFER|VMS_POINTER|VMS_ALLOC,               \
+    .offset       = offsetof(_state, _field),                        \
+    .start        = (_start),                                        \
+}
+
 #define VMSTATE_BUFFER_UNSAFE_INFO(_field, _state, _version, _info, _size) { \
     .name       = (stringify(_field)),                               \
     .version_id = (_version),                                        \
diff --git a/include/monitor/monitor.h b/include/monitor/monitor.h
index 3d6929d6c..47606d04a 100644
--- a/include/monitor/monitor.h
+++ b/include/monitor/monitor.h
@@ -27,10 +27,10 @@ int monitor_suspend(Monitor *mon);
 void monitor_resume(Monitor *mon);
 
 int monitor_read_bdrv_key_start(Monitor *mon, BlockDriverState *bs,
-                                BlockDriverCompletionFunc *completion_cb,
+                                BlockCompletionFunc *completion_cb,
                                 void *opaque);
 int monitor_read_block_device_key(Monitor *mon, const char *device,
-                                  BlockDriverCompletionFunc *completion_cb,
+                                  BlockCompletionFunc *completion_cb,
                                   void *opaque);
 
 int monitor_get_fd(Monitor *mon, const char *fdname, Error **errp);
@@ -64,7 +64,7 @@ AddfdInfo *monitor_fdset_add_fd(int fd, bool has_fdset_id, int64_t fdset_id,
                                 Error **errp);
 int monitor_fdset_get_fd(int64_t fdset_id, int flags);
 int monitor_fdset_dup_fd_add(int64_t fdset_id, int dup_fd);
-int monitor_fdset_dup_fd_remove(int dup_fd);
+void monitor_fdset_dup_fd_remove(int dup_fd);
 int monitor_fdset_dup_fd_find(int dup_fd);
 
 #endif /* !MONITOR_H */
diff --git a/include/net/net.h b/include/net/net.h
index ed594f9bd..008d61004 100644
--- a/include/net/net.h
+++ b/include/net/net.h
@@ -36,8 +36,7 @@ typedef struct NICConf {
 #define DEFINE_NIC_PROPERTIES(_state, _conf)                            \
     DEFINE_PROP_MACADDR("mac",   _state, _conf.macaddr),                \
     DEFINE_PROP_VLAN("vlan",     _state, _conf.peers),                   \
-    DEFINE_PROP_NETDEV("netdev", _state, _conf.peers),                   \
-    DEFINE_PROP_INT32("bootindex", _state, _conf.bootindex, -1)
+    DEFINE_PROP_NETDEV("netdev", _state, _conf.peers)
 
 
 /* Net clients */
diff --git a/include/qapi/qmp/qerror.h b/include/qapi/qmp/qerror.h
index 902d1a7a1..0ca6cbd0e 100644
--- a/include/qapi/qmp/qerror.h
+++ b/include/qapi/qmp/qerror.h
@@ -154,16 +154,4 @@ void qerror_report_err(Error *err);
 #define QERR_UNSUPPORTED \
     ERROR_CLASS_GENERIC_ERROR, "this feature or command is not currently supported"
 
-#define QERR_SOCKET_CONNECT_FAILED \
-    ERROR_CLASS_GENERIC_ERROR, "Failed to connect to socket"
-
-#define QERR_SOCKET_LISTEN_FAILED \
-    ERROR_CLASS_GENERIC_ERROR, "Failed to set socket to listening mode"
-
-#define QERR_SOCKET_BIND_FAILED \
-    ERROR_CLASS_GENERIC_ERROR, "Failed to bind socket"
-
-#define QERR_SOCKET_CREATE_FAILED \
-    ERROR_CLASS_GENERIC_ERROR, "Failed to create socket"
-
 #endif /* QERROR_H */
diff --git a/include/qapi/util.h b/include/qapi/util.h
new file mode 100644
index 000000000..de9238bf9
--- /dev/null
+++ b/include/qapi/util.h
@@ -0,0 +1,17 @@
+/*
+ * QAPI util functions
+ *
+ * Copyright Fujitsu, Inc. 2014
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#ifndef QAPI_UTIL_H
+#define QAPI_UTIL_H
+
+int qapi_enum_parse(const char *lookup[], const char *buf,
+                    int max, int def, Error **errp);
+
+#endif
diff --git a/include/qapi/visitor-impl.h b/include/qapi/visitor-impl.h
index ecc018319..09bb0fd40 100644
--- a/include/qapi/visitor-impl.h
+++ b/include/qapi/visitor-impl.h
@@ -55,6 +55,8 @@ struct Visitor
     void (*type_int64)(Visitor *v, int64_t *obj, const char *name, Error **errp);
     /* visit_type_size() falls back to (*type_uint64)() if type_size is unset */
     void (*type_size)(Visitor *v, uint64_t *obj, const char *name, Error **errp);
+    bool (*start_union)(Visitor *v, bool data_present, Error **errp);
+    void (*end_union)(Visitor *v, bool data_present, Error **errp);
 };
 
 void input_type_enum(Visitor *v, int *obj, const char *strings[],
diff --git a/include/qapi/visitor.h b/include/qapi/visitor.h
index 4a0178fa4..5934f59ad 100644
--- a/include/qapi/visitor.h
+++ b/include/qapi/visitor.h
@@ -58,5 +58,7 @@ void visit_type_size(Visitor *v, uint64_t *obj, const char *name, Error **errp);
 void visit_type_bool(Visitor *v, bool *obj, const char *name, Error **errp);
 void visit_type_str(Visitor *v, char **obj, const char *name, Error **errp);
 void visit_type_number(Visitor *v, double *obj, const char *name, Error **errp);
+bool visit_start_union(Visitor *v, bool data_present, Error **errp);
+void visit_end_union(Visitor *v, bool data_present, Error **errp);
 
 #endif
diff --git a/include/qemu-common.h b/include/qemu-common.h
index 6ef828223..f8622141a 100644
--- a/include/qemu-common.h
+++ b/include/qemu-common.h
@@ -41,6 +41,7 @@
 #include <assert.h>
 #include <signal.h>
 #include "glib-compat.h"
+#include "qemu/option.h"
 
 #ifdef _WIN32
 #include "sysemu/os-win32.h"
@@ -104,9 +105,16 @@ static inline char *realpath(const char *path, char *resolved_path)
 }
 #endif
 
+void cpu_ticks_init(void);
+
 /* icount */
-void configure_icount(const char *option);
+void configure_icount(QemuOpts *opts, Error **errp);
 extern int use_icount;
+extern int icount_align_option;
+/* drift information for info jit command */
+extern int64_t max_delay;
+extern int64_t max_advance;
+void dump_drift_info(FILE *f, fprintf_function cpu_fprintf);
 
 #include "qemu/osdep.h"
 #include "qemu/bswap.h"
@@ -182,6 +190,9 @@ int64_t strtosz_suffix_unit(const char *nptr, char **end,
 /* used to print char* safely */
 #define STR_OR_NULL(str) ((str) ? (str) : "null")
 
+/* id.c */
+bool id_wellformed(const char *id);
+
 /* path.c */
 void init_paths(const char *prefix);
 const char *path(const char *pathname);
@@ -346,7 +357,6 @@ char *qemu_find_file(int type, const char *name);
 void os_setup_early_signal_handling(void);
 char *os_find_datadir(void);
 void os_parse_cmd_args(int index, const char *optarg);
-void os_pidfile_error(void);
 
 /* Convert a byte between binary and BCD.  */
 static inline uint8_t to_bcd(uint8_t val)
diff --git a/include/qemu/bitmap.h b/include/qemu/bitmap.h
index 1babd5d81..f0273c965 100644
--- a/include/qemu/bitmap.h
+++ b/include/qemu/bitmap.h
@@ -12,7 +12,11 @@
 #ifndef BITMAP_H
 #define BITMAP_H
 
-#include "qemu-common.h"
+#include <glib.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include "qemu/osdep.h"
 #include "qemu/bitops.h"
 
 /*
@@ -88,10 +92,19 @@ int slow_bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1,
 int slow_bitmap_intersects(const unsigned long *bitmap1,
                            const unsigned long *bitmap2, long bits);
 
-static inline unsigned long *bitmap_new(long nbits)
+static inline unsigned long *bitmap_try_new(long nbits)
 {
     long len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
-    return g_malloc0(len);
+    return g_try_malloc0(len);
+}
+
+static inline unsigned long *bitmap_new(long nbits)
+{
+    unsigned long *ptr = bitmap_try_new(nbits);
+    if (ptr == NULL) {
+        abort();
+    }
+    return ptr;
 }
 
 static inline void bitmap_zero(unsigned long *dst, long nbits)
diff --git a/include/qemu/bitops.h b/include/qemu/bitops.h
index 7e2d5c996..181bd4606 100644
--- a/include/qemu/bitops.h
+++ b/include/qemu/bitops.h
@@ -12,7 +12,9 @@
 #ifndef BITOPS_H
 #define BITOPS_H
 
-#include "qemu-common.h"
+#include <stdint.h>
+#include <assert.h>
+
 #include "host-utils.h"
 
 #define BITS_PER_BYTE           CHAR_BIT
diff --git a/include/qemu/compiler.h b/include/qemu/compiler.h
index 155b35896..ac7c4c441 100644
--- a/include/qemu/compiler.h
+++ b/include/qemu/compiler.h
@@ -24,6 +24,12 @@
 #define QEMU_WARN_UNUSED_RESULT
 #endif
 
+#if QEMU_GNUC_PREREQ(4, 3)
+#define QEMU_ARTIFICIAL __attribute__((always_inline, artificial))
+#else
+#define QEMU_ARTIFICIAL
+#endif
+
 #if defined(_WIN32)
 # define QEMU_PACKED __attribute__((gcc_struct, packed))
 #else
diff --git a/include/qemu/error-report.h b/include/qemu/error-report.h
index 000eae395..7ab235590 100644
--- a/include/qemu/error-report.h
+++ b/include/qemu/error-report.h
@@ -38,6 +38,7 @@ void error_vprintf(const char *fmt, va_list ap) GCC_FMT_ATTR(1, 0);
 void error_printf(const char *fmt, ...) GCC_FMT_ATTR(1, 2);
 void error_printf_unless_qmp(const char *fmt, ...) GCC_FMT_ATTR(1, 2);
 void error_set_progname(const char *argv0);
+void error_vreport(const char *fmt, va_list ap) GCC_FMT_ATTR(1, 0);
 void error_report(const char *fmt, ...) GCC_FMT_ATTR(1, 2);
 const char *error_get_progname(void);
 extern bool enable_timestamp_msg;
diff --git a/include/qemu/main-loop.h b/include/qemu/main-loop.h
index 6f0200a7a..62c68c0f3 100644
--- a/include/qemu/main-loop.h
+++ b/include/qemu/main-loop.h
@@ -42,7 +42,7 @@
  *
  * In the case of QEMU tools, this will also start/initialize timers.
  */
-int qemu_init_main_loop(void);
+int qemu_init_main_loop(Error **errp);
 
 /**
  * main_loop_wait: Run one iteration of the main loop.
diff --git a/include/qemu/osdep.h b/include/qemu/osdep.h
index 8480d523f..b3300cc23 100644
--- a/include/qemu/osdep.h
+++ b/include/qemu/osdep.h
@@ -5,6 +5,7 @@
 #include <stdarg.h>
 #include <stddef.h>
 #include <stdbool.h>
+#include <stdint.h>
 #include <sys/types.h>
 #ifdef __OpenBSD__
 #include <sys/signal.h>
@@ -68,6 +69,12 @@ typedef signed int              int_fast16_t;
 #define MAX(a, b) (((a) > (b)) ? (a) : (b))
 #endif
 
+/* Minimum function that returns zero only iff both values are zero.
+ * Intended for use with unsigned values only. */
+#ifndef MIN_NON_ZERO
+#define MIN_NON_ZERO(a, b) (((a) != 0 && (a) < (b)) ? (a) : (b))
+#endif
+
 #ifndef ROUND_UP
 #define ROUND_UP(n,d) (((n) + (d) - 1) & -(d))
 #endif
@@ -95,8 +102,9 @@ typedef signed int              int_fast16_t;
 #define qemu_printf printf
 
 int qemu_daemon(int nochdir, int noclose);
+void *qemu_try_memalign(size_t alignment, size_t size);
 void *qemu_memalign(size_t alignment, size_t size);
-void *qemu_anon_ram_alloc(size_t size);
+void *qemu_anon_ram_alloc(size_t size, uint64_t *align);
 void qemu_vfree(void *ptr);
 void qemu_anon_ram_free(void *ptr, size_t size);
 
@@ -245,11 +253,7 @@ char *qemu_get_exec_dir(void);
  * Search the auxiliary vector for @type, returning the value
  * or 0 if @type is not present.
  */
-#if defined(CONFIG_GETAUXVAL) || defined(__linux__)
 unsigned long qemu_getauxval(unsigned long type);
-#else
-static inline unsigned long qemu_getauxval(unsigned long type) { return 0; }
-#endif
 
 void qemu_set_tty_echo(int fd, bool echo);
 
diff --git a/include/qemu/sockets.h b/include/qemu/sockets.h
index fdbb19678..f47dae614 100644
--- a/include/qemu/sockets.h
+++ b/include/qemu/sockets.h
@@ -47,7 +47,7 @@ int recv_all(int fd, void *buf, int len1, bool single_read);
 /* callback function for nonblocking connect
  * valid fd on success, negative error code on failure
  */
-typedef void NonBlockingConnectHandler(int fd, void *opaque);
+typedef void NonBlockingConnectHandler(int fd, Error *errp, void *opaque);
 
 InetSocketAddress *inet_parse(const char *str, Error **errp);
 int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp);
diff --git a/include/qemu/timer.h b/include/qemu/timer.h
index 7f9a074c2..5f5210d54 100644
--- a/include/qemu/timer.h
+++ b/include/qemu/timer.h
@@ -745,6 +745,8 @@ static inline int64_t get_clock(void)
 /* icount */
 int64_t cpu_get_icount(void);
 int64_t cpu_get_clock(void);
+int64_t cpu_get_clock_offset(void);
+int64_t cpu_icount_to_ns(int64_t icount);
 
 /*******************************************/
 /* host CPU ticks (if available) */
diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h
index 5f20b0e26..34751778a 100644
--- a/include/qemu/typedefs.h
+++ b/include/qemu/typedefs.h
@@ -32,9 +32,11 @@ typedef struct MemoryMappingList MemoryMappingList;
 
 typedef struct QEMUMachine QEMUMachine;
 typedef struct MachineClass MachineClass;
+typedef struct MachineState MachineState;
 typedef struct NICInfo NICInfo;
 typedef struct HCIInfo HCIInfo;
 typedef struct AudioState AudioState;
+typedef struct BlockBackend BlockBackend;
 typedef struct BlockDriverState BlockDriverState;
 typedef struct DriveInfo DriveInfo;
 typedef struct DisplayState DisplayState;
@@ -70,6 +72,7 @@ typedef struct SSIBus SSIBus;
 typedef struct EventNotifier EventNotifier;
 typedef struct VirtIODevice VirtIODevice;
 typedef struct QEMUSGList QEMUSGList;
+typedef struct QEMUSizedBuffer QEMUSizedBuffer;
 typedef struct SHPCDevice SHPCDevice;
 typedef struct FWCfgState FWCfgState;
 typedef struct PcGuestInfo PcGuestInfo;
diff --git a/include/qom/cpu.h b/include/qom/cpu.h
index 1aafbf5f3..2098f1cb5 100644
--- a/include/qom/cpu.h
+++ b/include/qom/cpu.h
@@ -95,9 +95,15 @@ struct TranslationBlock;
  * @get_phys_page_debug: Callback for obtaining a physical address.
  * @gdb_read_register: Callback for letting GDB read a register.
  * @gdb_write_register: Callback for letting GDB write a register.
+ * @debug_excp_handler: Callback for handling debug exceptions.
  * @vmsd: State description for migration.
  * @gdb_num_core_regs: Number of core registers accessible to GDB.
  * @gdb_core_xml_file: File name for core registers GDB XML description.
+ * @gdb_stop_before_watchpoint: Indicates whether GDB expects the CPU to stop
+ *           before the insn which triggers a watchpoint rather than after it.
+ * @cpu_exec_enter: Callback for cpu_exec preparation.
+ * @cpu_exec_exit: Callback for cpu_exec cleanup.
+ * @cpu_exec_interrupt: Callback for processing interrupts in cpu_exec.
  *
  * Represents a CPU family or model.
  */
@@ -134,6 +140,7 @@ typedef struct CPUClass {
     hwaddr (*get_phys_page_debug)(CPUState *cpu, vaddr addr);
     int (*gdb_read_register)(CPUState *cpu, uint8_t *buf, int reg);
     int (*gdb_write_register)(CPUState *cpu, uint8_t *buf, int reg);
+    void (*debug_excp_handler)(CPUState *cpu);
 
     int (*write_elf64_note)(WriteCoreDumpFunction f, CPUState *cpu,
                             int cpuid, void *opaque);
@@ -147,6 +154,11 @@ typedef struct CPUClass {
     const struct VMStateDescription *vmsd;
     int gdb_num_core_regs;
     const char *gdb_core_xml_file;
+    bool gdb_stop_before_watchpoint;
+
+    void (*cpu_exec_enter)(CPUState *cpu);
+    void (*cpu_exec_exit)(CPUState *cpu);
+    bool (*cpu_exec_interrupt)(CPUState *cpu, int interrupt_request);
 } CPUClass;
 
 #ifdef HOST_WORDS_BIGENDIAN
@@ -169,7 +181,8 @@ typedef struct CPUBreakpoint {
 
 typedef struct CPUWatchpoint {
     vaddr vaddr;
-    vaddr len_mask;
+    vaddr len;
+    vaddr hitaddr;
     int flags; /* BP_* */
     QTAILQ_ENTRY(CPUWatchpoint) entry;
 } CPUWatchpoint;
@@ -622,9 +635,12 @@ void cpu_single_step(CPUState *cpu, int enabled);
 #define BP_MEM_WRITE          0x02
 #define BP_MEM_ACCESS         (BP_MEM_READ | BP_MEM_WRITE)
 #define BP_STOP_BEFORE_ACCESS 0x04
-#define BP_WATCHPOINT_HIT     0x08
+/* 0x08 currently unused */
 #define BP_GDB                0x10
 #define BP_CPU                0x20
+#define BP_WATCHPOINT_HIT_READ 0x40
+#define BP_WATCHPOINT_HIT_WRITE 0x80
+#define BP_WATCHPOINT_HIT (BP_WATCHPOINT_HIT_READ | BP_WATCHPOINT_HIT_WRITE)
 
 int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
                           CPUBreakpoint **breakpoint);
diff --git a/include/qom/object.h b/include/qom/object.h
index 8a05a81a9..89c309296 100644
--- a/include/qom/object.h
+++ b/include/qom/object.h
@@ -338,6 +338,7 @@ typedef struct ObjectProperty
 {
     gchar *name;
     gchar *type;
+    gchar *description;
     ObjectPropertyAccessor *get;
     ObjectPropertyAccessor *set;
     ObjectPropertyResolve *resolve;
@@ -1275,6 +1276,19 @@ void object_property_add_alias(Object *obj, const char *name,
                                Error **errp);
 
 /**
+ * object_property_set_description:
+ * @obj: the object owning the property
+ * @name: the name of the property
+ * @description: the description of the property on the object
+ * @errp: if an error occurs, a pointer to an area to store the error
+ *
+ * Set an object property's description.
+ *
+ */
+void object_property_set_description(Object *obj, const char *name,
+                                     const char *description, Error **errp);
+
+/**
  * object_child_foreach:
  * @obj: the object whose children will be navigated
  * @fn: the iterator function to be called
diff --git a/include/sysemu/accel.h b/include/sysemu/accel.h
new file mode 100644
index 000000000..997720f36
--- /dev/null
+++ b/include/sysemu/accel.h
@@ -0,0 +1,62 @@
+/* QEMU accelerator interfaces
+ *
+ * Copyright (c) 2014 Red Hat Inc
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef HW_ACCEL_H
+#define HW_ACCEL_H
+
+#include "qemu/typedefs.h"
+#include "qom/object.h"
+
+typedef struct AccelState {
+    /*< private >*/
+    Object parent_obj;
+} AccelState;
+
+typedef struct AccelClass {
+    /*< private >*/
+    ObjectClass parent_class;
+    /*< public >*/
+
+    const char *opt_name;
+    const char *name;
+    int (*available)(void);
+    int (*init_machine)(MachineState *ms);
+    bool *allowed;
+} AccelClass;
+
+#define TYPE_ACCEL "accel"
+
+#define ACCEL_CLASS_SUFFIX  "-" TYPE_ACCEL
+#define ACCEL_CLASS_NAME(a) (a ACCEL_CLASS_SUFFIX)
+
+#define ACCEL_CLASS(klass) \
+    OBJECT_CLASS_CHECK(AccelClass, (klass), TYPE_ACCEL)
+#define ACCEL(obj) \
+    OBJECT_CHECK(AccelState, (obj), TYPE_ACCEL)
+#define ACCEL_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(AccelClass, (obj), TYPE_ACCEL)
+
+extern int tcg_tb_size;
+
+int configure_accelerator(MachineState *ms);
+
+#endif
diff --git a/include/sysemu/arch_init.h b/include/sysemu/arch_init.h
index 182d48d8c..54b36c16c 100644
--- a/include/sysemu/arch_init.h
+++ b/include/sysemu/arch_init.h
@@ -6,22 +6,23 @@
 
 enum {
     QEMU_ARCH_ALL = -1,
-    QEMU_ARCH_ALPHA = 1,
-    QEMU_ARCH_ARM = 2,
-    QEMU_ARCH_CRIS = 4,
-    QEMU_ARCH_I386 = 8,
-    QEMU_ARCH_M68K = 16,
-    QEMU_ARCH_LM32 = 32,
-    QEMU_ARCH_MICROBLAZE = 64,
-    QEMU_ARCH_MIPS = 128,
-    QEMU_ARCH_PPC = 256,
-    QEMU_ARCH_S390X = 512,
-    QEMU_ARCH_SH4 = 1024,
-    QEMU_ARCH_SPARC = 2048,
-    QEMU_ARCH_XTENSA = 4096,
-    QEMU_ARCH_OPENRISC = 8192,
-    QEMU_ARCH_UNICORE32 = 0x4000,
-    QEMU_ARCH_MOXIE = 0x8000,
+    QEMU_ARCH_ALPHA = (1 << 0),
+    QEMU_ARCH_ARM = (1 << 1),
+    QEMU_ARCH_CRIS = (1 << 2),
+    QEMU_ARCH_I386 = (1 << 3),
+    QEMU_ARCH_M68K = (1 << 4),
+    QEMU_ARCH_LM32 = (1 << 5),
+    QEMU_ARCH_MICROBLAZE = (1 << 6),
+    QEMU_ARCH_MIPS = (1 << 7),
+    QEMU_ARCH_PPC = (1 << 8),
+    QEMU_ARCH_S390X = (1 << 9),
+    QEMU_ARCH_SH4 = (1 << 10),
+    QEMU_ARCH_SPARC = (1 << 11),
+    QEMU_ARCH_XTENSA = (1 << 12),
+    QEMU_ARCH_OPENRISC = (1 << 13),
+    QEMU_ARCH_UNICORE32 = (1 << 14),
+    QEMU_ARCH_MOXIE = (1 << 15),
+    QEMU_ARCH_TRICORE = (1 << 16),
 };
 
 extern const uint32_t arch_type;
@@ -32,7 +33,6 @@ void do_smbios_option(QemuOpts *opts);
 void ram_mig_init(void);
 void cpudef_init(void);
 void audio_init(void);
-int tcg_available(void);
 int kvm_available(void);
 int xen_available(void);
 
diff --git a/include/sysemu/block-backend.h b/include/sysemu/block-backend.h
new file mode 100644
index 000000000..52d13c1c0
--- /dev/null
+++ b/include/sysemu/block-backend.h
@@ -0,0 +1,142 @@
+/*
+ * QEMU Block backends
+ *
+ * Copyright (C) 2014 Red Hat, Inc.
+ *
+ * Authors:
+ *  Markus Armbruster <armbru@redhat.com>,
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1
+ * or later.  See the COPYING.LIB file in the top-level directory.
+ */
+
+#ifndef BLOCK_BACKEND_H
+#define BLOCK_BACKEND_H
+
+#include "qemu/typedefs.h"
+#include "qapi/error.h"
+
+/*
+ * TODO Have to include block/block.h for a bunch of block layer
+ * types.  Unfortunately, this pulls in the whole BlockDriverState
+ * API, which we don't want used by many BlockBackend users.  Some of
+ * the types belong here, and the rest should be split into a common
+ * header and one for the BlockDriverState API.
+ */
+#include "block/block.h"
+
+/* Callbacks for block device models */
+typedef struct BlockDevOps {
+    /*
+     * Runs when virtual media changed (monitor commands eject, change)
+     * Argument load is true on load and false on eject.
+     * Beware: doesn't run when a host device's physical media
+     * changes.  Sure would be useful if it did.
+     * Device models with removable media must implement this callback.
+     */
+    void (*change_media_cb)(void *opaque, bool load);
+    /*
+     * Runs when an eject request is issued from the monitor, the tray
+     * is closed, and the medium is locked.
+     * Device models that do not implement is_medium_locked will not need
+     * this callback.  Device models that can lock the medium or tray might
+     * want to implement the callback and unlock the tray when "force" is
+     * true, even if they do not support eject requests.
+     */
+    void (*eject_request_cb)(void *opaque, bool force);
+    /*
+     * Is the virtual tray open?
+     * Device models implement this only when the device has a tray.
+     */
+    bool (*is_tray_open)(void *opaque);
+    /*
+     * Is the virtual medium locked into the device?
+     * Device models implement this only when device has such a lock.
+     */
+    bool (*is_medium_locked)(void *opaque);
+    /*
+     * Runs when the size changed (e.g. monitor command block_resize)
+     */
+    void (*resize_cb)(void *opaque);
+} BlockDevOps;
+
+BlockBackend *blk_new(const char *name, Error **errp);
+BlockBackend *blk_new_with_bs(const char *name, Error **errp);
+void blk_ref(BlockBackend *blk);
+void blk_unref(BlockBackend *blk);
+const char *blk_name(BlockBackend *blk);
+BlockBackend *blk_by_name(const char *name);
+BlockBackend *blk_next(BlockBackend *blk);
+
+BlockDriverState *blk_bs(BlockBackend *blk);
+
+void blk_hide_on_behalf_of_do_drive_del(BlockBackend *blk);
+
+void blk_iostatus_enable(BlockBackend *blk);
+int blk_attach_dev(BlockBackend *blk, void *dev);
+void blk_attach_dev_nofail(BlockBackend *blk, void *dev);
+void blk_detach_dev(BlockBackend *blk, void *dev);
+void *blk_get_attached_dev(BlockBackend *blk);
+void blk_set_dev_ops(BlockBackend *blk, const BlockDevOps *ops, void *opaque);
+int blk_read(BlockBackend *blk, int64_t sector_num, uint8_t *buf,
+             int nb_sectors);
+int blk_read_unthrottled(BlockBackend *blk, int64_t sector_num, uint8_t *buf,
+                         int nb_sectors);
+int blk_write(BlockBackend *blk, int64_t sector_num, const uint8_t *buf,
+              int nb_sectors);
+BlockAIOCB *blk_aio_write_zeroes(BlockBackend *blk, int64_t sector_num,
+                                 int nb_sectors, BdrvRequestFlags flags,
+                                 BlockCompletionFunc *cb, void *opaque);
+int blk_pread(BlockBackend *blk, int64_t offset, void *buf, int count);
+int blk_pwrite(BlockBackend *blk, int64_t offset, const void *buf, int count);
+int64_t blk_getlength(BlockBackend *blk);
+void blk_get_geometry(BlockBackend *blk, uint64_t *nb_sectors_ptr);
+BlockAIOCB *blk_aio_readv(BlockBackend *blk, int64_t sector_num,
+                          QEMUIOVector *iov, int nb_sectors,
+                          BlockCompletionFunc *cb, void *opaque);
+BlockAIOCB *blk_aio_writev(BlockBackend *blk, int64_t sector_num,
+                           QEMUIOVector *iov, int nb_sectors,
+                           BlockCompletionFunc *cb, void *opaque);
+BlockAIOCB *blk_aio_flush(BlockBackend *blk,
+                          BlockCompletionFunc *cb, void *opaque);
+BlockAIOCB *blk_aio_discard(BlockBackend *blk,
+                            int64_t sector_num, int nb_sectors,
+                            BlockCompletionFunc *cb, void *opaque);
+void blk_aio_cancel(BlockAIOCB *acb);
+void blk_aio_cancel_async(BlockAIOCB *acb);
+int blk_aio_multiwrite(BlockBackend *blk, BlockRequest *reqs, int num_reqs);
+int blk_ioctl(BlockBackend *blk, unsigned long int req, void *buf);
+BlockAIOCB *blk_aio_ioctl(BlockBackend *blk, unsigned long int req, void *buf,
+                          BlockCompletionFunc *cb, void *opaque);
+int blk_flush(BlockBackend *blk);
+int blk_flush_all(void);
+void blk_drain_all(void);
+BlockdevOnError blk_get_on_error(BlockBackend *blk, bool is_read);
+BlockErrorAction blk_get_error_action(BlockBackend *blk, bool is_read,
+                                      int error);
+void blk_error_action(BlockBackend *blk, BlockErrorAction action,
+                      bool is_read, int error);
+int blk_is_read_only(BlockBackend *blk);
+int blk_is_sg(BlockBackend *blk);
+int blk_enable_write_cache(BlockBackend *blk);
+void blk_set_enable_write_cache(BlockBackend *blk, bool wce);
+int blk_is_inserted(BlockBackend *blk);
+void blk_lock_medium(BlockBackend *blk, bool locked);
+void blk_eject(BlockBackend *blk, bool eject_flag);
+int blk_get_flags(BlockBackend *blk);
+void blk_set_guest_block_size(BlockBackend *blk, int align);
+void *blk_blockalign(BlockBackend *blk, size_t size);
+bool blk_op_is_blocked(BlockBackend *blk, BlockOpType op, Error **errp);
+void blk_op_unblock(BlockBackend *blk, BlockOpType op, Error *reason);
+void blk_op_block_all(BlockBackend *blk, Error *reason);
+void blk_op_unblock_all(BlockBackend *blk, Error *reason);
+AioContext *blk_get_aio_context(BlockBackend *blk);
+void blk_set_aio_context(BlockBackend *blk, AioContext *new_context);
+void blk_io_plug(BlockBackend *blk);
+void blk_io_unplug(BlockBackend *blk);
+BlockAcctStats *blk_get_stats(BlockBackend *blk);
+
+void *blk_aio_get(const AIOCBInfo *aiocb_info, BlockBackend *blk,
+                  BlockCompletionFunc *cb, void *opaque);
+
+#endif
diff --git a/include/sysemu/blockdev.h b/include/sysemu/blockdev.h
index 23a5d10c6..09d1e30b3 100644
--- a/include/sysemu/blockdev.h
+++ b/include/sysemu/blockdev.h
@@ -14,8 +14,8 @@
 #include "qapi/error.h"
 #include "qemu/queue.h"
 
-void blockdev_mark_auto_del(BlockDriverState *bs);
-void blockdev_auto_del(BlockDriverState *bs);
+void blockdev_mark_auto_del(BlockBackend *blk);
+void blockdev_auto_del(BlockBackend *blk);
 
 typedef enum {
     IF_DEFAULT = -1,            /* for use with drive_add() only */
@@ -30,14 +30,12 @@ typedef enum {
 } BlockInterfaceType;
 
 struct DriveInfo {
-    BlockDriverState *bdrv;
-    char *id;
     const char *devaddr;
     BlockInterfaceType type;
     int bus;
     int unit;
     int auto_del;               /* see blockdev_mark_auto_del() */
-    bool enable_auto_del;       /* Only for legacy drive_new() */
+    bool is_default;            /* Added by default_drive() ?  */
     int media_cd;
     int cyls, heads, secs, trans;
     QemuOpts *opts;
@@ -45,17 +43,23 @@ struct DriveInfo {
     QTAILQ_ENTRY(DriveInfo) next;
 };
 
+DriveInfo *blk_legacy_dinfo(BlockBackend *blk);
+DriveInfo *blk_set_legacy_dinfo(BlockBackend *blk, DriveInfo *dinfo);
+BlockBackend *blk_by_legacy_dinfo(DriveInfo *dinfo);
+
+void override_max_devs(BlockInterfaceType type, int max_devs);
+
 DriveInfo *drive_get(BlockInterfaceType type, int bus, int unit);
+bool drive_check_orphaned(void);
 DriveInfo *drive_get_by_index(BlockInterfaceType type, int index);
 int drive_get_max_bus(BlockInterfaceType type);
+int drive_get_max_devs(BlockInterfaceType type);
 DriveInfo *drive_get_next(BlockInterfaceType type);
-DriveInfo *drive_get_by_blockdev(BlockDriverState *bs);
 
 QemuOpts *drive_def(const char *optstr);
 QemuOpts *drive_add(BlockInterfaceType type, int index, const char *file,
                     const char *optstr);
 DriveInfo *drive_new(QemuOpts *arg, BlockInterfaceType block_default_type);
-void drive_del(DriveInfo *dinfo);
 
 /* device-hotplug */
 
diff --git a/include/sysemu/char.h b/include/sysemu/char.h
index 0bbd631e7..832b7fead 100644
--- a/include/sysemu/char.h
+++ b/include/sysemu/char.h
@@ -344,8 +344,7 @@ bool chr_is_ringbuf(const CharDriverState *chr);
 
 QemuOpts *qemu_chr_parse_compat(const char *label, const char *filename);
 
-void register_char_driver(const char *name, CharDriverState *(*open)(QemuOpts *));
-void register_char_driver_qapi(const char *name, ChardevBackendKind kind,
+void register_char_driver(const char *name, ChardevBackendKind kind,
         void (*parse)(QemuOpts *opts, ChardevBackend *backend, Error **errp));
 
 /* add an eventfd to the qemu devices that are polled */
@@ -358,6 +357,9 @@ CharDriverState *qemu_char_get_next_serial(void);
 /* msmouse */
 CharDriverState *qemu_chr_open_msmouse(void);
 
+/* testdev.c */
+CharDriverState *chr_testdev_init(void);
+
 /* baum.c */
 CharDriverState *chr_baum_init(void);
 
diff --git a/include/sysemu/cpus.h b/include/sysemu/cpus.h
index 4f790810b..3f162a9e0 100644
--- a/include/sysemu/cpus.h
+++ b/include/sysemu/cpus.h
@@ -10,6 +10,7 @@ void cpu_stop_current(void);
 void cpu_synchronize_all_states(void);
 void cpu_synchronize_all_post_reset(void);
 void cpu_synchronize_all_post_init(void);
+void cpu_clean_all_dirty(void);
 
 void qtest_clock_warp(int64_t dest);
 
diff --git a/include/sysemu/dma.h b/include/sysemu/dma.h
index 00f21f3da..3f2f4c89e 100644
--- a/include/sysemu/dma.h
+++ b/include/sysemu/dma.h
@@ -15,6 +15,7 @@
 #include "exec/address-spaces.h"
 #include "hw/hw.h"
 #include "block/block.h"
+#include "block/accounting.h"
 #include "sysemu/kvm.h"
 
 typedef struct ScatterGatherEntry ScatterGatherEntry;
@@ -196,24 +197,24 @@ void qemu_sglist_add(QEMUSGList *qsg, dma_addr_t base, dma_addr_t len);
 void qemu_sglist_destroy(QEMUSGList *qsg);
 #endif
 
-typedef BlockDriverAIOCB *DMAIOFunc(BlockDriverState *bs, int64_t sector_num,
-                                 QEMUIOVector *iov, int nb_sectors,
-                                 BlockDriverCompletionFunc *cb, void *opaque);
-
-BlockDriverAIOCB *dma_bdrv_io(BlockDriverState *bs,
-                              QEMUSGList *sg, uint64_t sector_num,
-                              DMAIOFunc *io_func, BlockDriverCompletionFunc *cb,
-                              void *opaque, DMADirection dir);
-BlockDriverAIOCB *dma_bdrv_read(BlockDriverState *bs,
-                                QEMUSGList *sg, uint64_t sector,
-                                BlockDriverCompletionFunc *cb, void *opaque);
-BlockDriverAIOCB *dma_bdrv_write(BlockDriverState *bs,
-                                 QEMUSGList *sg, uint64_t sector,
-                                 BlockDriverCompletionFunc *cb, void *opaque);
+typedef BlockAIOCB *DMAIOFunc(BlockBackend *blk, int64_t sector_num,
+                              QEMUIOVector *iov, int nb_sectors,
+                              BlockCompletionFunc *cb, void *opaque);
+
+BlockAIOCB *dma_blk_io(BlockBackend *blk,
+                       QEMUSGList *sg, uint64_t sector_num,
+                       DMAIOFunc *io_func, BlockCompletionFunc *cb,
+                       void *opaque, DMADirection dir);
+BlockAIOCB *dma_blk_read(BlockBackend *blk,
+                         QEMUSGList *sg, uint64_t sector,
+                         BlockCompletionFunc *cb, void *opaque);
+BlockAIOCB *dma_blk_write(BlockBackend *blk,
+                          QEMUSGList *sg, uint64_t sector,
+                          BlockCompletionFunc *cb, void *opaque);
 uint64_t dma_buf_read(uint8_t *ptr, int32_t len, QEMUSGList *sg);
 uint64_t dma_buf_write(uint8_t *ptr, int32_t len, QEMUSGList *sg);
 
-void dma_acct_start(BlockDriverState *bs, BlockAcctCookie *cookie,
+void dma_acct_start(BlockBackend *blk, BlockAcctCookie *cookie,
                     QEMUSGList *sg, enum BlockAcctType type);
 
 #endif
diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
index 174ea36af..22e42ef23 100644
--- a/include/sysemu/kvm.h
+++ b/include/sysemu/kvm.h
@@ -163,8 +163,7 @@ extern KVMState *kvm_state;
 
 /* external API */
 
-int kvm_init(MachineClass *mc);
-
+bool kvm_has_free_slot(MachineState *ms);
 int kvm_has_sync_mmu(void);
 int kvm_has_vcpu_events(void);
 int kvm_has_robust_singlestep(void);
@@ -303,6 +302,8 @@ bool kvm_arch_stop_on_emulation_error(CPUState *cpu);
 
 int kvm_check_extension(KVMState *s, unsigned int extension);
 
+int kvm_vm_check_extension(KVMState *s, unsigned int extension);
+
 #define kvm_vm_enable_cap(s, capability, cap_flags, ...)             \
     ({                                                               \
         struct kvm_enable_cap cap = {                                \
@@ -348,6 +349,7 @@ int kvm_physical_memory_addr_from_host(KVMState *s, void *ram_addr,
 void kvm_cpu_synchronize_state(CPUState *cpu);
 void kvm_cpu_synchronize_post_reset(CPUState *cpu);
 void kvm_cpu_synchronize_post_init(CPUState *cpu);
+void kvm_cpu_clean_state(CPUState *cpu);
 
 /* generic hooks - to be moved/refactored once there are more users */
 
@@ -372,6 +374,13 @@ static inline void cpu_synchronize_post_init(CPUState *cpu)
     }
 }
 
+static inline void cpu_clean_state(CPUState *cpu)
+{
+    if (kvm_enabled()) {
+        kvm_cpu_clean_state(cpu);
+    }
+}
+
 int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg);
 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg);
 void kvm_irqchip_release_virq(KVMState *s, int virq);
diff --git a/include/sysemu/qtest.h b/include/sysemu/qtest.h
index 95c9ade77..05473b75a 100644
--- a/include/sysemu/qtest.h
+++ b/include/sysemu/qtest.h
@@ -26,7 +26,6 @@ static inline bool qtest_enabled(void)
 
 bool qtest_driver(void);
 
-int qtest_init_accel(MachineClass *mc);
 void qtest_init(const char *qtest_chrdev, const char *qtest_log, Error **errp);
 
 static inline int qtest_available(void)
diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h
index d8539fd60..9fea3bc3a 100644
--- a/include/sysemu/sysemu.h
+++ b/include/sysemu/sysemu.h
@@ -130,6 +130,7 @@ extern int no_shutdown;
 extern int semihosting_enabled;
 extern int old_param;
 extern int boot_menu;
+extern bool boot_strict;
 extern uint8_t *boot_splash_filedata;
 extern size_t boot_splash_filedata_size;
 extern uint8_t qemu_extra_params_fw[2];
@@ -160,6 +161,7 @@ typedef struct node_info {
 extern NodeInfo numa_info[MAX_NODES];
 void set_numa_nodes(void);
 void set_numa_modes(void);
+void query_numa_node_mem(uint64_t node_mem[]);
 extern QemuOptsList qemu_numa_opts;
 int numa_init_func(QemuOpts *opts, void *opaque);
 
@@ -183,9 +185,6 @@ void do_pci_device_hot_remove(Monitor *mon, const QDict *qdict);
 /* generic hotplug */
 void drive_hot_add(Monitor *mon, const QDict *qdict);
 
-/* CPU hotplug */
-void qemu_register_cpu_added_notifier(Notifier *notifier);
-
 /* pcie aer error injection */
 void pcie_aer_inject_error_print(Monitor *mon, const QObject *data);
 int do_pcie_aer_inject_error(Monitor *mon,
@@ -212,6 +211,11 @@ void add_boot_device_path(int32_t bootindex, DeviceState *dev,
 char *get_boot_devices_list(size_t *size, bool ignore_suffixes);
 
 DeviceState *get_boot_device(uint32_t position);
+void check_boot_index(int32_t bootindex, Error **errp);
+void del_boot_device_path(DeviceState *dev, const char *suffix);
+void device_add_bootindex_property(Object *obj, int32_t *bootindex,
+                                   const char *name, const char *suffix,
+                                   DeviceState *dev, Error **errp);
 
 QemuOpts *qemu_get_machine_opts(void);
 
diff --git a/include/sysemu/tpm.h b/include/sysemu/tpm.h
index 13febddbe..9b81ce918 100644
--- a/include/sysemu/tpm.h
+++ b/include/sysemu/tpm.h
@@ -20,4 +20,11 @@ int tpm_config_parse(QemuOptsList *opts_list, const char *optarg);
 int tpm_init(void);
 void tpm_cleanup(void);
 
+#define TYPE_TPM_TIS                "tpm-tis"
+
+static inline bool tpm_find(void)
+{
+    return object_resolve_path_type("", TYPE_TPM_TIS, NULL);
+}
+
 #endif /* QEMU_TPM_H */
diff --git a/include/trace-tcg.h b/include/trace-tcg.h
new file mode 100644
index 000000000..6f6bdbb44
--- /dev/null
+++ b/include/trace-tcg.h
@@ -0,0 +1,7 @@
+#ifndef TRACE_TCG_H
+#define TRACE_TCG_H
+
+#include "trace/generated-tcg-tracers.h"
+#include "trace/generated-events.h"
+
+#endif  /* TRACE_TCG_H */
diff --git a/include/trace.h b/include/trace.h
index c15f49812..44a1f1f8c 100644
--- a/include/trace.h
+++ b/include/trace.h
@@ -2,5 +2,6 @@
 #define TRACE_H
 
 #include "trace/generated-tracers.h"
+#include "trace/generated-events.h"
 
 #endif  /* TRACE_H */
diff --git a/include/ui/console.h b/include/ui/console.h
index 845526ed0..22ef8ca6b 100644
--- a/include/ui/console.h
+++ b/include/ui/console.h
@@ -102,8 +102,7 @@ struct QemuConsoleClass {
     ObjectClass parent_class;
 };
 
-#define QEMU_BIG_ENDIAN_FLAG    0x01
-#define QEMU_ALLOCATED_FLAG     0x02
+#define QEMU_ALLOCATED_FLAG     0x01
 
 struct PixelFormat {
     uint8_t bits_per_pixel;
@@ -119,8 +118,6 @@ struct DisplaySurface {
     pixman_format_code_t format;
     pixman_image_t *image;
     uint8_t flags;
-
-    struct PixelFormat pf;
 };
 
 typedef struct QemuUIInfo {
@@ -188,9 +185,13 @@ struct DisplayChangeListener {
 };
 
 DisplayState *init_displaystate(void);
-DisplaySurface* qemu_create_displaysurface_from(int width, int height, int bpp,
-                                                int linesize, uint8_t *data,
-                                                bool byteswap);
+DisplaySurface *qemu_create_displaysurface_from(int width, int height,
+                                                pixman_format_code_t format,
+                                                int linesize, uint8_t *data);
+DisplaySurface *qemu_create_displaysurface_guestmem(int width, int height,
+                                                    pixman_format_code_t format,
+                                                    int linesize,
+                                                    uint64_t addr);
 PixelFormat qemu_different_endianness_pixelformat(int bpp);
 PixelFormat qemu_default_pixelformat(int bpp);
 
@@ -199,10 +200,12 @@ void qemu_free_displaysurface(DisplaySurface *surface);
 
 static inline int is_surface_bgr(DisplaySurface *surface)
 {
-    if (surface->pf.bits_per_pixel == 32 && surface->pf.rshift == 0)
+    if (PIXMAN_FORMAT_BPP(surface->format) == 32 &&
+        PIXMAN_FORMAT_TYPE(surface->format) == PIXMAN_TYPE_ABGR) {
         return 1;
-    else
+    } else {
         return 0;
+    }
 }
 
 static inline int is_buffer_shared(DisplaySurface *surface)
@@ -228,6 +231,10 @@ void dpy_text_resize(QemuConsole *con, int w, int h);
 void dpy_mouse_set(QemuConsole *con, int x, int y, int on);
 void dpy_cursor_define(QemuConsole *con, QEMUCursor *cursor);
 bool dpy_cursor_define_supported(QemuConsole *con);
+void dpy_gfx_update_dirty(QemuConsole *con,
+                          MemoryRegion *address_space,
+                          uint64_t base,
+                          bool invalidate);
 
 static inline int surface_stride(DisplaySurface *s)
 {
@@ -285,6 +292,9 @@ typedef struct GraphicHwOps {
 QemuConsole *graphic_console_init(DeviceState *dev, uint32_t head,
                                   const GraphicHwOps *ops,
                                   void *opaque);
+void graphic_console_set_hwops(QemuConsole *con,
+                               const GraphicHwOps *hw_ops,
+                               void *opaque);
 
 void graphic_hw_update(QemuConsole *con);
 void graphic_hw_invalidate(QemuConsole *con);
diff --git a/include/ui/qemu-pixman.h b/include/ui/qemu-pixman.h
index ba970f813..381969d97 100644
--- a/include/ui/qemu-pixman.h
+++ b/include/ui/qemu-pixman.h
@@ -33,6 +33,8 @@
 
 /* -------------------------------------------------------------------- */
 
+PixelFormat qemu_pixelformat_from_pixman(pixman_format_code_t format);
+pixman_format_code_t qemu_default_pixman_format(int bpp, bool native_endian);
 int qemu_pixman_get_type(int rshift, int gshift, int bshift);
 pixman_format_code_t qemu_pixman_get_format(PixelFormat *pf);
 
@@ -40,6 +42,8 @@ pixman_image_t *qemu_pixman_linebuf_create(pixman_format_code_t format,
                                            int width);
 void qemu_pixman_linebuf_fill(pixman_image_t *linebuf, pixman_image_t *fb,
                               int width, int x, int y);
+void qemu_pixman_linebuf_copy(pixman_image_t *fb, int width, int x, int y,
+                              pixman_image_t *linebuf);
 pixman_image_t *qemu_pixman_mirror_create(pixman_format_code_t format,
                                           pixman_image_t *image);
 void qemu_pixman_image_unref(pixman_image_t *image);
diff --git a/ioport.c b/ioport.c
index 3d91e79ed..783a3ae67 100644
--- a/ioport.c
+++ b/ioport.c
@@ -149,6 +149,14 @@ void portio_list_set_flush_coalesced(PortioList *piolist)
 
 void portio_list_destroy(PortioList *piolist)
 {
+    MemoryRegionPortioList *mrpio;
+    unsigned i;
+
+    for (i = 0; i < piolist->nr; ++i) {
+        mrpio = container_of(piolist->regions[i], MemoryRegionPortioList, mr);
+        object_unparent(OBJECT(&mrpio->mr));
+        g_free(mrpio);
+    }
     g_free(piolist->regions);
 }
 
@@ -291,8 +299,5 @@ void portio_list_del(PortioList *piolist)
     for (i = 0; i < piolist->nr; ++i) {
         mrpio = container_of(piolist->regions[i], MemoryRegionPortioList, mr);
         memory_region_del_subregion(piolist->address_space, &mrpio->mr);
-        memory_region_destroy(&mrpio->mr);
-        g_free(mrpio);
-        piolist->regions[i] = NULL;
     }
 }
diff --git a/iothread.c b/iothread.c
index d9403cf69..342a23fcb 100644
--- a/iothread.c
+++ b/iothread.c
@@ -17,6 +17,7 @@
 #include "block/aio.h"
 #include "sysemu/iothread.h"
 #include "qmp-commands.h"
+#include "qemu/error-report.h"
 
 #define IOTHREADS_PATH "/objects"
 
@@ -53,6 +54,9 @@ static void iothread_instance_finalize(Object *obj)
 {
     IOThread *iothread = IOTHREAD(obj);
 
+    if (!iothread->ctx) {
+        return;
+    }
     iothread->stopping = true;
     aio_notify(iothread->ctx);
     qemu_thread_join(&iothread->thread);
@@ -63,11 +67,16 @@ static void iothread_instance_finalize(Object *obj)
 
 static void iothread_complete(UserCreatable *obj, Error **errp)
 {
+    Error *local_error = NULL;
     IOThread *iothread = IOTHREAD(obj);
 
     iothread->stopping = false;
-    iothread->ctx = aio_context_new();
     iothread->thread_id = -1;
+    iothread->ctx = aio_context_new(&local_error);
+    if (!iothread->ctx) {
+        error_propagate(errp, local_error);
+        return;
+    }
 
     qemu_mutex_init(&iothread->init_done_lock);
     qemu_cond_init(&iothread->init_done_cond);
diff --git a/kvm-all.c b/kvm-all.c
index 1402f4f42..937bc9d90 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -25,6 +25,7 @@
 #include "qemu/option.h"
 #include "qemu/config-file.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/accel.h"
 #include "hw/hw.h"
 #include "hw/pci/msi.h"
 #include "hw/s390x/adapter.h"
@@ -44,10 +45,6 @@
 #include <sys/eventfd.h>
 #endif
 
-#ifdef CONFIG_VALGRIND_H
-#include <valgrind/memcheck.h>
-#endif
-
 /* KVM uses PAGE_SIZE in its definition of COALESCED_MMIO_MAX */
 #define PAGE_SIZE TARGET_PAGE_SIZE
 
@@ -76,6 +73,8 @@ typedef struct kvm_dirty_log KVMDirtyLog;
 
 struct KVMState
 {
+    AccelState parent_obj;
+
     KVMSlot *slots;
     int nr_slots;
     int fd;
@@ -110,6 +109,11 @@ struct KVMState
 #endif
 };
 
+#define TYPE_KVM_ACCEL ACCEL_CLASS_NAME("kvm")
+
+#define KVM_STATE(obj) \
+    OBJECT_CHECK(KVMState, (obj), TYPE_KVM_ACCEL)
+
 KVMState *kvm_state;
 bool kvm_kernel_irqchip;
 bool kvm_async_interrupts_allowed;
@@ -128,7 +132,7 @@ static const KVMCapabilityInfo kvm_required_capabilites[] = {
     KVM_CAP_LAST_INFO
 };
 
-static KVMSlot *kvm_alloc_slot(KVMState *s)
+static KVMSlot *kvm_get_free_slot(KVMState *s)
 {
     int i;
 
@@ -138,6 +142,22 @@ static KVMSlot *kvm_alloc_slot(KVMState *s)
         }
     }
 
+    return NULL;
+}
+
+bool kvm_has_free_slot(MachineState *ms)
+{
+    return kvm_get_free_slot(KVM_STATE(ms->accelerator));
+}
+
+static KVMSlot *kvm_alloc_slot(KVMState *s)
+{
+    KVMSlot *slot = kvm_get_free_slot(s);
+
+    if (slot) {
+        return slot;
+    }
+
     fprintf(stderr, "%s: no free slot available\n", __func__);
     abort();
 }
@@ -493,6 +513,19 @@ int kvm_check_extension(KVMState *s, unsigned int extension)
     return ret;
 }
 
+int kvm_vm_check_extension(KVMState *s, unsigned int extension)
+{
+    int ret;
+
+    ret = kvm_vm_ioctl(s, KVM_CHECK_EXTENSION, extension);
+    if (ret < 0) {
+        /* VM wide version not implemented, use global one instead */
+        ret = kvm_check_extension(s, extension);
+    }
+
+    return ret;
+}
+
 static int kvm_set_ioeventfd_mmio(int fd, hwaddr addr, uint32_t val,
                                   bool assign, uint32_t size, bool datamatch)
 {
@@ -617,8 +650,10 @@ static void kvm_set_phys_mem(MemoryRegionSection *section, bool add)
     unsigned delta;
 
     /* kvm works in page size chunks, but the function may be called
-       with sub-page size and unaligned start address. */
-    delta = TARGET_PAGE_ALIGN(size) - size;
+       with sub-page size and unaligned start address. Pad the start
+       address to next and truncate size to previous page boundary. */
+    delta = (TARGET_PAGE_SIZE - (start_addr & ~TARGET_PAGE_MASK));
+    delta &= ~TARGET_PAGE_MASK;
     if (delta > size) {
         return;
     }
@@ -1368,8 +1403,9 @@ static int kvm_max_vcpus(KVMState *s)
     return (ret) ? ret : kvm_recommended_vcpus(s);
 }
 
-int kvm_init(MachineClass *mc)
+static int kvm_init(MachineState *ms)
 {
+    MachineClass *mc = MACHINE_GET_CLASS(ms);
     static const char upgrade_note[] =
         "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
         "(see http://sourceforge.net/projects/kvm).\n";
@@ -1388,7 +1424,7 @@ int kvm_init(MachineClass *mc)
     int i, type = 0;
     const char *kvm_type;
 
-    s = g_malloc0(sizeof(KVMState));
+    s = KVM_STATE(ms->accelerator);
 
     /*
      * On systems where the kernel can support different base page
@@ -1577,7 +1613,6 @@ err:
         close(s->fd);
     }
     g_free(s->slots);
-    g_free(s);
 
     return ret;
 }
@@ -1669,18 +1704,37 @@ void kvm_cpu_synchronize_state(CPUState *cpu)
     }
 }
 
-void kvm_cpu_synchronize_post_reset(CPUState *cpu)
+static void do_kvm_cpu_synchronize_post_reset(void *arg)
 {
+    CPUState *cpu = arg;
+
     kvm_arch_put_registers(cpu, KVM_PUT_RESET_STATE);
     cpu->kvm_vcpu_dirty = false;
 }
 
-void kvm_cpu_synchronize_post_init(CPUState *cpu)
+void kvm_cpu_synchronize_post_reset(CPUState *cpu)
 {
+    run_on_cpu(cpu, do_kvm_cpu_synchronize_post_reset, cpu);
+}
+
+static void do_kvm_cpu_synchronize_post_init(void *arg)
+{
+    CPUState *cpu = arg;
+
     kvm_arch_put_registers(cpu, KVM_PUT_FULL_STATE);
     cpu->kvm_vcpu_dirty = false;
 }
 
+void kvm_cpu_synchronize_post_init(CPUState *cpu)
+{
+    run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, cpu);
+}
+
+void kvm_cpu_clean_state(CPUState *cpu)
+{
+    cpu->kvm_vcpu_dirty = false;
+}
+
 int kvm_cpu_exec(CPUState *cpu)
 {
     struct kvm_run *run = cpu->kvm_run;
@@ -1724,7 +1778,8 @@ int kvm_cpu_exec(CPUState *cpu)
             }
             fprintf(stderr, "error: kvm run failed %s\n",
                     strerror(-run_ret));
-            abort();
+            ret = -1;
+            break;
         }
 
         trace_kvm_run_exit(cpu->cpu_index, run->exit_reason);
@@ -1926,9 +1981,6 @@ int kvm_has_intx_set_mask(void)
 
 void kvm_setup_guest_memory(void *start, size_t size)
 {
-#ifdef CONFIG_VALGRIND_H
-    VALGRIND_MAKE_MEM_DEFINED(start, size);
-#endif
     if (!kvm_has_sync_mmu()) {
         int ret = qemu_madvise(start, size, QEMU_MADV_DONTFORK);
 
@@ -2199,3 +2251,25 @@ int kvm_get_one_reg(CPUState *cs, uint64_t id, void *target)
     }
     return r;
 }
+
+static void kvm_accel_class_init(ObjectClass *oc, void *data)
+{
+    AccelClass *ac = ACCEL_CLASS(oc);
+    ac->name = "KVM";
+    ac->init_machine = kvm_init;
+    ac->allowed = &kvm_allowed;
+}
+
+static const TypeInfo kvm_accel_type = {
+    .name = TYPE_KVM_ACCEL,
+    .parent = TYPE_ACCEL,
+    .class_init = kvm_accel_class_init,
+    .instance_size = sizeof(KVMState),
+};
+
+static void kvm_type_init(void)
+{
+    type_register_static(&kvm_accel_type);
+}
+
+type_init(kvm_type_init);
diff --git a/kvm-stub.c b/kvm-stub.c
index 8e7737caa..7ba90c546 100644
--- a/kvm-stub.c
+++ b/kvm-stub.c
@@ -35,11 +35,6 @@ int kvm_init_vcpu(CPUState *cpu)
     return -ENOSYS;
 }
 
-int kvm_init(MachineClass *mc)
-{
-    return -ENOSYS;
-}
-
 void kvm_flush_coalesced_mmio_buffer(void)
 {
 }
@@ -152,4 +147,9 @@ int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
 {
     return -ENOSYS;
 }
+
+bool kvm_has_free_slot(MachineState *ms)
+{
+    return false;
+}
 #endif
diff --git a/libcacard/cac.c b/libcacard/cac.c
index ae8c3784b..f38fdcedd 100644
--- a/libcacard/cac.c
+++ b/libcacard/cac.c
@@ -115,6 +115,7 @@ cac_applet_pki_process_apdu(VCard *card, VCardAPDU *apdu,
     VCardAppletPrivate *applet_private;
     int size, next;
     unsigned char *sign_buffer;
+    bool retain_sign_buffer = FALSE;
     vcard_7816_status_t status;
     VCardStatus ret = VCARD_FAIL;
 
@@ -178,6 +179,7 @@ cac_applet_pki_process_apdu(VCard *card, VCardAPDU *apdu,
             pki_applet->sign_buffer = sign_buffer;
             pki_applet->sign_buffer_len = size;
             *response = vcard_make_response(VCARD7816_STATUS_SUCCESS);
+            retain_sign_buffer = TRUE;
             break;
         case 0x00:
             /* we now have the whole buffer, do the operation, result will be
@@ -200,9 +202,11 @@ cac_applet_pki_process_apdu(VCard *card, VCardAPDU *apdu,
                                 VCARD7816_STATUS_ERROR_P1_P2_INCORRECT);
             break;
         }
-        g_free(sign_buffer);
-        pki_applet->sign_buffer = NULL;
-        pki_applet->sign_buffer_len = 0;
+        if (!retain_sign_buffer) {
+            g_free(sign_buffer);
+            pki_applet->sign_buffer = NULL;
+            pki_applet->sign_buffer_len = 0;
+        }
         ret = VCARD_DONE;
         break;
     case CAC_READ_BUFFER:
diff --git a/libcacard/vcard.c b/libcacard/vcard.c
index 87ad5166a..d140a8ed1 100644
--- a/libcacard/vcard.c
+++ b/libcacard/vcard.c
@@ -250,6 +250,11 @@ void
 vcard_select_applet(VCard *card, int channel, VCardApplet *applet)
 {
     assert(channel < MAX_CHANNEL);
+
+    /* If using an emulated card, make sure to log out of any already logged in
+     * session. */
+    vcard_emul_logout(card);
+
     card->current_applet[channel] = applet;
     /* reset the applet */
     if (applet && applet->reset_applet) {
diff --git a/libcacard/vcard_emul.h b/libcacard/vcard_emul.h
index 963563f86..f09ee98dc 100644
--- a/libcacard/vcard_emul.h
+++ b/libcacard/vcard_emul.h
@@ -40,6 +40,7 @@ int vcard_emul_get_login_count(VCard *card);
 /* login into the card, return the 7816 status word (sw2 || sw1) */
 vcard_7816_status_t vcard_emul_login(VCard *card, unsigned char *pin,
                                      int pin_len);
+void vcard_emul_logout(VCard *card);
 
 /*
  * key functions
diff --git a/libcacard/vcard_emul_nss.c b/libcacard/vcard_emul_nss.c
index f1bba57c2..950edee06 100644
--- a/libcacard/vcard_emul_nss.c
+++ b/libcacard/vcard_emul_nss.c
@@ -286,10 +286,10 @@ vcard_emul_rsa_op(VCard *card, VCardKey *key,
             }
         }
         if ((i < buffer_size) && (buffer[i] == 0)) {
-            /* yes, we have a properly formated PKCS #1 signature */
+            /* yes, we have a properly formatted PKCS #1 signature */
             /*
              * NOTE: even if we accidentally got an encrypt buffer, which
-             * through shear luck started with 00, 01, ff, 00, it won't matter
+             * through sheer luck started with 00, 01, ff, 00, it won't matter
              * because the resulting Sign operation will effectively decrypt
              * the real buffer.
              */
@@ -401,7 +401,7 @@ vcard_emul_login(VCard *card, unsigned char *pin, int pin_len)
 }
 
 void
-vcard_emul_reset(VCard *card, VCardPower power)
+vcard_emul_logout(VCard *card)
 {
     PK11SlotInfo *slot;
 
@@ -409,16 +409,24 @@ vcard_emul_reset(VCard *card, VCardPower power)
         return;
     }
 
+    slot = vcard_emul_card_get_slot(card);
+    if (PK11_IsLoggedIn(slot, NULL)) {
+        PK11_Logout(slot); /* NOTE: ignoring SECStatus return value */
+    }
+}
+
+void
+vcard_emul_reset(VCard *card, VCardPower power)
+{
     /*
      * if we reset the card (either power on or power off), we lose our login
      * state
      */
+    vcard_emul_logout(card);
+
     /* TODO: we may also need to send insertion/removal events? */
-    slot = vcard_emul_card_get_slot(card);
-    PK11_Logout(slot); /* NOTE: ignoring SECStatus return value */
 }
 
-
 static VReader *
 vcard_emul_find_vreader_from_slot(PK11SlotInfo *slot)
 {
diff --git a/libcacard/vscclient.c b/libcacard/vscclient.c
index 80111df00..fa6041de9 100644
--- a/libcacard/vscclient.c
+++ b/libcacard/vscclient.c
@@ -597,7 +597,7 @@ connect_to_qemu(
     const char *port
 ) {
     struct addrinfo hints;
-    struct addrinfo *server;
+    struct addrinfo *server = NULL;
     int ret, sock;
 
     sock = socket(AF_INET, SOCK_STREAM, 0);
@@ -629,9 +629,14 @@ connect_to_qemu(
     if (verbose) {
         printf("Connected (sizeof Header=%zd)!\n", sizeof(VSCMsgHeader));
     }
+
+    freeaddrinfo(server);
     return sock;
 
 cleanup_socket:
+    if (server) {
+        freeaddrinfo(server);
+    }
     closesocket(sock);
     return -1;
 }
diff --git a/libdecnumber/decNumber.c b/libdecnumber/decNumber.c
index a30632f94..58211e7af 100644
--- a/libdecnumber/decNumber.c
+++ b/libdecnumber/decNumber.c
@@ -5275,8 +5275,8 @@ static decNumber * decMultiplyOp(decNumber *res, const decNumber *lhs,
 /* 4. The working precisions for the static buffers are twice the     */
 /*    obvious size to allow for calls from decNumberPower.	      */
 /* ------------------------------------------------------------------ */
-decNumber * decExpOp(decNumber *res, const decNumber *rhs,
-			 decContext *set, uInt *status) {
+static decNumber *decExpOp(decNumber *res, const decNumber *rhs,
+                           decContext *set, uInt *status) {
   uInt ignore=0;		   /* working status */
   Int h;			   /* adjusted exponent for 0.xxxx */
   Int p;			   /* working precision */
@@ -5563,7 +5563,8 @@ decNumber * decExpOp(decNumber *res, const decNumber *rhs,
 /*	     where x is truncated (NB) into the range 10 through 99,  */
 /*	     and then c = k>>2 and e = k&3.			      */
 /* ------------------------------------------------------------------ */
-const uShort LNnn[90]={9016,  8652,  8316,  8008,  7724,  7456,	 7208,
+static const uShort LNnn[90] = {
+  9016,  8652,  8316,  8008,  7724,  7456,  7208,
   6972,	 6748,	6540,  6340,  6148,  5968,  5792,  5628,  5464,	 5312,
   5164,	 5020,	4884,  4748,  4620,  4496,  4376,  4256,  4144,	 4032,
  39233, 38181, 37157, 36157, 35181, 34229, 33297, 32389, 31501, 30629,
@@ -5635,8 +5636,8 @@ const uShort LNnn[90]={9016,  8652,  8316,  8008,  7724,  7456,	 7208,
 /* 5. The static buffers are larger than might be expected to allow   */
 /*    for calls from decNumberPower.				      */
 /* ------------------------------------------------------------------ */
-decNumber * decLnOp(decNumber *res, const decNumber *rhs,
-		    decContext *set, uInt *status) {
+static decNumber *decLnOp(decNumber *res, const decNumber *rhs,
+                          decContext *set, uInt *status) {
   uInt ignore=0;		   /* working status accumulator */
   uInt needbytes;		   /* for space calculations */
   Int residue;			   /* rounding residue */
@@ -6052,9 +6053,9 @@ static decNumber * decQuantizeOp(decNumber *res, const decNumber *lhs,
 /* The emphasis here is on speed for common cases, and avoiding	      */
 /* coefficient comparison if possible.				      */
 /* ------------------------------------------------------------------ */
-decNumber * decCompareOp(decNumber *res, const decNumber *lhs,
-			 const decNumber *rhs, decContext *set,
-			 Flag op, uInt *status) {
+static decNumber *decCompareOp(decNumber *res, const decNumber *lhs,
+                               const decNumber *rhs, decContext *set,
+                               Flag op, uInt *status) {
   #if DECSUBSET
   decNumber *alloclhs=NULL;	   /* non-NULL if rounded lhs allocated */
   decNumber *allocrhs=NULL;	   /* .., rhs */
@@ -6086,11 +6087,11 @@ decNumber * decCompareOp(decNumber *res, const decNumber *lhs,
 
     /* If total ordering then handle differing signs 'up front' */
     if (op==COMPTOTAL) {		/* total ordering */
-      if (decNumberIsNegative(lhs) & !decNumberIsNegative(rhs)) {
+      if (decNumberIsNegative(lhs) && !decNumberIsNegative(rhs)) {
 	result=-1;
 	break;
 	}
-      if (!decNumberIsNegative(lhs) & decNumberIsNegative(rhs)) {
+      if (!decNumberIsNegative(lhs) && decNumberIsNegative(rhs)) {
 	result=+1;
 	break;
 	}
diff --git a/linux-headers/asm-arm/kvm.h b/linux-headers/asm-arm/kvm.h
index e6ebdd347..09ee408c1 100644
--- a/linux-headers/asm-arm/kvm.h
+++ b/linux-headers/asm-arm/kvm.h
@@ -25,6 +25,7 @@
 
 #define __KVM_HAVE_GUEST_DEBUG
 #define __KVM_HAVE_IRQ_LINE
+#define __KVM_HAVE_READONLY_MEM
 
 #define KVM_REG_SIZE(id)						\
 	(1U << (((id) & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT))
@@ -173,6 +174,7 @@ struct kvm_arch_memory_slot {
 #define   KVM_DEV_ARM_VGIC_CPUID_MASK	(0xffULL << KVM_DEV_ARM_VGIC_CPUID_SHIFT)
 #define   KVM_DEV_ARM_VGIC_OFFSET_SHIFT	0
 #define   KVM_DEV_ARM_VGIC_OFFSET_MASK	(0xffffffffULL << KVM_DEV_ARM_VGIC_OFFSET_SHIFT)
+#define KVM_DEV_ARM_VGIC_GRP_NR_IRQS	3
 
 /* KVM_IRQ_LINE irq field index values */
 #define KVM_ARM_IRQ_TYPE_SHIFT		24
diff --git a/linux-headers/asm-arm64/kvm.h b/linux-headers/asm-arm64/kvm.h
index e633ff8cd..8e38878c8 100644
--- a/linux-headers/asm-arm64/kvm.h
+++ b/linux-headers/asm-arm64/kvm.h
@@ -37,6 +37,7 @@
 
 #define __KVM_HAVE_GUEST_DEBUG
 #define __KVM_HAVE_IRQ_LINE
+#define __KVM_HAVE_READONLY_MEM
 
 #define KVM_REG_SIZE(id)						\
 	(1U << (((id) & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT))
@@ -159,6 +160,7 @@ struct kvm_arch_memory_slot {
 #define   KVM_DEV_ARM_VGIC_CPUID_MASK	(0xffULL << KVM_DEV_ARM_VGIC_CPUID_SHIFT)
 #define   KVM_DEV_ARM_VGIC_OFFSET_SHIFT	0
 #define   KVM_DEV_ARM_VGIC_OFFSET_MASK	(0xffffffffULL << KVM_DEV_ARM_VGIC_OFFSET_SHIFT)
+#define KVM_DEV_ARM_VGIC_GRP_NR_IRQS	3
 
 /* KVM_IRQ_LINE irq field index values */
 #define KVM_ARM_IRQ_TYPE_SHIFT		24
diff --git a/linux-headers/asm-mips/kvm_para.h b/linux-headers/asm-mips/kvm_para.h
index 14fab8f0b..dbb2464f3 100644
--- a/linux-headers/asm-mips/kvm_para.h
+++ b/linux-headers/asm-mips/kvm_para.h
@@ -1 +1,5 @@
-#include <asm-generic/kvm_para.h>
+#ifndef _ASM_MIPS_KVM_PARA_H
+#define _ASM_MIPS_KVM_PARA_H
+
+
+#endif /* _ASM_MIPS_KVM_PARA_H */
diff --git a/linux-headers/asm-powerpc/kvm.h b/linux-headers/asm-powerpc/kvm.h
index 2bc4a9409..ab4d4732c 100644
--- a/linux-headers/asm-powerpc/kvm.h
+++ b/linux-headers/asm-powerpc/kvm.h
@@ -476,6 +476,11 @@ struct kvm_get_htab_header {
 
 /* FP and vector status/control registers */
 #define KVM_REG_PPC_FPSCR	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x80)
+/*
+ * VSCR register is documented as a 32-bit register in the ISA, but it can
+ * only be accesses via a vector register. Expose VSCR as a 32-bit register
+ * even though the kernel represents it as a 128-bit vector.
+ */
 #define KVM_REG_PPC_VSCR	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x81)
 
 /* Virtual processor areas */
@@ -548,6 +553,7 @@ struct kvm_get_htab_header {
 
 #define KVM_REG_PPC_VRSAVE	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0xb4)
 #define KVM_REG_PPC_LPCR	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0xb5)
+#define KVM_REG_PPC_LPCR_64	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xb5)
 #define KVM_REG_PPC_PPR		(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xb6)
 
 /* Architecture compatibility level */
@@ -555,6 +561,8 @@ struct kvm_get_htab_header {
 
 #define KVM_REG_PPC_DABRX	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0xb8)
 #define KVM_REG_PPC_WORT	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xb9)
+#define KVM_REG_PPC_SPRG9	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xba)
+#define KVM_REG_PPC_DBSR	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0xbb)
 
 /* Transactional Memory checkpointed state:
  * This is all GPRs, all VSX regs and a subset of SPRs
diff --git a/linux-headers/asm-s390/kvm.h b/linux-headers/asm-s390/kvm.h
index 98bedf3c1..d36b2fa10 100644
--- a/linux-headers/asm-s390/kvm.h
+++ b/linux-headers/asm-s390/kvm.h
@@ -111,12 +111,22 @@ struct kvm_guest_debug_arch {
 #define KVM_SYNC_GPRS   (1UL << 1)
 #define KVM_SYNC_ACRS   (1UL << 2)
 #define KVM_SYNC_CRS    (1UL << 3)
+#define KVM_SYNC_ARCH0  (1UL << 4)
+#define KVM_SYNC_PFAULT (1UL << 5)
 /* definition of registers in kvm_run */
 struct kvm_sync_regs {
 	__u64 prefix;	/* prefix register */
 	__u64 gprs[16];	/* general purpose registers */
 	__u32 acrs[16];	/* access registers */
 	__u64 crs[16];	/* control registers */
+	__u64 todpr;	/* tod programmable register [ARCH0] */
+	__u64 cputm;	/* cpu timer [ARCH0] */
+	__u64 ckc;	/* clock comparator [ARCH0] */
+	__u64 pp;	/* program parameter [ARCH0] */
+	__u64 gbea;	/* guest breaking-event address [ARCH0] */
+	__u64 pft;	/* pfault token [PFAULT] */
+	__u64 pfs;	/* pfault select [PFAULT] */
+	__u64 pfc;	/* pfault compare [PFAULT] */
 };
 
 #define KVM_REG_S390_TODPR	(KVM_REG_S390 | KVM_REG_SIZE_U32 | 0x1)
diff --git a/linux-headers/asm-x86/kvm.h b/linux-headers/asm-x86/kvm.h
index d3a87780c..d7dcef58a 100644
--- a/linux-headers/asm-x86/kvm.h
+++ b/linux-headers/asm-x86/kvm.h
@@ -23,7 +23,10 @@
 #define GP_VECTOR 13
 #define PF_VECTOR 14
 #define MF_VECTOR 16
+#define AC_VECTOR 17
 #define MC_VECTOR 18
+#define XM_VECTOR 19
+#define VE_VECTOR 20
 
 /* Select x86 specific features in <linux/kvm.h> */
 #define __KVM_HAVE_PIT
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index f5d2c38de..12045a11c 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -162,7 +162,7 @@ struct kvm_pit_config {
 #define KVM_EXIT_TPR_ACCESS       12
 #define KVM_EXIT_S390_SIEIC       13
 #define KVM_EXIT_S390_RESET       14
-#define KVM_EXIT_DCR              15
+#define KVM_EXIT_DCR              15 /* deprecated */
 #define KVM_EXIT_NMI              16
 #define KVM_EXIT_INTERNAL_ERROR   17
 #define KVM_EXIT_OSI              18
@@ -268,7 +268,7 @@ struct kvm_run {
 			__u64 trans_exc_code;
 			__u32 pgm_code;
 		} s390_ucontrol;
-		/* KVM_EXIT_DCR */
+		/* KVM_EXIT_DCR (deprecated) */
 		struct {
 			__u32 dcrn;
 			__u32 data;
@@ -399,13 +399,18 @@ struct kvm_vapic_addr {
 	__u64 vapic_addr;
 };
 
-/* for KVM_SET_MPSTATE */
+/* for KVM_SET_MP_STATE */
 
+/* not all states are valid on all architectures */
 #define KVM_MP_STATE_RUNNABLE          0
 #define KVM_MP_STATE_UNINITIALIZED     1
 #define KVM_MP_STATE_INIT_RECEIVED     2
 #define KVM_MP_STATE_HALTED            3
 #define KVM_MP_STATE_SIPI_RECEIVED     4
+#define KVM_MP_STATE_STOPPED           5
+#define KVM_MP_STATE_CHECK_STOP        6
+#define KVM_MP_STATE_OPERATING         7
+#define KVM_MP_STATE_LOAD              8
 
 struct kvm_mp_state {
 	__u32 mp_state;
@@ -649,9 +654,7 @@ struct kvm_ppc_smmu_info {
 #endif
 /* Bug in KVM_SET_USER_MEMORY_REGION fixed: */
 #define KVM_CAP_DESTROY_MEMORY_REGION_WORKS 21
-#ifdef __KVM_HAVE_USER_NMI
 #define KVM_CAP_USER_NMI 22
-#endif
 #ifdef __KVM_HAVE_GUEST_DEBUG
 #define KVM_CAP_SET_GUEST_DEBUG 23
 #endif
@@ -733,9 +736,7 @@ struct kvm_ppc_smmu_info {
 #define KVM_CAP_PPC_GET_SMMU_INFO 78
 #define KVM_CAP_S390_COW 79
 #define KVM_CAP_PPC_ALLOC_HTAB 80
-#ifdef __KVM_HAVE_READONLY_MEM
 #define KVM_CAP_READONLY_MEM 81
-#endif
 #define KVM_CAP_IRQFD_RESAMPLE 82
 #define KVM_CAP_PPC_BOOKE_WATCHDOG 83
 #define KVM_CAP_PPC_HTAB_FD 84
@@ -758,6 +759,8 @@ struct kvm_ppc_smmu_info {
 #define KVM_CAP_VM_ATTRIBUTES 101
 #define KVM_CAP_ARM_PSCI_0_2 102
 #define KVM_CAP_PPC_FIXUP_HCALL 103
+#define KVM_CAP_PPC_ENABLE_HCALL 104
+#define KVM_CAP_CHECK_EXTENSION_VM 105
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -940,15 +943,25 @@ struct kvm_device_attr {
 	__u64	addr;		/* userspace address of attr data */
 };
 
-#define KVM_DEV_TYPE_FSL_MPIC_20	1
-#define KVM_DEV_TYPE_FSL_MPIC_42	2
-#define KVM_DEV_TYPE_XICS		3
-#define KVM_DEV_TYPE_VFIO		4
 #define  KVM_DEV_VFIO_GROUP			1
 #define   KVM_DEV_VFIO_GROUP_ADD			1
 #define   KVM_DEV_VFIO_GROUP_DEL			2
-#define KVM_DEV_TYPE_ARM_VGIC_V2	5
-#define KVM_DEV_TYPE_FLIC		6
+
+enum kvm_device_type {
+	KVM_DEV_TYPE_FSL_MPIC_20	= 1,
+#define KVM_DEV_TYPE_FSL_MPIC_20	KVM_DEV_TYPE_FSL_MPIC_20
+	KVM_DEV_TYPE_FSL_MPIC_42,
+#define KVM_DEV_TYPE_FSL_MPIC_42	KVM_DEV_TYPE_FSL_MPIC_42
+	KVM_DEV_TYPE_XICS,
+#define KVM_DEV_TYPE_XICS		KVM_DEV_TYPE_XICS
+	KVM_DEV_TYPE_VFIO,
+#define KVM_DEV_TYPE_VFIO		KVM_DEV_TYPE_VFIO
+	KVM_DEV_TYPE_ARM_VGIC_V2,
+#define KVM_DEV_TYPE_ARM_VGIC_V2	KVM_DEV_TYPE_ARM_VGIC_V2
+	KVM_DEV_TYPE_FLIC,
+#define KVM_DEV_TYPE_FLIC		KVM_DEV_TYPE_FLIC
+	KVM_DEV_TYPE_MAX,
+};
 
 /*
  * ioctls for VM fds
@@ -1086,7 +1099,7 @@ struct kvm_s390_ucas_mapping {
 #define KVM_S390_INITIAL_RESET    _IO(KVMIO,   0x97)
 #define KVM_GET_MP_STATE          _IOR(KVMIO,  0x98, struct kvm_mp_state)
 #define KVM_SET_MP_STATE          _IOW(KVMIO,  0x99, struct kvm_mp_state)
-/* Available with KVM_CAP_NMI */
+/* Available with KVM_CAP_USER_NMI */
 #define KVM_NMI                   _IO(KVMIO,   0x9a)
 /* Available with KVM_CAP_SET_GUEST_DEBUG */
 #define KVM_SET_GUEST_DEBUG       _IOW(KVMIO,  0x9b, struct kvm_guest_debug)
diff --git a/linux-headers/linux/kvm_para.h b/linux-headers/linux/kvm_para.h
index 2dff7838b..e61661edf 100644
--- a/linux-headers/linux/kvm_para.h
+++ b/linux-headers/linux/kvm_para.h
@@ -20,6 +20,9 @@
 #define KVM_HC_FEATURES			3
 #define KVM_HC_PPC_MAP_MAGIC_PAGE	4
 #define KVM_HC_KICK_CPU			5
+#define KVM_HC_MIPS_GET_CLOCK_FREQ	6
+#define KVM_HC_MIPS_EXIT_VM		7
+#define KVM_HC_MIPS_CONSOLE_OUTPUT	8
 
 /*
  * hypercalls use architecture specific
diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h
index 26c218e69..0f21aa626 100644
--- a/linux-headers/linux/vfio.h
+++ b/linux-headers/linux/vfio.h
@@ -30,6 +30,12 @@
  */
 #define VFIO_DMA_CC_IOMMU		4
 
+/* Check if EEH is supported */
+#define VFIO_EEH			5
+
+/* Two-stage IOMMU */
+#define VFIO_TYPE1_NESTING_IOMMU	6	/* Implies v2 */
+
 /*
  * The IOCTL interface is designed for extensibility by embedding the
  * structure length (argsz) and flags into structures passed between
@@ -455,6 +461,37 @@ struct vfio_iommu_spapr_tce_info {
 
 #define VFIO_IOMMU_SPAPR_TCE_GET_INFO	_IO(VFIO_TYPE, VFIO_BASE + 12)
 
+/*
+ * EEH PE operation struct provides ways to:
+ * - enable/disable EEH functionality;
+ * - unfreeze IO/DMA for frozen PE;
+ * - read PE state;
+ * - reset PE;
+ * - configure PE.
+ */
+struct vfio_eeh_pe_op {
+	__u32 argsz;
+	__u32 flags;
+	__u32 op;
+};
+
+#define VFIO_EEH_PE_DISABLE		0	/* Disable EEH functionality */
+#define VFIO_EEH_PE_ENABLE		1	/* Enable EEH functionality  */
+#define VFIO_EEH_PE_UNFREEZE_IO		2	/* Enable IO for frozen PE   */
+#define VFIO_EEH_PE_UNFREEZE_DMA	3	/* Enable DMA for frozen PE  */
+#define VFIO_EEH_PE_GET_STATE		4	/* PE state retrieval        */
+#define  VFIO_EEH_PE_STATE_NORMAL	0	/* PE in functional state    */
+#define  VFIO_EEH_PE_STATE_RESET	1	/* PE reset in progress      */
+#define  VFIO_EEH_PE_STATE_STOPPED	2	/* Stopped DMA and IO        */
+#define  VFIO_EEH_PE_STATE_STOPPED_DMA	4	/* Stopped DMA only          */
+#define  VFIO_EEH_PE_STATE_UNAVAIL	5	/* State unavailable         */
+#define VFIO_EEH_PE_RESET_DEACTIVATE	5	/* Deassert PE reset         */
+#define VFIO_EEH_PE_RESET_HOT		6	/* Assert hot reset          */
+#define VFIO_EEH_PE_RESET_FUNDAMENTAL	7	/* Assert fundamental reset  */
+#define VFIO_EEH_PE_CONFIGURE		8	/* PE configuration          */
+
+#define VFIO_EEH_PE_OP			_IO(VFIO_TYPE, VFIO_BASE + 21)
+
 /* ***************************************************************** */
 
 #endif /* VFIO_H */
diff --git a/linux-user/aarch64/syscall.h b/linux-user/aarch64/syscall.h
index 18f44a8a4..dc72a15c5 100644
--- a/linux-user/aarch64/syscall.h
+++ b/linux-user/aarch64/syscall.h
@@ -8,3 +8,6 @@ struct target_pt_regs {
 #define UNAME_MACHINE "aarch64"
 #define UNAME_MINIMUM_RELEASE "3.8.0"
 #define TARGET_CLONE_BACKWARDS
+#define TARGET_MINSIGSTKSZ       2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
diff --git a/linux-user/alpha/syscall.h b/linux-user/alpha/syscall.h
index ed13d9a71..245cff254 100644
--- a/linux-user/alpha/syscall.h
+++ b/linux-user/alpha/syscall.h
@@ -252,3 +252,6 @@ struct target_pt_regs {
 #define TARGET_UAC_NOPRINT		1
 #define TARGET_UAC_NOFIX		2
 #define TARGET_UAC_SIGBUS		4
+#define TARGET_MINSIGSTKSZ              4096
+#define TARGET_MLOCKALL_MCL_CURRENT     0x2000
+#define TARGET_MLOCKALL_MCL_FUTURE      0x4000
diff --git a/linux-user/arm/syscall.h b/linux-user/arm/syscall.h
index e0d2cc3e5..3844a9611 100644
--- a/linux-user/arm/syscall.h
+++ b/linux-user/arm/syscall.h
@@ -44,3 +44,7 @@ struct target_pt_regs {
 #define UNAME_MINIMUM_RELEASE "2.6.32"
 
 #define TARGET_CLONE_BACKWARDS
+
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
diff --git a/linux-user/arm/syscall_nr.h b/linux-user/arm/syscall_nr.h
index bef847cfa..7d7be7cfe 100644
--- a/linux-user/arm/syscall_nr.h
+++ b/linux-user/arm/syscall_nr.h
@@ -350,7 +350,7 @@
 #define TARGET_NR_vmsplice			(343)
 #define TARGET_NR_move_pages			(344)
 #define TARGET_NR_getcpu			(345)
-					/* 346 for epoll_pwait */
+#define TARGET_NR_epoll_pwait                   (346)
 #define TARGET_NR_kexec_load			(347)
 #define TARGET_NR_utimensat			(348)
 #define TARGET_NR_signalfd			(349)
diff --git a/linux-user/cris/syscall.h b/linux-user/cris/syscall.h
index f5783c055..2957b0d6a 100644
--- a/linux-user/cris/syscall.h
+++ b/linux-user/cris/syscall.h
@@ -39,5 +39,8 @@ struct target_pt_regs {
 };
 
 #define TARGET_CLONE_BACKWARDS2
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
 
 #endif
diff --git a/linux-user/elfload.c b/linux-user/elfload.c
index 60777fecf..e2596a420 100644
--- a/linux-user/elfload.c
+++ b/linux-user/elfload.c
@@ -824,8 +824,6 @@ static uint32_t get_elf_hwcap2(void)
         NEW_AUX_ENT(AT_IGNOREPPC, AT_IGNOREPPC);        \
     } while (0)
 
-static inline uint32_t get_ppc64_abi(struct image_info *infop);
-
 static inline void init_thread(struct target_pt_regs *_regs, struct image_info *infop)
 {
     _regs->gpr[1] = infop->start_stack;
@@ -1205,13 +1203,6 @@ static inline void init_thread(struct target_pt_regs *regs, struct image_info *i
 
 #include "elf.h"
 
-#ifdef TARGET_PPC
-static inline uint32_t get_ppc64_abi(struct image_info *infop)
-{
-  return infop->elf_flags & EF_PPC64_ABI;
-}
-#endif
-
 struct exec
 {
     unsigned int a_info;   /* Use macros N_MAGIC, etc for access */
@@ -1548,7 +1539,6 @@ static abi_ulong create_elf_tables(abi_ulong p, int argc, int envc,
      * Generate 16 random bytes for userspace PRNG seeding (not
      * cryptically secure but it's not the aim of QEMU).
      */
-    srand((unsigned int) time(NULL));
     for (i = 0; i < 16; i++) {
         k_rand_bytes[i] = rand();
     }
@@ -1830,11 +1820,11 @@ static void load_elf_image(const char *image_name, int image_fd,
     loaddr = -1, hiaddr = 0;
     for (i = 0; i < ehdr->e_phnum; ++i) {
         if (phdr[i].p_type == PT_LOAD) {
-            abi_ulong a = phdr[i].p_vaddr;
+            abi_ulong a = phdr[i].p_vaddr - phdr[i].p_offset;
             if (a < loaddr) {
                 loaddr = a;
             }
-            a += phdr[i].p_memsz;
+            a = phdr[i].p_vaddr + phdr[i].p_memsz;
             if (a > hiaddr) {
                 hiaddr = a;
             }
@@ -2364,9 +2354,9 @@ struct elf_note_info {
 };
 
 struct vm_area_struct {
-    abi_ulong   vma_start;  /* start vaddr of memory region */
-    abi_ulong   vma_end;    /* end vaddr of memory region */
-    abi_ulong   vma_flags;  /* protection etc. flags for the region */
+    target_ulong   vma_start;  /* start vaddr of memory region */
+    target_ulong   vma_end;    /* end vaddr of memory region */
+    abi_ulong      vma_flags;  /* protection etc. flags for the region */
     QTAILQ_ENTRY(vm_area_struct) vma_link;
 };
 
@@ -2377,13 +2367,13 @@ struct mm_struct {
 
 static struct mm_struct *vma_init(void);
 static void vma_delete(struct mm_struct *);
-static int vma_add_mapping(struct mm_struct *, abi_ulong,
-                           abi_ulong, abi_ulong);
+static int vma_add_mapping(struct mm_struct *, target_ulong,
+                           target_ulong, abi_ulong);
 static int vma_get_mapping_count(const struct mm_struct *);
 static struct vm_area_struct *vma_first(const struct mm_struct *);
 static struct vm_area_struct *vma_next(struct vm_area_struct *);
 static abi_ulong vma_dump_size(const struct vm_area_struct *);
-static int vma_walker(void *priv, abi_ulong start, abi_ulong end,
+static int vma_walker(void *priv, target_ulong start, target_ulong end,
                       unsigned long flags);
 
 static void fill_elf_header(struct elfhdr *, int, uint16_t, uint32_t);
@@ -2475,8 +2465,8 @@ static void vma_delete(struct mm_struct *mm)
     g_free(mm);
 }
 
-static int vma_add_mapping(struct mm_struct *mm, abi_ulong start,
-                           abi_ulong end, abi_ulong flags)
+static int vma_add_mapping(struct mm_struct *mm, target_ulong start,
+                           target_ulong end, abi_ulong flags)
 {
     struct vm_area_struct *vma;
 
@@ -2544,7 +2534,7 @@ static abi_ulong vma_dump_size(const struct vm_area_struct *vma)
     return (vma->vma_end - vma->vma_start);
 }
 
-static int vma_walker(void *priv, abi_ulong start, abi_ulong end,
+static int vma_walker(void *priv, target_ulong start, target_ulong end,
                       unsigned long flags)
 {
     struct mm_struct *mm = (struct mm_struct *)priv;
diff --git a/linux-user/i386/syscall.h b/linux-user/i386/syscall.h
index 9bfc1ad8f..906aaac0b 100644
--- a/linux-user/i386/syscall.h
+++ b/linux-user/i386/syscall.h
@@ -147,3 +147,6 @@ struct target_vm86plus_struct {
 #define UNAME_MINIMUM_RELEASE "2.6.32"
 
 #define TARGET_CLONE_BACKWARDS
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
diff --git a/linux-user/ioctls.h b/linux-user/ioctls.h
index 609b27cf0..e67265510 100644
--- a/linux-user/ioctls.h
+++ b/linux-user/ioctls.h
@@ -78,7 +78,8 @@
      IOCTL(BLKRAGET, IOC_R, MK_PTR(TYPE_LONG))
      IOCTL(BLKSSZGET, IOC_R, MK_PTR(TYPE_LONG))
      IOCTL(BLKBSZGET, IOC_R, MK_PTR(TYPE_INT))
-     IOCTL(BLKPG, IOC_W, MK_PTR(MK_STRUCT(STRUCT_blkpg_ioctl_arg)))
+     IOCTL_SPECIAL(BLKPG, IOC_W, do_ioctl_blkpg,
+                   MK_PTR(MK_STRUCT(STRUCT_blkpg_ioctl_arg)))
 #ifdef FIBMAP
      IOCTL(FIBMAP, IOC_W | IOC_R, MK_PTR(TYPE_LONG))
 #endif
diff --git a/linux-user/m68k/syscall.h b/linux-user/m68k/syscall.h
index 889eaf732..9218493a4 100644
--- a/linux-user/m68k/syscall.h
+++ b/linux-user/m68k/syscall.h
@@ -18,4 +18,8 @@ struct target_pt_regs {
 #define UNAME_MACHINE "m68k"
 #define UNAME_MINIMUM_RELEASE "2.6.32"
 
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
+
 void do_m68k_simcall(CPUM68KState *, int);
diff --git a/linux-user/main.c b/linux-user/main.c
index b453a3985..5c14c1e87 100644
--- a/linux-user/main.c
+++ b/linux-user/main.c
@@ -309,7 +309,6 @@ void cpu_loop(CPUX86State *env)
                                           env->regs[8],
                                           env->regs[9],
                                           0, 0);
-            env->eip = env->exception_next_eip;
             break;
 #endif
         case EXCP0B_NOSEG:
@@ -1007,7 +1006,6 @@ void cpu_loop(CPUARMState *env)
     CPUState *cs = CPU(arm_env_get_cpu(env));
     int trapnr, sig;
     target_siginfo_t info;
-    uint32_t addr;
 
     for (;;) {
         cpu_exec_start(cs);
@@ -1043,12 +1041,11 @@ void cpu_loop(CPUARMState *env)
             /* fall through for segv */
         case EXCP_PREFETCH_ABORT:
         case EXCP_DATA_ABORT:
-            addr = env->exception.vaddress;
             info.si_signo = SIGSEGV;
             info.si_errno = 0;
             /* XXX: check env->error_code */
             info.si_code = TARGET_SEGV_MAPERR;
-            info._sifields._sigfault._addr = addr;
+            info._sifields._sigfault._addr = env->exception.vaddress;
             queue_signal(env, info.si_signo, &info);
             break;
         case EXCP_DEBUG:
@@ -3459,8 +3456,7 @@ CPUArchState *cpu_copy(CPUArchState *env)
         cpu_breakpoint_insert(new_cpu, bp->pc, bp->flags, NULL);
     }
     QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
-        cpu_watchpoint_insert(new_cpu, wp->vaddr, (~wp->len_mask) + 1,
-                              wp->flags, NULL);
+        cpu_watchpoint_insert(new_cpu, wp->vaddr, wp->len, wp->flags, NULL);
     }
 #endif
 
@@ -3548,6 +3544,17 @@ static void handle_arg_pagesize(const char *arg)
     }
 }
 
+static void handle_arg_randseed(const char *arg)
+{
+    unsigned long long seed;
+
+    if (parse_uint_full(arg, &seed, 0) != 0 || seed > UINT_MAX) {
+        fprintf(stderr, "Invalid seed number: %s\n", arg);
+        exit(1);
+    }
+    srand(seed);
+}
+
 static void handle_arg_gdb(const char *arg)
 {
     gdbstub_port = atoi(arg);
@@ -3676,6 +3683,8 @@ static const struct qemu_argument arg_table[] = {
      "",           "run in singlestep mode"},
     {"strace",     "QEMU_STRACE",      false, handle_arg_strace,
      "",           "log system calls"},
+    {"seed",       "QEMU_RAND_SEED",   true,  handle_arg_randseed,
+     "",           "Seed for pseudo-random number generator"},
     {"version",    "QEMU_VERSION",     false, handle_arg_version,
      "",           "display version information and exit"},
     {NULL, NULL, false, NULL, NULL, NULL}
@@ -3858,6 +3867,8 @@ int main(int argc, char **argv, char **envp)
     cpudef_setup(); /* parse cpu definitions in target config file (TBD) */
 #endif
 
+    srand(time(NULL));
+
     optind = parse_args(argc, argv);
 
     /* Zero out regs */
@@ -3928,6 +3939,10 @@ int main(int argc, char **argv, char **envp)
         do_strace = 1;
     }
 
+    if (getenv("QEMU_RAND_SEED")) {
+        handle_arg_randseed(getenv("QEMU_RAND_SEED"));
+    }
+
     target_environ = envlist_to_environ(envlist, NULL);
     envlist_free(envlist);
 
diff --git a/linux-user/microblaze/syscall.h b/linux-user/microblaze/syscall.h
index 5b5f6b447..3c1ed27c0 100644
--- a/linux-user/microblaze/syscall.h
+++ b/linux-user/microblaze/syscall.h
@@ -49,5 +49,8 @@ struct target_pt_regs {
 };
 
 #define TARGET_CLONE_BACKWARDS
+#define TARGET_MINSIGSTKSZ      2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
 
 #endif
diff --git a/linux-user/mips/syscall.h b/linux-user/mips/syscall.h
index 5bc56962a..35ca23b16 100644
--- a/linux-user/mips/syscall.h
+++ b/linux-user/mips/syscall.h
@@ -228,3 +228,6 @@ struct target_pt_regs {
 #define UNAME_MINIMUM_RELEASE "2.6.32"
 
 #define TARGET_CLONE_BACKWARDS
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
diff --git a/linux-user/mips64/syscall.h b/linux-user/mips64/syscall.h
index a7f5a5802..6733107dd 100644
--- a/linux-user/mips64/syscall.h
+++ b/linux-user/mips64/syscall.h
@@ -225,3 +225,6 @@ struct target_pt_regs {
 #define UNAME_MINIMUM_RELEASE "2.6.32"
 
 #define TARGET_CLONE_BACKWARDS
+#define TARGET_MINSIGSTKSZ      2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
diff --git a/linux-user/openrisc/syscall.h b/linux-user/openrisc/syscall.h
index c3b36da83..8ac03656d 100644
--- a/linux-user/openrisc/syscall.h
+++ b/linux-user/openrisc/syscall.h
@@ -23,3 +23,7 @@ struct target_pt_regs {
 
 #define UNAME_MACHINE "openrisc"
 #define UNAME_MINIMUM_RELEASE "2.6.32"
+
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
diff --git a/linux-user/ppc/syscall.h b/linux-user/ppc/syscall.h
index db92bbee1..0daf5cd2d 100644
--- a/linux-user/ppc/syscall.h
+++ b/linux-user/ppc/syscall.h
@@ -69,3 +69,7 @@ struct target_revectored_struct {
 #define UNAME_MINIMUM_RELEASE "2.6.32"
 
 #define TARGET_CLONE_BACKWARDS
+
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 0x2000
+#define TARGET_MLOCKALL_MCL_FUTURE  0x4000
diff --git a/linux-user/ppc/target_cpu.h b/linux-user/ppc/target_cpu.h
index 9cc0c3ba9..26f4ba297 100644
--- a/linux-user/ppc/target_cpu.h
+++ b/linux-user/ppc/target_cpu.h
@@ -38,4 +38,14 @@ static inline void cpu_set_tls(CPUPPCState *env, target_ulong newtls)
 #endif
 }
 
+#ifndef EF_PPC64_ABI
+#define EF_PPC64_ABI           0x3
+#endif
+
+static inline uint32_t get_ppc64_abi(struct image_info *infop)
+{
+  return infop->elf_flags & EF_PPC64_ABI;
+}
+
+
 #endif
diff --git a/linux-user/s390x/syscall.h b/linux-user/s390x/syscall.h
index aaad512d4..35f170af2 100644
--- a/linux-user/s390x/syscall.h
+++ b/linux-user/s390x/syscall.h
@@ -24,3 +24,6 @@ struct target_pt_regs {
 #define UNAME_MINIMUM_RELEASE "2.6.32"
 
 #define TARGET_CLONE_BACKWARDS2
+#define TARGET_MINSIGSTKSZ        2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
diff --git a/linux-user/sh4/syscall.h b/linux-user/sh4/syscall.h
index ccd2216e3..7aa4f239c 100644
--- a/linux-user/sh4/syscall.h
+++ b/linux-user/sh4/syscall.h
@@ -11,3 +11,7 @@ struct target_pt_regs {
 
 #define UNAME_MACHINE "sh4"
 #define UNAME_MINIMUM_RELEASE "2.6.32"
+
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
diff --git a/linux-user/signal.c b/linux-user/signal.c
index 1141054be..e11b20887 100644
--- a/linux-user/signal.c
+++ b/linux-user/signal.c
@@ -617,6 +617,15 @@ abi_long do_sigaltstack(abi_ulong uss_addr, abi_ulong uoss_addr, abi_ulong sp)
     {
         struct target_sigaltstack *uss;
         struct target_sigaltstack ss;
+        size_t minstacksize = TARGET_MINSIGSTKSZ;
+
+#if defined(TARGET_PPC64)
+        /* ELF V2 for PPC64 has a 4K minimum stack size for signal handlers */
+        struct image_info *image = ((TaskState *)thread_cpu->opaque)->info;
+        if (get_ppc64_abi(image) > 1) {
+            minstacksize = 4096;
+        }
+#endif
 
 	ret = -TARGET_EFAULT;
         if (!lock_user_struct(VERIFY_READ, uss, uss_addr, 1)) {
@@ -642,8 +651,9 @@ abi_long do_sigaltstack(abi_ulong uss_addr, abi_ulong uoss_addr, abi_ulong sp)
             ss.ss_sp = 0;
 	} else {
             ret = -TARGET_ENOMEM;
-            if (ss.ss_size < MINSIGSTKSZ)
+            if (ss.ss_size < minstacksize) {
                 goto out;
+            }
 	}
 
         target_sigaltstack_used.ss_sp = ss.ss_sp;
@@ -4315,15 +4325,7 @@ badframe:
     return 0;
 }
 
-#elif defined(TARGET_PPC) && !defined(TARGET_PPC64)
-
-/* FIXME: Many of the structures are defined for both PPC and PPC64, but
-   the signal handling is different enough that we haven't implemented
-   support for PPC64 yet.  Hence the restriction above.
-
-   There are various #if'd blocks for code for TARGET_PPC64.  These
-   blocks should go away so that we can successfully run 32-bit and
-   64-bit binaries on a QEMU configured for PPC64.  */
+#elif defined(TARGET_PPC)
 
 /* Size of dummy stack frame allocated when calling signal handler.
    See arch/powerpc/include/asm/ptrace.h.  */
@@ -4333,6 +4335,33 @@ badframe:
 #define SIGNAL_FRAMESIZE 64
 #endif
 
+/* See arch/powerpc/include/asm/ucontext.h.  Only used for 32-bit PPC;
+   on 64-bit PPC, sigcontext and mcontext are one and the same.  */
+struct target_mcontext {
+    target_ulong mc_gregs[48];
+    /* Includes fpscr.  */
+    uint64_t mc_fregs[33];
+    target_ulong mc_pad[2];
+    /* We need to handle Altivec and SPE at the same time, which no
+       kernel needs to do.  Fortunately, the kernel defines this bit to
+       be Altivec-register-large all the time, rather than trying to
+       twiddle it based on the specific platform.  */
+    union {
+        /* SPE vector registers.  One extra for SPEFSCR.  */
+        uint32_t spe[33];
+        /* Altivec vector registers.  The packing of VSCR and VRSAVE
+           varies depending on whether we're PPC64 or not: PPC64 splits
+           them apart; PPC32 stuffs them together.  */
+#if defined(TARGET_PPC64)
+#define QEMU_NVRREG 34
+#else
+#define QEMU_NVRREG 33
+#endif
+        ppc_avr_t altivec[QEMU_NVRREG];
+#undef QEMU_NVRREG
+    } mc_vregs __attribute__((__aligned__(16)));
+};
+
 /* See arch/powerpc/include/asm/sigcontext.h.  */
 struct target_sigcontext {
     target_ulong _unused[4];
@@ -4343,7 +4372,9 @@ struct target_sigcontext {
     target_ulong handler;
     target_ulong oldmask;
     target_ulong regs;      /* struct pt_regs __user * */
-    /* TODO: PPC64 includes extra bits here.  */
+#if defined(TARGET_PPC64)
+    struct target_mcontext mcontext;
+#endif
 };
 
 /* Indices for target_mcontext.mc_gregs, below.
@@ -4398,32 +4429,6 @@ enum {
     TARGET_PT_REGS_COUNT = 44
 };
 
-/* See arch/powerpc/include/asm/ucontext.h.  Only used for 32-bit PPC;
-   on 64-bit PPC, sigcontext and mcontext are one and the same.  */
-struct target_mcontext {
-    target_ulong mc_gregs[48];
-    /* Includes fpscr.  */
-    uint64_t mc_fregs[33];
-    target_ulong mc_pad[2];
-    /* We need to handle Altivec and SPE at the same time, which no
-       kernel needs to do.  Fortunately, the kernel defines this bit to
-       be Altivec-register-large all the time, rather than trying to
-       twiddle it based on the specific platform.  */
-    union {
-        /* SPE vector registers.  One extra for SPEFSCR.  */
-        uint32_t spe[33];
-        /* Altivec vector registers.  The packing of VSCR and VRSAVE
-           varies depending on whether we're PPC64 or not: PPC64 splits
-           them apart; PPC32 stuffs them together.  */
-#if defined(TARGET_PPC64)
-#define QEMU_NVRREG 34
-#else
-#define QEMU_NVRREG 33
-#endif
-        ppc_avr_t altivec[QEMU_NVRREG];
-#undef QEMU_NVRREG
-    } mc_vregs __attribute__((__aligned__(16)));
-};
 
 struct target_ucontext {
     target_ulong tuc_flags;
@@ -4437,7 +4442,7 @@ struct target_ucontext {
     target_sigset_t tuc_sigmask;
 #if defined(TARGET_PPC64)
     target_sigset_t unused[15]; /* Allow for uc_sigmask growth */
-    struct target_sigcontext tuc_mcontext;
+    struct target_sigcontext tuc_sigcontext;
 #else
     int32_t tuc_maskext[30];
     int32_t tuc_pad2[3];
@@ -4452,12 +4457,41 @@ struct target_sigframe {
     int32_t abigap[56];
 };
 
+#if defined(TARGET_PPC64)
+
+#define TARGET_TRAMP_SIZE 6
+
+struct target_rt_sigframe {
+        /* sys_rt_sigreturn requires the ucontext be the first field */
+        struct target_ucontext uc;
+        target_ulong  _unused[2];
+        uint32_t trampoline[TARGET_TRAMP_SIZE];
+        target_ulong pinfo; /* struct siginfo __user * */
+        target_ulong puc; /* void __user * */
+        struct target_siginfo info;
+        /* 64 bit ABI allows for 288 bytes below sp before decrementing it. */
+        char abigap[288];
+} __attribute__((aligned(16)));
+
+#else
+
 struct target_rt_sigframe {
     struct target_siginfo info;
     struct target_ucontext uc;
     int32_t abigap[56];
 };
 
+#endif
+
+#if defined(TARGET_PPC64)
+
+struct target_func_ptr {
+    target_ulong entry;
+    target_ulong toc;
+};
+
+#endif
+
 /* We use the mc_pad field for the signal return trampoline.  */
 #define tramp mc_pad
 
@@ -4481,8 +4515,7 @@ static target_ulong get_sigframe(struct target_sigaction *ka,
     return newsp;
 }
 
-static void save_user_regs(CPUPPCState *env, struct target_mcontext *frame,
-                          int sigret)
+static void save_user_regs(CPUPPCState *env, struct target_mcontext *frame)
 {
     target_ulong msr = env->msr;
     int i;
@@ -4549,11 +4582,14 @@ static void save_user_regs(CPUPPCState *env, struct target_mcontext *frame,
 
     /* Store MSR.  */
     __put_user(msr, &frame->mc_gregs[TARGET_PT_MSR]);
+}
 
+static void encode_trampoline(int sigret, uint32_t *tramp)
+{
     /* Set up the sigreturn trampoline: li r0,sigret; sc.  */
     if (sigret) {
-        __put_user(0x38000000UL | sigret, &frame->tramp[0]);
-        __put_user(0x44000002UL, &frame->tramp[1]);
+        __put_user(0x38000000 | sigret, &tramp[0]);
+        __put_user(0x44000002, &tramp[1]);
     }
 }
 
@@ -4645,6 +4681,9 @@ static void setup_frame(int sig, struct target_sigaction *ka,
     target_ulong frame_addr, newsp;
     int err = 0;
     int signal;
+#if defined(TARGET_PPC64)
+    struct image_info *image = ((TaskState *)thread_cpu->opaque)->info;
+#endif
 
     frame_addr = get_sigframe(ka, env, sizeof(*frame));
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 1))
@@ -4655,7 +4694,7 @@ static void setup_frame(int sig, struct target_sigaction *ka,
 
     __put_user(ka->_sa_handler, &sc->handler);
     __put_user(set->sig[0], &sc->oldmask);
-#if defined(TARGET_PPC64)
+#if TARGET_ABI_BITS == 64
     __put_user(set->sig[0] >> 32, &sc->_unused[3]);
 #else
     __put_user(set->sig[1], &sc->_unused[3]);
@@ -4664,7 +4703,10 @@ static void setup_frame(int sig, struct target_sigaction *ka,
     __put_user(sig, &sc->signal);
 
     /* Save user regs.  */
-    save_user_regs(env, &frame->mctx, TARGET_NR_sigreturn);
+    save_user_regs(env, &frame->mctx);
+
+    /* Construct the trampoline code on the stack. */
+    encode_trampoline(TARGET_NR_sigreturn, (uint32_t *)&frame->mctx.tramp);
 
     /* The kernel checks for the presence of a VDSO here.  We don't
        emulate a vdso, so use a sigreturn system call.  */
@@ -4684,7 +4726,24 @@ static void setup_frame(int sig, struct target_sigaction *ka,
     env->gpr[1] = newsp;
     env->gpr[3] = signal;
     env->gpr[4] = frame_addr + offsetof(struct target_sigframe, sctx);
+
+#if defined(TARGET_PPC64)
+    if (get_ppc64_abi(image) < 2) {
+        /* ELFv1 PPC64 function pointers are pointers to OPD entries. */
+        struct target_func_ptr *handler =
+            (struct target_func_ptr *)g2h(ka->_sa_handler);
+        env->nip = tswapl(handler->entry);
+        env->gpr[2] = tswapl(handler->toc);
+    } else {
+        /* ELFv2 PPC64 function pointers are entry points, but R12
+         * must also be set */
+        env->nip = tswapl((target_ulong) ka->_sa_handler);
+        env->gpr[12] = env->nip;
+    }
+#else
     env->nip = (target_ulong) ka->_sa_handler;
+#endif
+
     /* Signal handlers are entered in big-endian mode.  */
     env->msr &= ~MSR_LE;
 
@@ -4702,10 +4761,14 @@ static void setup_rt_frame(int sig, struct target_sigaction *ka,
                            target_sigset_t *set, CPUPPCState *env)
 {
     struct target_rt_sigframe *rt_sf;
-    struct target_mcontext *frame;
+    uint32_t *trampptr = 0;
+    struct target_mcontext *mctx = 0;
     target_ulong rt_sf_addr, newsp = 0;
     int i, err = 0;
     int signal;
+#if defined(TARGET_PPC64)
+    struct image_info *image = ((TaskState *)thread_cpu->opaque)->info;
+#endif
 
     rt_sf_addr = get_sigframe(ka, env, sizeof(*rt_sf));
     if (!lock_user_struct(VERIFY_WRITE, rt_sf, rt_sf_addr, 1))
@@ -4723,25 +4786,35 @@ static void setup_rt_frame(int sig, struct target_sigaction *ka,
                &rt_sf->uc.tuc_stack.ss_flags);
     __put_user(target_sigaltstack_used.ss_size,
                &rt_sf->uc.tuc_stack.ss_size);
+#if !defined(TARGET_PPC64)
     __put_user(h2g (&rt_sf->uc.tuc_mcontext),
                &rt_sf->uc.tuc_regs);
+#endif
     for(i = 0; i < TARGET_NSIG_WORDS; i++) {
         __put_user(set->sig[i], &rt_sf->uc.tuc_sigmask.sig[i]);
     }
 
-    frame = &rt_sf->uc.tuc_mcontext;
-    save_user_regs(env, frame, TARGET_NR_rt_sigreturn);
+#if defined(TARGET_PPC64)
+    mctx = &rt_sf->uc.tuc_sigcontext.mcontext;
+    trampptr = &rt_sf->trampoline[0];
+#else
+    mctx = &rt_sf->uc.tuc_mcontext;
+    trampptr = (uint32_t *)&rt_sf->uc.tuc_mcontext.tramp;
+#endif
+
+    save_user_regs(env, mctx);
+    encode_trampoline(TARGET_NR_rt_sigreturn, trampptr);
 
     /* The kernel checks for the presence of a VDSO here.  We don't
        emulate a vdso, so use a sigreturn system call.  */
-    env->lr = (target_ulong) h2g(frame->tramp);
+    env->lr = (target_ulong) h2g(trampptr);
 
     /* Turn off all fp exceptions.  */
     env->fpscr = 0;
 
     /* Create a stack frame for the caller of the handler.  */
     newsp = rt_sf_addr - (SIGNAL_FRAMESIZE + 16);
-    __put_user(env->gpr[1], (target_ulong *)(uintptr_t) newsp);
+    err |= put_user(env->gpr[1], newsp, target_ulong);
 
     if (err)
         goto sigsegv;
@@ -4752,7 +4825,24 @@ static void setup_rt_frame(int sig, struct target_sigaction *ka,
     env->gpr[4] = (target_ulong) h2g(&rt_sf->info);
     env->gpr[5] = (target_ulong) h2g(&rt_sf->uc);
     env->gpr[6] = (target_ulong) h2g(rt_sf);
+
+#if defined(TARGET_PPC64)
+    if (get_ppc64_abi(image) < 2) {
+        /* ELFv1 PPC64 function pointers are pointers to OPD entries. */
+        struct target_func_ptr *handler =
+            (struct target_func_ptr *)g2h(ka->_sa_handler);
+        env->nip = tswapl(handler->entry);
+        env->gpr[2] = tswapl(handler->toc);
+    } else {
+        /* ELFv2 PPC64 function pointers are entry points, but R12
+         * must also be set */
+        env->nip = tswapl((target_ulong) ka->_sa_handler);
+        env->gpr[12] = env->nip;
+    }
+#else
     env->nip = (target_ulong) ka->_sa_handler;
+#endif
+
     /* Signal handlers are entered in big-endian mode.  */
     env->msr &= ~MSR_LE;
 
@@ -4779,7 +4869,7 @@ long do_sigreturn(CPUPPCState *env)
         goto sigsegv;
 
 #if defined(TARGET_PPC64)
-    set.sig[0] = sc->oldmask + ((long)(sc->_unused[3]) << 32);
+    set.sig[0] = sc->oldmask + ((uint64_t)(sc->_unused[3]) << 32);
 #else
     __get_user(set.sig[0], &sc->oldmask);
     __get_user(set.sig[1], &sc->_unused[3]);
@@ -4817,10 +4907,11 @@ static int do_setcontext(struct target_ucontext *ucp, CPUPPCState *env, int sig)
         return 1;
 
 #if defined(TARGET_PPC64)
-    fprintf (stderr, "do_setcontext: not implemented\n");
-    return 0;
+    mcp_addr = h2g(ucp) +
+        offsetof(struct target_ucontext, tuc_sigcontext.mcontext);
 #else
     __get_user(mcp_addr, &ucp->tuc_regs);
+#endif
 
     if (!lock_user_struct(VERIFY_READ, mcp, mcp_addr, 1))
         return 1;
@@ -4831,7 +4922,6 @@ static int do_setcontext(struct target_ucontext *ucp, CPUPPCState *env, int sig)
 
     unlock_user_struct(mcp, mcp_addr, 1);
     return 0;
-#endif
 }
 
 long do_rt_sigreturn(CPUPPCState *env)
diff --git a/linux-user/sparc/syscall.h b/linux-user/sparc/syscall.h
index 9549ea0a2..58573b92e 100644
--- a/linux-user/sparc/syscall.h
+++ b/linux-user/sparc/syscall.h
@@ -15,3 +15,6 @@ struct target_pt_regs {
  * and copy_thread().
  */
 #define TARGET_CLONE_BACKWARDS
+#define TARGET_MINSIGSTKSZ      4096
+#define TARGET_MLOCKALL_MCL_CURRENT 0x2000
+#define TARGET_MLOCKALL_MCL_FUTURE  0x4000
diff --git a/linux-user/sparc64/syscall.h b/linux-user/sparc64/syscall.h
index 82b1680cb..8398d3f46 100644
--- a/linux-user/sparc64/syscall.h
+++ b/linux-user/sparc64/syscall.h
@@ -16,3 +16,6 @@ struct target_pt_regs {
  * and copy_thread().
  */
 #define TARGET_CLONE_BACKWARDS
+#define TARGET_MINSIGSTKSZ      4096
+#define TARGET_MLOCKALL_MCL_CURRENT 0x2000
+#define TARGET_MLOCKALL_MCL_FUTURE  0x4000
diff --git a/linux-user/strace.list b/linux-user/strace.list
index fcb258d34..aa0cd735c 100644
--- a/linux-user/strace.list
+++ b/linux-user/strace.list
@@ -1185,6 +1185,9 @@
 #ifdef TARGET_NR_set_mempolicy
 { TARGET_NR_set_mempolicy, "set_mempolicy" , NULL, NULL, NULL },
 #endif
+#ifdef TARGET_NR_setns
+{ TARGET_NR_setns, "setns" , NULL, NULL, NULL },
+#endif
 #ifdef TARGET_NR_setpgid
 { TARGET_NR_setpgid, "setpgid" , NULL, NULL, NULL },
 #endif
@@ -1404,6 +1407,15 @@
 #ifdef TARGET_NR_timer_settime
 { TARGET_NR_timer_settime, "timer_settime" , NULL, NULL, NULL },
 #endif
+#ifdef TARGET_NR_timerfd_create
+{ TARGET_NR_timerfd_create, "timerfd_create" , NULL, NULL, NULL },
+#endif
+#ifdef TARGET_NR_timerfd_gettime
+{ TARGET_NR_timerfd_gettime, "timerfd_gettime" , NULL, NULL, NULL },
+#endif
+#ifdef TARGET_NR_timerfd_settime
+{ TARGET_NR_timerfd_settime, "timerfd_settime" , NULL, NULL, NULL },
+#endif
 #ifdef TARGET_NR_times
 { TARGET_NR_times, "times" , NULL, NULL, NULL },
 #endif
diff --git a/linux-user/syscall.c b/linux-user/syscall.c
index a50229d0d..aaac6a25c 100644
--- a/linux-user/syscall.c
+++ b/linux-user/syscall.c
@@ -66,6 +66,9 @@ int __clone2(int (*fn)(void *), void *child_stack_base,
 #include <linux/wireless.h>
 #include <linux/icmp.h>
 #include "qemu-common.h"
+#ifdef CONFIG_TIMERFD
+#include <sys/timerfd.h>
+#endif
 #ifdef TARGET_GPROF
 #include <sys/gmon.h>
 #endif
@@ -251,6 +254,12 @@ _syscall2(int, capget, struct __user_cap_header_struct *, header,
           struct __user_cap_data_struct *, data);
 _syscall2(int, capset, struct __user_cap_header_struct *, header,
           struct __user_cap_data_struct *, data);
+#if defined(TARGET_NR_ioprio_get) && defined(__NR_ioprio_get)
+_syscall2(int, ioprio_get, int, which, int, who)
+#endif
+#if defined(TARGET_NR_ioprio_set) && defined(__NR_ioprio_set)
+_syscall3(int, ioprio_set, int, which, int, who, int, ioprio)
+#endif
 
 static bitmask_transtbl fcntl_flags_tbl[] = {
   { TARGET_O_ACCMODE,   TARGET_O_WRONLY,    O_ACCMODE,   O_WRONLY,    },
@@ -294,7 +303,6 @@ static int sys_getcwd1(char *buf, size_t size)
   return strlen(buf)+1;
 }
 
-#ifdef TARGET_NR_openat
 static int sys_openat(int dirfd, const char *pathname, int flags, mode_t mode)
 {
   /*
@@ -306,7 +314,6 @@ static int sys_openat(int dirfd, const char *pathname, int flags, mode_t mode)
   }
   return (openat(dirfd, pathname, flags));
 }
-#endif
 
 #ifdef TARGET_NR_utimensat
 #ifdef CONFIG_UTIMENSAT
@@ -1798,6 +1805,7 @@ static struct iovec *lock_iovec(int type, abi_ulong target_addr,
     abi_ulong total_len, max_len;
     int i;
     int err = 0;
+    bool bad_address = false;
 
     if (count == 0) {
         errno = 0;
@@ -1838,9 +1846,20 @@ static struct iovec *lock_iovec(int type, abi_ulong target_addr,
             vec[i].iov_base = 0;
         } else {
             vec[i].iov_base = lock_user(type, base, len, copy);
+            /* If the first buffer pointer is bad, this is a fault.  But
+             * subsequent bad buffers will result in a partial write; this
+             * is realized by filling the vector with null pointers and
+             * zero lengths. */
             if (!vec[i].iov_base) {
-                err = EFAULT;
-                goto fail;
+                if (i == 0) {
+                    err = EFAULT;
+                    goto fail;
+                } else {
+                    bad_address = true;
+                }
+            }
+            if (bad_address) {
+                len = 0;
             }
             if (len > max_len - total_len) {
                 len = max_len - total_len;
@@ -2419,9 +2438,13 @@ struct target_semid_ds
 {
   struct target_ipc_perm sem_perm;
   abi_ulong sem_otime;
+#if !defined(TARGET_PPC64)
   abi_ulong __unused1;
+#endif
   abi_ulong sem_ctime;
+#if !defined(TARGET_PPC64)
   abi_ulong __unused2;
+#endif
   abi_ulong sem_nsems;
   abi_ulong __unused3;
   abi_ulong __unused4;
@@ -2643,9 +2666,18 @@ static inline abi_long do_semctl(int semid, int semnum, int cmd,
     switch( cmd ) {
 	case GETVAL:
 	case SETVAL:
-            arg.val = tswap32(target_su.val);
+            /* In 64 bit cross-endian situations, we will erroneously pick up
+             * the wrong half of the union for the "val" element.  To rectify
+             * this, the entire 8-byte structure is byteswapped, followed by
+	     * a swap of the 4 byte val field. In other cases, the data is
+	     * already in proper host byte order. */
+	    if (sizeof(target_su.val) != (sizeof(target_su.buf))) {
+		target_su.buf = tswapal(target_su.buf);
+		arg.val = tswap32(target_su.val);
+	    } else {
+		arg.val = target_su.val;
+	    }
             ret = get_errno(semctl(semid, semnum, cmd, arg));
-            target_su.val = tswap32(arg.val);
             break;
 	case GETALL:
 	case SETALL:
@@ -2861,15 +2893,23 @@ struct target_msgbuf {
 };
 
 static inline abi_long do_msgsnd(int msqid, abi_long msgp,
-                                 unsigned int msgsz, int msgflg)
+                                 ssize_t msgsz, int msgflg)
 {
     struct target_msgbuf *target_mb;
     struct msgbuf *host_mb;
     abi_long ret = 0;
 
+    if (msgsz < 0) {
+        return -TARGET_EINVAL;
+    }
+
     if (!lock_user_struct(VERIFY_READ, target_mb, msgp, 0))
         return -TARGET_EFAULT;
     host_mb = malloc(msgsz+sizeof(long));
+    if (!host_mb) {
+        unlock_user_struct(target_mb, msgp, 0);
+        return -TARGET_ENOMEM;
+    }
     host_mb->mtype = (abi_long) tswapal(target_mb->mtype);
     memcpy(host_mb->mtext, target_mb->mtext, msgsz);
     ret = get_errno(msgsnd(msqid, host_mb, msgsz, msgflg));
@@ -3112,8 +3152,8 @@ static inline abi_long do_shmdt(abi_ulong shmaddr)
 #ifdef TARGET_NR_ipc
 /* ??? This only works with linear mappings.  */
 /* do_ipc() must return target values and target errnos. */
-static abi_long do_ipc(unsigned int call, int first,
-                       int second, int third,
+static abi_long do_ipc(unsigned int call, abi_long first,
+                       abi_long second, abi_long third,
                        abi_long ptr, abi_long fifth)
 {
     int version;
@@ -3131,9 +3171,15 @@ static abi_long do_ipc(unsigned int call, int first,
         ret = get_errno(semget(first, second, third));
         break;
 
-    case IPCOP_semctl:
-        ret = do_semctl(first, second, third, (union target_semun)(abi_ulong) ptr);
+    case IPCOP_semctl: {
+        /* The semun argument to semctl is passed by value, so dereference the
+         * ptr argument. */
+        abi_ulong atptr;
+        get_user_ual(atptr, ptr);
+        ret = do_semctl(first, second, third,
+                (union target_semun) atptr);
         break;
+    }
 
     case IPCOP_msgget:
         ret = get_errno(msgget(first, second));
@@ -3652,6 +3698,59 @@ out:
     return ret;
 }
 
+static abi_long do_ioctl_blkpg(const IOCTLEntry *ie, uint8_t *buf_temp, int fd,
+                               abi_long cmd, abi_long arg)
+{
+    void *argptr;
+    int target_size;
+    const argtype *arg_type = ie->arg_type;
+    const argtype part_arg_type[] = { MK_STRUCT(STRUCT_blkpg_partition) };
+    abi_long ret;
+
+    struct blkpg_ioctl_arg *host_blkpg = (void*)buf_temp;
+    struct blkpg_partition host_part;
+
+    /* Read and convert blkpg */
+    arg_type++;
+    target_size = thunk_type_size(arg_type, 0);
+    argptr = lock_user(VERIFY_READ, arg, target_size, 1);
+    if (!argptr) {
+        ret = -TARGET_EFAULT;
+        goto out;
+    }
+    thunk_convert(buf_temp, argptr, arg_type, THUNK_HOST);
+    unlock_user(argptr, arg, 0);
+
+    switch (host_blkpg->op) {
+    case BLKPG_ADD_PARTITION:
+    case BLKPG_DEL_PARTITION:
+        /* payload is struct blkpg_partition */
+        break;
+    default:
+        /* Unknown opcode */
+        ret = -TARGET_EINVAL;
+        goto out;
+    }
+
+    /* Read and convert blkpg->data */
+    arg = (abi_long)(uintptr_t)host_blkpg->data;
+    target_size = thunk_type_size(part_arg_type, 0);
+    argptr = lock_user(VERIFY_READ, arg, target_size, 1);
+    if (!argptr) {
+        ret = -TARGET_EFAULT;
+        goto out;
+    }
+    thunk_convert(&host_part, argptr, part_arg_type, THUNK_HOST);
+    unlock_user(argptr, arg, 0);
+
+    /* Swizzle the data pointer to our local copy and call! */
+    host_blkpg->data = &host_part;
+    ret = get_errno(ioctl(fd, ie->host_cmd, host_blkpg));
+
+out:
+    return ret;
+}
+
 static abi_long do_ioctl_rt(const IOCTLEntry *ie, uint8_t *buf_temp,
                                 int fd, abi_long cmd, abi_long arg)
 {
@@ -4914,6 +5013,47 @@ static inline abi_long host_to_target_itimerspec(abi_ulong target_addr,
     return 0;
 }
 
+static inline abi_long target_to_host_sigevent(struct sigevent *host_sevp,
+                                               abi_ulong target_addr)
+{
+    struct target_sigevent *target_sevp;
+
+    if (!lock_user_struct(VERIFY_READ, target_sevp, target_addr, 1)) {
+        return -TARGET_EFAULT;
+    }
+
+    /* This union is awkward on 64 bit systems because it has a 32 bit
+     * integer and a pointer in it; we follow the conversion approach
+     * used for handling sigval types in signal.c so the guest should get
+     * the correct value back even if we did a 64 bit byteswap and it's
+     * using the 32 bit integer.
+     */
+    host_sevp->sigev_value.sival_ptr =
+        (void *)(uintptr_t)tswapal(target_sevp->sigev_value.sival_ptr);
+    host_sevp->sigev_signo =
+        target_to_host_signal(tswap32(target_sevp->sigev_signo));
+    host_sevp->sigev_notify = tswap32(target_sevp->sigev_notify);
+    host_sevp->_sigev_un._tid = tswap32(target_sevp->_sigev_un._tid);
+
+    unlock_user_struct(target_sevp, target_addr, 1);
+    return 0;
+}
+
+#if defined(TARGET_NR_mlockall)
+static inline int target_to_host_mlockall_arg(int arg)
+{
+    int result = 0;
+
+    if (arg & TARGET_MLOCKALL_MCL_CURRENT) {
+        result |= MCL_CURRENT;
+    }
+    if (arg & TARGET_MLOCKALL_MCL_FUTURE) {
+        result |= MCL_FUTURE;
+    }
+    return result;
+}
+#endif
+
 #if defined(TARGET_NR_stat64) || defined(TARGET_NR_newfstatat)
 static inline abi_long host_to_target_stat64(void *cpu_env,
                                              abi_ulong target_addr,
@@ -5082,6 +5222,7 @@ static int open_self_cmdline(void *cpu_env, int fd)
 
         if (word_skipped) {
             if (write(fd, cp_buf, nb_read) != nb_read) {
+                close(fd_orig);
                 return -1;
             }
         }
@@ -5092,10 +5233,8 @@ static int open_self_cmdline(void *cpu_env, int fd)
 
 static int open_self_maps(void *cpu_env, int fd)
 {
-#if defined(TARGET_ARM) || defined(TARGET_M68K) || defined(TARGET_UNICORE32)
     CPUState *cpu = ENV_GET_CPU((CPUArchState *)cpu_env);
     TaskState *ts = cpu->opaque;
-#endif
     FILE *fp;
     char *line = NULL;
     size_t len = 0;
@@ -5118,13 +5257,18 @@ static int open_self_maps(void *cpu_env, int fd)
         if ((fields < 10) || (fields > 11)) {
             continue;
         }
-        if (!strncmp(path, "[stack]", 7)) {
-            continue;
-        }
-        if (h2g_valid(min) && h2g_valid(max)) {
+        if (h2g_valid(min)) {
+            int flags = page_get_flags(h2g(min));
+            max = h2g_valid(max - 1) ? max : (uintptr_t)g2h(GUEST_ADDR_MAX);
+            if (page_check_range(h2g(min), max - min, flags) == -1) {
+                continue;
+            }
+            if (h2g(min) == ts->info->stack_limit) {
+                pstrcpy(path, sizeof(path), "      [stack]");
+            }
             dprintf(fd, TARGET_ABI_FMT_lx "-" TARGET_ABI_FMT_lx
                     " %c%c%c%c %08" PRIx64 " %02x:%02x %d %s%s\n",
-                    h2g(min), h2g(max), flag_r, flag_w,
+                    h2g(min), h2g(max - 1) + 1, flag_r, flag_w,
                     flag_x, flag_p, offset, dev_maj, dev_min, inode,
                     path[0] ? "         " : "", path);
         }
@@ -5133,14 +5277,6 @@ static int open_self_maps(void *cpu_env, int fd)
     free(line);
     fclose(fp);
 
-#if defined(TARGET_ARM) || defined(TARGET_M68K) || defined(TARGET_UNICORE32)
-    dprintf(fd, "%08llx-%08llx rw-p %08llx 00:00 0          [stack]\n",
-                (unsigned long long)ts->info->stack_limit,
-                (unsigned long long)(ts->info->start_stack +
-                                     (TARGET_PAGE_SIZE - 1)) & TARGET_PAGE_MASK,
-                (unsigned long long)0);
-#endif
-
     return 0;
 }
 
@@ -5279,7 +5415,7 @@ static int open_net_route(void *cpu_env, int fd)
 }
 #endif
 
-static int do_open(void *cpu_env, const char *pathname, int flags, mode_t mode)
+static int do_openat(void *cpu_env, int dirfd, const char *pathname, int flags, mode_t mode)
 {
     struct fake_open {
         const char *filename;
@@ -5300,7 +5436,7 @@ static int do_open(void *cpu_env, const char *pathname, int flags, mode_t mode)
 
     if (is_proc_myself(pathname, "exe")) {
         int execfd = qemu_getauxval(AT_EXECFD);
-        return execfd ? execfd : get_errno(open(exec_path, flags, mode));
+        return execfd ? execfd : get_errno(sys_openat(dirfd, exec_path, flags, mode));
     }
 
     for (fake_open = fakes; fake_open->filename; fake_open++) {
@@ -5334,7 +5470,28 @@ static int do_open(void *cpu_env, const char *pathname, int flags, mode_t mode)
         return fd;
     }
 
-    return get_errno(open(path(pathname), flags, mode));
+    return get_errno(sys_openat(dirfd, path(pathname), flags, mode));
+}
+
+#define TIMER_MAGIC 0x0caf0000
+#define TIMER_MAGIC_MASK 0xffff0000
+
+/* Convert QEMU provided timer ID back to internal 16bit index format */
+static target_timer_t get_timer_id(abi_long arg)
+{
+    target_timer_t timerid = arg;
+
+    if ((timerid & TIMER_MAGIC_MASK) != TIMER_MAGIC) {
+        return -TARGET_EINVAL;
+    }
+
+    timerid &= 0xffff;
+
+    if (timerid >= ARRAY_SIZE(g_posix_timers)) {
+        return -TARGET_EINVAL;
+    }
+
+    return timerid;
 }
 
 /* do_syscall() should always have a single exit point at the end so
@@ -5409,22 +5566,19 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
     case TARGET_NR_open:
         if (!(p = lock_user_string(arg1)))
             goto efault;
-        ret = get_errno(do_open(cpu_env, p,
-                                target_to_host_bitmask(arg2, fcntl_flags_tbl),
-                                arg3));
+        ret = get_errno(do_openat(cpu_env, AT_FDCWD, p,
+                                  target_to_host_bitmask(arg2, fcntl_flags_tbl),
+                                  arg3));
         unlock_user(p, arg1, 0);
         break;
-#if defined(TARGET_NR_openat) && defined(__NR_openat)
     case TARGET_NR_openat:
         if (!(p = lock_user_string(arg2)))
             goto efault;
-        ret = get_errno(sys_openat(arg1,
-                                   path(p),
-                                   target_to_host_bitmask(arg3, fcntl_flags_tbl),
-                                   arg4));
+        ret = get_errno(do_openat(cpu_env, arg1, p,
+                                  target_to_host_bitmask(arg3, fcntl_flags_tbl),
+                                  arg4));
         unlock_user(p, arg2, 0);
         break;
-#endif
     case TARGET_NR_close:
         ret = get_errno(close(arg1));
         break;
@@ -6620,11 +6774,22 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             p2 = lock_user(VERIFY_WRITE, arg2, arg3, 0);
             if (!p || !p2) {
                 ret = -TARGET_EFAULT;
+            } else if (!arg3) {
+                /* Short circuit this for the magic exe check. */
+                ret = -TARGET_EINVAL;
             } else if (is_proc_myself((const char *)p, "exe")) {
                 char real[PATH_MAX], *temp;
                 temp = realpath(exec_path, real);
-                ret = temp == NULL ? get_errno(-1) : strlen(real) ;
-                snprintf((char *)p2, arg3, "%s", real);
+                /* Return value is # of bytes that we wrote to the buffer. */
+                if (temp == NULL) {
+                    ret = get_errno(-1);
+                } else {
+                    /* Don't worry about sign mismatch as earlier mapping
+                     * logic would have thrown a bad address error. */
+                    ret = MIN(strlen(real), arg3);
+                    /* We cannot NUL terminate the string. */
+                    memcpy(p2, real, ret);
+                }
             } else {
                 ret = get_errno(readlink(path(p), p2, arg3));
             }
@@ -6763,7 +6928,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
 #endif
 #ifdef TARGET_NR_mlockall
     case TARGET_NR_mlockall:
-        ret = get_errno(mlockall(arg1));
+        ret = get_errno(mlockall(target_to_host_mlockall_arg(arg1)));
         break;
 #endif
 #ifdef TARGET_NR_munlockall
@@ -7679,6 +7844,9 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             struct sched_param *target_schp;
             struct sched_param schp;
 
+            if (arg2 == 0) {
+                return -TARGET_EINVAL;
+            }
             if (!lock_user_struct(VERIFY_READ, target_schp, arg2, 1))
                 goto efault;
             schp.sched_priority = tswap32(target_schp->sched_priority);
@@ -7690,6 +7858,10 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         {
             struct sched_param *target_schp;
             struct sched_param schp;
+
+            if (arg2 == 0) {
+                return -TARGET_EINVAL;
+            }
             ret = get_errno(sched_getparam(arg1, &schp));
             if (!is_error(ret)) {
                 if (!lock_user_struct(VERIFY_WRITE, target_schp, arg2, 0))
@@ -7703,6 +7875,9 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         {
             struct sched_param *target_schp;
             struct sched_param schp;
+            if (arg3 == 0) {
+                return -TARGET_EINVAL;
+            }
             if (!lock_user_struct(VERIFY_READ, target_schp, arg3, 1))
                 goto efault;
             schp.sched_priority = tswap32(target_schp->sched_priority);
@@ -7727,7 +7902,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             struct timespec ts;
             ret = get_errno(sched_rr_get_interval(arg1, &ts));
             if (!is_error(ret)) {
-                host_to_target_timespec(arg2, &ts);
+                ret = host_to_target_timespec(arg2, &ts);
             }
         }
         break;
@@ -8966,6 +9141,14 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         ret = get_errno(clock_nanosleep(arg1, arg2, &ts, arg4 ? &ts : NULL));
         if (arg4)
             host_to_target_timespec(arg4, &ts);
+
+#if defined(TARGET_PPC)
+        /* clock_nanosleep is odd in that it returns positive errno values.
+         * On PPC, CR0 bit 3 should be set in such a situation. */
+        if (ret) {
+            ((CPUPPCState *)cpu_env)->crf[0] |= 1;
+        }
+#endif
         break;
     }
 #endif
@@ -9062,12 +9245,16 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
 #if defined(TARGET_NR_mq_open) && defined(__NR_mq_open)
     case TARGET_NR_mq_open:
         {
-            struct mq_attr posix_mq_attr;
+            struct mq_attr posix_mq_attr, *attrp;
 
             p = lock_user_string(arg1 - 1);
-            if (arg4 != 0)
+            if (arg4 != 0) {
                 copy_from_user_mq_attr (&posix_mq_attr, arg4);
-            ret = get_errno(mq_open(p, arg2, arg3, &posix_mq_attr));
+                attrp = &posix_mq_attr;
+            } else {
+                attrp = 0;
+            }
+            ret = get_errno(mq_open(p, arg2, arg3, attrp));
             unlock_user (p, arg1, 0);
         }
         break;
@@ -9413,8 +9600,6 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         /* args: clockid_t clockid, struct sigevent *sevp, timer_t *timerid */
 
         struct sigevent host_sevp = { {0}, }, *phost_sevp = NULL;
-        struct target_sigevent *ptarget_sevp;
-        struct target_timer_t *ptarget_timer;
 
         int clkid = arg1;
         int timer_index = next_free_host_timer();
@@ -9425,25 +9610,20 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             timer_t *phtimer = g_posix_timers  + timer_index;
 
             if (arg2) {
-                if (!lock_user_struct(VERIFY_READ, ptarget_sevp, arg2, 1)) {
-                    goto efault;
-                }
-
-                host_sevp.sigev_signo = tswap32(ptarget_sevp->sigev_signo);
-                host_sevp.sigev_notify = tswap32(ptarget_sevp->sigev_notify);
-
                 phost_sevp = &host_sevp;
+                ret = target_to_host_sigevent(phost_sevp, arg2);
+                if (ret != 0) {
+                    break;
+                }
             }
 
             ret = get_errno(timer_create(clkid, phost_sevp, phtimer));
             if (ret) {
                 phtimer = NULL;
             } else {
-                if (!lock_user_struct(VERIFY_WRITE, ptarget_timer, arg3, 1)) {
+                if (put_user(TIMER_MAGIC | timer_index, arg3, target_timer_t)) {
                     goto efault;
                 }
-                ptarget_timer->ptr = tswap32(0xcafe0000 | timer_index);
-                unlock_user_struct(ptarget_timer, arg3, 1);
             }
         }
         break;
@@ -9455,11 +9635,14 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
     {
         /* args: timer_t timerid, int flags, const struct itimerspec *new_value,
          * struct itimerspec * old_value */
-        arg1 &= 0xffff;
-        if (arg3 == 0 || arg1 < 0 || arg1 >= ARRAY_SIZE(g_posix_timers)) {
+        target_timer_t timerid = get_timer_id(arg1);
+
+        if (timerid < 0) {
+            ret = timerid;
+        } else if (arg3 == 0) {
             ret = -TARGET_EINVAL;
         } else {
-            timer_t htimer = g_posix_timers[arg1];
+            timer_t htimer = g_posix_timers[timerid];
             struct itimerspec hspec_new = {{0},}, hspec_old = {{0},};
 
             target_to_host_itimerspec(&hspec_new, arg3);
@@ -9475,13 +9658,14 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
     case TARGET_NR_timer_gettime:
     {
         /* args: timer_t timerid, struct itimerspec *curr_value */
-        arg1 &= 0xffff;
-        if (!arg2) {
-            return -TARGET_EFAULT;
-        } else if (arg1 < 0 || arg1 >= ARRAY_SIZE(g_posix_timers)) {
-            ret = -TARGET_EINVAL;
+        target_timer_t timerid = get_timer_id(arg1);
+
+        if (timerid < 0) {
+            ret = timerid;
+        } else if (!arg2) {
+            ret = -TARGET_EFAULT;
         } else {
-            timer_t htimer = g_posix_timers[arg1];
+            timer_t htimer = g_posix_timers[timerid];
             struct itimerspec hspec;
             ret = get_errno(timer_gettime(htimer, &hspec));
 
@@ -9497,11 +9681,12 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
     case TARGET_NR_timer_getoverrun:
     {
         /* args: timer_t timerid */
-        arg1 &= 0xffff;
-        if (arg1 < 0 || arg1 >= ARRAY_SIZE(g_posix_timers)) {
-            ret = -TARGET_EINVAL;
+        target_timer_t timerid = get_timer_id(arg1);
+
+        if (timerid < 0) {
+            ret = timerid;
         } else {
-            timer_t htimer = g_posix_timers[arg1];
+            timer_t htimer = g_posix_timers[timerid];
             ret = get_errno(timer_getoverrun(htimer));
         }
         break;
@@ -9512,18 +9697,86 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
     case TARGET_NR_timer_delete:
     {
         /* args: timer_t timerid */
-        arg1 &= 0xffff;
-        if (arg1 < 0 || arg1 >= ARRAY_SIZE(g_posix_timers)) {
-            ret = -TARGET_EINVAL;
+        target_timer_t timerid = get_timer_id(arg1);
+
+        if (timerid < 0) {
+            ret = timerid;
         } else {
-            timer_t htimer = g_posix_timers[arg1];
+            timer_t htimer = g_posix_timers[timerid];
             ret = get_errno(timer_delete(htimer));
-            g_posix_timers[arg1] = 0;
+            g_posix_timers[timerid] = 0;
         }
         break;
     }
 #endif
 
+#if defined(TARGET_NR_timerfd_create) && defined(CONFIG_TIMERFD)
+    case TARGET_NR_timerfd_create:
+        ret = get_errno(timerfd_create(arg1,
+                target_to_host_bitmask(arg2, fcntl_flags_tbl)));
+        break;
+#endif
+
+#if defined(TARGET_NR_timerfd_gettime) && defined(CONFIG_TIMERFD)
+    case TARGET_NR_timerfd_gettime:
+        {
+            struct itimerspec its_curr;
+
+            ret = get_errno(timerfd_gettime(arg1, &its_curr));
+
+            if (arg2 && host_to_target_itimerspec(arg2, &its_curr)) {
+                goto efault;
+            }
+        }
+        break;
+#endif
+
+#if defined(TARGET_NR_timerfd_settime) && defined(CONFIG_TIMERFD)
+    case TARGET_NR_timerfd_settime:
+        {
+            struct itimerspec its_new, its_old, *p_new;
+
+            if (arg3) {
+                if (target_to_host_itimerspec(&its_new, arg3)) {
+                    goto efault;
+                }
+                p_new = &its_new;
+            } else {
+                p_new = NULL;
+            }
+
+            ret = get_errno(timerfd_settime(arg1, arg2, p_new, &its_old));
+
+            if (arg4 && host_to_target_itimerspec(arg4, &its_old)) {
+                goto efault;
+            }
+        }
+        break;
+#endif
+
+#if defined(TARGET_NR_ioprio_get) && defined(__NR_ioprio_get)
+    case TARGET_NR_ioprio_get:
+        ret = get_errno(ioprio_get(arg1, arg2));
+        break;
+#endif
+
+#if defined(TARGET_NR_ioprio_set) && defined(__NR_ioprio_set)
+    case TARGET_NR_ioprio_set:
+        ret = get_errno(ioprio_set(arg1, arg2, arg3));
+        break;
+#endif
+
+#if defined(TARGET_NR_setns) && defined(CONFIG_SETNS)
+    case TARGET_NR_setns:
+        ret = get_errno(setns(arg1, arg2));
+        break;
+#endif
+#if defined(TARGET_NR_unshare) && defined(CONFIG_SETNS)
+    case TARGET_NR_unshare:
+        ret = get_errno(unshare(arg1));
+        break;
+#endif
+
     default:
     unimplemented:
         gemu_log("qemu: Unsupported syscall: %d\n", num);
diff --git a/linux-user/syscall_defs.h b/linux-user/syscall_defs.h
index c9e632390..ebb3be119 100644
--- a/linux-user/syscall_defs.h
+++ b/linux-user/syscall_defs.h
@@ -2564,10 +2564,7 @@ struct target_ucred {
 
 #endif
 
-
-struct target_timer_t {
-    abi_ulong ptr;
-};
+typedef int32_t target_timer_t;
 
 #define TARGET_SIGEV_MAX_SIZE 64
 
diff --git a/linux-user/syscall_types.h b/linux-user/syscall_types.h
index 9d0c92d05..1fd4ee0bf 100644
--- a/linux-user/syscall_types.h
+++ b/linux-user/syscall_types.h
@@ -252,4 +252,4 @@ STRUCT(blkpg_ioctl_arg,
        TYPE_INT, /* op */
        TYPE_INT, /* flags */
        TYPE_INT, /* datalen */
-       MK_PTR(MK_STRUCT(STRUCT_blkpg_partition))) /* data */
+       TYPE_PTRVOID) /* data */
diff --git a/linux-user/unicore32/syscall.h b/linux-user/unicore32/syscall.h
index f7e55254c..385a97562 100644
--- a/linux-user/unicore32/syscall.h
+++ b/linux-user/unicore32/syscall.h
@@ -53,4 +53,8 @@ struct target_pt_regs {
 #define UNAME_MACHINE "UniCore-II"
 #define UNAME_MINIMUM_RELEASE "2.6.32"
 
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
+
 #endif /* __UC32_SYSCALL_H__ */
diff --git a/linux-user/x86_64/syscall.h b/linux-user/x86_64/syscall.h
index e03b5a0cf..88b3c3fe3 100644
--- a/linux-user/x86_64/syscall.h
+++ b/linux-user/x86_64/syscall.h
@@ -97,3 +97,6 @@ struct target_msqid64_ds {
 #define TARGET_ARCH_SET_FS 0x1002
 #define TARGET_ARCH_GET_FS 0x1003
 #define TARGET_ARCH_GET_GS 0x1004
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
diff --git a/main-loop.c b/main-loop.c
index 3cc79f82f..981bcb5f8 100644
--- a/main-loop.c
+++ b/main-loop.c
@@ -84,6 +84,11 @@ static int qemu_signal_init(void)
     sigaddset(&set, SIGIO);
     sigaddset(&set, SIGALRM);
     sigaddset(&set, SIGBUS);
+    /* SIGINT cannot be handled via signalfd, so that ^C can be used
+     * to interrupt QEMU when it is being run under gdb.  SIGHUP and
+     * SIGTERM are also handled asynchronously, even though it is not
+     * strictly necessary, because they use the same handler as SIGINT.
+     */
     pthread_sigmask(SIG_BLOCK, &set, NULL);
 
     sigdelset(&set, SIG_IPI);
@@ -126,10 +131,11 @@ void qemu_notify_event(void)
 
 static GArray *gpollfds;
 
-int qemu_init_main_loop(void)
+int qemu_init_main_loop(Error **errp)
 {
     int ret;
     GSource *src;
+    Error *local_error = NULL;
 
     init_clocks();
 
@@ -138,8 +144,12 @@ int qemu_init_main_loop(void)
         return ret;
     }
 
+    qemu_aio_context = aio_context_new(&local_error);
+    if (!qemu_aio_context) {
+        error_propagate(errp, local_error);
+        return -EMFILE;
+    }
     gpollfds = g_array_new(FALSE, FALSE, sizeof(GPollFD));
-    qemu_aio_context = aio_context_new();
     src = aio_get_g_source(qemu_aio_context);
     g_source_attach(src, NULL);
     g_source_unref(src);
diff --git a/memory.c b/memory.c
index 64d717619..15cf9ebd8 100644
--- a/memory.c
+++ b/memory.c
@@ -56,8 +56,7 @@ static void memory_init(void)
 typedef struct AddrRange AddrRange;
 
 /*
- * Note using signed integers limits us to physical addresses at most
- * 63 bits wide.  They are needed for negative offsetting in aliases
+ * Note that signed integers are needed for negative offsetting in aliases
  * (large MemoryRegion::alias_offset).
  */
 struct AddrRange {
@@ -877,30 +876,6 @@ static char *memory_region_escape_name(const char *name)
     return escaped;
 }
 
-static void object_property_add_child_array(Object *owner,
-                                            const char *name,
-                                            Object *child)
-{
-    int i;
-    char *base_name = memory_region_escape_name(name);
-
-    for (i = 0; ; i++) {
-        char *full_name = g_strdup_printf("%s[%d]", base_name, i);
-        Error *local_err = NULL;
-
-        object_property_add_child(owner, full_name, child, &local_err);
-        g_free(full_name);
-        if (!local_err) {
-            break;
-        }
-
-        error_free(local_err);
-    }
-
-    g_free(base_name);
-}
-        
-
 void memory_region_init(MemoryRegion *mr,
                         Object *owner,
                         const char *name,
@@ -918,8 +893,12 @@ void memory_region_init(MemoryRegion *mr,
     mr->name = g_strdup(name);
 
     if (name) {
-        object_property_add_child_array(owner, name, OBJECT(mr));
+        char *escaped_name = memory_region_escape_name(name);
+        char *name_array = g_strdup_printf("%s[*]", escaped_name);
+        object_property_add_child(owner, name_array, OBJECT(mr), &error_abort);
         object_unref(OBJECT(mr));
+        g_free(name_array);
+        g_free(escaped_name);
     }
 }
 
@@ -1163,13 +1142,14 @@ void memory_region_init_io(MemoryRegion *mr,
 void memory_region_init_ram(MemoryRegion *mr,
                             Object *owner,
                             const char *name,
-                            uint64_t size)
+                            uint64_t size,
+                            Error **errp)
 {
     memory_region_init(mr, owner, name, size);
     mr->ram = true;
     mr->terminates = true;
     mr->destructor = memory_region_destructor_ram;
-    mr->ram_addr = qemu_ram_alloc(size, mr);
+    mr->ram_addr = qemu_ram_alloc(size, mr, errp);
 }
 
 #ifdef __linux__
@@ -1199,7 +1179,15 @@ void memory_region_init_ram_ptr(MemoryRegion *mr,
     mr->ram = true;
     mr->terminates = true;
     mr->destructor = memory_region_destructor_ram_from_ptr;
-    mr->ram_addr = qemu_ram_alloc_from_ptr(size, ptr, mr);
+
+    /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL.  */
+    assert(ptr != NULL);
+    mr->ram_addr = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_abort);
+}
+
+void memory_region_set_skip_dump(MemoryRegion *mr)
+{
+    mr->skip_dump = true;
 }
 
 void memory_region_init_alias(MemoryRegion *mr,
@@ -1221,7 +1209,8 @@ void memory_region_init_rom_device(MemoryRegion *mr,
                                    const MemoryRegionOps *ops,
                                    void *opaque,
                                    const char *name,
-                                   uint64_t size)
+                                   uint64_t size,
+                                   Error **errp)
 {
     memory_region_init(mr, owner, name, size);
     mr->ops = ops;
@@ -1229,7 +1218,7 @@ void memory_region_init_rom_device(MemoryRegion *mr,
     mr->terminates = true;
     mr->rom_device = true;
     mr->destructor = memory_region_destructor_rom_device;
-    mr->ram_addr = qemu_ram_alloc(size, mr);
+    mr->ram_addr = qemu_ram_alloc(size, mr, errp);
 }
 
 void memory_region_init_iommu(MemoryRegion *mr,
@@ -1264,12 +1253,6 @@ static void memory_region_finalize(Object *obj)
     g_free(mr->ioeventfds);
 }
 
-void memory_region_destroy(MemoryRegion *mr)
-{
-    object_unparent(OBJECT(mr));
-}
-
-
 Object *memory_region_owner(MemoryRegion *mr)
 {
     Object *obj = OBJECT(mr);
@@ -1314,8 +1297,12 @@ uint64_t memory_region_size(MemoryRegion *mr)
     return int128_get64(mr->size);
 }
 
-const char *memory_region_name(MemoryRegion *mr)
+const char *memory_region_name(const MemoryRegion *mr)
 {
+    if (!mr->name) {
+        ((MemoryRegion *)mr)->name =
+            object_get_canonical_path_component(OBJECT(mr));
+    }
     return mr->name;
 }
 
@@ -1324,6 +1311,11 @@ bool memory_region_is_ram(MemoryRegion *mr)
     return mr->ram;
 }
 
+bool memory_region_is_skip_dump(MemoryRegion *mr)
+{
+    return mr->skip_dump;
+}
+
 bool memory_region_is_logging(MemoryRegion *mr)
 {
     return mr->dirty_log_mask;
@@ -1757,6 +1749,11 @@ ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr)
     return mr->ram_addr;
 }
 
+uint64_t memory_region_get_alignment(const MemoryRegion *mr)
+{
+    return mr->align;
+}
+
 static int cmp_flatrange_addr(const void *addr_, const void *fr_)
 {
     const AddrRange *addr = addr_;
@@ -1980,7 +1977,6 @@ typedef struct MemoryRegionList MemoryRegionList;
 
 struct MemoryRegionList {
     const MemoryRegion *mr;
-    bool printed;
     QTAILQ_ENTRY(MemoryRegionList) queue;
 };
 
@@ -2010,7 +2006,7 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f,
 
         /* check if the alias is already in the queue */
         QTAILQ_FOREACH(ml, alias_print_queue, queue) {
-            if (ml->mr == mr->alias && !ml->printed) {
+            if (ml->mr == mr->alias) {
                 found = true;
             }
         }
@@ -2018,7 +2014,6 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f,
         if (!found) {
             ml = g_new(MemoryRegionList, 1);
             ml->mr = mr->alias;
-            ml->printed = false;
             QTAILQ_INSERT_TAIL(alias_print_queue, ml, queue);
         }
         mon_printf(f, TARGET_FMT_plx "-" TARGET_FMT_plx
@@ -2033,8 +2028,8 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f,
                    mr->romd_mode ? 'R' : '-',
                    !mr->readonly && !(mr->rom_device && mr->romd_mode) ? 'W'
                                                                        : '-',
-                   mr->name,
-                   mr->alias->name,
+                   memory_region_name(mr),
+                   memory_region_name(mr->alias),
                    mr->alias_offset,
                    mr->alias_offset
                    + (int128_nz(mr->size) ?
@@ -2052,7 +2047,7 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f,
                    mr->romd_mode ? 'R' : '-',
                    !mr->readonly && !(mr->rom_device && mr->romd_mode) ? 'W'
                                                                        : '-',
-                   mr->name);
+                   memory_region_name(mr));
     }
 
     QTAILQ_INIT(&submr_print_queue);
@@ -2100,10 +2095,8 @@ void mtree_info(fprintf_function mon_printf, void *f)
     mon_printf(f, "aliases\n");
     /* print aliased regions */
     QTAILQ_FOREACH(ml, &ml_head, queue) {
-        if (!ml->printed) {
-            mon_printf(f, "%s\n", ml->mr->name);
-            mtree_print_mr(mon_printf, f, ml->mr, 0, 0, &ml_head);
-        }
+        mon_printf(f, "%s\n", memory_region_name(ml->mr));
+        mtree_print_mr(mon_printf, f, ml->mr, 0, 0, &ml_head);
     }
 
     QTAILQ_FOREACH_SAFE(ml, &ml_head, queue, ml2) {
diff --git a/memory_mapping.c b/memory_mapping.c
index 87a6ed5c8..7b69801cb 100644
--- a/memory_mapping.c
+++ b/memory_mapping.c
@@ -203,7 +203,8 @@ static void guest_phys_blocks_region_add(MemoryListener *listener,
     GuestPhysBlock *predecessor;
 
     /* we only care about RAM */
-    if (!memory_region_is_ram(section->mr)) {
+    if (!memory_region_is_ram(section->mr) ||
+        memory_region_is_skip_dump(section->mr)) {
         return;
     }
 
diff --git a/migration-rdma.c b/migration-rdma.c
index d99812c45..b32dbdfcc 100644
--- a/migration-rdma.c
+++ b/migration-rdma.c
@@ -2523,7 +2523,7 @@ static void *qemu_rdma_data_init(const char *host_port, Error **errp)
 /*
  * QEMUFile interface to the control channel.
  * SEND messages for control only.
- * pc.ram is handled with regular RDMA messages.
+ * VM's ram is handled with regular RDMA messages.
  */
 static int qemu_rdma_put_buffer(void *opaque, const uint8_t *buf,
                                 int64_t pos, int size)
@@ -2539,7 +2539,7 @@ static int qemu_rdma_put_buffer(void *opaque, const uint8_t *buf,
 
     /*
      * Push out any writes that
-     * we're queued up for pc.ram.
+     * we're queued up for VM's ram.
      */
     ret = qemu_rdma_write_flush(f, rdma);
     if (ret < 0) {
diff --git a/migration-tcp.c b/migration-tcp.c
index 2e34517bb..91c9cf381 100644
--- a/migration-tcp.c
+++ b/migration-tcp.c
@@ -33,12 +33,12 @@
     do { } while (0)
 #endif
 
-static void tcp_wait_for_connect(int fd, void *opaque)
+static void tcp_wait_for_connect(int fd, Error *err, void *opaque)
 {
     MigrationState *s = opaque;
 
     if (fd < 0) {
-        DPRINTF("migrate connect error\n");
+        DPRINTF("migrate connect error: %s\n", error_get_pretty(err));
         s->file = NULL;
         migrate_fd_error(s);
     } else {
diff --git a/migration-unix.c b/migration-unix.c
index 0a5f8a133..1cdadfbc8 100644
--- a/migration-unix.c
+++ b/migration-unix.c
@@ -33,12 +33,12 @@
     do { } while (0)
 #endif
 
-static void unix_wait_for_connect(int fd, void *opaque)
+static void unix_wait_for_connect(int fd, Error *err, void *opaque)
 {
     MigrationState *s = opaque;
 
     if (fd < 0) {
-        DPRINTF("migrate connect error\n");
+        DPRINTF("migrate connect error: %s\n", error_get_pretty(err));
         s->file = NULL;
         migrate_fd_error(s);
     } else {
diff --git a/migration.c b/migration.c
index 8d675b31a..c49a05a16 100644
--- a/migration.c
+++ b/migration.c
@@ -103,7 +103,6 @@ static void process_incoming_migration_co(void *opaque)
     }
     qemu_announce_self();
 
-    bdrv_clear_incoming_migration_all();
     /* Make sure all file formats flush their mutable metadata */
     bdrv_invalidate_cache_all(&local_err);
     if (local_err) {
diff --git a/monitor.c b/monitor.c
index 5bc70a642..f1031a1e3 100644
--- a/monitor.c
+++ b/monitor.c
@@ -25,7 +25,6 @@
 #include "hw/hw.h"
 #include "monitor/qdev.h"
 #include "hw/usb.h"
-#include "hw/pcmcia.h"
 #include "hw/i386/pc.h"
 #include "hw/pci/pci.h"
 #include "sysemu/watchdog.h"
@@ -206,7 +205,7 @@ struct Monitor {
     ReadLineState *rs;
     MonitorControl *mc;
     CPUState *mon_cpu;
-    BlockDriverCompletionFunc *password_completion_cb;
+    BlockCompletionFunc *password_completion_cb;
     void *password_opaque;
     mon_cmd_t *cmd_table;
     QError *error;
@@ -886,19 +885,12 @@ static void do_trace_event_set_state(Monitor *mon, const QDict *qdict)
 {
     const char *tp_name = qdict_get_str(qdict, "name");
     bool new_state = qdict_get_bool(qdict, "option");
+    Error *local_err = NULL;
 
-    bool found = false;
-    TraceEvent *ev = NULL;
-    while ((ev = trace_event_pattern(tp_name, ev)) != NULL) {
-        found = true;
-        if (!trace_event_get_state_static(ev)) {
-            monitor_printf(mon, "event \"%s\" is not traceable\n", tp_name);
-        } else {
-            trace_event_set_state_dynamic(ev, new_state);
-        }
-    }
-    if (!trace_event_is_pattern(tp_name) && !found) {
-        monitor_printf(mon, "unknown event name \"%s\"\n", tp_name);
+    qmp_trace_event_set_state(tp_name, new_state, true, true, &local_err);
+    if (local_err) {
+        qerror_report_err(local_err);
+        error_free(local_err);
     }
 }
 
@@ -1047,6 +1039,7 @@ static void do_info_registers(Monitor *mon, const QDict *qdict)
 static void do_info_jit(Monitor *mon, const QDict *qdict)
 {
     dump_exec_info((FILE *)mon, monitor_fprintf);
+    dump_drift_info((FILE *)mon, monitor_fprintf);
 }
 
 static void do_info_history(Monitor *mon, const QDict *qdict)
@@ -1078,7 +1071,15 @@ static void do_info_cpu_stats(Monitor *mon, const QDict *qdict)
 
 static void do_trace_print_events(Monitor *mon, const QDict *qdict)
 {
-    trace_print_events((FILE *)mon, &monitor_fprintf);
+    TraceEventInfoList *events = qmp_trace_event_get_state("*", NULL);
+    TraceEventInfoList *elem;
+
+    for (elem = events; elem != NULL; elem = elem->next) {
+        monitor_printf(mon, "%s : state %u\n",
+                       elem->value->name,
+                       elem->value->state == TRACE_EVENT_STATE_ENABLED ? 1 : 0);
+    }
+    qapi_free_TraceEventInfoList(events);
 }
 
 static int client_migrate_info(Monitor *mon, const QDict *qdict,
@@ -1947,7 +1948,10 @@ static void do_info_numa(Monitor *mon, const QDict *qdict)
 {
     int i;
     CPUState *cpu;
+    uint64_t *node_mem;
 
+    node_mem = g_new0(uint64_t, nb_numa_nodes);
+    query_numa_node_mem(node_mem);
     monitor_printf(mon, "%d nodes\n", nb_numa_nodes);
     for (i = 0; i < nb_numa_nodes; i++) {
         monitor_printf(mon, "node %d cpus:", i);
@@ -1958,8 +1962,9 @@ static void do_info_numa(Monitor *mon, const QDict *qdict)
         }
         monitor_printf(mon, "\n");
         monitor_printf(mon, "node %d size: %" PRId64 " MB\n", i,
-            numa_info[i].node_mem >> 20);
+                       node_mem[i] >> 20);
     }
+    g_free(node_mem);
 }
 
 #ifdef CONFIG_PROFILER
@@ -2541,8 +2546,10 @@ static int monitor_fdset_dup_fd_find_remove(int dup_fd, bool remove)
                     if (QLIST_EMPTY(&mon_fdset->dup_fds)) {
                         monitor_fdset_cleanup(mon_fdset);
                     }
+                    return -1;
+                } else {
+                    return mon_fdset->id;
                 }
-                return mon_fdset->id;
             }
         }
     }
@@ -2554,9 +2561,9 @@ int monitor_fdset_dup_fd_find(int dup_fd)
     return monitor_fdset_dup_fd_find_remove(dup_fd, false);
 }
 
-int monitor_fdset_dup_fd_remove(int dup_fd)
+void monitor_fdset_dup_fd_remove(int dup_fd)
 {
-    return monitor_fdset_dup_fd_find_remove(dup_fd, true);
+    monitor_fdset_dup_fd_find_remove(dup_fd, true);
 }
 
 int monitor_handle_fd_param(Monitor *mon, const char *fdname)
@@ -2788,13 +2795,6 @@ static mon_cmd_t info_cmds[] = {
         .mhandler.cmd = hmp_info_status,
     },
     {
-        .name       = "pcmcia",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show guest PCMCIA status",
-        .mhandler.cmd = pcmcia_info,
-    },
-    {
         .name       = "mice",
         .args_type  = "",
         .params     = "",
@@ -2918,6 +2918,13 @@ static mon_cmd_t info_cmds[] = {
         .mhandler.cmd = hmp_info_memdev,
     },
     {
+        .name       = "memory-devices",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show memory devices",
+        .mhandler.cmd = hmp_info_memory_devices,
+    },
+    {
         .name       = NULL,
     },
 };
@@ -2965,7 +2972,7 @@ static target_long monitor_get_ccr (const struct MonitorDef *md, int val)
 
     u = 0;
     for (i = 0; i < 8; i++)
-        u |= env->crf[i] << (32 - (4 * i));
+        u |= env->crf[i] << (32 - (4 * (i + 1)));
 
     return u;
 }
@@ -4205,24 +4212,6 @@ static void file_completion(Monitor *mon, const char *input)
     closedir(ffs);
 }
 
-typedef struct MonitorBlockComplete {
-    Monitor *mon;
-    const char *input;
-} MonitorBlockComplete;
-
-static void block_completion_it(void *opaque, BlockDriverState *bs)
-{
-    const char *name = bdrv_get_device_name(bs);
-    MonitorBlockComplete *mbc = opaque;
-    Monitor *mon = mbc->mon;
-    const char *input = mbc->input;
-
-    if (input[0] == '\0' ||
-        !strncmp(name, (char *)input, strlen(input))) {
-        readline_add_completion(mon->rs, name);
-    }
-}
-
 static const char *next_arg_type(const char *typestr)
 {
     const char *p = strchr(typestr, ':');
@@ -4329,23 +4318,26 @@ void object_add_completion(ReadLineState *rs, int nb_args, const char *str)
     g_slist_free(list);
 }
 
-static void device_del_bus_completion(ReadLineState *rs,  BusState *bus,
-                                      const char *str, size_t len)
+static void peripheral_device_del_completion(ReadLineState *rs,
+                                             const char *str, size_t len)
 {
-    BusChild *kid;
+    Object *peripheral = container_get(qdev_get_machine(), "/peripheral");
+    GSList *list, *item;
+
+    list = qdev_build_hotpluggable_device_list(peripheral);
+    if (!list) {
+        return;
+    }
 
-    QTAILQ_FOREACH(kid, &bus->children, sibling) {
-        DeviceState *dev = kid->child;
-        BusState *dev_child;
+    for (item = list; item; item = g_slist_next(item)) {
+        DeviceState *dev = item->data;
 
         if (dev->id && !strncmp(str, dev->id, len)) {
             readline_add_completion(rs, dev->id);
         }
-
-        QLIST_FOREACH(dev_child, &dev->child_bus, sibling) {
-            device_del_bus_completion(rs, dev_child, str, len);
-        }
     }
+
+    g_slist_free(list);
 }
 
 void chardev_remove_completion(ReadLineState *rs, int nb_args, const char *str)
@@ -4420,7 +4412,7 @@ void device_del_completion(ReadLineState *rs, int nb_args, const char *str)
 
     len = strlen(str);
     readline_set_completion_index(rs, len);
-    device_del_bus_completion(rs, sysbus_get_default(), str, len);
+    peripheral_device_del_completion(rs, str, len);
 }
 
 void object_del_completion(ReadLineState *rs, int nb_args, const char *str)
@@ -4520,16 +4512,15 @@ void netdev_del_completion(ReadLineState *rs, int nb_args, const char *str)
 
 void watchdog_action_completion(ReadLineState *rs, int nb_args, const char *str)
 {
+    int i;
+
     if (nb_args != 2) {
         return;
     }
     readline_set_completion_index(rs, strlen(str));
-    add_completion_option(rs, str, "reset");
-    add_completion_option(rs, str, "shutdown");
-    add_completion_option(rs, str, "poweroff");
-    add_completion_option(rs, str, "pause");
-    add_completion_option(rs, str, "debug");
-    add_completion_option(rs, str, "none");
+    for (i = 0; WatchdogExpirationAction_lookup[i]; i++) {
+        add_completion_option(rs, str, WatchdogExpirationAction_lookup[i]);
+    }
 }
 
 void migrate_set_capability_completion(ReadLineState *rs, int nb_args,
@@ -4661,9 +4652,9 @@ static void monitor_find_completion_by_table(Monitor *mon,
 {
     const char *cmdname;
     int i;
-    const char *ptype, *str;
+    const char *ptype, *str, *name;
     const mon_cmd_t *cmd;
-    MonitorBlockComplete mbs;
+    BlockDriverState *bs;
 
     if (nb_args <= 1) {
         /* command completion */
@@ -4715,10 +4706,14 @@ static void monitor_find_completion_by_table(Monitor *mon,
             break;
         case 'B':
             /* block device name completion */
-            mbs.mon = mon;
-            mbs.input = str;
             readline_set_completion_index(mon->rs, strlen(str));
-            bdrv_iterate(block_completion_it, &mbs);
+            for (bs = bdrv_next(NULL); bs; bs = bdrv_next(bs)) {
+                name = bdrv_get_device_name(bs);
+                if (str[0] == '\0' ||
+                    !strncmp(name, str, strlen(str))) {
+                    readline_add_completion(mon->rs, name);
+                }
+            }
             break;
         case 's':
         case 'S':
@@ -4745,8 +4740,11 @@ static void monitor_find_completion(void *opaque,
         return;
     }
 #ifdef DEBUG_COMPLETION
-    for (i = 0; i < nb_args; i++) {
-        monitor_printf(mon, "arg%d = '%s'\n", i, args[i]);
+    {
+        int i;
+        for (i = 0; i < nb_args; i++) {
+            monitor_printf(mon, "arg%d = '%s'\n", i, args[i]);
+        }
     }
 #endif
 
@@ -5243,6 +5241,7 @@ static void monitor_event(void *opaque, int event)
         monitor_printf(mon, "QEMU %s monitor - type 'help' for more "
                        "information\n", QEMU_VERSION);
         if (!mon->mux_out) {
+            readline_restart(mon->rs);
             readline_show_prompt(mon->rs);
         }
         mon->reset_seen = 1;
@@ -5374,7 +5373,7 @@ ReadLineState *monitor_get_rs(Monitor *mon)
 }
 
 int monitor_read_bdrv_key_start(Monitor *mon, BlockDriverState *bs,
-                                BlockDriverCompletionFunc *completion_cb,
+                                BlockCompletionFunc *completion_cb,
                                 void *opaque)
 {
     int err;
@@ -5406,7 +5405,7 @@ int monitor_read_bdrv_key_start(Monitor *mon, BlockDriverState *bs,
 }
 
 int monitor_read_block_device_key(Monitor *mon, const char *device,
-                                  BlockDriverCompletionFunc *completion_cb,
+                                  BlockCompletionFunc *completion_cb,
                                   void *opaque)
 {
     BlockDriverState *bs;
diff --git a/nbd.c b/nbd.c
index e7d1ceec4..a7bce4511 100644
--- a/nbd.c
+++ b/nbd.c
@@ -18,6 +18,7 @@
 
 #include "block/nbd.h"
 #include "block/block.h"
+#include "block/block_int.h"
 
 #include "block/coroutine.h"
 
@@ -107,6 +108,8 @@ struct NBDExport {
     uint32_t nbdflags;
     QTAILQ_HEAD(, NBDClient) clients;
     QTAILQ_ENTRY(NBDExport) next;
+
+    AioContext *ctx;
 };
 
 static QTAILQ_HEAD(, NBDExport) exports = QTAILQ_HEAD_INITIALIZER(exports);
@@ -123,6 +126,8 @@ struct NBDClient {
     CoMutex send_lock;
     Coroutine *send_coroutine;
 
+    bool can_read;
+
     QTAILQ_ENTRY(NBDClient) next;
     int nb_requests;
     bool closing;
@@ -130,6 +135,10 @@ struct NBDClient {
 
 /* That's all folks */
 
+static void nbd_set_handlers(NBDClient *client);
+static void nbd_unset_handlers(NBDClient *client);
+static void nbd_update_can_read(NBDClient *client);
+
 ssize_t nbd_wr_sync(int fd, void *buffer, size_t size, bool do_read)
 {
     size_t offset = 0;
@@ -156,7 +165,7 @@ ssize_t nbd_wr_sync(int fd, void *buffer, size_t size, bool do_read)
             err = socket_error();
 
             /* recoverable error */
-            if (err == EINTR || (offset > 0 && err == EAGAIN)) {
+            if (err == EINTR || (offset > 0 && (err == EAGAIN || err == EWOULDBLOCK))) {
                 continue;
             }
 
@@ -862,7 +871,7 @@ void nbd_client_put(NBDClient *client)
          */
         assert(client->closing);
 
-        qemu_set_fd_handler2(client->sock, NULL, NULL, NULL, NULL);
+        nbd_unset_handlers(client);
         close(client->sock);
         client->sock = -1;
         if (client->exp) {
@@ -898,6 +907,7 @@ static NBDRequest *nbd_request_get(NBDClient *client)
 
     assert(client->nb_requests <= MAX_NBD_REQUESTS - 1);
     client->nb_requests++;
+    nbd_update_can_read(client);
 
     req = g_slice_new0(NBDRequest);
     nbd_client_get(client);
@@ -914,12 +924,39 @@ static void nbd_request_put(NBDRequest *req)
     }
     g_slice_free(NBDRequest, req);
 
-    if (client->nb_requests-- == MAX_NBD_REQUESTS) {
-        qemu_notify_event();
-    }
+    client->nb_requests--;
+    nbd_update_can_read(client);
     nbd_client_put(client);
 }
 
+static void bs_aio_attached(AioContext *ctx, void *opaque)
+{
+    NBDExport *exp = opaque;
+    NBDClient *client;
+
+    TRACE("Export %s: Attaching clients to AIO context %p\n", exp->name, ctx);
+
+    exp->ctx = ctx;
+
+    QTAILQ_FOREACH(client, &exp->clients, next) {
+        nbd_set_handlers(client);
+    }
+}
+
+static void bs_aio_detach(void *opaque)
+{
+    NBDExport *exp = opaque;
+    NBDClient *client;
+
+    TRACE("Export %s: Detaching clients from AIO context %p\n", exp->name, exp->ctx);
+
+    QTAILQ_FOREACH(client, &exp->clients, next) {
+        nbd_unset_handlers(client);
+    }
+
+    exp->ctx = NULL;
+}
+
 NBDExport *nbd_export_new(BlockDriverState *bs, off_t dev_offset,
                           off_t size, uint32_t nbdflags,
                           void (*close)(NBDExport *))
@@ -932,7 +969,15 @@ NBDExport *nbd_export_new(BlockDriverState *bs, off_t dev_offset,
     exp->nbdflags = nbdflags;
     exp->size = size == -1 ? bdrv_getlength(bs) : size;
     exp->close = close;
+    exp->ctx = bdrv_get_aio_context(bs);
     bdrv_ref(bs);
+    bdrv_add_aio_context_notifier(bs, bs_aio_attached, bs_aio_detach, exp);
+    /*
+     * NBD exports are used for non-shared storage migration.  Make sure
+     * that BDRV_O_INCOMING is cleared and the image is ready for write
+     * access since the export could be available before migration handover.
+     */
+    bdrv_invalidate_cache(bs, NULL);
     return exp;
 }
 
@@ -980,6 +1025,8 @@ void nbd_export_close(NBDExport *exp)
     nbd_export_set_name(exp, NULL);
     nbd_export_put(exp);
     if (exp->bs) {
+        bdrv_remove_aio_context_notifier(exp->bs, bs_aio_attached,
+                                         bs_aio_detach, exp);
         bdrv_unref(exp->bs);
         exp->bs = NULL;
     }
@@ -1023,10 +1070,6 @@ void nbd_export_close_all(void)
     }
 }
 
-static int nbd_can_read(void *opaque);
-static void nbd_read(void *opaque);
-static void nbd_restart_write(void *opaque);
-
 static ssize_t nbd_co_send_reply(NBDRequest *req, struct nbd_reply *reply,
                                  int len)
 {
@@ -1035,9 +1078,8 @@ static ssize_t nbd_co_send_reply(NBDRequest *req, struct nbd_reply *reply,
     ssize_t rc, ret;
 
     qemu_co_mutex_lock(&client->send_lock);
-    qemu_set_fd_handler2(csock, nbd_can_read, nbd_read,
-                         nbd_restart_write, client);
     client->send_coroutine = qemu_coroutine_self();
+    nbd_set_handlers(client);
 
     if (!len) {
         rc = nbd_send_reply(csock, reply);
@@ -1054,7 +1096,7 @@ static ssize_t nbd_co_send_reply(NBDRequest *req, struct nbd_reply *reply,
     }
 
     client->send_coroutine = NULL;
-    qemu_set_fd_handler2(csock, nbd_can_read, nbd_read, NULL, client);
+    nbd_set_handlers(client);
     qemu_co_mutex_unlock(&client->send_lock);
     return rc;
 }
@@ -1067,6 +1109,8 @@ static ssize_t nbd_co_receive_request(NBDRequest *req, struct nbd_request *reque
     ssize_t rc;
 
     client->recv_coroutine = qemu_coroutine_self();
+    nbd_update_can_read(client);
+
     rc = nbd_receive_request(csock, request);
     if (rc < 0) {
         if (rc != -EAGAIN) {
@@ -1108,6 +1152,8 @@ static ssize_t nbd_co_receive_request(NBDRequest *req, struct nbd_request *reque
 
 out:
     client->recv_coroutine = NULL;
+    nbd_update_can_read(client);
+
     return rc;
 }
 
@@ -1259,13 +1305,6 @@ out:
     nbd_client_close(client);
 }
 
-static int nbd_can_read(void *opaque)
-{
-    NBDClient *client = opaque;
-
-    return client->recv_coroutine || client->nb_requests < MAX_NBD_REQUESTS;
-}
-
 static void nbd_read(void *opaque)
 {
     NBDClient *client = opaque;
@@ -1284,6 +1323,37 @@ static void nbd_restart_write(void *opaque)
     qemu_coroutine_enter(client->send_coroutine, NULL);
 }
 
+static void nbd_set_handlers(NBDClient *client)
+{
+    if (client->exp && client->exp->ctx) {
+        aio_set_fd_handler(client->exp->ctx, client->sock,
+                           client->can_read ? nbd_read : NULL,
+                           client->send_coroutine ? nbd_restart_write : NULL,
+                           client);
+    }
+}
+
+static void nbd_unset_handlers(NBDClient *client)
+{
+    if (client->exp && client->exp->ctx) {
+        aio_set_fd_handler(client->exp->ctx, client->sock, NULL, NULL, NULL);
+    }
+}
+
+static void nbd_update_can_read(NBDClient *client)
+{
+    bool can_read = client->recv_coroutine ||
+                    client->nb_requests < MAX_NBD_REQUESTS;
+
+    if (can_read != client->can_read) {
+        client->can_read = can_read;
+        nbd_set_handlers(client);
+
+        /* There is no need to invoke aio_notify(), since aio_set_fd_handler()
+         * in nbd_set_handlers() will have taken care of that */
+    }
+}
+
 NBDClient *nbd_client_new(NBDExport *exp, int csock,
                           void (*close)(NBDClient *))
 {
@@ -1292,13 +1362,14 @@ NBDClient *nbd_client_new(NBDExport *exp, int csock,
     client->refcount = 1;
     client->exp = exp;
     client->sock = csock;
+    client->can_read = true;
     if (nbd_send_negotiate(client)) {
         g_free(client);
         return NULL;
     }
     client->close = close;
     qemu_co_mutex_init(&client->send_lock);
-    qemu_set_fd_handler2(csock, nbd_can_read, nbd_read, NULL, client);
+    nbd_set_handlers(client);
 
     if (exp) {
         QTAILQ_INSERT_TAIL(&exp->clients, client, next);
diff --git a/net/l2tpv3.c b/net/l2tpv3.c
index 528d95b64..3b805a7a4 100644
--- a/net/l2tpv3.c
+++ b/net/l2tpv3.c
@@ -516,7 +516,7 @@ static void net_l2tpv3_cleanup(NetClientState *nc)
     qemu_purge_queued_packets(nc);
     l2tpv3_read_poll(s, false);
     l2tpv3_write_poll(s, false);
-    if (s->fd > 0) {
+    if (s->fd >= 0) {
         close(s->fd);
     }
     destroy_vector(s->msgvec, MAX_L2TPV3_MSGCNT, IOVSIZE);
@@ -660,7 +660,6 @@ int net_init_l2tpv3(const NetClientOptions *opts,
     if (fd == -1) {
         fd = -errno;
         error_report("l2tpv3_open : socket creation failed, errno = %d", -fd);
-        freeaddrinfo(result);
         goto outerr;
     }
     if (bind(fd, (struct sockaddr *) result->ai_addr, result->ai_addrlen)) {
@@ -746,7 +745,7 @@ int net_init_l2tpv3(const NetClientOptions *opts,
     return 0;
 outerr:
     qemu_del_net_client(nc);
-    if (fd > 0) {
+    if (fd >= 0) {
         close(fd);
     }
     if (result) {
diff --git a/net/net.c b/net/net.c
index 6d930ea63..7acc162b4 100644
--- a/net/net.c
+++ b/net/net.c
@@ -41,12 +41,14 @@
 #include "qapi-visit.h"
 #include "qapi/opts-visitor.h"
 #include "qapi/dealloc-visitor.h"
+#include "sysemu/sysemu.h"
 
 /* Net bridge is currently not supported for W32. */
 #if !defined(_WIN32)
 # define CONFIG_NET_BRIDGE
 #endif
 
+static VMChangeStateEntry *net_change_state_entry;
 static QTAILQ_HEAD(, NetClientState) net_clients;
 
 const char *host_net_devices[] = {
@@ -452,6 +454,12 @@ void qemu_set_vnet_hdr_len(NetClientState *nc, int len)
 
 int qemu_can_send_packet(NetClientState *sender)
 {
+    int vm_running = runstate_is_running();
+
+    if (!vm_running) {
+        return 0;
+    }
+
     if (!sender->peer) {
         return 1;
     }
@@ -504,7 +512,8 @@ void qemu_purge_queued_packets(NetClientState *nc)
     qemu_net_queue_purge(nc->peer->incoming_queue, nc);
 }
 
-void qemu_flush_queued_packets(NetClientState *nc)
+static
+void qemu_flush_or_purge_queued_packets(NetClientState *nc, bool purge)
 {
     nc->receive_disabled = 0;
 
@@ -518,9 +527,17 @@ void qemu_flush_queued_packets(NetClientState *nc)
          * the file descriptor (for tap, for example).
          */
         qemu_notify_event();
+    } else if (purge) {
+        /* Unable to empty the queue, purge remaining packets */
+        qemu_net_queue_purge(nc->incoming_queue, nc);
     }
 }
 
+void qemu_flush_queued_packets(NetClientState *nc)
+{
+    qemu_flush_or_purge_queued_packets(nc, false);
+}
+
 static ssize_t qemu_send_packet_async_with_flags(NetClientState *sender,
                                                  unsigned flags,
                                                  const uint8_t *buf, int size,
@@ -1168,6 +1185,22 @@ void qmp_set_link(const char *name, bool up, Error **errp)
     }
 }
 
+static void net_vm_change_state_handler(void *opaque, int running,
+                                        RunState state)
+{
+    /* Complete all queued packets, to guarantee we don't modify
+     * state later when VM is not running.
+     */
+    if (!running) {
+        NetClientState *nc;
+        NetClientState *tmp;
+
+        QTAILQ_FOREACH_SAFE(nc, &net_clients, next, tmp) {
+            qemu_flush_or_purge_queued_packets(nc, true);
+        }
+    }
+}
+
 void net_cleanup(void)
 {
     NetClientState *nc;
@@ -1183,6 +1216,8 @@ void net_cleanup(void)
             qemu_del_net_client(nc);
         }
     }
+
+    qemu_del_vm_change_state_handler(net_change_state_entry);
 }
 
 void net_check_clients(void)
@@ -1268,6 +1303,9 @@ int net_init_clients(void)
 #endif
     }
 
+    net_change_state_entry =
+        qemu_add_vm_change_state_handler(net_vm_change_state_handler, NULL);
+
     QTAILQ_INIT(&net_clients);
 
     if (qemu_opts_foreach(qemu_find_opts("netdev"), net_init_netdev, NULL, 1) == -1)
diff --git a/net/queue.c b/net/queue.c
index 859d02a13..f94831871 100644
--- a/net/queue.c
+++ b/net/queue.c
@@ -233,6 +233,9 @@ void qemu_net_queue_purge(NetQueue *queue, NetClientState *from)
         if (packet->sender == from) {
             QTAILQ_REMOVE(&queue->packets, packet, entry);
             queue->nq_count--;
+            if (packet->sent_cb) {
+                packet->sent_cb(packet->sender, 0);
+            }
             g_free(packet);
         }
     }
diff --git a/net/slirp.c b/net/slirp.c
index 647039ec3..377d7ef8c 100644
--- a/net/slirp.c
+++ b/net/slirp.c
@@ -345,8 +345,7 @@ void net_slirp_hostfwd_remove(Monitor *mon, const QDict *qdict)
 
     host_port = atoi(p);
 
-    err = slirp_remove_hostfwd(QTAILQ_FIRST(&slirp_stacks)->slirp, is_udp,
-                               host_addr, host_port);
+    err = slirp_remove_hostfwd(s->slirp, is_udp, host_addr, host_port);
 
     monitor_printf(mon, "host forwarding rule for %s %s\n", src_str,
                    err ? "not found" : "removed");
@@ -524,15 +523,21 @@ static int slirp_smb(SlirpState* s, const char *exported_dir,
     fprintf(f,
             "[global]\n"
             "private dir=%s\n"
-            "socket address=127.0.0.1\n"
+            "interfaces=127.0.0.1\n"
+            "bind interfaces only=yes\n"
             "pid directory=%s\n"
             "lock directory=%s\n"
             "state directory=%s\n"
+            "cache directory=%s\n"
             "ncalrpc dir=%s/ncalrpc\n"
             "log file=%s/log.smbd\n"
             "smb passwd file=%s/smbpasswd\n"
             "security = user\n"
             "map to guest = Bad User\n"
+            "load printers = no\n"
+            "printing = bsd\n"
+            "disable spoolss = yes\n"
+            "usershare max shares = 0\n"
             "[qemu]\n"
             "path=%s\n"
             "read only=no\n"
@@ -545,13 +550,14 @@ static int slirp_smb(SlirpState* s, const char *exported_dir,
             s->smb_dir,
             s->smb_dir,
             s->smb_dir,
+            s->smb_dir,
             exported_dir,
             passwd->pw_name
             );
     fclose(f);
 
-    snprintf(smb_cmdline, sizeof(smb_cmdline), "%s -s %s",
-             CONFIG_SMBD_COMMAND, smb_conf);
+    snprintf(smb_cmdline, sizeof(smb_cmdline), "%s -l %s -s %s",
+             CONFIG_SMBD_COMMAND, s->smb_dir, smb_conf);
 
     if (slirp_add_exec(s->slirp, 0, smb_cmdline, &vserver_addr, 139) < 0 ||
         slirp_add_exec(s->slirp, 0, smb_cmdline, &vserver_addr, 445) < 0) {
@@ -637,17 +643,16 @@ static int slirp_guestfwd(SlirpState *s, const char *config_str,
         goto fail_syntax;
     }
 
-    fwd = g_malloc(sizeof(struct GuestFwd));
     snprintf(buf, sizeof(buf), "guestfwd.tcp.%d", port);
 
     if ((strlen(p) > 4) && !strncmp(p, "cmd:", 4)) {
         if (slirp_add_exec(s->slirp, 0, &p[4], &server, port) < 0) {
             error_report("conflicting/invalid host:port in guest forwarding "
                          "rule '%s'", config_str);
-            g_free(fwd);
             return -1;
         }
     } else {
+        fwd = g_malloc(sizeof(struct GuestFwd));
         fwd->hd = qemu_chr_new(buf, p, NULL);
         if (!fwd->hd) {
             error_report("could not open guest forwarding device '%s'", buf);
diff --git a/net/socket.c b/net/socket.c
index fb21e20a5..68a93cd7e 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -352,7 +352,7 @@ static NetSocketState *net_socket_fd_init_dgram(NetClientState *peer,
 {
     struct sockaddr_in saddr;
     int newfd;
-    socklen_t saddr_len;
+    socklen_t saddr_len = sizeof(saddr);
     NetClientState *nc;
     NetSocketState *s;
 
@@ -389,11 +389,6 @@ static NetSocketState *net_socket_fd_init_dgram(NetClientState *peer,
 
     nc = qemu_new_net_client(&net_dgram_socket_info, peer, model, name);
 
-    snprintf(nc->info_str, sizeof(nc->info_str),
-            "socket: fd=%d (%s mcast=%s:%d)",
-            fd, is_connected ? "cloned" : "",
-            inet_ntoa(saddr.sin_addr), ntohs(saddr.sin_port));
-
     s = DO_UPCAST(NetSocketState, nc, nc);
 
     s->fd = fd;
@@ -404,6 +399,12 @@ static NetSocketState *net_socket_fd_init_dgram(NetClientState *peer,
     /* mcast: save bound address as dst */
     if (is_connected) {
         s->dgram_dst = saddr;
+        snprintf(nc->info_str, sizeof(nc->info_str),
+                 "socket: fd=%d (cloned mcast=%s:%d)",
+                 fd, inet_ntoa(saddr.sin_addr), ntohs(saddr.sin_port));
+    } else {
+        snprintf(nc->info_str, sizeof(nc->info_str),
+                 "socket: fd=%d", fd);
     }
 
     return s;
diff --git a/net/tap-bsd.c b/net/tap-bsd.c
index 90f8a0227..bf91bd03f 100644
--- a/net/tap-bsd.c
+++ b/net/tap-bsd.c
@@ -27,12 +27,13 @@
 #include "sysemu/sysemu.h"
 #include "qemu/error-report.h"
 
-#ifdef __NetBSD__
+#if defined(__NetBSD__) || defined(__FreeBSD__)
 #include <sys/ioctl.h>
 #include <net/if.h>
 #include <net/if_tap.h>
 #endif
 
+#ifndef __FreeBSD__
 int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
              int vnet_hdr_required, int mq_required)
 {
@@ -108,6 +109,73 @@ int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
     return fd;
 }
 
+#else /* __FreeBSD__ */
+
+#define PATH_NET_TAP "/dev/tap"
+
+int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
+             int vnet_hdr_required, int mq_required)
+{
+    int fd, s, ret;
+    struct ifreq ifr;
+
+    TFR(fd = open(PATH_NET_TAP, O_RDWR));
+    if (fd < 0) {
+        error_report("could not open %s: %s", PATH_NET_TAP, strerror(errno));
+        return -1;
+    }
+
+    memset(&ifr, 0, sizeof(ifr));
+
+    ret = ioctl(fd, TAPGIFNAME, (void *)&ifr);
+    if (ret < 0) {
+        error_report("could not get tap interface name");
+        goto error;
+    }
+
+    if (ifname[0] != '\0') {
+        /* User requested the interface to have a specific name */
+        s = socket(AF_LOCAL, SOCK_DGRAM, 0);
+        if (s < 0) {
+            error_report("could not open socket to set interface name");
+            goto error;
+        }
+        ifr.ifr_data = ifname;
+        ret = ioctl(s, SIOCSIFNAME, (void *)&ifr);
+        close(s);
+        if (ret < 0) {
+            error_report("could not set tap interface name");
+            goto error;
+        }
+    } else {
+        pstrcpy(ifname, ifname_size, ifr.ifr_name);
+    }
+
+    if (*vnet_hdr) {
+        /* BSD doesn't have IFF_VNET_HDR */
+        *vnet_hdr = 0;
+
+        if (vnet_hdr_required && !*vnet_hdr) {
+            error_report("vnet_hdr=1 requested, but no kernel "
+                         "support for IFF_VNET_HDR available");
+            goto error;
+        }
+    }
+    if (mq_required) {
+        error_report("mq_required requested, but not kernel support"
+                     "for IFF_MULTI_QUEUE available");
+        goto error;
+    }
+
+    fcntl(fd, F_SETFL, O_NONBLOCK);
+    return fd;
+
+error:
+    close(fd);
+    return -1;
+}
+#endif /* __FreeBSD__ */
+
 int tap_set_sndbuf(int fd, const NetdevTapOptions *tap)
 {
     return 0;
diff --git a/net/tap.c b/net/tap.c
index a40f7f023..bde6b58b1 100644
--- a/net/tap.c
+++ b/net/tap.c
@@ -598,7 +598,6 @@ static int net_init_tap_one(const NetdevTapOptions *tap, NetClientState *peer,
 
     s = net_tap_fd_init(peer, model, name, fd, vnet_hdr);
     if (!s) {
-        close(fd);
         return -1;
     }
 
@@ -797,6 +796,7 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
         if (net_init_tap_one(tap, peer, "bridge", name, ifname,
                              script, downscript, vhostfdname,
                              vnet_hdr, fd)) {
+            close(fd);
             return -1;
         }
     } else {
@@ -824,6 +824,7 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
             if (queues > 1 && i == 0 && !tap->has_ifname) {
                 if (tap_fd_get_ifname(fd, ifname)) {
                     error_report("Fail to get ifname");
+                    close(fd);
                     return -1;
                 }
             }
@@ -832,6 +833,7 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
                                  i >= 1 ? "no" : script,
                                  i >= 1 ? "no" : downscript,
                                  vhostfdname, vnet_hdr, fd)) {
+                close(fd);
                 return -1;
             }
         }
diff --git a/net/tap_int.h b/net/tap_int.h
index 86bb224bc..79afdf2d5 100644
--- a/net/tap_int.h
+++ b/net/tap_int.h
@@ -29,9 +29,6 @@
 #include "qemu-common.h"
 #include "qapi-types.h"
 
-#define DEFAULT_NETWORK_SCRIPT "/etc/qemu-ifup"
-#define DEFAULT_NETWORK_DOWN_SCRIPT "/etc/qemu-ifdown"
-
 int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
              int vnet_hdr_required, int mq_required);
 
diff --git a/numa.c b/numa.c
index 7bf7834b7..afd28666b 100644
--- a/numa.c
+++ b/numa.c
@@ -35,6 +35,7 @@
 #include "hw/boards.h"
 #include "sysemu/hostmem.h"
 #include "qmp-commands.h"
+#include "hw/mem/pc-dimm.h"
 
 QemuOptsList qemu_numa_opts = {
     .name = "numa",
@@ -210,8 +211,8 @@ void set_numa_nodes(void)
             numa_total += numa_info[i].node_mem;
         }
         if (numa_total != ram_size) {
-            error_report("total memory for NUMA nodes (%" PRIu64 ")"
-                         " should equal RAM size (" RAM_ADDR_FMT ")",
+            error_report("total memory for NUMA nodes (0x%" PRIx64 ")"
+                         " should equal RAM size (0x" RAM_ADDR_FMT ")",
                          numa_total, ram_size);
             exit(1);
         }
@@ -263,14 +264,14 @@ static void allocate_system_memory_nonnuma(MemoryRegion *mr, Object *owner,
         if (err) {
             qerror_report_err(err);
             error_free(err);
-            memory_region_init_ram(mr, owner, name, ram_size);
+            memory_region_init_ram(mr, owner, name, ram_size, &error_abort);
         }
 #else
         fprintf(stderr, "-mem-path not supported on this host\n");
         exit(1);
 #endif
     } else {
-        memory_region_init_ram(mr, owner, name, ram_size);
+        memory_region_init_ram(mr, owner, name, ram_size, &error_abort);
     }
     vmstate_register_ram_global(mr);
 }
@@ -315,13 +316,51 @@ void memory_region_allocate_system_memory(MemoryRegion *mr, Object *owner,
     }
 }
 
+static void numa_stat_memory_devices(uint64_t node_mem[])
+{
+    MemoryDeviceInfoList *info_list = NULL;
+    MemoryDeviceInfoList **prev = &info_list;
+    MemoryDeviceInfoList *info;
+
+    qmp_pc_dimm_device_list(qdev_get_machine(), &prev);
+    for (info = info_list; info; info = info->next) {
+        MemoryDeviceInfo *value = info->value;
+
+        if (value) {
+            switch (value->kind) {
+            case MEMORY_DEVICE_INFO_KIND_DIMM:
+                node_mem[value->dimm->node] += value->dimm->size;
+                break;
+            default:
+                break;
+            }
+        }
+    }
+    qapi_free_MemoryDeviceInfoList(info_list);
+}
+
+void query_numa_node_mem(uint64_t node_mem[])
+{
+    int i;
+
+    if (nb_numa_nodes <= 0) {
+        return;
+    }
+
+    numa_stat_memory_devices(node_mem);
+    for (i = 0; i < nb_numa_nodes; i++) {
+        node_mem[i] += numa_info[i].node_mem;
+    }
+}
+
 static int query_memdev(Object *obj, void *opaque)
 {
     MemdevList **list = opaque;
+    MemdevList *m = NULL;
     Error *err = NULL;
 
     if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
-        MemdevList *m = g_malloc0(sizeof(*m));
+        m = g_malloc0(sizeof(*m));
 
         m->value = g_malloc0(sizeof(*m->value));
 
@@ -369,13 +408,16 @@ static int query_memdev(Object *obj, void *opaque)
 
     return 0;
 error:
+    g_free(m->value);
+    g_free(m);
+
     return -1;
 }
 
 MemdevList *qmp_query_memdev(Error **errp)
 {
     Object *obj;
-    MemdevList *list = NULL, *m;
+    MemdevList *list = NULL;
 
     obj = object_resolve_path("/objects", NULL);
     if (obj == NULL) {
@@ -389,11 +431,6 @@ MemdevList *qmp_query_memdev(Error **errp)
     return list;
 
 error:
-    while (list) {
-        m = list;
-        list = list->next;
-        g_free(m->value);
-        g_free(m);
-    }
+    qapi_free_MemdevList(list);
     return NULL;
 }
diff --git a/os-posix.c b/os-posix.c
index cb2a7f7ad..ba091f153 100644
--- a/os-posix.c
+++ b/os-posix.c
@@ -47,7 +47,7 @@
 static struct passwd *user_pwd;
 static const char *chroot_dir;
 static int daemonize;
-static int fds[2];
+static int daemon_pipe;
 
 void os_setup_early_signal_handling(void)
 {
@@ -204,45 +204,45 @@ static void change_root(void)
 void os_daemonize(void)
 {
     if (daemonize) {
-	pid_t pid;
-
-	if (pipe(fds) == -1)
-	    exit(1);
-
-	pid = fork();
-	if (pid > 0) {
-	    uint8_t status;
-	    ssize_t len;
-
-	    close(fds[1]);
-
-	again:
-            len = read(fds[0], &status, 1);
-            if (len == -1 && (errno == EINTR))
-                goto again;
-
-            if (len != 1)
-                exit(1);
-            else if (status == 1) {
-                fprintf(stderr, "Could not acquire pidfile: %s\n", strerror(errno));
-                exit(1);
-            } else
-                exit(0);
-	} else if (pid < 0)
+        pid_t pid;
+        int fds[2];
+
+        if (pipe(fds) == -1) {
             exit(1);
+        }
+
+        pid = fork();
+        if (pid > 0) {
+            uint8_t status;
+            ssize_t len;
 
-	close(fds[0]);
-	qemu_set_cloexec(fds[1]);
+            close(fds[1]);
 
-	setsid();
+            do {
+                len = read(fds[0], &status, 1);
+            } while (len < 0 && errno == EINTR);
 
-	pid = fork();
-	if (pid > 0)
-	    exit(0);
-	else if (pid < 0)
-	    exit(1);
+            /* only exit successfully if our child actually wrote
+             * a one-byte zero to our pipe, upon successful init */
+            exit(len == 1 && status == 0 ? 0 : 1);
+
+        } else if (pid < 0) {
+            exit(1);
+        }
 
-	umask(027);
+        close(fds[0]);
+        daemon_pipe = fds[1];
+        qemu_set_cloexec(daemon_pipe);
+
+        setsid();
+
+        pid = fork();
+        if (pid > 0) {
+            exit(0);
+        } else if (pid < 0) {
+            exit(1);
+        }
+        umask(027);
 
         signal(SIGTSTP, SIG_IGN);
         signal(SIGTTOU, SIG_IGN);
@@ -255,47 +255,36 @@ void os_setup_post(void)
     int fd = 0;
 
     if (daemonize) {
-	uint8_t status = 0;
-	ssize_t len;
-
-    again1:
-	len = write(fds[1], &status, 1);
-	if (len == -1 && (errno == EINTR))
-	    goto again1;
-
-	if (len != 1)
-	    exit(1);
-
         if (chdir("/")) {
             perror("not able to chdir to /");
             exit(1);
         }
-	TFR(fd = qemu_open("/dev/null", O_RDWR));
-	if (fd == -1)
-	    exit(1);
+        TFR(fd = qemu_open("/dev/null", O_RDWR));
+        if (fd == -1) {
+            exit(1);
+        }
     }
 
     change_root();
     change_process_uid();
 
     if (daemonize) {
+        uint8_t status = 0;
+        ssize_t len;
+
         dup2(fd, 0);
         dup2(fd, 1);
         dup2(fd, 2);
 
         close(fd);
-    }
-}
 
-void os_pidfile_error(void)
-{
-    if (daemonize) {
-        uint8_t status = 1;
-        if (write(fds[1], &status, 1) != 1) {
-            perror("daemonize. Writing to pipe\n");
+        do {        
+            len = write(daemon_pipe, &status, 1);
+        } while (len < 0 && errno == EINTR);
+        if (len != 1) {
+            exit(1);
         }
-    } else
-        fprintf(stderr, "Could not acquire pid file: %s\n", strerror(errno));
+    }
 }
 
 void os_set_line_buffering(void)
diff --git a/os-win32.c b/os-win32.c
index 5f95caac1..c0daf8e18 100644
--- a/os-win32.c
+++ b/os-win32.c
@@ -104,11 +104,6 @@ void os_parse_cmd_args(int index, const char *optarg)
     return;
 }
 
-void os_pidfile_error(void)
-{
-    fprintf(stderr, "Could not acquire pid file: %s\n", strerror(errno));
-}
-
 int qemu_create_pidfile(const char *filename)
 {
     char buffer[128];
diff --git a/pc-bios/QEMU,tcx.bin b/pc-bios/QEMU,tcx.bin
index eed108f3f..d79cc1fea 100644
--- a/pc-bios/QEMU,tcx.bin
+++ b/pc-bios/QEMU,tcx.bin
diff --git a/pc-bios/bios-256k.bin b/pc-bios/bios-256k.bin
index 09686a381..fab9da2b3 100644
--- a/pc-bios/bios-256k.bin
+++ b/pc-bios/bios-256k.bin
diff --git a/pc-bios/bios.bin b/pc-bios/bios.bin
index 2314027c3..8c718e1b9 100644
--- a/pc-bios/bios.bin
+++ b/pc-bios/bios.bin
diff --git a/pc-bios/linuxboot.bin b/pc-bios/linuxboot.bin
index e7c36694f..130103fb7 100644
--- a/pc-bios/linuxboot.bin
+++ b/pc-bios/linuxboot.bin
diff --git a/pc-bios/openbios-ppc b/pc-bios/openbios-ppc
index 0f2fc3a0a..994052f14 100644
--- a/pc-bios/openbios-ppc
+++ b/pc-bios/openbios-ppc
diff --git a/pc-bios/openbios-sparc32 b/pc-bios/openbios-sparc32
index 8917b558e..6d5a381a7 100644
--- a/pc-bios/openbios-sparc32
+++ b/pc-bios/openbios-sparc32
diff --git a/pc-bios/openbios-sparc64 b/pc-bios/openbios-sparc64
index cf72a59ee..61bd46bf4 100644
--- a/pc-bios/openbios-sparc64
+++ b/pc-bios/openbios-sparc64
diff --git a/pc-bios/optionrom/linuxboot.S b/pc-bios/optionrom/linuxboot.S
index 748c83116..5bc0af08e 100644
--- a/pc-bios/optionrom/linuxboot.S
+++ b/pc-bios/optionrom/linuxboot.S
@@ -76,14 +76,45 @@ boot_kernel:
 
 
 copy_kernel:
+	/* Compute initrd address */
+	mov		$0xe801, %ax
+	xor		%cx, %cx
+	xor		%dx, %dx
+	int		$0x15
+
+	/* Output could be in AX/BX or CX/DX */
+	or		%cx, %cx
+	jnz		1f
+	or		%dx, %dx
+	jnz		1f
+	mov		%ax, %cx
+	mov		%bx, %dx
+1:
+
+	or		%dx, %dx
+	jnz		2f
+	addw		$1024, %cx            /* add 1 MB */
+	movzwl		%cx, %edi
+	shll		$10, %edi             /* convert to bytes */
+	jmp		3f
+
+2:
+	addw		$16777216 >> 16, %dx  /* add 16 MB */
+	movzwl		%dx, %edi
+	shll		$16, %edi             /* convert to bytes */
+
+3:
+	read_fw         FW_CFG_INITRD_SIZE
+	subl            %eax, %edi
+	andl            $-4096, %edi          /* EDI = start of initrd */
 
 	/* We need to load the kernel into memory we can't access in 16 bit
 	   mode, so let's get into 32 bit mode, write the kernel and jump
 	   back again. */
 
 	/* Reserve space on the stack for our GDT descriptor. */
-	mov		%esp, %ebp
-	sub		$16, %esp
+	mov             %esp, %ebp
+	sub             $16, %esp
 
 	/* Now create the GDT descriptor */
 	movw		$((3 * 8) - 1), -16(%bp)
@@ -108,10 +139,18 @@ copy_kernel:
 	/* We're now running in 16-bit CS, but 32-bit ES! */
 
 	/* Load kernel and initrd */
+	pushl		%edi
+	read_fw_blob_addr32_edi(FW_CFG_INITRD)
 	read_fw_blob_addr32(FW_CFG_KERNEL)
-	read_fw_blob_addr32(FW_CFG_INITRD)
 	read_fw_blob_addr32(FW_CFG_CMDLINE)
-	read_fw_blob_addr32(FW_CFG_SETUP)
+
+	read_fw		FW_CFG_SETUP_ADDR
+	mov		%eax, %edi
+	mov		%eax, %ebx
+	read_fw_blob_addr32_edi(FW_CFG_SETUP)
+
+	/* Update the header with the initrd address we chose above */
+	popl		%es:0x218(%ebx)
 
 	/* And now jump into Linux! */
 	mov		$0, %eax
diff --git a/pc-bios/optionrom/optionrom.h b/pc-bios/optionrom/optionrom.h
index ce436085d..f1a9021ec 100644
--- a/pc-bios/optionrom/optionrom.h
+++ b/pc-bios/optionrom/optionrom.h
@@ -51,8 +51,6 @@
 .endm
 
 #define read_fw_blob_pre(var)				\
-	read_fw		var ## _ADDR;			\
-	mov		%eax, %edi;			\
 	read_fw		var ## _SIZE;			\
 	mov		%eax, %ecx;			\
 	mov		$var ## _DATA, %ax;		\
@@ -68,6 +66,8 @@
  * Clobbers:	%eax, %edx, %es, %ecx, %edi
  */
 #define read_fw_blob(var)				\
+	read_fw		var ## _ADDR;			\
+	mov		%eax, %edi;			\
 	read_fw_blob_pre(var);				\
 	/* old as(1) doesn't like this insn so emit the bytes instead: \
 	rep insb	(%dx), %es:(%edi);		\
@@ -80,7 +80,22 @@
  *
  * Clobbers:	%eax, %edx, %es, %ecx, %edi
  */
-#define read_fw_blob_addr32(var)				\
+#define read_fw_blob_addr32(var)			\
+	read_fw		var ## _ADDR;			\
+	mov		%eax, %edi;			\
+	read_fw_blob_pre(var);				\
+	/* old as(1) doesn't like this insn so emit the bytes instead: \
+	addr32 rep insb	(%dx), %es:(%edi);		\
+	*/						\
+	.dc.b		0x67,0xf3,0x6c
+
+/*
+ * Read a blob from the fw_cfg device in forced addr32 mode, address is in %edi.
+ * Requires _SIZE and _DATA values for the parameter.
+ *
+ * Clobbers:	%eax, %edx, %edi, %es, %ecx
+ */
+#define read_fw_blob_addr32_edi(var)			\
 	read_fw_blob_pre(var);				\
 	/* old as(1) doesn't like this insn so emit the bytes instead: \
 	addr32 rep insb	(%dx), %es:(%edi);		\
diff --git a/pc-bios/petalogix-s3adsp1800.dtb b/pc-bios/petalogix-s3adsp1800.dtb
index 93c5973fd..8ac80f2f2 100644
--- a/pc-bios/petalogix-s3adsp1800.dtb
+++ b/pc-bios/petalogix-s3adsp1800.dtb
diff --git a/pc-bios/s390-ccw.img b/pc-bios/s390-ccw.img
index e3ea0d566..44873ad18 100644
--- a/pc-bios/s390-ccw.img
+++ b/pc-bios/s390-ccw.img
diff --git a/pc-bios/s390-ccw/bootmap.c b/pc-bios/s390-ccw/bootmap.c
index f1756796d..115d8bbac 100644
--- a/pc-bios/s390-ccw/bootmap.c
+++ b/pc-bios/s390-ccw/bootmap.c
@@ -40,11 +40,6 @@ static void jump_to_IPL_2(void)
     ResetInfo *current = 0;
 
     void (*ipl)(void) = (void *) (uint64_t) current->ipl_continue;
-    debug_print_addr("set IPL addr to", ipl);
-
-    /* Ensure the guest output starts fresh */
-    sclp_print("\n");
-
     *current = save;
     ipl(); /* should not return */
 }
@@ -64,6 +59,11 @@ static void jump_to_IPL_code(uint64_t address)
     current->ipl_addr = (uint32_t) (uint64_t) &jump_to_IPL_2;
     current->ipl_continue = address & 0x7fffffff;
 
+    debug_print_int("set IPL addr to", current->ipl_continue);
+
+    /* Ensure the guest output starts fresh */
+    sclp_print("\n");
+
     /*
      * HACK ALERT.
      * We use the load normal reset to keep r15 unchanged. jump_to_IPL_2
@@ -93,11 +93,23 @@ static inline void verify_boot_info(BootInfo *bip)
                "Bad block size in zIPL section of the 1st record.");
 }
 
-static bool eckd_valid_address(BootMapPointer *p)
+static block_number_t eckd_block_num(BootMapPointer *p)
 {
+    const uint64_t sectors = virtio_get_sectors();
+    const uint64_t heads = virtio_get_heads();
     const uint64_t cylinder = p->eckd.cylinder
                             + ((p->eckd.head & 0xfff0) << 12);
     const uint64_t head = p->eckd.head & 0x000f;
+    const block_number_t block = sectors * heads * cylinder
+                               + sectors * head
+                               + p->eckd.sector
+                               - 1; /* block nr starts with zero */
+    return block;
+}
+
+static bool eckd_valid_address(BootMapPointer *p)
+{
+    const uint64_t head = p->eckd.head & 0x000f;
 
     if (head >= virtio_get_heads()
         ||  p->eckd.sector > virtio_get_sectors()
@@ -105,27 +117,14 @@ static bool eckd_valid_address(BootMapPointer *p)
         return false;
     }
 
-    if (!virtio_guessed_disk_nature() && cylinder >= virtio_get_cylinders()) {
+    if (!virtio_guessed_disk_nature() &&
+        eckd_block_num(p) >= virtio_get_blocks()) {
         return false;
     }
 
     return true;
 }
 
-static block_number_t eckd_block_num(BootMapPointer *p)
-{
-    const uint64_t sectors = virtio_get_sectors();
-    const uint64_t heads = virtio_get_heads();
-    const uint64_t cylinder = p->eckd.cylinder
-                            + ((p->eckd.head & 0xfff0) << 12);
-    const uint64_t head = p->eckd.head & 0x000f;
-    const block_number_t block = sectors * heads * cylinder
-                               + sectors * head
-                               + p->eckd.sector
-                               - 1; /* block nr starts with zero */
-    return block;
-}
-
 static block_number_t load_eckd_segments(block_number_t blk, uint64_t *address)
 {
     block_number_t block_nr;
@@ -223,7 +222,6 @@ static void ipl_eckd_cdl(void)
 
     memset(sec, FREE_SPACE_FILLER, sizeof(sec));
     read_block(1, ipl2, "Cannot read IPL2 record at block 1");
-    IPL_assert(magic_match(ipl2, IPL2_MAGIC), "No IPL2 record");
 
     mbr = &ipl2->u.x.mbr;
     IPL_assert(magic_match(mbr, ZIPL_MAGIC), "No zIPL section in IPL2 record.");
@@ -247,12 +245,10 @@ static void ipl_eckd_cdl(void)
     /* no return */
 }
 
-static void ipl_eckd_ldl(ECKD_IPL_mode_t mode)
+static void print_eckd_ldl_msg(ECKD_IPL_mode_t mode)
 {
     LDL_VTOC *vlbl = (void *)sec; /* already read, 3rd block */
     char msg[4] = { '?', '.', '\n', '\0' };
-    block_number_t block_nr;
-    BootInfo *bip;
 
     sclp_print((mode == ECKD_CMS) ? "CMS" : "LDL");
     sclp_print(" version ");
@@ -272,12 +268,27 @@ static void ipl_eckd_ldl(ECKD_IPL_mode_t mode)
     }
     sclp_print(msg);
     print_volser(vlbl->volser);
+}
+
+static void ipl_eckd_ldl(ECKD_IPL_mode_t mode)
+{
+    block_number_t block_nr;
+    BootInfo *bip = (void *)(sec + 0x70); /* BootInfo is MBR for LDL */
+
+    if (mode != ECKD_LDL_UNLABELED) {
+        print_eckd_ldl_msg(mode);
+    }
 
     /* DO NOT read BootMap pointer (only one, xECKD) at block #2 */
 
     memset(sec, FREE_SPACE_FILLER, sizeof(sec));
-    read_block(0, sec, "Cannot read block 0");
-    bip = (void *)(sec + 0x70); /* "boot info" is "eckd mbr" for LDL */
+    read_block(0, sec, "Cannot read block 0 to grab boot info.");
+    if (mode == ECKD_LDL_UNLABELED) {
+        if (!magic_match(bip->magic, ZIPL_MAGIC)) {
+            return; /* not applicable layout */
+        }
+        sclp_print("unlabeled LDL.\n");
+    }
     verify_boot_info(bip);
 
     block_nr = eckd_block_num((void *)&(bip->bp.ipl.bm_ptr.eckd.bptr));
@@ -285,17 +296,23 @@ static void ipl_eckd_ldl(ECKD_IPL_mode_t mode)
     /* no return */
 }
 
-static void ipl_eckd(ECKD_IPL_mode_t mode)
+static void print_eckd_msg(void)
 {
-    switch (mode) {
-    case ECKD_CDL:
-        ipl_eckd_cdl(); /* no return */
-    case ECKD_CMS:
-    case ECKD_LDL:
-        ipl_eckd_ldl(mode); /* no return */
-    default:
-        virtio_panic("\n! Unknown ECKD IPL mode !\n");
+    char msg[] = "Using ECKD scheme (block size *****), ";
+    char *p = &msg[34], *q = &msg[30];
+    int n = virtio_get_block_size();
+
+    /* Fill in the block size and show up the message */
+    if (n > 0 && n <= 99999) {
+        while (n) {
+            *p-- = '0' + (n % 10);
+            n /= 10;
+        }
+        while (p >= q) {
+            *p-- = ' ';
+        }
     }
+    sclp_print(msg);
 }
 
 /***********************************************************************
@@ -447,14 +464,13 @@ void zipl_load(void)
     }
 
     /* We have failed to follow the SCSI scheme, so */
-    sclp_print("Using ECKD scheme.\n");
     if (virtio_guessed_disk_nature()) {
         sclp_print("Using guessed DASD geometry.\n");
         virtio_assume_eckd();
     }
-
+    print_eckd_msg();
     if (magic_match(mbr->magic, IPL1_MAGIC)) {
-        ipl_eckd(ECKD_CDL); /* no return */
+        ipl_eckd_cdl(); /* no return */
     }
 
     /* LDL/CMS? */
@@ -462,11 +478,18 @@ void zipl_load(void)
     read_block(2, vlbl, "Cannot read block 2");
 
     if (magic_match(vlbl->magic, CMS1_MAGIC)) {
-        ipl_eckd(ECKD_CMS); /* no return */
+        ipl_eckd_ldl(ECKD_CMS); /* no return */
     }
     if (magic_match(vlbl->magic, LNX1_MAGIC)) {
-        ipl_eckd(ECKD_LDL); /* no return */
+        ipl_eckd_ldl(ECKD_LDL); /* no return */
     }
 
-    virtio_panic("\n* invalid MBR magic *\n");
+    ipl_eckd_ldl(ECKD_LDL_UNLABELED); /* it still may return */
+    /*
+     * Ok, it is not a LDL by any means.
+     * It still might be a CDL with zero record keys for IPL1 and IPL2
+     */
+    ipl_eckd_cdl();
+
+    virtio_panic("\n* this can never happen *\n");
 }
diff --git a/pc-bios/s390-ccw/bootmap.h b/pc-bios/s390-ccw/bootmap.h
index 30ef22fe6..6a4823d54 100644
--- a/pc-bios/s390-ccw/bootmap.h
+++ b/pc-bios/s390-ccw/bootmap.h
@@ -257,9 +257,9 @@ typedef struct IplVolumeLabel {
 
 typedef enum {
     ECKD_NO_IPL,
-    ECKD_CDL,
     ECKD_CMS,
     ECKD_LDL,
+    ECKD_LDL_UNLABELED,
 } ECKD_IPL_mode_t;
 
 /* utility code below */
diff --git a/pc-bios/s390-ccw/virtio.c b/pc-bios/s390-ccw/virtio.c
index 31b23b086..c0540d1cd 100644
--- a/pc-bios/s390-ccw/virtio.c
+++ b/pc-bios/s390-ccw/virtio.c
@@ -275,12 +275,14 @@ void virtio_assume_scsi(void)
 {
     guessed_disk_nature = true;
     blk_cfg.blk_size = 512;
+    blk_cfg.physical_block_exp = 0;
 }
 
 void virtio_assume_eckd(void)
 {
     guessed_disk_nature = true;
     blk_cfg.blk_size = 4096;
+    blk_cfg.physical_block_exp = 0;
 
     /* this must be here to calculate code segment position */
     blk_cfg.geometry.heads = 15;
@@ -290,36 +292,52 @@ void virtio_assume_eckd(void)
 bool virtio_disk_is_scsi(void)
 {
     if (guessed_disk_nature) {
-        return (blk_cfg.blk_size  == 512);
+        return (virtio_get_block_size()  == 512);
     }
     return (blk_cfg.geometry.heads == 255)
         && (blk_cfg.geometry.sectors == 63)
-        && (blk_cfg.blk_size  == 512);
+        && (virtio_get_block_size()  == 512);
+}
+
+/*
+ * Other supported value pairs, if any, would need to be added here.
+ * Note: head count is always 15.
+ */
+static inline u8 virtio_eckd_sectors_for_block_size(int size)
+{
+    switch (size) {
+    case 512:
+        return 49;
+    case 1024:
+        return 33;
+    case 2048:
+        return 21;
+    case 4096:
+        return 12;
+    }
+    return 0;
 }
 
 bool virtio_disk_is_eckd(void)
 {
+    const int block_size = virtio_get_block_size();
+
     if (guessed_disk_nature) {
-        return (blk_cfg.blk_size  == 4096);
+        return (block_size  == 4096);
     }
     return (blk_cfg.geometry.heads == 15)
-        && (blk_cfg.geometry.sectors == 12)
-        && (blk_cfg.blk_size  == 4096);
+        && (blk_cfg.geometry.sectors ==
+            virtio_eckd_sectors_for_block_size(block_size));
 }
 
 bool virtio_ipl_disk_is_valid(void)
 {
-    return blk_cfg.blk_size && (virtio_disk_is_scsi() || virtio_disk_is_eckd());
+    return virtio_disk_is_scsi() || virtio_disk_is_eckd();
 }
 
 int virtio_get_block_size(void)
 {
-    return blk_cfg.blk_size;
-}
-
-uint16_t virtio_get_cylinders(void)
-{
-    return blk_cfg.geometry.cylinders;
+    return blk_cfg.blk_size << blk_cfg.physical_block_exp;
 }
 
 uint8_t virtio_get_heads(void)
@@ -332,6 +350,12 @@ uint8_t virtio_get_sectors(void)
     return blk_cfg.geometry.sectors;
 }
 
+uint64_t virtio_get_blocks(void)
+{
+    return blk_cfg.capacity /
+           (virtio_get_block_size() / VIRTIO_SECTOR_SIZE);
+}
+
 void virtio_setup_block(struct subchannel_id schid)
 {
     struct vq_info_block info;
diff --git a/pc-bios/s390-ccw/virtio.h b/pc-bios/s390-ccw/virtio.h
index f1fb1b08f..c23466b8d 100644
--- a/pc-bios/s390-ccw/virtio.h
+++ b/pc-bios/s390-ccw/virtio.h
@@ -192,9 +192,9 @@ extern bool virtio_disk_is_scsi(void);
 extern bool virtio_disk_is_eckd(void);
 extern bool virtio_ipl_disk_is_valid(void);
 extern int virtio_get_block_size(void);
-extern uint16_t virtio_get_cylinders(void);
 extern uint8_t virtio_get_heads(void);
 extern uint8_t virtio_get_sectors(void);
+extern uint64_t virtio_get_blocks(void);
 extern int virtio_read_many(ulong sector, void *load_addr, int sec_num);
 
 #define VIRTIO_SECTOR_SIZE 512
diff --git a/pc-bios/vgabios-cirrus.bin b/pc-bios/vgabios-cirrus.bin
index 57a5f954a..0c4d25346 100644
--- a/pc-bios/vgabios-cirrus.bin
+++ b/pc-bios/vgabios-cirrus.bin
diff --git a/pc-bios/vgabios-qxl.bin b/pc-bios/vgabios-qxl.bin
index ed79993ad..4e08e1362 100644
--- a/pc-bios/vgabios-qxl.bin
+++ b/pc-bios/vgabios-qxl.bin
diff --git a/pc-bios/vgabios-stdvga.bin b/pc-bios/vgabios-stdvga.bin
index d3579b4fb..e5e5b14e4 100644
--- a/pc-bios/vgabios-stdvga.bin
+++ b/pc-bios/vgabios-stdvga.bin
diff --git a/pc-bios/vgabios-vmware.bin b/pc-bios/vgabios-vmware.bin
index f89845e75..cf2576d34 100644
--- a/pc-bios/vgabios-vmware.bin
+++ b/pc-bios/vgabios-vmware.bin
diff --git a/pc-bios/vgabios.bin b/pc-bios/vgabios.bin
index d3038f418..bad187dc2 100644
--- a/pc-bios/vgabios.bin
+++ b/pc-bios/vgabios.bin
diff --git a/pixman/.gitignore b/pixman/.gitignore
index 96d222bbe..0f114966c 100644
--- a/pixman/.gitignore
+++ b/pixman/.gitignore
@@ -26,24 +26,58 @@ stamp-h?
 config.h
 config.h.in
 .*.swp
-pixman/pixman-combine32.c
-pixman/pixman-combine32.h
-pixman/pixman-combine64.c
-pixman/pixman-combine64.h
+demos/alpha-test
+demos/checkerboard
+demos/clip-in
+demos/clip-test
+demos/composite-test
+demos/conical-test
+demos/convolution-test
+demos/gradient-test
+demos/linear-gradient
+demos/quad2quad
+demos/radial-test
+demos/scale
+demos/screen-test
+demos/srgb-test
+demos/srgb-trap-test
+demos/trap-test
+demos/tri-test
+pixman/pixman-srgb.c
 pixman/pixman-version.h
+test/a1-trap-test
+test/affine-test
+test/alpha-loop
+test/alphamap
 test/alpha-test
 test/blitters-test
 test/clip-in
 test/clip-test
+test/combiner-test
 test/composite
 test/composite-test
+test/composite-traps-test
 test/convolution-test
 test/fetch-test
+test/glyph-test
+test/gradient-crash-test
 test/gradient-test
+test/infinite-loop
+test/lowlevel-blt-bench
 test/oob-test
+test/pdf-op-test
+test/prng-test
+test/radial-perf-test
+test/region-contains-test
 test/region-test
+test/region-translate
+test/region-translate-test
+test/rotate-test
+test/scaling-crash-test
+test/scaling-helpers-test
 test/scaling-test
 test/screen-test
+test/stress-test
 test/trap-crasher
 test/trap-test
 test/window-test
@@ -52,3 +86,4 @@ test/window-test
 *.lib
 *.ilk
 *.obj
+*.exe
diff --git a/pixman/CODING_STYLE b/pixman/CODING_STYLE
index 126f1a981..9f5171d10 100644
--- a/pixman/CODING_STYLE
+++ b/pixman/CODING_STYLE
@@ -19,9 +19,6 @@ not
 
 
 
-Specific guidelines:
-
-
 Indentation
 ===========
 
@@ -93,7 +90,7 @@ or like this:
          * It extends over multiple lines
 	 */
 
-Generally comments should say things that is clear from the code
+Generally comments should say things that aren't clear from the code
 itself. If too many comments say obvious things, then people will just
 stop reading all comments, including the good ones.
 
@@ -152,21 +149,6 @@ Whitespace
 
 	if (condition) foo (); else bar ();	/* Yuck! */
 
-* Do eliminate trailing whitespace (space or tab characters) on any
-  line. Also, avoid putting initial or final blank lines into any
-  file, and never use multiple blank lines instead of a single blank
-  line.
-
-* Do enable the default git pre-commit hook that detect trailing
-  whitespace for you and help you to avoid corrupting cairo's tree
-  with it. Do that as follows:
-
-	chmod a+x .git/hooks/pre-commit
-
-* You might also find the git-stripspace utility helpful which acts as
-  a filter to remove trailing whitespace as well as initial, final,
-  and duplicate blank lines.
-
 
 Function Definitions
 ====================
@@ -174,7 +156,7 @@ Function Definitions
 Function definitions should take the following form:
 
 	void
-	my_function (argument)
+	my_function (int argument)
 	{
 	    do_my_things ();
 	}
@@ -214,3 +196,4 @@ popular editors:
  * vim:sw=4:sts=4:ts=8:tw=78:fo=tcroq:cindent:cino=\:0,(0
  * vim:isk=a-z,A-Z,48-57,_,.,-,>
  */
+
diff --git a/pixman/COPYING b/pixman/COPYING
index b0571e6a6..6168dea56 100644
--- a/pixman/COPYING
+++ b/pixman/COPYING
@@ -1,40 +1,42 @@
-The following is the 'standard copyright' agreed upon by most contributors,
-and is currently the canonical license, though a modification is currently
-under discussion.  Copyright holders of new code should use this license
-statement where possible, and append their name to this list.  
+The following is the MIT license, agreed upon by most contributors.
+Copyright holders of new code should use this license statement where
+possible. They may also add themselves to the list below.
 
-Copyright 1987, 1988, 1989, 1998  The Open Group
-Copyright 1987, 1988, 1989 Digital Equipment Corporation
-Copyright 1999, 2004, 2008 Keith Packard
-Copyright 2000 SuSE, Inc.
-Copyright 2000 Keith Packard, member of The XFree86 Project, Inc.
-Copyright 2004, 2005, 2007, 2008 Red Hat, Inc.
-Copyright 2004 Nicholas Miell
-Copyright 2005 Lars Knoll & Zack Rusin, Trolltech
-Copyright 2005 Trolltech AS
-Copyright 2007 Luca Barbato
-Copyright 2008 Aaron Plattner, NVIDIA Corporation
-Copyright 2008 Rodrigo Kumpera
-Copyright 2008 André Tupinambá
-Copyright 2008 Mozilla Corporation
-Copyright 2008 Frederic Plourde
-Copyright 2009 Sun Microsystems, Inc.
-
-Permission is hereby granted, free of charge, to any person obtaining a
-copy of this software and associated documentation files (the "Software"),
-to deal in the Software without restriction, including without limitation
-the rights to use, copy, modify, merge, publish, distribute, sublicense,
-and/or sell copies of the Software, and to permit persons to whom the
-Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice (including the next
-paragraph) shall be included in all copies or substantial portions of the
-Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
+/*
+ * Copyright 1987, 1988, 1989, 1998  The Open Group
+ * Copyright 1987, 1988, 1989 Digital Equipment Corporation
+ * Copyright 1999, 2004, 2008 Keith Packard
+ * Copyright 2000 SuSE, Inc.
+ * Copyright 2000 Keith Packard, member of The XFree86 Project, Inc.
+ * Copyright 2004, 2005, 2007, 2008, 2009, 2010 Red Hat, Inc.
+ * Copyright 2004 Nicholas Miell
+ * Copyright 2005 Lars Knoll & Zack Rusin, Trolltech
+ * Copyright 2005 Trolltech AS
+ * Copyright 2007 Luca Barbato
+ * Copyright 2008 Aaron Plattner, NVIDIA Corporation
+ * Copyright 2008 Rodrigo Kumpera
+ * Copyright 2008 André Tupinambá
+ * Copyright 2008 Mozilla Corporation
+ * Copyright 2008 Frederic Plourde
+ * Copyright 2009, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2009, 2010 Nokia Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
diff --git a/pixman/Makefile.am b/pixman/Makefile.am
index 63b08c1fb..5137c9ea3 100644
--- a/pixman/Makefile.am
+++ b/pixman/Makefile.am
@@ -1,4 +1,4 @@
-SUBDIRS = pixman test
+SUBDIRS = pixman demos test
 
 pkgconfigdir=$(libdir)/pkgconfig
 pkgconfig_DATA=pixman-1.pc
@@ -10,16 +10,20 @@ snapshot:
 	test -d "$(srcdir)/.git" && distdir=$$distdir-`cd "$(srcdir)" && git rev-parse HEAD | cut -c 1-6`; \
 	$(MAKE) $(AM_MAKEFLAGS) distdir="$$distdir" dist
 
-GPGKEY=6FF7C1A8
+GPGKEY=3892336E
 USERNAME=$$USER
-RELEASE_OR_SNAPSHOT = $$(if test "x$(CAIRO_VERSION_MINOR)" = "x$$(echo "$(CAIRO_VERSION_MINOR)/2*2" | bc)" ; then echo release; else echo snapshot; fi)
+RELEASE_OR_SNAPSHOT = $$(if test "x$(PIXMAN_VERSION_MINOR)" = "x$$(echo "$(PIXMAN_VERSION_MINOR)/2*2" | bc)" ; then echo release; else echo snapshot; fi)
 RELEASE_CAIRO_HOST =	$(USERNAME)@cairographics.org
-RELEASE_CAIRO_DIR =	/srv/cairo.freedesktop.org/www/releases
-RELEASE_CAIRO_URL = 	http://cairographics.org/releases
+RELEASE_CAIRO_DIR =	/srv/cairo.freedesktop.org/www/$(RELEASE_OR_SNAPSHOT)s
+RELEASE_CAIRO_URL = 	http://cairographics.org/$(RELEASE_OR_SNAPSHOT)s
 RELEASE_XORG_URL =	http://xorg.freedesktop.org/archive/individual/lib
 RELEASE_XORG_HOST =	$(USERNAME)@xorg.freedesktop.org
 RELEASE_XORG_DIR =	/srv/xorg.freedesktop.org/archive/individual/lib
-RELEASE_ANNOUNCE_LIST = cairo-announce@cairographics.org, xorg-announce@lists.freedesktop.org
+RELEASE_ANNOUNCE_LIST = cairo-announce@cairographics.org, xorg-announce@lists.freedesktop.org, pixman@lists.freedesktop.org
+
+EXTRA_DIST =				\
+	Makefile.win32			\
+	Makefile.win32.common
 
 tar_gz = $(PACKAGE)-$(VERSION).tar.gz
 tar_bz2 = $(PACKAGE)-$(VERSION).tar.bz2
@@ -83,11 +87,12 @@ release-tag:
 	git tag -u $(GPGKEY) -m "$(PACKAGE) $(VERSION) release" $(PACKAGE)-$(VERSION)
 
 release-upload: release-check $(tar_gz) $(tar_bz2) $(sha1_tgz) $(sha1_tbz2) $(md5_tgz) $(gpg_file)
-	mkdir -p releases
 	scp $(tar_gz) $(sha1_tgz) $(gpg_file) $(RELEASE_CAIRO_HOST):$(RELEASE_CAIRO_DIR)
 	scp $(tar_gz) $(tar_bz2) $(RELEASE_XORG_HOST):$(RELEASE_XORG_DIR)
 	ssh $(RELEASE_CAIRO_HOST) "rm -f $(RELEASE_CAIRO_DIR)/LATEST-$(PACKAGE)-[0-9]* && ln -s $(tar_gz) $(RELEASE_CAIRO_DIR)/LATEST-$(PACKAGE)-$(VERSION)"
 
+RELEASE_TYPE = $$(if test "x$(PIXMAN_VERSION_MINOR)" = "x$$(echo "$(PIXMAN_VERSION_MINOR)/2*2" | bc)" ; then echo "stable release in the" ; else echo "development snapshot leading up to a stable"; fi)
+
 release-publish-message: $(HASHFILES) ensure-prev
 	@echo "Please follow the instructions in RELEASING to push stuff out and"
 	@echo "send out the announcement mails.  Here is the excerpt you need:"
@@ -95,7 +100,7 @@ release-publish-message: $(HASHFILES) ensure-prev
 	@echo "Lists:  $(RELEASE_ANNOUNCE_LIST)"
 	@echo "Subject: [ANNOUNCE] $(PACKAGE) release $(VERSION) now available"
 	@echo "============================== CUT HERE =============================="
-	@echo "A new $(PACKAGE) release $(VERSION) is now available"
+	@echo "A new $(PACKAGE) release $(VERSION) is now available. This is a $(RELEASE_TYPE)"
 	@echo ""
 	@echo "tar.gz:"
 	@echo "	$(RELEASE_CAIRO_URL)/$(tar_gz)"
@@ -116,7 +121,7 @@ release-publish-message: $(HASHFILES) ensure-prev
 	@echo ""
 	@echo "GPG signature:"
 	@echo "	$(RELEASE_CAIRO_URL)/$(gpg_file)"
-	@echo "	(signed by `git config --get user.name` <`git config --get user.email`>)"
+	@echo "	(signed by`gpg --list-keys $(GPGKEY) | grep uid | cut -b4- | tr -s " "`)"
 	@echo ""
 	@echo "Git:"
 	@echo "	git://git.freedesktop.org/git/pixman"
diff --git a/pixman/Makefile.win32 b/pixman/Makefile.win32
new file mode 100644
index 000000000..c3ca3bc59
--- /dev/null
+++ b/pixman/Makefile.win32
@@ -0,0 +1,25 @@
+default: all
+
+top_srcdir = .
+include $(top_srcdir)/Makefile.win32.common
+
+all: pixman test
+
+pixman:
+	@$(MAKE) -C pixman -f Makefile.win32
+
+test:
+	@$(MAKE) -C test -f Makefile.win32
+
+clean_r:
+	@$(MAKE) -C pixman -f Makefile.win32 clean
+	@$(MAKE) -C test   -f Makefile.win32 clean
+
+check:
+	@$(MAKE) -C test -f Makefile.win32 check
+
+
+clean: clean_r
+
+
+.PHONY: all pixman test clean check
diff --git a/pixman/Makefile.win32.common b/pixman/Makefile.win32.common
new file mode 100644
index 000000000..777f94ce2
--- /dev/null
+++ b/pixman/Makefile.win32.common
@@ -0,0 +1,56 @@
+LIBRARY = pixman-1
+
+CC = cl
+LD = link
+AR = lib
+PERL = perl
+
+ifeq ($(top_builddir),)
+top_builddir = $(top_srcdir)
+endif
+
+CFG_VAR = $(CFG)
+ifeq ($(CFG_VAR),)
+CFG_VAR = release
+endif
+
+ifeq ($(CFG_VAR),debug)
+CFG_CFLAGS  = -MDd -Od -Zi
+CFG_LDFLAGS = -DEBUG
+else
+CFG_CFLAGS  = -MD -O2
+CFG_LDFLAGS =
+endif
+
+# Package definitions, to be used instead of those provided in config.h
+PKG_CFLAGS  = -DPACKAGE=$(LIBRARY) -DPACKAGE_VERSION="" -DPACKAGE_BUGREPORT=""
+
+BASE_CFLAGS = -nologo -I. -I$(top_srcdir) -I$(top_srcdir)/pixman
+
+PIXMAN_CFLAGS  = $(BASE_CFLAGS) $(PKG_CFLAGS) $(CFG_CFLAGS) $(CFLAGS)
+PIXMAN_LDFLAGS = -nologo $(CFG_LDFLAGS) $(LDFLAGS)
+PIXMAN_ARFLAGS = -nologo $(LDFLAGS)
+
+
+inform:
+ifneq ($(CFG),release)
+ifneq ($(CFG),debug)
+ifneq ($(CFG),)
+	@echo "Invalid specified configuration option: "$(CFG)"."
+	@echo
+	@echo "Possible choices for configuration are 'release' and 'debug'"
+	@exit 1
+endif
+	@echo "Using default RELEASE configuration... (use CFG=release or CFG=debug)"
+endif
+endif
+
+
+$(CFG_VAR)/%.obj: %.c $(libpixman_headers)
+	@mkdir -p $(CFG_VAR)
+	@$(CC) -c $(PIXMAN_CFLAGS) -Fo"$@" $<
+
+clean: inform
+	@$(RM) $(CFG_VAR)/*.{exe,ilk,lib,obj,pdb} $(BUILT_SOURCES) || exit 0
+
+.PHONY: inform clean
diff --git a/pixman/README b/pixman/README
index 60dff4561..6d8cfd8ad 100644
--- a/pixman/README
+++ b/pixman/README
@@ -1,22 +1,116 @@
-pixman is a library that provides low-level pixel manipulation
+Pixman is a library that provides low-level pixel manipulation
 features such as image compositing and trapezoid rasterization.
 
-Please submit bugs & patches to the libpixman bugzilla:
+Questions, bug reports and patches should be directed to the pixman
+mailing list:
+
+        http://lists.freedesktop.org/mailman/listinfo/pixman
+
+You can also file bugs at
 
         https://bugs.freedesktop.org/enter_bug.cgi?product=pixman
 
-All questions regarding this software should be directed to the pixman
-mailing list:
+For real time discussions about pixman, feel free to join the IRC
+channels #cairo and #xorg-devel on the FreeNode IRC network.
 
-        http://lists.freedesktop.org/mailman/listinfo/pixman
 
-The master development code repository can be found at:
+Contributing
+------------
+
+In order to contribute to pixman, you will need a working knowledge of
+the git version control system. For a quick getting started guide,
+there is the "Everyday Git With 20 Commands Or So guide"
+
+        http://www.kernel.org/pub/software/scm/git/docs/everyday.html
+
+from the Git homepage. For more in depth git documentation, see the
+resources on the Git community documentation page:
+
+        http://git-scm.com/documentation
+
+Pixman uses the infrastructure from the freedesktop.org umbrella
+project. For instructions about how to use the git service on
+freedesktop.org, see:
+
+        http://www.freedesktop.org/wiki/Infrastructure/git/Developers
+
+The Pixman master repository can be found at:
 
 	git://anongit.freedesktop.org/git/pixman
 
-	http://gitweb.freedesktop.org/?p=pixman;a=summary
+and browsed on the web here:
+
+	http://cgit.freedesktop.org/pixman/
+
+
+Sending patches
+---------------
+
+The general workflow for sending patches is to first make sure that
+git can send mail on your system. Then, 
+
+ - create a branch off of master in your local git repository
+
+ - make your changes as one or more commits
+
+ - use the 
+
+        git send-email
+
+   command to send the patch series to pixman@lists.freedesktop.org.
+
+In order for your patches to be accepted, please consider the
+following guidelines:
+
+ - This link:
+
+        http://www.kernel.org/pub/software/scm/git/docs/user-manual.html#patch-series
+
+   describes how what a good patch series is, and to create one with
+   git.
+
+ - At each point in the series, pixman should compile and the test
+   suite should pass.
+
+   The exception here is if you are changing the test suite to
+   demonstrate a bug. In this case, make one commit that makes the
+   test suite fail due to the bug, and then another commit that fixes
+   the bug.
+
+   You can run the test suite with 
+
+        make check
+
+   It will take around two minutes to run on a modern PC.
+
+ - Follow the coding style described in the CODING_STYLE file
+
+ - For bug fixes, include an update to the test suite to make sure
+   the bug doesn't reappear.
+
+ - For new features, add tests of the feature to the test
+   suite. Also, add a program demonstrating the new feature to the
+   demos/ directory.
+
+ - Write descriptive commit messages. Useful information to include:
+        - Benchmark results, before and after
+	- Description of the bug that was fixed
+	- Detailed rationale for any new API
+	- Alternative approaches that were rejected (and why they
+          don't work)
+	- If review comments were incorporated, a brief version
+          history describing what those changes were.
 
-For more information on the git code manager, see:
+ - For big patch series, send an introductory email with an overall
+   description of the patch series, including benchmarks and
+   motivation. Each commit message should still be descriptive and
+   include enough information to understand why this particular commit
+   was necessary.
 
-	http://wiki.x.org/wiki/GitPage
+Pixman has high standards for code quality and so almost everybody
+should expect to have the first versions of their patches rejected.
 
+If you think that the reviewers are wrong about something, or that the
+guidelines above are wrong, feel free to discuss the issue on the
+list. The purpose of the guidelines and code review is to ensure high
+code quality; it is not an exercise in compliance.
diff --git a/pixman/RELEASING b/pixman/RELEASING
index 7ddcef49f..657857de2 100644
--- a/pixman/RELEASING
+++ b/pixman/RELEASING
@@ -11,7 +11,7 @@ Here are the steps to follow to create a new pixman release:
 	git log master...origin		(no output; note: *3* dots)
 
 2) Increment pixman_(major|minor|micro) in configure.ac according to
-   the directions in that file. Use git commit to record this.
+   the directions in that file.
 
 3) Make sure that new version works, including
 
@@ -23,7 +23,7 @@ Here are the steps to follow to create a new pixman release:
 	- the cairo test suite hasn't gained any new failures compared
 	  to last pixman version.
 
-4) Use "git commit" to record any changes made in step 3.
+4) Use "git commit" to record the changes made in step 2 and 3.
 
 5) Generate and publish the tar files by running 
 
@@ -55,3 +55,5 @@ Here are the steps to follow to create a new pixman release:
    You must use "--tags" here; otherwise the new tag will not
    be pushed out.
 
+8) Change the topic of the #cairo IRC channel on freenode to advertise
+   the new version.
diff --git a/pixman/TODO b/pixman/TODO
deleted file mode 100644
index 4434ec7cb..000000000
--- a/pixman/TODO
+++ /dev/null
@@ -1,271 +0,0 @@
-  - Testing
-    - Test implementations against each other
-    - Test both with and without the operator strength reduction.
-      They shold be identical.
-
-  - SSE 2 issues:
-
-      - Use MM_HINT_NTA instead of MM_HINT_T0
-
-      - Use of fbCompositeOver_x888x8x8888sse2()
-
-  - Update the RLEASING file
-
-  - Things to keep in mind if breaking ABI:
-
-      - There should be a guard #ifndef I_AM_EITHER_CAIRO_OR_THE_X_SERVER
-
-      - X server will require 16.16 essentially forever. Can we get
-        the required precision by simply adding offset_x/y to the
-        relevant rendering API?
-
-      - Get rid of workaround for X server bug.
-
-      - pixman_image_set_indexed() should copy its argument, and X
-        should be ported over to use a pixman_image as the
-        representation of a Picture, rather than creating one on each
-        operation.
-
-      - We should get rid of pixman_set_static_pointers()
-
-      - We should get rid of the various trapezoid helper functions().
-        (They only exist because they are theoretically available to
-        drivers).
-
-      - 16 bit regions should be deleted
-
-      - There should only be one trap rasterization API.
-
-      - The PIXMAN_g8/c8/etc formats should use the A channel
-        to indicate the actual depth. That way PIXMAN_x4c4 and PIXMAN_c8
-	won't collide.
-
-  - Maybe bite the bullet and make configure.ac generate a pixman-types.h
-    file that can be included from pixman.h to avoid the #ifdef magic
-    in pixman.h
-
-  - Make pixman_region_point_in() survive a NULL box, then fix up
-    pixman-compose.c
-
-      - Possibly look into inlining the fetch functions
-
-  - There is a bug with source clipping demonstrated by clip-test in the
-    test directory. If we interprete source clipping as given in
-    destination coordinates, which is probably the only sane choice,
-    then the result should have two red bars down the sides.
-    
-  - Test suite
-
-  - Add a general way of dealing with architecture specific
-    fast-paths.  The current idea is to have each operation that can
-    be optimized is called through a function pointer that is
-    initially set to an initialization function that is responsible for
-    setting the function pointer to the appropriate fast-path.
-
-  - Go through things marked FIXME
-
-  - Add calls to prepare and finish access where necessary.  grep for
-    ACCESS_MEM, and make sure they are correctly wrapped in prepare
-    and finish.
-
-  - restore READ/WRITE in the fbcompose combiners since they sometimes
-    store directly to destination drawables.
-
-  - It probably makes sense to move the more strange X region API
-    into pixman as well, but guarded with PIXMAN_XORG_COMPATIBILITY
-
-  - Reinstate the FbBits typedef? At the moment we don't
-    even have the FbBits type; we just use uint32_t everywhere.
-
-    Keith says in bug 2335:
-
-        The 64-bit code in fb (pixman) is probably broken; it hasn't been
-        used in quite some time as PCI (and AGP) is 32-bits wide, so
-        doing things 64-bits at a time is a net loss.  To quickly fix
-        this, I suggest just using 32-bit datatypes by setting
-        IC_SHIFT to 5 for all machines.
-
-  - Consider optimizing the 8/16 bit solid fills in pixman-util.c by
-    storing more than one value at a time.
-
-  - Add an image cache to prevent excessive malloc/free. Note that pixman
-    needs to be thread safe when used from cairo.
-
-  - Moving to 24.8 coordinates. This is tricky because X is still
-    defined as 16.16 and will be basically forever. It's possible we
-    could do this by adding extra offset_x/y parameters to the
-    trapezoid calls. The X server could then just call the API with
-    (0, 0). Cairo would have to make sure that the delta *within* a
-    batch of trapezoids does not exceed 16 bit.
-
-  - Consider adding actual backends. Brain dump:
-
-    A backend is something that knows how to
-
-      - Create images
-      - Composite three images
-      - Rasterize trapezoids
-      - Do solid fills and blits
-
-    These operations are provided by a vtable that the backend will
-    create when it is initialized. Initial backends:
-
-      - VMX
-      - SSE2
-      - MMX
-      - Plain Old C
-
-    When the SIMD backends are initialized, they will be passed a
-    pointer to the Plain Old C backend that they can use for fallback
-    purposes.
-
-    Images would gain a vtable as well that would contain things like
-
-      - Read scanline
-      - Write scanline
-
-    (Or even read_patch/write_patch as suggested by Keith a while
-    back).
-
-    This could simplify the compositing code considerably.
-
-  - Review the pixman_format_code_t enum to make sure it will support
-    future formats. Some formats we will probably need:
-
-    	   ARGB/ABGR with 16/32/64 bit integer/floating channels
-	   YUV2,
-	   YV12
-
-    Also we may need the ability to distinguish between PICT_c8 and
-    PICT_x4c4. (This could be done by interpreting the A channel as
-    the depth for TYPE_COLOR and TYPE_GRAY formats).
-
-    A possibility may be to reserve the two top bits and make them
-    encode "number of places to shift the channel widths given" Since
-    these bits are 00 at the moment everything will continue to work,
-    but these additional widths will be allowed:
-
-    	     All even widths between 18-32
-	     All multiples of four widths between 33 and 64
-	     All multiples of eight between 64 and 128
-
-    This means things like r21g22b21 won't work - is that worth
-    worrying about? I don't think so. And of course the bpp field
-    can't handle a depth of over 256, so > 64 bit channels arent'
-    really all that useful.
-
-    We could reserve one extra bit to indicate floating point, but
-    we may also just add 
-
-       	   PIXMAN_TYPE_ARGB_FLOAT
-	   PIXMAN_TYPE_BGRA_FLOAT
-	   PIXMAN_TYPE_A_FLOAT
-    
-    image types. With five bits we can support up to 32 different
-    format types, which should be enough for everybody, even if we
-    decide to support all the various video formats here:
-
-    	        http://www.fourcc.org/yuv.php
-
-    It may make sense to have a PIXMAN_TYPE_YUV, and then use the
-    channel bits to specify the exact subtype.
-
-    Another possibility is to add 
-
-      	  PIXMAN_TYPE_ARGB_W
-	  PIXMAN_TYPE_ARGB_WW
-    
-    where the channel widths would get 16 and 32 added to them,
-    respectively.
-
-    What about color spaces such a linear vs. srGB etc.?
-
-
-done:
-
-- Use pixmanFillsse2 and pixmanBltsse2
-
-- Be consistent about calling sse2 sse2
-
-- Rename "SSE" to "MMX_EXTENSIONS". (Deleted mmx extensions).
-
-- Commented-out uses of fbCompositeCopyAreasse2()
-
-- Consider whether calling regions region16 is really such a great
-  idea. Vlad wants 32 bit regions for Cairo. This will break X server
-  ABI, but should otherwise be mostly harmless, though a
-  pixman_region_get_boxes16() may be useful.
-
-- Altivec signal issue (Company has fix, there is also a patch by
-  dwmw2 in rawhide).
-
-- Behdad's MMX issue - see list
-
-- SSE2 issues:
-    - Crashes in Mozilla because of unaligned stack. Possible fixes
-        - Make use of gcc 4.2 feature to align the stack
-        - Write some sort of trampoline that aligns the stack
-          before calling SSE functions.
-
-- Get rid of the switch-of-doom; replace it with a big table
-  describing the various fast paths.
-
-- Make source clipping optional.
-    - done: source clipping happens through an indirection.
-        still needs to make the indirection settable. (And call it
-        from X)
-
-- Run cairo test suite; fix bugs
-	- one bug in source-scale-clip
-
- - Remove the warning suppression in the ACCESS_MEM macro and fix the
-    warnings that are real
-	- irrelevant now.
-
-- make the wrapper functions global instead of image specific
-	- this won't work since pixman is linked to both fb and wfb
-
-- Add non-mmx solid fill
-
-- Make sure the endian-ness macros are defined correctly.
-
-- The rectangles in a region probably shouldn't be returned const as
-  the X server will be changing them.
-
-- Right now we _always_ have a clip region, which is empty by default.
-  Why does this work at all? It probably doesn't. The server
-  distinguishes two cases, one where nothing is clipped (CT_NONE), and
-  one where there is a clip region (CT_REGION).
-
-- Default clip region should be the full image
-
-  - Test if pseudo color still works. It does, but it also shows that
-    copying a pixman_indexed_t on every composite operation is not
-    going to fly. So, for now set_indexed() does not copy the 
-    indexed table. 
-
-    Also just the malloc() to allocate a pixman image shows up pretty
-    high.
-
-    Options include
-
-      - Make all the setters not copy their arguments
-
-      - Possibly combined with going back to the stack allocated 
-        approach that we already use for regions.
-
-      - Keep a cached pixman_image_t around for every picture. It would
-        have to be kept uptodate every time something changes about the
-        picture.
-
-      - Break the X server ABI and simply have the relevant parameter
-        stored in the pixman image. This would have the additional benefits
-        that:
-
-          - We can get rid of the annoying repeat field which is duplicated
-            elsewhere.
-
-          - We can use pixman_color_t and pixman_gradient_stop_t
-            etc. instead of the types that are defined in
-            renderproto.h
-
diff --git a/pixman/autogen.sh b/pixman/autogen.sh
index 354f254e4..fc34bd55c 100755
--- a/pixman/autogen.sh
+++ b/pixman/autogen.sh
@@ -9,4 +9,6 @@ cd $srcdir
 autoreconf -v --install || exit 1
 cd $ORIGDIR || exit $?
 
-$srcdir/configure "$@"
+if test -z "$NOCONFIGURE"; then
+    $srcdir/configure "$@"
+fi
diff --git a/pixman/configure.ac b/pixman/configure.ac
index c9269f4ec..dce76b3b2 100644
--- a/pixman/configure.ac
+++ b/pixman/configure.ac
@@ -26,8 +26,8 @@ AC_PREREQ([2.57])
 #
 #   - The version in git has an odd MICRO version number
 #
-#   - Released versions both development and stable have an even MICRO 
-#     version number
+#   - Released versions, both development and stable, have an
+#     even MICRO version number
 #
 #   - Released development versions have an odd MINOR number
 #
@@ -53,18 +53,18 @@ AC_PREREQ([2.57])
 #
 
 m4_define([pixman_major], 0)
-m4_define([pixman_minor], 18)
-m4_define([pixman_micro], 4)
+m4_define([pixman_minor], 32)
+m4_define([pixman_micro], 6)
 
 m4_define([pixman_version],[pixman_major.pixman_minor.pixman_micro])
 
-AC_INIT(pixman, pixman_version, "pixman@lists.freedesktop.org", pixman)
+AC_INIT(pixman, pixman_version, [pixman@lists.freedesktop.org], pixman)
 AM_INIT_AUTOMAKE([foreign dist-bzip2])
 
 # Suppress verbose compile lines
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
-AM_CONFIG_HEADER(config.h)
+AC_CONFIG_HEADERS(config.h)
 
 AC_CANONICAL_HOST
 
@@ -77,6 +77,72 @@ AC_CHECK_FUNCS([getisax])
 AC_C_BIGENDIAN
 AC_C_INLINE
 
+dnl PIXMAN_LINK_WITH_ENV(env-setup, program, true-action, false-action)
+dnl
+dnl Compiles and links the given program in the environment setup by env-setup
+dnl and executes true-action on success and false-action on failure.
+AC_DEFUN([PIXMAN_LINK_WITH_ENV],[dnl
+	save_CFLAGS="$CFLAGS"
+	save_LDFLAGS="$LDFLAGS"
+	save_LIBS="$LIBS"
+	CFLAGS=""
+	LDFLAGS=""
+	LIBS=""
+	$1
+	CFLAGS="$save_CFLAGS $CFLAGS"
+	LDFLAGS="$save_LDFLAGS $LDFLAGS"
+	LIBS="$save_LIBS $LIBS"
+	AC_LINK_IFELSE(
+		[AC_LANG_SOURCE([$2])],
+		[pixman_cc_stderr=`test -f conftest.err && cat conftest.err`
+		 pixman_cc_flag=yes],
+		[pixman_cc_stderr=`test -f conftest.err && cat conftest.err`
+		 pixman_cc_flag=no])
+
+	if test "x$pixman_cc_stderr" != "x"; then
+		pixman_cc_flag=no
+	fi
+
+	if test "x$pixman_cc_flag" = "xyes"; then
+		ifelse([$3], , :, [$3])
+	else
+		ifelse([$4], , :, [$4])
+	fi
+	CFLAGS="$save_CFLAGS"
+	LDFLAGS="$save_LDFLAGS"
+	LIBS="$save_LIBS"
+])
+
+dnl Find a -Werror for catching warnings.
+WERROR=
+for w in -Werror -errwarn; do
+    if test "z$WERROR" = "z"; then
+        AC_MSG_CHECKING([whether the compiler supports $w])
+        PIXMAN_LINK_WITH_ENV(
+		[CFLAGS=$w],
+		[int main(int c, char **v) { (void)c; (void)v; return 0; }],
+		[WERROR=$w; yesno=yes], [yesno=no])
+	AC_MSG_RESULT($yesno)
+    fi
+done
+
+dnl PIXMAN_CHECK_CFLAG(flag, [program])
+dnl  Adds flag to CFLAGS if the given program links without warnings or errors.
+AC_DEFUN([PIXMAN_CHECK_CFLAG], [dnl
+	AC_MSG_CHECKING([whether the compiler supports $1])
+	PIXMAN_LINK_WITH_ENV(
+		[CFLAGS="$WERROR $1"],
+		[$2
+		 int main(int c, char **v) { (void)c; (void)v; return 0; }
+		],
+		[_yesno=yes],
+		[_yesno=no])
+	if test "x$_yesno" = xyes; then
+	   CFLAGS="$CFLAGS $1"
+	fi
+	AC_MSG_RESULT($_yesno)
+])
+
 AC_CHECK_SIZEOF(long)
 
 # Checks for Sun Studio compilers
@@ -87,7 +153,7 @@ AC_CHECK_DECL([__amd64], [AMD64_ABI="yes"], [AMD64_ABI="no"])
 # if we're using Sun Studio and neither the user nor a config.site
 # has set CFLAGS.
 if test $SUNCC = yes &&			\
-   test "$test_CFLAGS" == "" &&		\
+   test "x$test_CFLAGS" = "x" &&	\
    test "$CFLAGS" = "-g"
 then
   CFLAGS="-O -g"
@@ -115,61 +181,145 @@ AC_SUBST(PIXMAN_VERSION_MICRO)
 AC_SUBST(LT_VERSION_INFO)
 
 # Check for dependencies
-#PKG_CHECK_MODULES(DEP, x11)
 
-changequote(,)dnl
-if test "x$GCC" = "xyes"; then
+PIXMAN_CHECK_CFLAG([-Wall])
+PIXMAN_CHECK_CFLAG([-Wdeclaration-after-statement])
+PIXMAN_CHECK_CFLAG([-fno-strict-aliasing])
 
-  case " $CFLAGS " in
-  *[\ \	]-Wall[\ \	]*) ;;
-  *) CFLAGS="$CFLAGS -Wall" ;;
-  esac 
+dnl =========================================================================
+dnl OpenMP for the test suite?
+dnl
 
-  case " $CFLAGS " in
-  *[\ \	]-fno-strict-aliasing[\ \	]*) ;;
-  *) CFLAGS="$CFLAGS -fno-strict-aliasing" ;;
-  esac
+# Check for OpenMP support only when autoconf support that (require autoconf >=2.62)
+OPENMP_CFLAGS=
+m4_ifdef([AC_OPENMP], [AC_OPENMP])
 
-fi changequote([,])dnl
+if test "x$enable_openmp" = "xyes" && test "x$ac_cv_prog_c_openmp" = "xunsupported" ; then
+  AC_MSG_WARN([OpenMP support requested but found unsupported])
+fi
 
-AC_PATH_PROG(PERL, perl, no)
-if test "x$PERL" = xno; then
-    AC_MSG_ERROR([Perl is required to build pixman.])
+dnl May not fail to link without -Wall -Werror added
+dnl So try to link only when openmp is supported
+dnl ac_cv_prog_c_openmp is not defined when --disable-openmp is used
+if test "x$ac_cv_prog_c_openmp" != "xunsupported" && test "x$ac_cv_prog_c_openmp" != "x"; then
+  m4_define([openmp_test_program],[dnl
+  #include <stdio.h>
+
+  extern unsigned int lcg_seed;
+  #pragma omp threadprivate(lcg_seed)
+  unsigned int lcg_seed;
+
+  unsigned function(unsigned a, unsigned b)
+  {
+	lcg_seed ^= b;
+	return ((a + b) ^ a ) + lcg_seed;
+  }
+
+  int main(int argc, char **argv)
+  {
+	int i;
+	int n1 = 0, n2 = argc;
+	unsigned checksum = 0;
+	int verbose = argv != NULL;
+	unsigned (*test_function)(unsigned, unsigned);
+	test_function = function;
+	#pragma omp parallel for reduction(+:checksum) default(none) \
+					shared(n1, n2, test_function, verbose)
+	for (i = n1; i < n2; i++)
+	{
+		unsigned crc = test_function (i, 0);
+		if (verbose)
+			printf ("%d: %08X\n", i, crc);
+		checksum += crc;
+	}
+	printf("%u\n", checksum);
+	return 0;
+  }
+  ])
+
+  PIXMAN_LINK_WITH_ENV(
+	[CFLAGS="$OPENMP_CFLAGS" LDFLAGS="$OPENMP_CFLAGS"],
+	[openmp_test_program],
+	[have_openmp=yes],
+	[have_openmp=no])
+  if test "x$have_openmp" = "xyes" ; then
+    AC_DEFINE(USE_OPENMP, 1, [use OpenMP in the test suite])
+  fi
 fi
-AC_SUBST(PERL)
+AC_SUBST(OPENMP_CFLAGS)
 
 dnl =========================================================================
 dnl -fvisibility stuff
 
-have_gcc4=no
-AC_MSG_CHECKING(for -fvisibility)
-AC_COMPILE_IFELSE([
+PIXMAN_CHECK_CFLAG([-fvisibility=hidden], [dnl
 #if defined(__GNUC__) && (__GNUC__ >= 4)
+#ifdef _WIN32
+#error Have -fvisibility but it is ignored and generates a warning
+#endif
 #else
-error Need GCC 4.0 for visibility
+#error Need GCC 4.0 for visibility
 #endif
-int main () { return 0; } 
-], have_gcc4=yes)
+])
 
-if test "x$have_gcc4" = "xyes"; then
-   CFLAGS="$CFLAGS -fvisibility=hidden"
-fi
-AC_MSG_RESULT($have_gcc4)
-
-have_sunstudio8=no
-AC_MSG_CHECKING([for -xldscope (Sun compilers)])
-AC_COMPILE_IFELSE([
+PIXMAN_CHECK_CFLAG([-xldscope=hidden], [dnl
 #if defined(__SUNPRO_C) && (__SUNPRO_C >= 0x550)
 #else
-error Need Sun Studio 8 for visibility
+#error Need Sun Studio 8 for visibility
 #endif
-int main () { return 0; } 
-], have_sunstudio8=yes)
+])
+
+dnl ===========================================================================
+dnl Check for Loongson Multimedia Instructions
 
-if test "x$have_sunstudio8" = "xyes"; then
-   CFLAGS="$CFLAGS -xldscope=hidden"
+if test "x$LS_CFLAGS" = "x" ; then
+    LS_CFLAGS="-march=loongson2f"
 fi
-AC_MSG_RESULT($have_sunstudio8)
+
+have_loongson_mmi=no
+AC_MSG_CHECKING(whether to use Loongson MMI assembler)
+
+xserver_save_CFLAGS=$CFLAGS
+CFLAGS=" $LS_CFLAGS $CFLAGS -I$srcdir"
+AC_LINK_IFELSE([AC_LANG_SOURCE([[
+#ifndef __mips_loongson_vector_rev
+#error "Loongson Multimedia Instructions are only available on Loongson"
+#endif
+#if defined(__GNUC__) && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 4))
+#error "Need GCC >= 4.4 for Loongson MMI compilation"
+#endif
+#include "pixman/loongson-mmintrin.h"
+int main () {
+    union {
+        __m64 v;
+        char c[8];
+    } a = { .c = {1, 2, 3, 4, 5, 6, 7, 8} };
+    int b = 4;
+    __m64 c = _mm_srli_pi16 (a.v, b);
+    return 0;
+}]])], have_loongson_mmi=yes)
+CFLAGS=$xserver_save_CFLAGS
+
+AC_ARG_ENABLE(loongson-mmi,
+   [AC_HELP_STRING([--disable-loongson-mmi],
+                   [disable Loongson MMI fast paths])],
+   [enable_loongson_mmi=$enableval], [enable_loongson_mmi=auto])
+
+if test $enable_loongson_mmi = no ; then
+   have_loongson_mmi=disabled
+fi
+
+if test $have_loongson_mmi = yes ; then
+   AC_DEFINE(USE_LOONGSON_MMI, 1, [use Loongson Multimedia Instructions])
+else
+   LS_CFLAGS=
+fi
+
+AC_MSG_RESULT($have_loongson_mmi)
+if test $enable_loongson_mmi = yes && test $have_loongson_mmi = no ; then
+   AC_MSG_ERROR([Loongson MMI not detected])
+fi
+
+AM_CONDITIONAL(USE_LOONGSON_MMI, test $have_loongson_mmi = yes)
 
 dnl ===========================================================================
 dnl Check for MMX
@@ -191,20 +341,34 @@ have_mmx_intrinsics=no
 AC_MSG_CHECKING(whether to use MMX intrinsics)
 xserver_save_CFLAGS=$CFLAGS
 CFLAGS="$MMX_CFLAGS $CFLAGS"
-AC_COMPILE_IFELSE([
+AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
 #if defined(__GNUC__) && (__GNUC__ < 3 || (__GNUC__ == 3 && __GNUC_MINOR__ < 4))
-error "Need GCC >= 3.4 for MMX intrinsics"
+#error "Need GCC >= 3.4 for MMX intrinsics"
 #endif
 #include <mmintrin.h>
 int main () {
     __m64 v = _mm_cvtsi32_si64 (1);
+    __m64 w;
+
+    /* Some versions of clang will choke on K */
+    asm ("pshufw %2, %1, %0\n\t"
+        : "=y" (w)
+        : "y" (v), "K" (5)
+    );
+
+    /* Some versions of clang will choke on this */
+    asm ("pmulhuw %1, %0\n\t"
+	: "+y" (w)
+	: "y" (v)
+    );
+
     return _mm_cvtsi64_si32 (v);
-}], have_mmx_intrinsics=yes)
+}]])], have_mmx_intrinsics=yes)
 CFLAGS=$xserver_save_CFLAGS
 
 AC_ARG_ENABLE(mmx,
    [AC_HELP_STRING([--disable-mmx],
-                   [disable MMX fast paths])],
+                   [disable x86 MMX fast paths])],
    [enable_mmx=$enableval], [enable_mmx=auto])
 
 if test $enable_mmx = no ; then
@@ -212,17 +376,17 @@ if test $enable_mmx = no ; then
 fi
 
 if test $have_mmx_intrinsics = yes ; then
-   AC_DEFINE(USE_MMX, 1, [use MMX compiler intrinsics])
+   AC_DEFINE(USE_X86_MMX, 1, [use x86 MMX compiler intrinsics])
 else
    MMX_CFLAGS=
 fi
 
 AC_MSG_RESULT($have_mmx_intrinsics)
 if test $enable_mmx = yes && test $have_mmx_intrinsics = no ; then
-   AC_MSG_ERROR([MMX intrinsics not detected])
+   AC_MSG_ERROR([x86 MMX intrinsics not detected])
 fi
 
-AM_CONDITIONAL(USE_MMX, test $have_mmx_intrinsics = yes)
+AM_CONDITIONAL(USE_X86_MMX, test $have_mmx_intrinsics = yes)
 
 dnl ===========================================================================
 dnl Check for SSE2
@@ -234,7 +398,7 @@ if test "x$SSE2_CFLAGS" = "x" ; then
          SSE2_CFLAGS="-xarch=sse2"
       fi
    else
-      SSE2_CFLAGS="-mmmx -msse2 -Winline"
+      SSE2_CFLAGS="-msse2 -Winline"
    fi
 fi
 
@@ -243,7 +407,7 @@ AC_MSG_CHECKING(whether to use SSE2 intrinsics)
 xserver_save_CFLAGS=$CFLAGS
 CFLAGS="$SSE2_CFLAGS $CFLAGS"
 
-AC_COMPILE_IFELSE([
+AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
 #if defined(__GNUC__) && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 2))
 #   if !defined(__amd64__) && !defined(__x86_64__)
 #      error "Need GCC >= 4.2 for SSE2 intrinsics on x86"
@@ -256,7 +420,7 @@ int main () {
     __m128i a = _mm_set1_epi32 (0), b = _mm_set1_epi32 (0), c;
 	c = _mm_xor_si128 (a, b);
     return 0;
-}], have_sse2_intrinsics=yes)
+}]])], have_sse2_intrinsics=yes)
 CFLAGS=$xserver_save_CFLAGS
 
 AC_ARG_ENABLE(sse2,
@@ -280,6 +444,50 @@ fi
 AM_CONDITIONAL(USE_SSE2, test $have_sse2_intrinsics = yes)
 
 dnl ===========================================================================
+dnl Check for SSSE3
+
+if test "x$SSSE3_CFLAGS" = "x" ; then
+    SSSE3_CFLAGS="-mssse3 -Winline"
+fi
+
+have_ssse3_intrinsics=no
+AC_MSG_CHECKING(whether to use SSSE3 intrinsics)
+xserver_save_CFLAGS=$CFLAGS
+CFLAGS="$SSSE3_CFLAGS $CFLAGS"
+
+AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
+#include <mmintrin.h>
+#include <xmmintrin.h>
+#include <emmintrin.h>
+#include <tmmintrin.h>
+int main () {
+    __m128i a = _mm_set1_epi32 (0), b = _mm_set1_epi32 (0), c;
+    c = _mm_maddubs_epi16 (a, b);
+    return 0;
+}]])], have_ssse3_intrinsics=yes)
+CFLAGS=$xserver_save_CFLAGS
+
+AC_ARG_ENABLE(ssse3,
+   [AC_HELP_STRING([--disable-ssse3],
+                   [disable SSSE3 fast paths])],
+   [enable_ssse3=$enableval], [enable_ssse3=auto])
+
+if test $enable_ssse3 = no ; then
+   have_ssse3_intrinsics=disabled
+fi
+
+if test $have_ssse3_intrinsics = yes ; then
+   AC_DEFINE(USE_SSSE3, 1, [use SSSE3 compiler intrinsics])
+fi
+
+AC_MSG_RESULT($have_ssse3_intrinsics)
+if test $enable_ssse3 = yes && test $have_ssse3_intrinsics = no ; then
+   AC_MSG_ERROR([SSSE3 intrinsics not detected])
+fi
+
+AM_CONDITIONAL(USE_SSSE3, test $have_ssse3_intrinsics = yes)
+
+dnl ===========================================================================
 dnl Other special flags needed when building code using MMX or SSE instructions
 case $host_os in
    solaris*)
@@ -293,7 +501,7 @@ case $host_os in
 	 hwcap_save_LDFLAGS="$LDFLAGS"
 	 HWCAP_LDFLAGS='-Wl,-M,$(srcdir)/solaris-hwcap.mapfile'
 	 LDFLAGS="$LDFLAGS -Wl,-M,pixman/solaris-hwcap.mapfile"
-	 AC_LINK_IFELSE([int main() { return 0; }],
+	 AC_LINK_IFELSE([AC_LANG_SOURCE([[int main() { return 0; }]])],
 			use_hwcap_mapfile=yes,
 			HWCAP_LDFLAGS="")
 	 LDFLAGS="$hwcap_save_LDFLAGS"
@@ -308,10 +516,13 @@ case $host_os in
       ;;
 esac
 
+AC_SUBST(LS_CFLAGS)
+AC_SUBST(IWMMXT_CFLAGS)
 AC_SUBST(MMX_CFLAGS)
 AC_SUBST(MMX_LDFLAGS)
 AC_SUBST(SSE2_CFLAGS)
 AC_SUBST(SSE2_LDFLAGS)
+AC_SUBST(SSSE3_CFLAGS)
 
 dnl ===========================================================================
 dnl Check for VMX/Altivec
@@ -325,16 +536,16 @@ have_vmx_intrinsics=no
 AC_MSG_CHECKING(whether to use VMX/Altivec intrinsics)
 xserver_save_CFLAGS=$CFLAGS
 CFLAGS="$VMX_CFLAGS $CFLAGS"
-AC_COMPILE_IFELSE([
+AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
 #if defined(__GNUC__) && (__GNUC__ < 3 || (__GNUC__ == 3 && __GNUC_MINOR__ < 4))
-error "Need GCC >= 3.4 for sane altivec support"
+#error "Need GCC >= 3.4 for sane altivec support"
 #endif
 #include <altivec.h>
 int main () {
     vector unsigned int v = vec_splat_u32 (1);
     v = vec_sub (v, v);
     return 0;
-}], have_vmx_intrinsics=yes)
+}]])], have_vmx_intrinsics=yes)
 CFLAGS=$xserver_save_CFLAGS
 
 AC_ARG_ENABLE(vmx,
@@ -367,7 +578,7 @@ have_arm_simd=no
 AC_MSG_CHECKING(whether to use ARM SIMD assembler)
 xserver_save_CFLAGS=$CFLAGS
 CFLAGS="-x assembler-with-cpp $CFLAGS"
-AC_COMPILE_IFELSE([[
+AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
 .text
 .arch armv6
 .object_arch armv4
@@ -377,7 +588,7 @@ AC_COMPILE_IFELSE([[
 #error EABI is required (to be sure that calling conventions are compatible)
 #endif
 pld [r0]
-uqadd8 r0, r0, r0]], have_arm_simd=yes)
+uqadd8 r0, r0, r0]])], have_arm_simd=yes)
 CFLAGS=$xserver_save_CFLAGS
 
 AC_ARG_ENABLE(arm-simd,
@@ -406,7 +617,7 @@ have_arm_neon=no
 AC_MSG_CHECKING(whether to use ARM NEON assembler)
 xserver_save_CFLAGS=$CFLAGS
 CFLAGS="-x assembler-with-cpp $CFLAGS"
-AC_COMPILE_IFELSE([[
+AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
 .text
 .fpu neon
 .arch armv7a
@@ -418,7 +629,7 @@ AC_COMPILE_IFELSE([[
 #error EABI is required (to be sure that calling conventions are compatible)
 #endif
 pld [r0]
-vmovn.u16 d0, q0]], have_arm_neon=yes)
+vmovn.u16 d0, q0]])], have_arm_neon=yes)
 CFLAGS=$xserver_save_CFLAGS
 
 AC_ARG_ENABLE(arm-neon,
@@ -441,17 +652,124 @@ if test $enable_arm_neon = yes && test $have_arm_neon = no ; then
    AC_MSG_ERROR([ARM NEON intrinsics not detected])
 fi
 
+dnl ===========================================================================
+dnl Check for IWMMXT
+
+AC_ARG_ENABLE(arm-iwmmxt,
+   [AC_HELP_STRING([--disable-arm-iwmmxt],
+                   [disable ARM IWMMXT fast paths])],
+   [enable_iwmmxt=$enableval], [enable_iwmmxt=auto])
+
+AC_ARG_ENABLE(arm-iwmmxt2,
+   [AC_HELP_STRING([--disable-arm-iwmmxt2],
+                   [build ARM IWMMXT fast paths with -march=iwmmxt instead of -march=iwmmxt2])],
+   [enable_iwmmxt2=$enableval], [enable_iwmmxt2=auto])
+
+if test "x$IWMMXT_CFLAGS" = "x" ; then
+   IWMMXT_CFLAGS="-flax-vector-conversions -Winline -march=iwmmxt"
+   if test $enable_iwmmxt2 != no ; then
+      IWMMXT_CFLAGS="${IWMMXT_CFLAGS}2"
+   fi
+fi
+
+have_iwmmxt_intrinsics=no
+AC_MSG_CHECKING(whether to use ARM IWMMXT intrinsics)
+xserver_save_CFLAGS=$CFLAGS
+CFLAGS="$CFLAGS $IWMMXT_CFLAGS"
+AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
+#ifndef __arm__
+#error "IWMMXT is only available on ARM"
+#endif
+#ifndef __IWMMXT__
+#error "IWMMXT not enabled (with -march=iwmmxt)"
+#endif
+#if defined(__GNUC__) && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8))
+#error "Need GCC >= 4.8 for IWMMXT intrinsics"
+#endif
+#include <mmintrin.h>
+int main () {
+	union {
+		__m64 v;
+		char c[8];
+	} a = { .c = {1, 2, 3, 4, 5, 6, 7, 8} };
+	int b = 4;
+	__m64 c = _mm_srli_si64 (a.v, b);
+}]])], have_iwmmxt_intrinsics=yes)
+CFLAGS=$xserver_save_CFLAGS
+
+if test $enable_iwmmxt = no ; then
+   have_iwmmxt_intrinsics=disabled
+fi
+
+if test $have_iwmmxt_intrinsics = yes ; then
+   AC_DEFINE(USE_ARM_IWMMXT, 1, [use ARM IWMMXT compiler intrinsics])
+else
+   IWMMXT_CFLAGS=
+fi
+
+AC_MSG_RESULT($have_iwmmxt_intrinsics)
+if test $enable_iwmmxt = yes && test $have_iwmmxt_intrinsics = no ; then
+   AC_MSG_ERROR([IWMMXT intrinsics not detected])
+fi
+
+AM_CONDITIONAL(USE_ARM_IWMMXT, test $have_iwmmxt_intrinsics = yes)
+
+dnl ==========================================================================
+dnl Check if assembler is gas compatible and supports MIPS DSPr2 instructions
+
+have_mips_dspr2=no
+AC_MSG_CHECKING(whether to use MIPS DSPr2 assembler)
+xserver_save_CFLAGS=$CFLAGS
+CFLAGS="-mdspr2 $CFLAGS"
+
+AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
+#if !(defined(__mips__) &&  __mips_isa_rev >= 2)
+#error MIPS DSPr2 is currently only available on MIPS32r2 platforms.
+#endif
+int
+main ()
+{
+    int c = 0, a = 0, b = 0;
+    __asm__ __volatile__ (
+        "precr.qb.ph %[c], %[a], %[b]          \n\t"
+        : [c] "=r" (c)
+        : [a] "r" (a), [b] "r" (b)
+    );
+    return c;
+}]])], have_mips_dspr2=yes)
+CFLAGS=$xserver_save_CFLAGS
+
+AC_ARG_ENABLE(mips-dspr2,
+   [AC_HELP_STRING([--disable-mips-dspr2],
+                   [disable MIPS DSPr2 fast paths])],
+   [enable_mips_dspr2=$enableval], [enable_mips_dspr2=auto])
+
+if test $enable_mips_dspr2 = no ; then
+   have_mips_dspr2=disabled
+fi
+
+if test $have_mips_dspr2 = yes ; then
+   AC_DEFINE(USE_MIPS_DSPR2, 1, [use MIPS DSPr2 assembly optimizations])
+fi
+
+AM_CONDITIONAL(USE_MIPS_DSPR2, test $have_mips_dspr2 = yes)
+
+AC_MSG_RESULT($have_mips_dspr2)
+if test $enable_mips_dspr2 = yes && test $have_mips_dspr2 = no ; then
+   AC_MSG_ERROR([MIPS DSPr2 instructions not detected])
+fi
+
 dnl =========================================================================================
 dnl Check for GNU-style inline assembly support
 
 have_gcc_inline_asm=no
 AC_MSG_CHECKING(whether to use GNU-style inline assembler)
-AC_COMPILE_IFELSE([
+AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
 int main () {
     /* Most modern architectures have a NOP instruction, so this is a fairly generic test. */
 	asm volatile ( "\tnop\n" : : : "cc", "memory" );
     return 0;
-}], have_gcc_inline_asm=yes)
+}]])], have_gcc_inline_asm=yes)
 
 AC_ARG_ENABLE(gcc-inline-asm,
    [AC_HELP_STRING([--disable-gcc-inline-asm],
@@ -474,6 +792,20 @@ fi
 AM_CONDITIONAL(USE_GCC_INLINE_ASM, test $have_gcc_inline_asm = yes)
 
 dnl ==============================================
+dnl Static test programs
+
+AC_ARG_ENABLE(static-testprogs,
+   [AC_HELP_STRING([--enable-static-testprogs],
+		   [build test programs as static binaries [default=no]])],
+   [enable_static_testprogs=$enableval], [enable_static_testprogs=no])
+
+TESTPROGS_EXTRA_LDFLAGS=
+if test "x$enable_static_testprogs" = "xyes" ; then
+   TESTPROGS_EXTRA_LDFLAGS="-all-static"
+fi
+AC_SUBST(TESTPROGS_EXTRA_LDFLAGS)
+
+dnl ==============================================
 dnl Timers
 
 AC_ARG_ENABLE(timers,
@@ -495,63 +827,112 @@ AC_ARG_ENABLE(gtk,
    [enable_gtk=$enableval], [enable_gtk=auto])
 
 PKG_PROG_PKG_CONFIG
+
+if test $enable_gtk = yes ; then
+   AC_CHECK_LIB([pixman-1], [pixman_version_string])
+   PKG_CHECK_MODULES(GTK, [gtk+-2.0 >= 2.16 pixman-1])
+fi
+
 if test $enable_gtk = auto ; then
-   PKG_CHECK_EXISTS([gtk+-2.0], [enable_gtk=yes], [enable_gtk=no])
+   AC_CHECK_LIB([pixman-1], [pixman_version_string], [enable_gtk=auto], [enable_gtk=no])
 fi
-if test $enable_gtk = yes ; then
-   PKG_CHECK_MODULES(GTK, [gtk+-2.0])
+
+if test $enable_gtk = auto ; then
+   PKG_CHECK_MODULES(GTK, [gtk+-2.0 >= 2.16 pixman-1], [enable_gtk=yes], [enable_gtk=no])
 fi
 
 AM_CONDITIONAL(HAVE_GTK, [test "x$enable_gtk" = xyes])
 
 AC_SUBST(GTK_CFLAGS)
 AC_SUBST(GTK_LIBS)
-AC_SUBST(DEP_CFLAGS)
-AC_SUBST(DEP_LIBS)
 
 dnl =====================================
-dnl posix_memalign 
+dnl posix_memalign, sigaction, alarm, gettimeofday
 
 AC_CHECK_FUNC(posix_memalign, have_posix_memalign=yes, have_posix_memalign=no)
 if test x$have_posix_memalign = xyes; then
    AC_DEFINE(HAVE_POSIX_MEMALIGN, 1, [Whether we have posix_memalign()])
 fi
 
-dnl =====================================
-dnl Thread local storage
+AC_CHECK_FUNC(sigaction, have_sigaction=yes, have_sigaction=no)
+if test x$have_sigaction = xyes; then
+   AC_DEFINE(HAVE_SIGACTION, 1, [Whether we have sigaction()])
+fi
 
-support_for__thread=no
+AC_CHECK_FUNC(alarm, have_alarm=yes, have_alarm=no)
+if test x$have_alarm = xyes; then
+   AC_DEFINE(HAVE_ALARM, 1, [Whether we have alarm()])
+fi
 
-AC_MSG_CHECKING(for __thread)
-AC_COMPILE_IFELSE([
-#ifdef __MINGW32__
-#error MinGW has broken __thread support
-#endif
-__thread int x ;
-int main () { return 0; }
-], support_for__thread=yes)
+AC_CHECK_HEADER([sys/mman.h],
+   [AC_DEFINE(HAVE_SYS_MMAN_H, [1], [Define to 1 if we have <sys/mman.h>])])
 
-if test $support_for__thread = yes; then 
-   AC_DEFINE([TOOLCHAIN_SUPPORTS__THREAD],[],[Whether the tool chain supports __thread])
+AC_CHECK_FUNC(mmap, have_mmap=yes, have_mmap=no)
+if test x$have_mmap = xyes; then
+   AC_DEFINE(HAVE_MMAP, 1, [Whether we have mmap()])
 fi
 
-AC_MSG_RESULT($support_for__thread)
+AC_CHECK_FUNC(mprotect, have_mprotect=yes, have_mprotect=no)
+if test x$have_mprotect = xyes; then
+   AC_DEFINE(HAVE_MPROTECT, 1, [Whether we have mprotect()])
+fi
 
-dnl posix tls
+AC_CHECK_FUNC(getpagesize, have_getpagesize=yes, have_getpagesize=no)
+if test x$have_getpagesize = xyes; then
+   AC_DEFINE(HAVE_GETPAGESIZE, 1, [Whether we have getpagesize()])
+fi
 
-if test $support_for__thread = no; then
+AC_CHECK_HEADER([fenv.h],
+   [AC_DEFINE(HAVE_FENV_H, [1], [Define to 1 if we have <fenv.h>])])
 
-support_for_pthread_setspecific=no
-   
-AC_MSG_CHECKING(for pthread_setspecific)
+AC_CHECK_LIB(m, feenableexcept, have_feenableexcept=yes, have_feenableexcept=no)
+if test x$have_feenableexcept = xyes; then
+   AC_DEFINE(HAVE_FEENABLEEXCEPT, 1, [Whether we have feenableexcept()])
+fi
 
-save_LDFLAGS=$LDFLAGS
+AC_CHECK_FUNC(gettimeofday, have_gettimeofday=yes, have_gettimeofday=no)
+AC_CHECK_HEADER(sys/time.h, have_sys_time_h=yes, have_sys_time_h=no)
+if test x$have_gettimeofday = xyes && test x$have_sys_time_h = xyes; then
+   AC_DEFINE(HAVE_GETTIMEOFDAY, 1, [Whether we have gettimeofday()])
+fi
 
-LDFLAGS="-pthread"
+dnl =====================================
+dnl Check for missing sqrtf() as, e.g., for Solaris 9
 
-AC_LINK_IFELSE([
-#include <pthread.h>
+AC_SEARCH_LIBS([sqrtf], [m], [],
+               [AC_DEFINE([sqrtf], [sqrt],
+                          [Define to sqrt if you do not have the `sqrtf' function.])])
+
+dnl =====================================
+dnl Thread local storage
+
+AC_MSG_CHECKING(for thread local storage (TLS) support)
+AC_CACHE_VAL(ac_cv_tls, [
+    ac_cv_tls=none
+    keywords="__thread __declspec(thread)"
+    for kw in $keywords ; do
+        AC_TRY_COMPILE([
+#if defined(__MINGW32__) && !(__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5))
+#error This MinGW version has broken __thread support
+#endif
+#ifdef __OpenBSD__
+#error OpenBSD has broken __thread support
+#endif
+
+int $kw test;], [], [ac_cv_tls=$kw; break])
+    done
+])
+AC_MSG_RESULT($ac_cv_tls)
+
+if test "$ac_cv_tls" != "none"; then
+    AC_DEFINE_UNQUOTED([TLS], $ac_cv_tls, [The compiler supported TLS storage class])
+fi
+
+dnl
+dnl posix tls
+dnl
 
+m4_define([pthread_test_program],AC_LANG_SOURCE([[dnl
 #include <stdlib.h>
 #include <pthread.h>
 
@@ -568,7 +949,7 @@ int
 main ()
 {
     void *value = NULL;
-    
+
     if (pthread_once (&once_control, make_key) != 0)
     {
 	value = NULL;
@@ -582,27 +963,166 @@ main ()
 	    pthread_setspecific (key, value);
 	}
     }
+    return 0;
 }
-], support_for_pthread_setspecific=yes);
+]]))
+
+AC_DEFUN([PIXMAN_CHECK_PTHREAD],[dnl
+    if test "z$support_for_pthreads" != "zyes"; then
+	PIXMAN_LINK_WITH_ENV(
+		[$1], [pthread_test_program],
+		[PTHREAD_CFLAGS="$CFLAGS"
+		 PTHREAD_LIBS="$LIBS"
+		 PTHREAD_LDFLAGS="$LDFLAGS"
+		 support_for_pthreads=yes])
+    fi
+])
+
+support_for_pthreads=no
+
+AC_MSG_CHECKING(for pthreads)
+
+PIXMAN_CHECK_PTHREAD([CFLAGS="-pthread"; LDFLAGS="-pthread"])
+PIXMAN_CHECK_PTHREAD([CFLAGS="-D_REENTRANT"; LIBS="-lpthread"])
+PIXMAN_CHECK_PTHREAD([CFLAGS="-D_REENTRANT"; LDFLAGS="-lroot"])
+    
+if test $support_for_pthreads = yes; then
+    AC_DEFINE([HAVE_PTHREADS], [], [Whether pthreads is supported])
+    if test $ac_cv_tls = none ; then
+        CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+    fi
+fi
 
-LDFLAGS=$save_LDFLAGS
+AC_MSG_RESULT($support_for_pthreads)
 
-if test $support_for_pthread_setspecific = yes; then
-   PTHREAD_LDFLAGS="-pthread"
-   AC_DEFINE([HAVE_PTHREAD_SETSPECIFIC], [], [Whether pthread_setspecific() is supported])
+AC_SUBST(TOOLCHAIN_SUPPORTS__THREAD)
+AC_SUBST(HAVE_PTHREADS)
+AC_SUBST(PTHREAD_LDFLAGS)
+AC_SUBST(PTHREAD_LIBS)
+AC_SUBST(PTHREAD_CFLAGS)
+
+dnl =====================================
+dnl __attribute__((constructor))
+
+support_for_attribute_constructor=no
+
+AC_MSG_CHECKING(for __attribute__((constructor)))
+AC_LINK_IFELSE([AC_LANG_SOURCE([[
+#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 7))
+/* attribute 'constructor' is supported since gcc 2.7, but some compilers
+ * may only pretend to be gcc, so let's try to actually use it
+ */
+static int x = 1;
+static void __attribute__((constructor)) constructor_function () { x = 0; }
+int main (void) { return x; }
+#else
+#error not gcc or gcc version is older than 2.7
+#endif
+]])], support_for_attribute_constructor=yes)
+
+if test x$support_for_attribute_constructor = xyes; then
+   AC_DEFINE([TOOLCHAIN_SUPPORTS_ATTRIBUTE_CONSTRUCTOR],
+             [],[Whether the tool chain supports __attribute__((constructor))])
 fi
 
-AC_MSG_RESULT($support_for_pthread_setspecific);
+AC_MSG_RESULT($support_for_attribute_constructor)
+AC_SUBST(TOOLCHAIN_SUPPORTS_ATTRIBUTE_CONSTRUCTOR)
+
+dnl =====================================
+dnl __float128
+
+support_for_float128=no
+
+AC_MSG_CHECKING(for __float128)
+AC_LINK_IFELSE([AC_LANG_SOURCE([[
+__float128 a = 1.0Q, b = 2.0Q; int main (void) { return a + b; }
+]])], support_for_float128=yes)
 
+if test x$support_for_float128 = xyes; then
+   AC_DEFINE([HAVE_FLOAT128], [], [Whether the tool chain supports __float128])
 fi
 
-AC_SUBST(TOOLCHAIN_SUPPORTS__THREAD)
-AC_SUBST(HAVE_PTHREAD_SETSPECIFIC)
-AC_SUBST(PTHREAD_LDFLAGS)
+AC_MSG_RESULT($support_for_float128)
+
+dnl =====================================
+dnl __builtin_clz
+
+support_for_builtin_clz=no
+
+AC_MSG_CHECKING(for __builtin_clz)
+AC_LINK_IFELSE([AC_LANG_SOURCE([[
+unsigned int x = 11; int main (void) { return __builtin_clz(x); }
+]])], support_for_builtin_clz=yes)
+
+if test x$support_for_builtin_clz = xyes; then
+   AC_DEFINE([HAVE_BUILTIN_CLZ], [], [Whether the compiler supports __builtin_clz])
+fi
+
+AC_MSG_RESULT($support_for_builtin_clz)
+
+dnl =====================================
+dnl GCC vector extensions
+
+support_for_gcc_vector_extensions=no
+
+AC_MSG_CHECKING(for GCC vector extensions)
+AC_LINK_IFELSE([AC_LANG_SOURCE([[
+unsigned int __attribute__ ((vector_size(16))) e, a, b;
+int main (void) { e = a - ((b << 27) + (b >> (32 - 27))) + 1; return e[0]; }
+]])], support_for_gcc_vector_extensions=yes)
+
+if test x$support_for_gcc_vector_extensions = xyes; then
+   AC_DEFINE([HAVE_GCC_VECTOR_EXTENSIONS], [],
+             [Whether the compiler supports GCC vector extensions])
+fi
+
+AC_MSG_RESULT($support_for_gcc_vector_extensions)
+
+dnl ==================
+dnl libpng
+
+AC_ARG_ENABLE(libpng, AS_HELP_STRING([--enable-libpng], [Build support for libpng (default: auto)]),
+                      [have_libpng=$enableval], [have_libpng=auto])
+
+case x$have_libpng in
+	xyes) PKG_CHECK_MODULES(PNG, [libpng]) ;;
+	xno) ;;
+	*) PKG_CHECK_MODULES(PNG, [libpng], have_libpng=yes, have_libpng=no) ;;
+esac
+
+if test x$have_libpng = xyes; then
+    AC_DEFINE([HAVE_LIBPNG], [1], [Whether we have libpng])
+fi
+
+AC_SUBST(HAVE_LIBPNG)
 
 AC_OUTPUT([pixman-1.pc
            pixman-1-uninstalled.pc
            Makefile
 	   pixman/Makefile
 	   pixman/pixman-version.h
+	   demos/Makefile
 	   test/Makefile])
+
+m4_if(m4_eval(pixman_minor % 2), [1], [
+   echo
+   echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
+   echo
+   echo "      Thanks for testing this development snapshot of pixman. Please"
+   echo "      report any problems you find, either by sending email to "
+   echo
+   echo "          pixman@lists.freedesktop.org"
+   echo
+   echo "      or by filing a bug at "
+   echo
+   echo "          https://bugs.freedesktop.org/enter_bug.cgi?product=pixman "
+   echo
+   echo "      If you are looking for a stable release of pixman, please note "
+   echo "      that stable releases have _even_ minor version numbers. Ie., "
+   echo "      pixman-0.]m4_eval(pixman_minor & ~1)[.x are stable releases, whereas pixman-$PIXMAN_VERSION_MAJOR.$PIXMAN_VERSION_MINOR.$PIXMAN_VERSION_MICRO is a "
+   echo "      development snapshot that may contain bugs and experimental "
+   echo "      features. "
+   echo 
+   echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
+   echo
+])
diff --git a/pixman/demos/Makefile.am b/pixman/demos/Makefile.am
new file mode 100644
index 000000000..e04743d7f
--- /dev/null
+++ b/pixman/demos/Makefile.am
@@ -0,0 +1,52 @@
+EXTRA_DIST = parrot.c parrot.jpg scale.ui
+
+if HAVE_GTK
+
+AM_CFLAGS = $(OPENMP_CFLAGS)
+AM_LDFLAGS = $(OPENMP_CFLAGS)
+
+LDADD = $(top_builddir)/pixman/libpixman-1.la -lm $(GTK_LIBS) $(PNG_LIBS)
+AM_CPPFLAGS = -I$(top_srcdir)/pixman -I$(top_builddir)/pixman $(GTK_CFLAGS) $(PNG_CFLAGS)
+
+GTK_UTILS = gtk-utils.c gtk-utils.h ../test/utils.c ../test/utils.h \
+            ../test/utils-prng.c ../test/utils-prng.h
+
+DEMOS =				\
+	clip-test		\
+	clip-in			\
+	composite-test		\
+	gradient-test		\
+	radial-test		\
+	linear-gradient		\
+	conical-test		\
+	alpha-test		\
+	screen-test		\
+	convolution-test	\
+	trap-test		\
+	tri-test		\
+	quad2quad		\
+	checkerboard		\
+	srgb-trap-test		\
+	srgb-test		\
+	scale
+
+gradient_test_SOURCES = gradient-test.c $(GTK_UTILS)
+alpha_test_SOURCES = alpha-test.c $(GTK_UTILS)
+composite_test_SOURCES = composite-test.c $(GTK_UTILS)
+clip_test_SOURCES = clip-test.c $(GTK_UTILS)
+clip_in_SOURCES = clip-in.c $(GTK_UTILS)
+trap_test_SOURCES = trap-test.c $(GTK_UTILS)
+screen_test_SOURCES = screen-test.c $(GTK_UTILS)
+convolution_test_SOURCES = convolution-test.c $(GTK_UTILS)
+radial_test_SOURCES = radial-test.c $(GTK_UTILS)
+linear_gradient_SOURCES = linear-gradient.c $(GTK_UTILS)
+conical_test_SOURCES = conical-test.c $(GTK_UTILS)
+tri_test_SOURCES = tri-test.c $(GTK_UTILS)
+checkerboard_SOURCES = checkerboard.c $(GTK_UTILS)
+srgb_test_SOURCES = srgb-test.c $(GTK_UTILS)
+srgb_trap_test_SOURCES = srgb-trap-test.c $(GTK_UTILS)
+scale_SOURCES = scale.c $(GTK_UTILS)
+
+noinst_PROGRAMS = $(DEMOS)
+
+endif
diff --git a/pixman/test/alpha-test.c b/pixman/demos/alpha-test.c
index 92c208142..54e30fad5 100644
--- a/pixman/test/alpha-test.c
+++ b/pixman/demos/alpha-test.c
@@ -40,10 +40,12 @@ main (int argc, char **argv)
     };
 #endif
 
+#if 0
     pixman_point_fixed_t c_inner;
     pixman_point_fixed_t c_outer;
     pixman_fixed_t r_inner;
     pixman_fixed_t r_outer;
+#endif
     
     for (i = 0; i < WIDTH * HEIGHT; ++i)
 	alpha[i] = 0x4f00004f; /* pale blue */
@@ -69,6 +71,7 @@ main (int argc, char **argv)
 					src,
 					WIDTH * 4);
     
+#if 0
     c_inner.x = pixman_double_to_fixed (50.0);
     c_inner.y = pixman_double_to_fixed (50.0);
     c_outer.x = pixman_double_to_fixed (50.0);
@@ -76,7 +79,6 @@ main (int argc, char **argv)
     r_inner = 0;
     r_outer = pixman_double_to_fixed (50.0);
     
-#if 0
     grad_img = pixman_image_create_conical_gradient (&c_inner, r_inner,
 						    stops, 2);
 #endif
diff --git a/pixman/demos/checkerboard.c b/pixman/demos/checkerboard.c
new file mode 100644
index 000000000..449fedb37
--- /dev/null
+++ b/pixman/demos/checkerboard.c
@@ -0,0 +1,71 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include "pixman.h"
+#include "gtk-utils.h"
+
+int
+main (int argc, char **argv)
+{
+#define WIDTH 400
+#define HEIGHT 400
+#define TILE_SIZE 25
+
+    pixman_image_t *checkerboard;
+    pixman_image_t *destination;
+#define D2F(d) (pixman_double_to_fixed(d))
+    pixman_transform_t trans = { {
+	    { D2F (-1.96830), D2F (-1.82250), D2F (512.12250)},
+	    { D2F (0.00000), D2F (-7.29000), D2F (1458.00000)},
+	    { D2F (0.00000), D2F (-0.00911), D2F (0.59231)},
+	}};
+    int i, j;
+
+    checkerboard = pixman_image_create_bits (PIXMAN_a8r8g8b8,
+					     WIDTH, HEIGHT,
+					     NULL, 0);
+
+    destination = pixman_image_create_bits (PIXMAN_a8r8g8b8,
+					    WIDTH, HEIGHT,
+					    NULL, 0);
+
+    for (i = 0; i < HEIGHT / TILE_SIZE; ++i)
+    {
+	for (j = 0; j < WIDTH / TILE_SIZE; ++j)
+	{
+	    double u = (double)(j + 1) / (WIDTH / TILE_SIZE);
+	    double v = (double)(i + 1) / (HEIGHT / TILE_SIZE);
+	    pixman_color_t black = { 0, 0, 0, 0xffff };
+	    pixman_color_t white = {
+		v * 0xffff,
+		u * 0xffff,
+		(1 - (double)u) * 0xffff,
+		0xffff };
+	    pixman_color_t *c;
+	    pixman_image_t *fill;
+
+	    if ((j & 1) != (i & 1))
+		c = &black;
+	    else
+		c = &white;
+
+	    fill = pixman_image_create_solid_fill (c);
+
+	    pixman_image_composite (PIXMAN_OP_SRC, fill, NULL, checkerboard,
+				    0, 0, 0, 0, j * TILE_SIZE, i * TILE_SIZE,
+				    TILE_SIZE, TILE_SIZE);
+	}
+    }
+
+    pixman_image_set_transform (checkerboard, &trans);
+    pixman_image_set_filter (checkerboard, PIXMAN_FILTER_BEST, NULL, 0);
+    pixman_image_set_repeat (checkerboard, PIXMAN_REPEAT_NONE);
+
+    pixman_image_composite (PIXMAN_OP_SRC,
+			    checkerboard, NULL, destination,
+			    0, 0, 0, 0, 0, 0,
+			    WIDTH, HEIGHT);
+
+    show_image (destination);
+
+    return 0;
+}
diff --git a/pixman/test/clip-in.c b/pixman/demos/clip-in.c
index 51579811f..51579811f 100644
--- a/pixman/test/clip-in.c
+++ b/pixman/demos/clip-in.c
diff --git a/pixman/test/clip-test.c b/pixman/demos/clip-test.c
index aa0df4482..aa0df4482 100644
--- a/pixman/test/clip-test.c
+++ b/pixman/demos/clip-test.c
diff --git a/pixman/test/composite-test.c b/pixman/demos/composite-test.c
index 5401abfdf..8213e2f9d 100644
--- a/pixman/test/composite-test.c
+++ b/pixman/demos/composite-test.c
@@ -3,9 +3,10 @@
 #include <stdio.h>
 #include "pixman.h"
 #include "gtk-utils.h"
+#include "parrot.c"
 
-#define WIDTH	60
-#define HEIGHT	60
+#define WIDTH	80
+#define HEIGHT	80
 
 typedef struct {
     const char *name;
@@ -87,29 +88,26 @@ int
 main (int argc, char **argv)
 {
 #define d2f pixman_double_to_fixed
-    
+
     GtkWidget *window, *swindow;
     GtkWidget *table;
     uint32_t *dest = malloc (WIDTH * HEIGHT * 4);
     uint32_t *src = malloc (WIDTH * HEIGHT * 4);
-    pixman_image_t *src_img;
+    pixman_image_t *gradient, *parrot;
     pixman_image_t *dest_img;
-    pixman_point_fixed_t p1 = { -10 << 0, 0 };
-    pixman_point_fixed_t p2 = { WIDTH << 16, (HEIGHT - 10) << 16 };
-    uint16_t full = 0xcfff;
-    uint16_t low  = 0x5000;
-    uint16_t alpha = 0xffff;
+    pixman_point_fixed_t p1 = { -10 << 16, 10 << 16 };
+    pixman_point_fixed_t p2 = { (WIDTH + 10) << 16, (HEIGHT - 10) << 16 };
+    uint16_t alpha = 0xdddd;
     pixman_gradient_stop_t stops[6] =
     {
-	{ d2f (0.0), { full, low, low, alpha } },
-	{ d2f (0.25), { full, full, low, alpha } },
-	{ d2f (0.4), { low, full, low, alpha } },
-	{ d2f (0.5), { low, full, full, alpha } },
-	{ d2f (0.8), { low, low, full, alpha } },
-	{ d2f (1.0), { full, low, full, alpha } },
+	{ d2f (0.0), { 0xf2f2, 0x8787, 0x7d7d, alpha } },
+	{ d2f (0.22), { 0xf3f3, 0xeaea, 0x8383, alpha } },
+	{ d2f (0.42), { 0x6b6b, 0xc0c0, 0x7777, alpha } },
+	{ d2f (0.57), { 0x4b4b, 0xc9c9, 0xf5f5, alpha } },
+	{ d2f (0.75), { 0x6a6a, 0x7f7f, 0xbebe, alpha } },
+	{ d2f (1.0), { 0xeded, 0x8282, 0xb0b0, alpha } },
     };
-	
-	    
+
     int i;
 
     gtk_init (&argc, &argv);
@@ -117,20 +115,20 @@ main (int argc, char **argv)
     window = gtk_window_new (GTK_WINDOW_TOPLEVEL);
 
     gtk_window_set_default_size (GTK_WINDOW (window), 800, 600);
-    
+
     g_signal_connect (window, "delete-event",
 		      G_CALLBACK (gtk_main_quit),
 		      NULL);
     table = gtk_table_new (G_N_ELEMENTS (operators) / 6, 6, TRUE);
 
-    src_img = pixman_image_create_linear_gradient (&p1, &p2, stops,
-						   sizeof (stops) / sizeof (stops[0]));
+    gradient = pixman_image_create_linear_gradient (&p1, &p2, stops, G_N_ELEMENTS (stops));
+    parrot = pixman_image_create_bits (PIXMAN_a8r8g8b8, WIDTH, HEIGHT, (uint32_t *)parrot_bits, WIDTH * 4);
+
+    pixman_image_set_repeat (gradient, PIXMAN_REPEAT_PAD);
 
-    pixman_image_set_repeat (src_img, PIXMAN_REPEAT_PAD);
-    
     dest_img = pixman_image_create_bits (PIXMAN_a8r8g8b8,
 					 WIDTH, HEIGHT,
-					 dest,
+					 NULL,
 					 WIDTH * 4);
     pixman_image_set_accessors (dest_img, reader, writer);
 
@@ -140,7 +138,6 @@ main (int argc, char **argv)
 	GdkPixbuf *pixbuf;
 	GtkWidget *vbox;
 	GtkWidget *label;
-	int j, k;
 
 	vbox = gtk_vbox_new (FALSE, 0);
 
@@ -148,14 +145,11 @@ main (int argc, char **argv)
 	gtk_box_pack_start (GTK_BOX (vbox), label, FALSE, FALSE, 6);
 	gtk_widget_show (label);
 
-	for (j = 0; j < HEIGHT; ++j)
-	{
-	    for (k = 0; k < WIDTH; ++k)
-		dest[j * WIDTH + k] = 0x7f6f6f00;
-	}
-	pixman_image_composite (operators[i].op, src_img, NULL, dest_img,
+	pixman_image_composite (PIXMAN_OP_SRC, gradient, NULL, dest_img,
+				0, 0, 0, 0, 0, 0, WIDTH, HEIGHT);
+	pixman_image_composite (operators[i].op, parrot, NULL, dest_img,
 				0, 0, 0, 0, 0, 0, WIDTH, HEIGHT);
-	pixbuf = pixbuf_from_argb32 (pixman_image_get_data (dest_img), TRUE,
+	pixbuf = pixbuf_from_argb32 (pixman_image_get_data (dest_img),
 				     WIDTH, HEIGHT, WIDTH * 4);
 	image = gtk_image_new_from_pixbuf (pixbuf);
 	gtk_box_pack_start (GTK_BOX (vbox), image, FALSE, FALSE, 0);
@@ -168,7 +162,7 @@ main (int argc, char **argv)
 	g_object_unref (pixbuf);
     }
 
-    pixman_image_unref (src_img);
+    pixman_image_unref (gradient);
     free (src);
     pixman_image_unref (dest_img);
     free (dest);
@@ -177,7 +171,7 @@ main (int argc, char **argv)
     gtk_scrolled_window_set_policy (GTK_SCROLLED_WINDOW (swindow),
 				    GTK_POLICY_AUTOMATIC,
 				    GTK_POLICY_AUTOMATIC);
-    
+
     gtk_scrolled_window_add_with_viewport (GTK_SCROLLED_WINDOW (swindow), table);
     gtk_widget_show (table);
 
diff --git a/pixman/demos/conical-test.c b/pixman/demos/conical-test.c
new file mode 100644
index 000000000..6b3243016
--- /dev/null
+++ b/pixman/demos/conical-test.c
@@ -0,0 +1,100 @@
+#include "../test/utils.h"
+#include "gtk-utils.h"
+
+#define SIZE 128
+#define GRADIENTS_PER_ROW 7
+#define NUM_ROWS ((NUM_GRADIENTS + GRADIENTS_PER_ROW - 1) / GRADIENTS_PER_ROW)
+#define WIDTH (SIZE * GRADIENTS_PER_ROW)
+#define HEIGHT (SIZE * NUM_ROWS)
+#define NUM_GRADIENTS 35
+
+#define double_to_color(x)					\
+    (((uint32_t) ((x)*65536)) - (((uint32_t) ((x)*65536)) >> 16))
+
+#define PIXMAN_STOP(offset,r,g,b,a)		\
+    { pixman_double_to_fixed (offset),		\
+	{					\
+	    double_to_color (r),		\
+		double_to_color (g),		\
+		double_to_color (b),		\
+		double_to_color (a)		\
+	}					\
+    }
+
+
+static const pixman_gradient_stop_t stops[] = {
+    PIXMAN_STOP (0.25,       1, 0, 0, 0.7),
+    PIXMAN_STOP (0.5,        1, 1, 0, 0.7),
+    PIXMAN_STOP (0.75,       0, 1, 0, 0.7),
+    PIXMAN_STOP (1.0,        0, 0, 1, 0.7)
+};
+
+#define NUM_STOPS (sizeof (stops) / sizeof (stops[0]))
+
+static pixman_image_t *
+create_conical (int index)
+{
+    pixman_point_fixed_t c;
+    double angle;
+
+    c.x = pixman_double_to_fixed (0);
+    c.y = pixman_double_to_fixed (0);
+
+    angle = (0.5 / NUM_GRADIENTS + index / (double)NUM_GRADIENTS) * 720 - 180;
+
+    return pixman_image_create_conical_gradient (
+	&c, pixman_double_to_fixed (angle), stops, NUM_STOPS);
+}
+
+int
+main (int argc, char **argv)
+{
+    pixman_transform_t transform;
+    pixman_image_t *src_img, *dest_img;
+    int i;
+
+    enable_divbyzero_exceptions ();
+
+    dest_img = pixman_image_create_bits (PIXMAN_a8r8g8b8,
+					 WIDTH, HEIGHT,
+					 NULL, 0);
+ 
+    draw_checkerboard (dest_img, 25, 0xffaaaaaa, 0xff888888);
+
+    pixman_transform_init_identity (&transform);
+
+    pixman_transform_translate (NULL, &transform,
+				pixman_double_to_fixed (0.5),
+				pixman_double_to_fixed (0.5));
+
+    pixman_transform_scale (NULL, &transform,
+			    pixman_double_to_fixed (SIZE),
+			    pixman_double_to_fixed (SIZE));
+    pixman_transform_translate (NULL, &transform,
+				pixman_double_to_fixed (0.5),
+				pixman_double_to_fixed (0.5));
+
+    for (i = 0; i < NUM_GRADIENTS; i++)
+    {
+	int column = i % GRADIENTS_PER_ROW;
+	int row = i / GRADIENTS_PER_ROW;
+
+	src_img = create_conical (i); 
+	pixman_image_set_repeat (src_img, PIXMAN_REPEAT_NORMAL);
+   
+	pixman_image_set_transform (src_img, &transform);
+	
+	pixman_image_composite32 (
+	    PIXMAN_OP_OVER, src_img, NULL,dest_img,
+	    0, 0, 0, 0, column * SIZE, row * SIZE,
+	    SIZE, SIZE);
+	
+	pixman_image_unref (src_img);
+    }
+
+    show_image (dest_img);
+
+    pixman_image_unref (dest_img);
+
+    return 0;
+}
diff --git a/pixman/test/convolution-test.c b/pixman/demos/convolution-test.c
index da284af7b..da284af7b 100644
--- a/pixman/test/convolution-test.c
+++ b/pixman/demos/convolution-test.c
diff --git a/pixman/test/gradient-test.c b/pixman/demos/gradient-test.c
index fc84844b0..e68f69a5f 100644
--- a/pixman/test/gradient-test.c
+++ b/pixman/demos/gradient-test.c
@@ -15,12 +15,11 @@ main (int argc, char **argv)
     int i;
     pixman_gradient_stop_t stops[2] =
 	{
-	    { pixman_int_to_fixed (0), { 0xffff, 0xeeee, 0xeeee, 0xeeee } },
-	    { pixman_int_to_fixed (1), { 0xffff, 0x1111, 0x1111, 0x1111 } }
+	    { pixman_int_to_fixed (0), { 0x0000, 0x0000, 0xffff, 0xffff } },
+	    { pixman_int_to_fixed (1), { 0xffff, 0x1111, 0x1111, 0xffff } }
 	};
-    pixman_point_fixed_t p1 = { pixman_double_to_fixed (0), 0 };
-    pixman_point_fixed_t p2 = { pixman_double_to_fixed (WIDTH / 8.),
-				pixman_int_to_fixed (0) };
+    pixman_point_fixed_t p1 = { pixman_double_to_fixed (50), 0 };
+    pixman_point_fixed_t p2 = { pixman_double_to_fixed (200), 0 };
 #if 0
     pixman_transform_t trans = {
 	{ { pixman_double_to_fixed (2), pixman_double_to_fixed (0.5), pixman_double_to_fixed (-100), },
@@ -36,19 +35,22 @@ main (int argc, char **argv)
     };
 #endif
 
+#if 0
     pixman_point_fixed_t c_inner;
     pixman_point_fixed_t c_outer;
     pixman_fixed_t r_inner;
     pixman_fixed_t r_outer;
+#endif
     
     for (i = 0; i < WIDTH * HEIGHT; ++i)
-	dest[i] = 0x4f00004f; /* pale blue */
+	dest[i] = 0xff00ff00;
     
     dest_img = pixman_image_create_bits (PIXMAN_a8r8g8b8,
 					 WIDTH, HEIGHT, 
 					 dest,
 					 WIDTH * 4);
 
+#if 0
     c_inner.x = pixman_double_to_fixed (50.0);
     c_inner.y = pixman_double_to_fixed (50.0);
     c_outer.x = pixman_double_to_fixed (50.0);
@@ -58,6 +60,7 @@ main (int argc, char **argv)
     
     src_img = pixman_image_create_conical_gradient (&c_inner, r_inner,
 						    stops, 2);
+#endif
 #if 0
     src_img = pixman_image_create_conical_gradient (&c_inner, r_inner,
 						    stops, 2);
@@ -70,7 +73,7 @@ main (int argc, char **argv)
 						    stops, 2);
     
     pixman_image_set_transform (src_img, &trans);
-    pixman_image_set_repeat (src_img, PIXMAN_REPEAT_PAD);
+    pixman_image_set_repeat (src_img, PIXMAN_REPEAT_NONE);
     
     pixman_image_composite (PIXMAN_OP_OVER, src_img, NULL, dest_img,
 			    0, 0, 0, 0, 0, 0, 10 * WIDTH, HEIGHT);
diff --git a/pixman/demos/gtk-utils.c b/pixman/demos/gtk-utils.c
new file mode 100644
index 000000000..32d4aecc7
--- /dev/null
+++ b/pixman/demos/gtk-utils.c
@@ -0,0 +1,179 @@
+#include <gtk/gtk.h>
+#include <config.h>
+#include "../test/utils.h"
+#include "gtk-utils.h"
+
+pixman_image_t *
+pixman_image_from_file (const char *filename, pixman_format_code_t format)
+{
+    GdkPixbuf *pixbuf;
+    pixman_image_t *image;
+    int width, height;
+    uint32_t *data, *d;
+    uint8_t *gdk_data;
+    int n_channels;
+    int j, i;
+    int stride;
+
+    if (!(pixbuf = gdk_pixbuf_new_from_file (filename, NULL)))
+	return NULL;
+
+    image = NULL;
+
+    width = gdk_pixbuf_get_width (pixbuf);
+    height = gdk_pixbuf_get_height (pixbuf);
+    n_channels = gdk_pixbuf_get_n_channels (pixbuf);
+    gdk_data = gdk_pixbuf_get_pixels (pixbuf);
+    stride = gdk_pixbuf_get_rowstride (pixbuf);
+
+    if (!(data = malloc (width * height * sizeof (uint32_t))))
+	goto out;
+
+    d = data;
+    for (j = 0; j < height; ++j)
+    {
+	uint8_t *gdk_line = gdk_data;
+
+	for (i = 0; i < width; ++i)
+	{
+	    int r, g, b, a;
+	    uint32_t pixel;
+
+	    r = gdk_line[0];
+	    g = gdk_line[1];
+	    b = gdk_line[2];
+
+	    if (n_channels == 4)
+		a = gdk_line[3];
+	    else
+		a = 0xff;
+
+	    r = (r * a + 127) / 255;
+	    g = (g * a + 127) / 255;
+	    b = (b * a + 127) / 255;
+
+	    pixel = (a << 24) | (r << 16) | (g << 8) | b;
+
+	    *d++ = pixel;
+	    gdk_line += n_channels;
+	}
+
+	gdk_data += stride;
+    }
+
+    image = pixman_image_create_bits (
+	format, width, height, data, width * 4);
+
+out:
+    g_object_unref (pixbuf);
+    return image;
+}
+
+GdkPixbuf *
+pixbuf_from_argb32 (uint32_t *bits,
+		    int width,
+		    int height,
+		    int stride)
+{
+    GdkPixbuf *pixbuf = gdk_pixbuf_new (GDK_COLORSPACE_RGB, TRUE,
+					8, width, height);
+    int p_stride = gdk_pixbuf_get_rowstride (pixbuf);
+    guint32 *p_bits = (guint32 *)gdk_pixbuf_get_pixels (pixbuf);
+    int i;
+
+    for (i = 0; i < height; ++i)
+    {
+	uint32_t *src_row = &bits[i * (stride / 4)];
+	uint32_t *dst_row = p_bits + i * (p_stride / 4);
+
+	a8r8g8b8_to_rgba_np (dst_row, src_row, width);
+    }
+
+    return pixbuf;
+}
+
+static gboolean
+on_expose (GtkWidget *widget, GdkEventExpose *expose, gpointer data)
+{
+    pixman_image_t *pimage = data;
+    int width = pixman_image_get_width (pimage);
+    int height = pixman_image_get_height (pimage);
+    int stride = pixman_image_get_stride (pimage);
+    cairo_surface_t *cimage;
+    cairo_format_t format;
+    cairo_t *cr;
+
+    if (pixman_image_get_format (pimage) == PIXMAN_x8r8g8b8)
+	format = CAIRO_FORMAT_RGB24;
+    else
+	format = CAIRO_FORMAT_ARGB32;
+
+    cimage = cairo_image_surface_create_for_data (
+	(uint8_t *)pixman_image_get_data (pimage),
+	format, width, height, stride);
+    
+    cr = gdk_cairo_create (widget->window);
+
+    cairo_rectangle (cr, 0, 0, width, height);
+    cairo_set_source_surface (cr, cimage, 0, 0);
+    cairo_fill (cr);
+
+    cairo_destroy (cr);
+    cairo_surface_destroy (cimage);
+    
+    return TRUE;
+}
+
+void
+show_image (pixman_image_t *image)
+{
+    GtkWidget *window;
+    int width, height;
+    int argc;
+    char **argv;
+    char *arg0 = g_strdup ("pixman-test-program");
+    pixman_format_code_t format;
+    pixman_image_t *copy;
+
+    argc = 1;
+    argv = (char **)&arg0;
+
+    gtk_init (&argc, &argv);
+    
+    window = gtk_window_new (GTK_WINDOW_TOPLEVEL);
+    width = pixman_image_get_width (image);
+    height = pixman_image_get_height (image);
+
+    gtk_window_set_default_size (GTK_WINDOW (window), width, height);
+
+    format = pixman_image_get_format (image);
+
+    /* We always display the image as if it contains sRGB data. That
+     * means that no conversion should take place when the image
+     * has the a8r8g8b8_sRGB format.
+     */
+    switch (format)
+    {
+    case PIXMAN_a8r8g8b8_sRGB:
+    case PIXMAN_a8r8g8b8:
+    case PIXMAN_x8r8g8b8:
+	copy = pixman_image_ref (image);
+	break;
+
+    default:
+	copy = pixman_image_create_bits (PIXMAN_a8r8g8b8,
+					 width, height, NULL, -1);
+	pixman_image_composite32 (PIXMAN_OP_SRC,
+				  image, NULL, copy,
+				  0, 0, 0, 0, 0, 0,
+				  width, height);
+	break;
+    }
+
+    g_signal_connect (window, "expose_event", G_CALLBACK (on_expose), copy);
+    g_signal_connect (window, "delete_event", G_CALLBACK (gtk_main_quit), NULL);
+    
+    gtk_widget_show (window);
+    
+    gtk_main ();
+}
diff --git a/pixman/test/gtk-utils.h b/pixman/demos/gtk-utils.h
index 2cb13bcf0..36be4def6 100644
--- a/pixman/test/gtk-utils.h
+++ b/pixman/demos/gtk-utils.h
@@ -6,8 +6,10 @@
 
 void show_image (pixman_image_t *image);
 
+pixman_image_t *
+pixman_image_from_file (const char *filename, pixman_format_code_t format);
+
 GdkPixbuf *pixbuf_from_argb32 (uint32_t *bits,
-		               gboolean has_alpha,
                                int width,
                                int height,
                                int stride);
diff --git a/pixman/demos/linear-gradient.c b/pixman/demos/linear-gradient.c
new file mode 100644
index 000000000..46433a6e5
--- /dev/null
+++ b/pixman/demos/linear-gradient.c
@@ -0,0 +1,50 @@
+#include "../test/utils.h"
+#include "gtk-utils.h"
+
+#define WIDTH 1024
+#define HEIGHT 640
+
+int
+main (int argc, char **argv)
+{
+    pixman_image_t *src_img, *dest_img;
+    pixman_gradient_stop_t stops[] = {
+        { 0x00000, { 0x0000, 0x0000, 0x4444, 0xdddd } },
+        { 0x10000, { 0xeeee, 0xeeee, 0x8888, 0xdddd } },
+#if 0
+        /* These colors make it very obvious that dithering
+         * is useful even for 8-bit gradients
+         */
+	{ 0x00000, { 0x6666, 0x3333, 0x3333, 0xffff } },
+	{ 0x10000, { 0x3333, 0x6666, 0x6666, 0xffff } },
+#endif
+    };
+    pixman_point_fixed_t p1, p2;
+
+    enable_divbyzero_exceptions ();
+
+    dest_img = pixman_image_create_bits (PIXMAN_x8r8g8b8,
+					 WIDTH, HEIGHT,
+					 NULL, 0);
+
+    p1.x = p1.y = 0x0000;
+    p2.x = WIDTH << 16;
+    p2.y = HEIGHT << 16;
+    
+    src_img = pixman_image_create_linear_gradient (&p1, &p2, stops, ARRAY_LENGTH (stops));
+
+    pixman_image_composite32 (PIXMAN_OP_OVER,
+			      src_img,
+			      NULL,
+			      dest_img,
+			      0, 0,
+			      0, 0,
+			      0, 0,
+			      WIDTH, HEIGHT);
+
+    show_image (dest_img);
+
+    pixman_image_unref (dest_img);
+
+    return 0;
+}
diff --git a/pixman/demos/parrot.c b/pixman/demos/parrot.c
new file mode 100644
index 000000000..60fd270e8
--- /dev/null
+++ b/pixman/demos/parrot.c
@@ -0,0 +1,1079 @@
+/* This parrot is a finger painting by Rubens LP:
+ *
+ *     http://www.flickr.com/photos/dorubens/4030604504/in/set-72157622586088192/
+ *
+ * Used here under Creative Commons Attribution. The artist's web site:
+ *
+ *     http://www.rubenslp.com.br/
+ *
+ */
+static const uint32_t parrot_bits[] =
+{
+    0x716f7070, 0x1c1b1b1b, 0x110f1010, 0x16151415, 0x14121313, 0x2c292b2b,
+    0x403e3f3f, 0x19181818, 0x06050605, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x02010101, 0x08070707,
+    0x05040404, 0x0a060908, 0x27262426, 0xb3b0b1b2, 0x99979897, 0x2b2a2929,
+    0x100f0f0f, 0x0f0d0e0e, 0x0e0e0c0d, 0x0d0d0b0c, 0x12111111, 0x10100e0f,
+    0x0e0e0c0d, 0x0e0e0c0d, 0x12101211, 0x13121212, 0x17151516, 0x100f0e0f,
+    0x15141414, 0x423f4042, 0x3b393a3a, 0x13121212, 0x16151515, 0x2b282b29,
+    0x13121112, 0x100f0f0f, 0x0f0d0f0e, 0x08070807, 0x0d0c0c0c, 0x0a090a09,
+    0x0e0e0c0d, 0x0c0c0a0b, 0x10100f0f, 0x0f0e0e0e, 0x07060706, 0x0d0c0d0c,
+    0x0e0d0e0d, 0x05040504, 0x08070807, 0x0c0b0c0b, 0x0d0c0d0c, 0x05040504,
+    0x110f1110, 0x08070707, 0x04030303, 0x09080808, 0x06050605, 0x01000000,
+    0x08070707, 0x06050505, 0x05040504, 0x100e100f, 0x0b0a0b0a, 0x01000100,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x04030403,
+    0x03020302, 0x0b0a0b0a, 0x14131313, 0x0e0d0d0d, 0x0e0d0e0d, 0x231f2222,
+    0x4d4b4b4d, 0xa7a5a6a6, 0x5b595a5a, 0x07060606, 0x00000000, 0x00000000,
+    0x01000100, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x02010201,
+    0x05040404, 0x07050706, 0x04020303, 0x403e3e3f, 0xb6b3b5b5, 0x84828283,
+    0x1a191819, 0x0e0d0d0d, 0x0d0c0b0c, 0x0f0f0d0e, 0x0e0d0d0d, 0x0f0f0e0e,
+    0x0e0e0c0d, 0x0c0b0a0b, 0x0b0a090a, 0x11111010, 0x100f0f0f, 0x100f0f0f,
+    0x1b19191a, 0x1f1e1e1e, 0x46434544, 0x3a37383a, 0x1c1b1a1b, 0x1e1d1d1d,
+    0x29272828, 0x19171818, 0x0e0d0d0d, 0x0f0e0e0e, 0x06050505, 0x0c0b0b0b,
+    0x100e100f, 0x09080908, 0x0c0c0a0b, 0x0f0f0e0e, 0x0c0c0a0b, 0x05040404,
+    0x08070807, 0x0c0b0c0b, 0x05040504, 0x06050605, 0x100e100f, 0x09080908,
+    0x09080908, 0x12101211, 0x09080908, 0x03020202, 0x08070707, 0x01000100,
+    0x04030403, 0x07060606, 0x08070707, 0x08070707, 0x0f0e0f0e, 0x0b0a0b0a,
+    0x03020302, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x03020202,
+    0x09080908, 0x05040504, 0x00000000, 0x00000000, 0x0b0a0b0a, 0x0f0e0f0e,
+    0x1a191a19, 0x77757576, 0xc3c1c2c2, 0x75737374, 0x1f1e1e1f, 0x06050505,
+    0x07030605, 0x00000000, 0x03020302, 0x00000000, 0x00000000, 0x00000000,
+    0x03020202, 0x04030403, 0x04030303, 0x02010101, 0x5b5a5959, 0xafacaeae,
+    0x5a575859, 0x1b19191a, 0x0e0d0d0d, 0x100e100f, 0x100f0f0f, 0x11101010,
+    0x12121111, 0x0c0c0a0b, 0x09090708, 0x0d0d0b0c, 0x0f0e0d0e, 0x0d0d0b0c,
+    0x14131313, 0x1c1b1b1b, 0x322f3132, 0x514f504f, 0x2d2b2b2b, 0x2e2b2c2e,
+    0x21202020, 0x201f1f1f, 0x15141414, 0x12101211, 0x0e0d0d0d, 0x08070807,
+    0x0b0a0a0a, 0x100e0f0f, 0x07060706, 0x0a090a09, 0x0f0f0d0e, 0x0c0c0a0b,
+    0x09090708, 0x0d0c0c0c, 0x0b0a0b0a, 0x06050605, 0x0b0a0b0a, 0x0c0b0c0b,
+    0x08070807, 0x07060706, 0x0f0e0f0e, 0x0a090a09, 0x01000000, 0x05040504,
+    0x03020202, 0x01000000, 0x08070707, 0x05040504, 0x09080908, 0x0d0c0d0c,
+    0x07060606, 0x04030403, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x02010201, 0x06050505, 0x08070807, 0x01000100, 0x00000000, 0x00000000,
+    0x0f0d0e0e, 0x2c2a2a2b, 0x9c9a9b9b, 0xcac7cac9, 0x4d4c4b4d, 0x1b19181a,
+    0x0a090909, 0x00000000, 0x02010201, 0x05040504, 0x00000000, 0x00000000,
+    0x00000000, 0x02010101, 0x06040605, 0x03020302, 0x100d0e10, 0x615f5f60,
+    0x9d9a9c9c, 0x32313131, 0x14131313, 0x0d0c0c0c, 0x0f0e0e0e, 0x0e0d0d0d,
+    0x100f0f0f, 0x11101010, 0x08070807, 0x06050405, 0x0f0e0e0e, 0x100e0f0f,
+    0x0d0d0c0c, 0x2a282929, 0x3d3c3b3c, 0x38373637, 0x4f4d4d4f, 0x19181718,
+    0x27262626, 0x14131313, 0x29272828, 0x2e2b2c2d, 0x201d1e20, 0x15121414,
+    0x04030403, 0x07060606, 0x0c0b0c0b, 0x0a090a09, 0x08070707, 0x0e0d0e0d,
+    0x0b0a0a0a, 0x07060606, 0x0c0b0b0b, 0x0a090909, 0x04030403, 0x0a090909,
+    0x0e0d0e0d, 0x0a090a09, 0x09080908, 0x0e0d0e0d, 0x07060706, 0x08070807,
+    0x08070807, 0x00000000, 0x01000000, 0x07060606, 0x04030403, 0x08070807,
+    0x0e0d0e0d, 0x07060706, 0x06050605, 0x01000100, 0x00000000, 0x00000000,
+    0x00000000, 0x02010201, 0x07060706, 0x01000100, 0x00000000, 0x00000000,
+    0x01000100, 0x01000100, 0x322e3131, 0xa9a8a8a8, 0xb9b8b8b8, 0x39383639,
+    0x1d1b1b1c, 0x0c0b0b0b, 0x04030303, 0x05040404, 0x07060706, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x06030605, 0x09060807, 0x0e0d0c0d,
+    0x605e5e5f, 0x99959898, 0x2e2c2c2e, 0x13121212, 0x0d0d0b0c, 0x11111010,
+    0x12121111, 0x13131212, 0x0f0e0e0e, 0x09080908, 0x03020302, 0x0c0b0b0b,
+    0x0e0d0d0d, 0x11101010, 0x38363737, 0x514f4f50, 0x34333134, 0x46434546,
+    0x24222124, 0x29262827, 0x04030303, 0x05040404, 0x14131313, 0x15151414,
+    0x100f100f, 0x07060706, 0x07060606, 0x0d0c0d0c, 0x0a090909, 0x07060706,
+    0x0f0e0f0e, 0x0c0b0c0b, 0x01000100, 0x0c0b0c0b, 0x0a090a09, 0x01000100,
+    0x08070707, 0x12101211, 0x0b0a0b0a, 0x06050605, 0x0f0e0f0e, 0x07060706,
+    0x04030403, 0x06050605, 0x02010201, 0x00000000, 0x05040504, 0x03020302,
+    0x06050605, 0x0d0c0d0c, 0x08070707, 0x07060706, 0x100e100f, 0x05040504,
+    0x01000100, 0x00000000, 0x02010201, 0x07060606, 0x03020202, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x06050605, 0xb8b6b6b7, 0xa4a2a3a3,
+    0x2c2b2b2b, 0x1c191a1b, 0x0e0c0d0d, 0x08070707, 0x35323433, 0x1a191919,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x04020403,
+    0x1b19191a, 0x68666667, 0x7d7a7c7c, 0x23212023, 0x1c191a1c, 0x0f0e0d0e,
+    0x11111010, 0x11111010, 0x10100f0f, 0x0c0b0b0b, 0x09080908, 0x04030403,
+    0x0e0d0e0d, 0x0e0d0d0d, 0x18171717, 0x52505150, 0x5d5a5b5c, 0x3b3a383a,
+    0x3d3a3b3d, 0x24212224, 0x29252729, 0x06050505, 0x04030303, 0x04030403,
+    0x06050505, 0x0c0b0b0b, 0x09080908, 0x04030303, 0x0d0c0c0c, 0x06050605,
+    0x05040504, 0x0c0b0b0b, 0x08070807, 0x06050605, 0x09080908, 0x0c0b0c0b,
+    0x05040504, 0x0a090909, 0x0e0d0e0d, 0x0a090a09, 0x09080908, 0x0f0e0e0e,
+    0x09080908, 0x04030403, 0x09080908, 0x02010201, 0x00000000, 0x07060706,
+    0x05040504, 0x07060606, 0x0f0d0f0e, 0x06050605, 0x08070807, 0x11101010,
+    0x0c0b0b0b, 0x11101010, 0x09080808, 0x03020302, 0x05040404, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x7c7a7b7b, 0x1d1c1c1d, 0x100f0f0f, 0x110e1010, 0x07070607, 0x5b585a59,
+    0x3e3b3d3c, 0x05040404, 0x00000000, 0x01010001, 0x00000000, 0x00000000,
+    0x02010101, 0x1d1b1b1c, 0x615f6060, 0x54525253, 0x1e1c1b1e, 0x18151617,
+    0x1b18181b, 0x1a171819, 0x1b181a1a, 0x1b18191a, 0x100e0f0f, 0x03020302,
+    0x07060606, 0x0d0c0c0c, 0x120f1111, 0x27252626, 0x73727272, 0x706e6f6f,
+    0x524f5152, 0x2f2c2d2f, 0x1e1c1d1d, 0x1f1c1e1e, 0x09070808, 0x03020202,
+    0x04030303, 0x03020302, 0x0b0a0a0a, 0x08070807, 0x02010101, 0x0b0a0b0a,
+    0x0b0a0b0a, 0x04030303, 0x0d0c0d0c, 0x09080808, 0x05040504, 0x0b0a0b0a,
+    0x08070807, 0x02010201, 0x0c0b0c0b, 0x0c0b0b0b, 0x0a090a09, 0x08070807,
+    0x100e100f, 0x06050605, 0x04030403, 0x07060706, 0x02010201, 0x00000000,
+    0x06050605, 0x03020302, 0x09080908, 0x0d0c0d0c, 0x0d0c0c0c, 0x0a090909,
+    0x0d0c0c0c, 0x15131314, 0x1b19191a, 0x1d1b1b1c, 0x11101010, 0x02010201,
+    0x01000100, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x02010201, 0x1c1b1b1b, 0x100f0f0f, 0x07060606, 0x0b0b090b,
+    0x8f898e8c, 0x37353636, 0x01010001, 0x00000000, 0x00000000, 0x01000000,
+    0x00000000, 0x00000000, 0x14121313, 0x57545557, 0x43414142, 0x211e1f21,
+    0x18161517, 0x18161518, 0x1f1d1c1f, 0x1b191a1a, 0x13111212, 0x0c0b0b0b,
+    0x02010101, 0x08070807, 0x0f0e0e0e, 0x14121313, 0x403e3e40, 0x8b888a8a,
+    0x68666767, 0x4c4a4a4c, 0x28252628, 0x23202123, 0x16141615, 0x03020202,
+    0x06050605, 0x04030403, 0x04030403, 0x0d0c0c0c, 0x0c0b0c0b, 0x00000000,
+    0x0a090a09, 0x0b0a0b0a, 0x03020302, 0x09080908, 0x0c0b0b0b, 0x04030403,
+    0x0c0b0c0b, 0x0b0a0b0a, 0x01000100, 0x09080908, 0x0f0e0e0e, 0x09080908,
+    0x0c0b0b0b, 0x0b0a0909, 0x03020202, 0x06050605, 0x08070707, 0x04030303,
+    0x00000000, 0x06050605, 0x02010201, 0x0c0b0c0b, 0x0f0d0e0e, 0x05040504,
+    0x0c0c0a0b, 0x100f0f0f, 0x0f0f0d0e, 0x14121313, 0x18161717, 0x100e0f0f,
+    0x02010101, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x0b0a0a0a, 0x0d0c0c0c, 0x08070707,
+    0x201f1f1f, 0xa29ca19f, 0x27262626, 0x00000000, 0x09080808, 0x05040404,
+    0x02010101, 0x03010302, 0x03020302, 0x1b19191a, 0x35323335, 0x39373738,
+    0x1c1a1a1c, 0x19161718, 0x1c1a191c, 0x211f1f21, 0x201d1d20, 0x14121313,
+    0x09080808, 0x04030403, 0x09080908, 0x0b0a0a0a, 0x1d1b1b1c, 0x4f4c4d4e,
+    0x87838685, 0x4a494749, 0x32303031, 0x1b1a1a1a, 0x1a191919, 0x13121312,
+    0x03020302, 0x09080908, 0x0d0c0d0c, 0x0d0c0d0c, 0x100e0f0f, 0x09080908,
+    0x01000000, 0x09080808, 0x09080808, 0x02010201, 0x0a090a09, 0x09080908,
+    0x04030403, 0x0c0b0c0b, 0x07060706, 0x00000000, 0x08070807, 0x0a090909,
+    0x09080808, 0x08070707, 0x0c0b090a, 0x03020000, 0x04030101, 0x06050405,
+    0x03020202, 0x00000000, 0x05040504, 0x04030403, 0x07060606, 0x100e100f,
+    0x07060706, 0x09080808, 0x11111010, 0x0d0d0b0c, 0x0e0e0c0d, 0x100f0f0f,
+    0x0f0e0f0e, 0x03020302, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x08070707, 0x10100f0f,
+    0x0c0a0b0b, 0x41403f40, 0x89858887, 0x1e1c1d1d, 0x00000000, 0x110f1010,
+    0x37353636, 0x1f1e1e1e, 0x05040404, 0x06050505, 0x201d1f1f, 0x322f3031,
+    0x33303132, 0x18151618, 0x19161719, 0x1f1d1c1f, 0x1c1a1a1b, 0x1c1a1a1b,
+    0x17151516, 0x09080808, 0x04030403, 0x09080908, 0x0f0d0e0e, 0x1a191819,
+    0x66646465, 0x77747676, 0x2b292a29, 0x07050706, 0x07060606, 0x27242726,
+    0x25232424, 0x08070707, 0x09080808, 0x11101010, 0x12111111, 0x15131414,
+    0x0b0a0b0a, 0x04030403, 0x04030203, 0x06050304, 0x01000000, 0x06050505,
+    0x07060606, 0x07060606, 0x09080808, 0x09080808, 0x02010101, 0x07060606,
+    0x100e0f0f, 0x09080808, 0x0c0b090a, 0x0a090708, 0x05040203, 0x03020000,
+    0x05040203, 0x04030102, 0x03020000, 0x05040304, 0x03020202, 0x09080808,
+    0x0a090a09, 0x08070807, 0x0a090a09, 0x110f1110, 0x0d0d0b0c, 0x0e0e0c0d,
+    0x11110f10, 0x0c0b0b0b, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x04030303,
+    0x0d0c0c0c, 0x0a090909, 0x504d4e4f, 0x78757776, 0x1a191919, 0x00000000,
+    0x16151515, 0x31303030, 0x312f3030, 0x100f0f0f, 0x0b0a0b0a, 0x14121313,
+    0x2c292a2c, 0x26232426, 0x19171619, 0x1b171a1b, 0x1f1c1d1f, 0x201d1d20,
+    0x0d0c0c0c, 0x0a090909, 0x06050505, 0x05040504, 0x0c0b0b0b, 0x0b0a0a0a,
+    0x22212121, 0x817d7f7e, 0x59575857, 0x17161616, 0x04030303, 0x01000000,
+    0x07060706, 0x0c0b0b0b, 0x02010201, 0x04030403, 0x0b0a0a0a, 0x201e1e1f,
+    0x17151616, 0x0d0c0c0c, 0x02010201, 0x04030303, 0x04030102, 0x04030101,
+    0x05040202, 0x06050304, 0x05040203, 0x09080707, 0x05040303, 0x03020102,
+    0x07060405, 0x09080607, 0x09080506, 0x0a090708, 0x0c0b090a, 0x06050303,
+    0x04030101, 0x06050302, 0x03020000, 0x03020001, 0x04030102, 0x05040202,
+    0x06050304, 0x0a090808, 0x07060505, 0x08070707, 0x0a090909, 0x0a090909,
+    0x09080808, 0x12111111, 0x09080808, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x03020202,
+    0x03020302, 0x09080908, 0x0a090a09, 0x110f1110, 0x66646465, 0x100e0f0f,
+    0x05030504, 0x1f1d1e1f, 0x25242424, 0x22212121, 0x0f0e0e0e, 0x17151616,
+    0x0f0e0e0e, 0x1e1d1d1d, 0x211e1e21, 0x1a18171a, 0x17151417, 0x1d1a1a1d,
+    0x201d1e20, 0x19161719, 0x00000000, 0x00000000, 0x01000100, 0x15121414,
+    0x16141415, 0x32303031, 0x78747776, 0x2f2e2d2d, 0x09080808, 0x06030605,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x09080808, 0x18171617, 0x11101010, 0x02010201, 0x01000000, 0x05040303,
+    0x03020000, 0x05040202, 0x06050203, 0x04030102, 0x06050304, 0x04030001,
+    0x03020001, 0x09090605, 0x09090607, 0x09080608, 0x16131216, 0x0a0a0709,
+    0x06050305, 0x05040204, 0x07060406, 0x03020002, 0x03020001, 0x04030102,
+    0x02010000, 0x06050304, 0x06050304, 0x03020001, 0x09080607, 0x100f0c0d,
+    0x0e0d0a0b, 0x0d0c090a, 0x0d0c0b0b, 0x05040303, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x03020302, 0x0b0a0a0a, 0x09080908, 0x0e0d0e0d, 0x0f0e0f0e,
+    0x0f0c0d0e, 0x07060606, 0x18171717, 0x27242526, 0x1d1b1c1c, 0x0b0a0a0a,
+    0x16141515, 0x0d0c0c0c, 0x17161617, 0x1a18191a, 0x1a17181a, 0x18151618,
+    0x1e1b1d1d, 0x201e1e1f, 0x17141516, 0x0e0b0c0e, 0x00000000, 0x00000000,
+    0x03020202, 0x15121314, 0x2f2c2d2e, 0x58565657, 0x18161717, 0x06030505,
+    0x01000100, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x08070707, 0x0c0b0b0b, 0x15131314, 0x07060606, 0x01000000,
+    0x03020001, 0x03020000, 0x05040202, 0x09080607, 0x05040102, 0x0b0a0809,
+    0x0e0c0b0c, 0x18121110, 0x271b1d19, 0x39262a25, 0x4f363b33, 0x6042483f,
+    0x3f2a2a1f, 0x36272314, 0x3a2c2615, 0x3a2e2717, 0x382e2617, 0x3c322b1c,
+    0x362e271b, 0x29221c12, 0x28231e17, 0x1815120f, 0x0d0b0909, 0x0b0a0808,
+    0x0c0c090b, 0x100f0c0d, 0x11100d0e, 0x100f0c0d, 0x03020001, 0x02010000,
+    0x03020000, 0x01000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x05040504, 0x03020302, 0x00000000, 0x0a090909, 0x0a090909, 0x08070707,
+    0x0c0b0c0b, 0x09080908, 0x09080908, 0x0a090909, 0x1b191a1a, 0x211e1f20,
+    0x14111214, 0x2c292a2b, 0x19171619, 0x1e1c1c1e, 0x1d1b1b1d, 0x14131213,
+    0x0d0c0c0c, 0x13121212, 0x13121212, 0x0b0a0b0a, 0x00000000, 0x00000000,
+    0x00000000, 0x04030403, 0x17151516, 0x1a191919, 0x2d2b2c2c, 0x04030303,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x03020302, 0x100e0f0f, 0x1c191b1b, 0x0f0e0f0e, 0x03020202,
+    0x00000000, 0x07050404, 0x09060507, 0x0a080606, 0x05050202, 0x09060607,
+    0x32272a2f, 0x5e43535c, 0x8f617882, 0xd18b9f91, 0xffacbb99, 0xffafb687,
+    0xffb3b37a, 0xffb6b071, 0xffb8ae69, 0xffb8ab61, 0xffbaaa5c, 0xffbca955,
+    0xffbea854, 0xffbfa958, 0xf9bba553, 0xefb29e4c, 0xe7ae9d56, 0xb88e7f49,
+    0xa27d7046, 0x7c5d5436, 0x4c3c3629, 0x1f1c1a1a, 0x12120f12, 0x08060404,
+    0x03010000, 0x03020000, 0x03020000, 0x03020000, 0x01000000, 0x00000000,
+    0x00000000, 0x02010201, 0x06050605, 0x02010201, 0x07060706, 0x0a090a09,
+    0x0b0a0a0a, 0x17141616, 0x0a090a09, 0x08070807, 0x05040504, 0x0e0d0d0d,
+    0x11101010, 0x0d0b0b0d, 0x4b49494a, 0x4f4c4e4e, 0x100e0f0f, 0x0f0e0e0e,
+    0x08070707, 0x0b0a0a0a, 0x0a090a09, 0x0d0c0d0c, 0x07060706, 0x00000000,
+    0x00000000, 0x00000000, 0x03020302, 0x0e0d0e0d, 0x1a181819, 0x0e0d0d0d,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x02010201, 0x01000000, 0x12111111, 0x14131313, 0x13121212,
+    0x03020102, 0x02010000, 0x02010000, 0x0e0c090b, 0x03030000, 0x0d0a090a,
+    0x4330393e, 0x9269808b, 0xd193b0b5, 0xf7a9c1ab, 0xffa8bea4, 0xffa5bda7,
+    0xffa9b998, 0xffafb27d, 0xffb6ae6c, 0xffb6ac65, 0xffb8aa5c, 0xffbda650,
+    0xffbda54d, 0xffbea54c, 0xffbfa54c, 0xffbea54c, 0xffbfa44b, 0xffbfa64c,
+    0xffbfa54c, 0xffbca34b, 0xffb09a46, 0xffa48f40, 0xeb988541, 0xb67d7040,
+    0x59463f2d, 0x07070506, 0x03030002, 0x04030002, 0x03020000, 0x03020000,
+    0x02010000, 0x01000000, 0x06050605, 0x08070707, 0x01000100, 0x06050605,
+    0x0a090a09, 0x07060706, 0x0e0c0d0d, 0x120f1111, 0x06050605, 0x0b0a0b0a,
+    0x05040404, 0x02020102, 0x12111111, 0x5b595a5a, 0x48464747, 0x06050505,
+    0x03020202, 0x02010201, 0x01000101, 0x00000000, 0x00000000, 0x01000100,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x05040504, 0x04030403,
+    0x0c0a0a0b, 0x06050505, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x100f0f0f, 0x22201f21,
+    0x14131112, 0x08080305, 0x02010000, 0x02010000, 0x00000000, 0x06040404,
+    0x62455259, 0xbf86a4b0, 0xf4abcccb, 0xfdadbd9b, 0xffadb891, 0xffabbb9b,
+    0xffacb686, 0xffb6ab66, 0xffbaa958, 0xffb9a85c, 0xffb6ac67, 0xffbaa95c,
+    0xffbba752, 0xffbda64e, 0xffc0a54d, 0xffc0a44c, 0xffbea64d, 0xffbfa64c,
+    0xffbea64c, 0xffbfa64c, 0xfebfa54a, 0xffbea54b, 0xffbba249, 0xffb39b45,
+    0xff9b8739, 0xff8f7b32, 0xf8907e37, 0xac756b43, 0x1c1a1711, 0x02020001,
+    0x02000000, 0x02010000, 0x03020000, 0x05040303, 0x07060606, 0x01000100,
+    0x07060706, 0x09080908, 0x07060706, 0x0d0c0d0c, 0x0b0a0a0a, 0x0f0c0d0e,
+    0x07060706, 0x04030403, 0x00000000, 0x0a090909, 0x33323232, 0x211f2021,
+    0x02020101, 0x00000000, 0x00000000, 0x00000000, 0x01000100, 0x03010202,
+    0x05040404, 0x03010202, 0x05030403, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x02010101, 0x03020001,
+    0x08070506, 0x12100e10, 0x0a080606, 0x06040203, 0x0b070707, 0x241b1c1f,
+    0x7752656c, 0xe9a1c9d6, 0xffb7ddea, 0xfcb7cbb7, 0xf9abb890, 0xfeb1b98a,
+    0xfeb4b175, 0xffb8ad69, 0xffb7ab65, 0xffb2af70, 0xffafb484, 0xffb1b380,
+    0xffb8aa5c, 0xffbea54e, 0xffbfa64b, 0xffc0a64d, 0xffc0a64b, 0xffbfa64b,
+    0xffc0a54d, 0xffbfa54a, 0xffbca34a, 0xffbba149, 0xffbba149, 0xffbaa04a,
+    0xffb59c44, 0xffb09845, 0xffab9543, 0xfe9d883a, 0xfe8d7931, 0xff88772f,
+    0xde837744, 0x4c3b372d, 0x06050303, 0x03020001, 0x03020000, 0x05040203,
+    0x03020202, 0x01000100, 0x07060706, 0x06050605, 0x0e0d0e0d, 0x0d0c0d0c,
+    0x04030403, 0x08070807, 0x03020202, 0x00000000, 0x00000000, 0x08070707,
+    0x06060505, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x06050505, 0x05040404, 0x04030302, 0x0a09080a, 0x03020302,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x01000000, 0x02010201, 0x04030303,
+    0x03020101, 0x02010000, 0x03020000, 0x04030002, 0x1c141617, 0x452d383c,
+    0x8858707b, 0xe89fc8d4, 0xffb6dadf, 0xffbbd3c8, 0xfebac5a4, 0xffbab885,
+    0xffb9ba89, 0xf7b0b68a, 0xfeb1b88a, 0xffacb484, 0xffaeb484, 0xffafb382,
+    0xffb7ab62, 0xffbfa547, 0xffc0a44b, 0xffbea64b, 0xffbca248, 0xffb9a049,
+    0xffb8a04a, 0xffb69f4d, 0xffb39c49, 0xffad9644, 0xffa89242, 0xffa58f3f,
+    0xffa18b3b, 0xffa08b39, 0xffa5903e, 0xffa38c3b, 0xff958035, 0xff907c34,
+    0xfe887531, 0xfe7e6c29, 0xff7c6a29, 0xe6837540, 0x5a4b473d, 0x08070506,
+    0x03020001, 0x04030102, 0x0a090909, 0x09080908, 0x04030403, 0x0e0d0e0d,
+    0x0e0d0e0d, 0x04030403, 0x04030403, 0x04030303, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x03010201, 0x01000100, 0x01000100,
+    0x05030403, 0x02010101, 0x01000100, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x03020302,
+    0x08070707, 0x06050304, 0x05040202, 0x02010000, 0x02010000, 0x5d3f4d51,
+    0xc37da2af, 0xeb8bbed2, 0xffacd1d2, 0xffb6d2cc, 0xffb9d5d0, 0xfeb7cab6,
+    0xffbac39f, 0xffb9c4a1, 0xfeb6c19a, 0xffb5b785, 0xffb5b279, 0xfeb1af76,
+    0xffb1b179, 0xffbaa859, 0xffbea64d, 0xffc0a74e, 0xffb9a248, 0xffa48d3a,
+    0xffab9752, 0xffb1a268, 0xffbeb282, 0xffcdc4a0, 0xffd2caa8, 0xffbfb79a,
+    0xffb9b197, 0xff938b6e, 0xffcbc3a7, 0xff9b9171, 0xff998b60, 0xff897737,
+    0xff83702a, 0xff82702b, 0xff7e6c2b, 0xff7a6827, 0xff766525, 0xff7a6828,
+    0xc9887c57, 0x17161211, 0x06050304, 0x05040203, 0x05040404, 0x00000000,
+    0x0b0a0b0a, 0x0a090a09, 0x07060706, 0x07060706, 0x02010101, 0x00000000,
+    0x00000000, 0x00000000, 0x01000100, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x03020202, 0x09080808, 0x01000100, 0x00000000, 0x00000000, 0x00000000,
+    0x02010201, 0x03020302, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x05050404, 0x0d0c0c0c, 0x08070507, 0x06050204, 0x07060305, 0x03030000,
+    0x38263032, 0xd482acb7, 0xff96c5c8, 0xff9ac1bf, 0xff9acbd5, 0xfe9cd2e5,
+    0xffb1d5d8, 0xffaec7b4, 0xffb1ba90, 0xffb6af72, 0xffb9aa5c, 0xffb7ac64,
+    0xffb1b07c, 0xffaeb27d, 0xffb0b178, 0xffa8b68e, 0xffa5c0aa, 0xffb29e4b,
+    0xffa48f3f, 0xffdfd7b7, 0xfff9f7f1, 0xfffaf8f4, 0xffe5e5e2, 0xffcacac8,
+    0xffd2d2d1, 0xffebebea, 0xffd8d8d7, 0xffb0b0ae, 0xfff2f2f1, 0xffc6c6c6,
+    0xffc3c2c1, 0xffe7e3da, 0xffada78c, 0xff8f7f45, 0xff786424, 0xff7a682c,
+    0xfe8b7c48, 0xff978a57, 0xff887a42, 0x805d5b4a, 0x13120f13, 0x0b0a0808,
+    0x09090607, 0x0d0c0c0c, 0x0a090a09, 0x02010201, 0x05040504, 0x01000100,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x01000100, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x03020202, 0x08070707, 0x05050405, 0x01000100, 0x00000000,
+    0x00000000, 0x00000000, 0x05040504, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x03020202, 0x0a090909, 0x05040202, 0x07070202, 0x07060302,
+    0x0e090a09, 0x89567077, 0xff94c4cc, 0xff8bbfc3, 0xfe92c7d5, 0xff91cfeb,
+    0xff9dd4ec, 0xffa0cfd8, 0xff9bbeb4, 0xffa9b487, 0xffb3ad68, 0xffafb074,
+    0xffadb486, 0xffa7b894, 0xffabb68f, 0xffa8b99b, 0xffa3bca9, 0xffa4b189,
+    0xffaba15b, 0xffb5a460, 0xffe5dfc8, 0xfffefefe, 0xfffdfdfe, 0xfffdfdfd,
+    0xffd2d2d3, 0xffeeeeee, 0xfff2f2f2, 0xffcecece, 0xffc9c9c9, 0xffaaaaaa,
+    0xfff7f7f7, 0xffe4e4e4, 0xffc5c5c4, 0xffe2e1e3, 0xffc3c2c5, 0xffdad5cc,
+    0xffa89d71, 0xffeceada, 0xfffffffe, 0xfffafafd, 0xfff3f2e7, 0x8978766e,
+    0x100f0c0f, 0x0c0b0909, 0x13121011, 0x0e0d0d0d, 0x00000000, 0x04030403,
+    0x03020302, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x01000100, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x04040303, 0x09090809, 0x09080808, 0x0f0e0d0f, 0x03030202, 0x02010101,
+    0x07050606, 0x00000000, 0x02010201, 0x08070807, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x02010101, 0x07060505, 0x04030102, 0x04040000,
+    0x02020000, 0x35242a2f, 0xc06b99ac, 0xff7bc1d9, 0xff79c0d5, 0xfe7bc3e1,
+    0xff8bc6d3, 0xffa8c5aa, 0xffb1bc85, 0xffafbb85, 0xffacbe95, 0xffb1b884,
+    0xffadbb94, 0xffa9bc9e, 0xffa6b99a, 0xffa2b99f, 0xff9db497, 0xff9ca57b,
+    0xffa09552, 0xffaf9b55, 0xffd4c8a2, 0xfff8f5ee, 0xffffffff, 0xfffefefc,
+    0xfffefefe, 0xfff1f1f1, 0xffe2e2e2, 0xff737373, 0xffdadada, 0xffe7e7e7,
+    0xffc2c2c2, 0xffc8c8c8, 0xffcfcfcf, 0xffececec, 0xffcececd, 0xffb2b2b2,
+    0xfff6f6f6, 0xffc1c2c4, 0xffe5e6e6, 0xfffcfbfc, 0xff8c8b8d, 0xfec5c5c3,
+    0xffffffff, 0xc8c7c7c7, 0x11110e0f, 0x0c0b080a, 0x0d0d090a, 0x0a090909,
+    0x04030403, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x01000100, 0x05040504, 0x01000100, 0x00000000, 0x00000000, 0x00000000,
+    0x06050505, 0x0c0b0b0c, 0x0a0a0909, 0x07070607, 0x02010101, 0x01000000,
+    0x03020202, 0x06050605, 0x08060707, 0x06050605, 0x04030403, 0x00000000,
+    0x00000000, 0x00000000, 0x03020101, 0x09080508, 0x0c0c0808, 0x05040203,
+    0x02020000, 0x06050303, 0x6745555a, 0xde6fabc2, 0xff6bc0e0, 0xff6cbee2,
+    0xfe5fbae1, 0xff7bbcb7, 0xffccca5e, 0xffedce35, 0xffeace3e, 0xffe7d146,
+    0xffebd13f, 0xffeece35, 0xffeccc3c, 0xffdccf56, 0xffcdd077, 0xffd6d373,
+    0xffdbcb60, 0xffdac24e, 0xffdfc451, 0xfff1ebcb, 0xffffffff, 0xffffffff,
+    0xffffffff, 0xffffffff, 0xffffffff, 0xfffcfcfc, 0xffdadada, 0xffb8b8b8,
+    0xffd9d9d9, 0xffd9d9d9, 0xffe5e5e5, 0xffe3e3e3, 0xffd3d3d3, 0xffdedfde,
+    0xffe3e3e3, 0xffd3d3d4, 0xffefefef, 0xffd8d8d7, 0xfff4f4f4, 0xffacabab,
+    0xff868686, 0xffffffff, 0xfefcfcfa, 0xe3a09f9f, 0x2d262325, 0x04040102,
+    0x05040102, 0x05040303, 0x01000101, 0x00000000, 0x02010201, 0x00000000,
+    0x01000000, 0x00000000, 0x06050505, 0x0d0c0c0c, 0x00000000, 0x00000000,
+    0x02010201, 0x19181718, 0x11100f11, 0x08080707, 0x04040303, 0x00000000,
+    0x00000000, 0x00000000, 0x05040504, 0x02010101, 0x07060706, 0x04030303,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x08070506, 0x11100c0f,
+    0x06040305, 0x00000000, 0x2b1e2223, 0xce89acb6, 0xf872bddb, 0xff5eb9e2,
+    0xff64bce3, 0xfe55b5e6, 0xff5eb6d3, 0xffc3ce6a, 0xffffda27, 0xfffeda25,
+    0xfffdda2e, 0xffffd928, 0xffffd926, 0xffffd925, 0xffffda25, 0xfffdd92e,
+    0xfffade3f, 0xfffeda33, 0xfffed933, 0xffffdc36, 0xffe2cd57, 0xffb6a878,
+    0xffb1afa7, 0xffb1b0b2, 0xffacacac, 0xffababab, 0xffefefef, 0xffffffff,
+    0xfffafafa, 0xfff7f7f7, 0xfff8f8f8, 0xfffbfbfb, 0xffe8e8e8, 0xffd6d6d6,
+    0xffececec, 0xfff5f5f5, 0xffededed, 0xfffcfcfc, 0xfffbfbfb, 0xfffdfdfd,
+    0xfffefefe, 0xffe7e7e7, 0xfff7f7f7, 0xffffffff, 0xfeb4b3b4, 0xff504c4d,
+    0x9a404242, 0x00000000, 0x02010000, 0x02010000, 0x00000000, 0x00000000,
+    0x00000000, 0x01000100, 0x01000100, 0x0a090909, 0x0c0b0b0b, 0x312f3030,
+    0x0d0c0c0c, 0x0e0d0d0d, 0x0c0b0b0b, 0x04010302, 0x03010201, 0x04040303,
+    0x04030303, 0x00000000, 0x00000000, 0x01000100, 0x06050605, 0x0d0c0d0c,
+    0x03020302, 0x00000000, 0x00000000, 0x00000000, 0x01000000, 0x05040404,
+    0x09080607, 0x09080605, 0x07060405, 0x29212022, 0xca85a8b2, 0xff79c1de,
+    0xfe4baad5, 0xff55b3db, 0xff57b6e0, 0xff62b8cd, 0xffbacb74, 0xffffda26,
+    0xfffed924, 0xfffed82a, 0xfffed92d, 0xfffed828, 0xfffed926, 0xffffd829,
+    0xfffcdb30, 0xfffdda34, 0xfffed92f, 0xfffdd92e, 0xfffddc2f, 0xfffdd829,
+    0xfffad734, 0xfff1d565, 0xffedece7, 0xfff5f5f6, 0xfffefefe, 0xfffdfdfd,
+    0xffb9b9b9, 0xffbebebe, 0xfff3f3f3, 0xfff8f8f8, 0xfffbfbfb, 0xfffdfdfd,
+    0xfff7f7f7, 0xffeaeaea, 0xffe6e6e6, 0xfff6f6f6, 0xfff7f7f7, 0xffc7c7c7,
+    0xfff8f8f8, 0xfffbfbfb, 0xfffefefe, 0xffffffff, 0xfef9f9f9, 0xfeaca9ab,
+    0xff3d393d, 0xff4d484c, 0xe9424242, 0x01010101, 0x04030101, 0x02010000,
+    0x00000000, 0x03020202, 0x0a090909, 0x00000000, 0x07060706, 0x0d0c0c0c,
+    0x28262727, 0x47454547, 0x08070707, 0x0e0d0d0d, 0x0f0f0d0e, 0x08070707,
+    0x07060606, 0x05040405, 0x03020203, 0x02010102, 0x02010201, 0x0a090a09,
+    0x09080908, 0x01000100, 0x00000000, 0x00000000, 0x00000000, 0x04030304,
+    0x0c0a0b0b, 0x0b0a0909, 0x0b0a0708, 0x09080608, 0x1d17171a, 0xaf6e8f97,
+    0xff79c2dd, 0xff49a8d3, 0xff4eaed8, 0xff55b6e2, 0xff6ebbc1, 0xffc9ce64,
+    0xffffda26, 0xfffdd927, 0xfffed926, 0xfffcd929, 0xfffddc40, 0xfffdda2f,
+    0xfffed926, 0xfffdda2b, 0xfffedc33, 0xfffdda2f, 0xfffdd92b, 0xffffd827,
+    0xfffdd827, 0xffffda27, 0xffe7d16d, 0xff959590, 0xff6f6b6e, 0xff8f8d8e,
+    0xffb0afaf, 0xffd2d1d1, 0xffdfdfdf, 0xffededed, 0xffc4c4c4, 0xffc8c8c8,
+    0xffd6d6d6, 0xffdbdbdb, 0xffd8d8d8, 0xffd4d4d4, 0xff9d9d9d, 0xff8e8d8e,
+    0xffb1b0b1, 0xfff7f7f7, 0xffe0e0e0, 0xfffcfdfd, 0xfffdfdfd, 0xffddddde,
+    0xfb9f9d9e, 0xfc2d292c, 0xff2e282c, 0xfe5a565b, 0xf63c383b, 0x54333333,
+    0x08080708, 0x07060304, 0x02010101, 0x02010201, 0x08070807, 0x06050505,
+    0x0d0c0c0c, 0x201e1f1f, 0x34313333, 0x18171617, 0x0e0e0d0e, 0x17171617,
+    0x1d1c1b1d, 0x02010101, 0x00000000, 0x02000001, 0x01000001, 0x03010103,
+    0x07060706, 0x09080908, 0x04030403, 0x00000000, 0x00000000, 0x00000000,
+    0x03020203, 0x09050607, 0x0b090909, 0x13111012, 0x09070607, 0x110e0d0f,
+    0xbc7899a6, 0xff77c1dd, 0xff4ba8d4, 0xfe4dadd6, 0xff56b7e1, 0xff78bcbb,
+    0xffd8d152, 0xffffda26, 0xffffd827, 0xfffcdd43, 0xfffcdc3f, 0xfffbdf5e,
+    0xfff9dd4e, 0xfffddc3f, 0xfffddb38, 0xfffed927, 0xfffdd928, 0xfffdd92d,
+    0xfffdd829, 0xfffed829, 0xfffed928, 0xfff5d430, 0xffa39036, 0xff534d49,
+    0xffaba8aa, 0xfffdfdfd, 0xffdadada, 0xffd8d7d7, 0xffc4c4c4, 0xffbfbfbf,
+    0xffc7c7c7, 0xffcecece, 0xffc3c3c3, 0xffbfbfbf, 0xffb8b8b8, 0xffb1b1b1,
+    0xffb8b8b8, 0xffdedede, 0xffb6b6b6, 0xfff1f1f1, 0xffffffff, 0xffc2c1c1,
+    0xff757274, 0xfe4b474b, 0xff2b252a, 0xff2e292d, 0xfe332e32, 0xff514c50,
+    0xfe514d50, 0xab606262, 0x1a1a1a1a, 0x0b0b0809, 0x04030203, 0x03020302,
+    0x0a090a09, 0x0e0d0d0d, 0x13121112, 0x29262729, 0x19171818, 0x06050605,
+    0x0d0c0c0d, 0x1b1a1a1a, 0x1a171819, 0x03010103, 0x01000000, 0x01000001,
+    0x02000002, 0x02000002, 0x00000000, 0x03020302, 0x06050605, 0x01000000,
+    0x00000000, 0x01000000, 0x04000002, 0x06000001, 0x0f07080a, 0x0d0a070a,
+    0x03030101, 0xbd789ba2, 0xff75c1de, 0xff4eacd5, 0xfe4bacd2, 0xff56b4e1,
+    0xff69bacf, 0xffbecb6f, 0xfffed928, 0xffffd822, 0xfffbdc3c, 0xfffbe068,
+    0xfff9e179, 0xfff8e37d, 0xfffcdf65, 0xfffcdd4d, 0xfffbdb37, 0xfffed928,
+    0xfffddb31, 0xfffdd932, 0xfffed82a, 0xfffeda29, 0xffffda2b, 0xfff5d333,
+    0xff82722d, 0xff2d2925, 0xff959298, 0xffffffff, 0xffffffff, 0xffffffff,
+    0xffffffff, 0xfff6f6f6, 0xfff0f0f0, 0xffe6e6e6, 0xffe2e2e2, 0xffe5e5e5,
+    0xffededed, 0xfff5f5f5, 0xfff7f7f7, 0xffb9b9b9, 0xffe4e4e3, 0xffe9e8e9,
+    0xff5c5a5d, 0xff342f33, 0xff2e292d, 0xff2d282c, 0xff2d282c, 0xff2e292d,
+    0xff322d31, 0xff464044, 0xff4a464a, 0xea8f8e8f, 0x5c524f52, 0x0c0c080a,
+    0x04030102, 0x07060706, 0x0e0d0e0d, 0x11101010, 0x16151515, 0x14131213,
+    0x05040404, 0x00000000, 0x22212121, 0x0e0d0c0e, 0x04020204, 0x01000001,
+    0x02000002, 0x01000001, 0x01000001, 0x03000002, 0x04030403, 0x02010101,
+    0x00000000, 0x03020302, 0x02000101, 0x05000001, 0x05000001, 0x08010203,
+    0x0f060407, 0x0c050407, 0xa069838b, 0xff89c6df, 0xfe59b3d7, 0xff4ba8d3,
+    0xff57b5e1, 0xff5ebad5, 0xff9fc28d, 0xfffad930, 0xffffd824, 0xfffdd830,
+    0xfffbdf57, 0xfff8e272, 0xfff6e589, 0xfff7e283, 0xfff6e382, 0xfff8df64,
+    0xfffddc4a, 0xfffdda2c, 0xfffdd82b, 0xfffacf26, 0xfffbd527, 0xfffed827,
+    0xfffed825, 0xfffdd225, 0xff917a39, 0xff352d33, 0xff302e30, 0xff3b373a,
+    0xff5b585b, 0xff7e7b7e, 0xffaeaeae, 0xffeaeaea, 0xfff4f3f4, 0xfff6f6f6,
+    0xfff5f5f5, 0xfff5f5f5, 0xfff6f6f6, 0xffd8d8d8, 0xffa7a7a7, 0xffefefef,
+    0xffe5e4e5, 0xff524d51, 0xff2b262b, 0xff2f2a2e, 0xff2f2a2e, 0xff2f2a2e,
+    0xff2e292d, 0xff2e292d, 0xff302b2f, 0xff433d41, 0xff484246, 0xff939192,
+    0x8c6c6b6d, 0x10100d0e, 0x07060405, 0x0d0c0c0c, 0x0e0d0e0d, 0x12111111,
+    0x11101010, 0x09080808, 0x07060706, 0x08070707, 0x08060608, 0x01000000,
+    0x01000001, 0x02000002, 0x01000001, 0x01000001, 0x08050608, 0x06050605,
+    0x05040404, 0x00000000, 0x03020202, 0x04020202, 0x05000001, 0x06000001,
+    0x0a020304, 0x0c040305, 0x0a030003, 0x764e5e64, 0xff97ccdb, 0xff80c0d6,
+    0xff5dafd0, 0xff52b3dd, 0xff53b3df, 0xff6ebac3, 0xffcbd06b, 0xffedd53c,
+    0xfffed824, 0xfffcdb3a, 0xfffae36a, 0xfff8e173, 0xfff8e387, 0xfff6e593,
+    0xfff7e27b, 0xfff9e170, 0xfffbdf57, 0xfffed92c, 0xfffdd528, 0xffdab123,
+    0xffc2a528, 0xfffcd82a, 0xffe1b824, 0xffb89326, 0xff40392e, 0xff726e71,
+    0xffbcbbbb, 0xffdad9da, 0xffdbdada, 0xffebeaea, 0xffc9c8c9, 0xffa5a4a4,
+    0xffa9a8a8, 0xffacabab, 0xffadabac, 0xffaeacad, 0xffbbbbbb, 0xffdadada,
+    0xffffffff, 0xffffffff, 0xff989798, 0xff2f2a2e, 0xff2e292d, 0xff2e292d,
+    0xff2f2a2e, 0xff2e292d, 0xff2f2a2e, 0xff2e292d, 0xff312c30, 0xff423d41,
+    0xfe474246, 0xff8b898b, 0x71434344, 0x12110e0f, 0x0c0b080a, 0x110f0f10,
+    0x0f0e0f0e, 0x13111312, 0x09080908, 0x03020302, 0x06050505, 0x100f0f0f,
+    0x01000001, 0x01000001, 0x01000001, 0x02000001, 0x03000103, 0x0d0a0b0d,
+    0x13101113, 0x03020202, 0x05040504, 0x02010101, 0x04010101, 0x06000001,
+    0x07000002, 0x08000103, 0x0e050406, 0x06000000, 0x50364144, 0xf8a7ccd2,
+    0xf599c5d0, 0xfd93c4d4, 0xff57b3d6, 0xff41a0cc, 0xff5ab6dc, 0xff82c4d1,
+    0xffafca90, 0xfff5d732, 0xffffd925, 0xfffedb33, 0xfffcdb3f, 0xfff9e170,
+    0xfff7e48e, 0xfff8e27c, 0xfff8e16d, 0xfff9df62, 0xfffbde52, 0xfffedb2b,
+    0xfffdc825, 0xff9c8129, 0xff74652a, 0xffb79b2e, 0xff7a6026, 0xff4d4533,
+    0xff97959a, 0xfffbfdfa, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
+    0xffffffff, 0xffffffff, 0xffffffff, 0xfffefefe, 0xfffbfbfb, 0xfffefffe,
+    0xffffffff, 0xffffffff, 0xfffefefe, 0xfff6f6f6, 0xff656365, 0xff2c272b,
+    0xff2e282c, 0xff2d282c, 0xff2e292d, 0xff2d282c, 0xff2f2a2e, 0xff2f2a2e,
+    0xff302b2f, 0xff423d41, 0xff3d383c, 0xff807e81, 0xa7676969, 0x13100d0e,
+    0x04030102, 0x0b0a090b, 0x0f0e0f0e, 0x09080908, 0x04030303, 0x0a090a09,
+    0x0f0e0e0e, 0x24222223, 0x01000001, 0x02000002, 0x01000001, 0x00000000,
+    0x04030303, 0x02010201, 0x02000002, 0x00000000, 0x05040504, 0x04030303,
+    0x04000001, 0x07000002, 0x07000002, 0x0c040404, 0x08020000, 0x01000000,
+    0x583b4549, 0xf7a1c8d2, 0xd38cabb5, 0xfc6ebbd8, 0xff3191be, 0xff50b1d8,
+    0xff63bde5, 0xff88c3b5, 0xffe4d449, 0xffffda29, 0xffffd826, 0xffffd829,
+    0xfffddc3a, 0xfff9e06d, 0xfff8e277, 0xfffbde4f, 0xfffcdb43, 0xfffddd49,
+    0xfff5d447, 0xffeaca44, 0xffc19930, 0xff4e4429, 0xff51472d, 0xff433828,
+    0xff302c2c, 0xff726d70, 0xfff8f8f8, 0xfffffefe, 0xfffdfdfd, 0xfffefdfe,
+    0xfffdfdfd, 0xfffefefe, 0xfffefefe, 0xfffdfdfd, 0xfffefefe, 0xfffefefe,
+    0xfffefefe, 0xfffefefe, 0xfffefefe, 0xfffdfdfd, 0xfffefefe, 0xffe3e2e2,
+    0xff524d50, 0xfe2c272b, 0xff2b282b, 0xfe2d282c, 0xff2e282c, 0xff2e292d,
+    0xff2e292d, 0xff2f2a2e, 0xff2e292d, 0xff3f3a3e, 0xff353034, 0xff767477,
+    0xbb757878, 0x110a0809, 0x05000000, 0x07030103, 0x05030403, 0x01000100,
+    0x0d0c0d0c, 0x0d0c0c0c, 0x201e1e1f, 0x25222325, 0x01000001, 0x02000002,
+    0x08050607, 0x0f0c0d0f, 0x02000002, 0x02000001, 0x01000000, 0x04000000,
+    0x01000000, 0x04030303, 0x04010201, 0x05000001, 0x07000002, 0x09040103,
+    0x04010100, 0x0e080709, 0xbf7c9aa0, 0x9b647c82, 0xc46597a8, 0xff3996bd,
+    0xff42a3cd, 0xff56b4e0, 0xff6ab9ca, 0xffcdd162, 0xffefd846, 0xfffcd829,
+    0xfffed929, 0xfffed925, 0xfffcda3b, 0xfffae273, 0xfffbdc40, 0xffffd929,
+    0xfffed927, 0xffffdb33, 0xffe5b32d, 0xffad8e3e, 0xff63583b, 0xff2f2a29,
+    0xff383129, 0xff2f292e, 0xff2c2a2b, 0xff817d81, 0xffffffff, 0xfffefefe,
+    0xfffefefe, 0xfffdfefd, 0xfffefefe, 0xfffefefe, 0xfffefefe, 0xfffefefe,
+    0xfffefefe, 0xfffefefe, 0xfffefefe, 0xfffefefe, 0xfffefefe, 0xfffefefe,
+    0xfffefefe, 0xffd4d3d3, 0xff413f41, 0xff2f292d, 0xf1464748, 0xff2d292d,
+    0xff2c292e, 0xff302b2e, 0xff2d282c, 0xff2e292d, 0xff2e292d, 0xff3a373a,
+    0xff332f33, 0xff787376, 0xb9727373, 0x12080508, 0x0a030003, 0x0c030204,
+    0x0b060608, 0x1c181a1b, 0x19161718, 0x16161515, 0x19181818, 0x12111111,
+    0x04020204, 0x0f0c0d0f, 0x0b08090b, 0x01000001, 0x02000001, 0x04000001,
+    0x07000002, 0x07000002, 0x06000002, 0x05000001, 0x03000000, 0x04010102,
+    0x05020102, 0x06040203, 0x0e0b0709, 0xae758d92, 0xa96d868c, 0x76445b64,
+    0xfa4fa0c4, 0xff399ac6, 0xff4bacd3, 0xff4eadd4, 0xff8bc2a5, 0xffc0cc75,
+    0xffe7d440, 0xffffda2b, 0xfffeda28, 0xfffdd927, 0xfffcdf53, 0xfffcdc45,
+    0xfffed825, 0xffffd825, 0xfffed629, 0xfffed731, 0xffd8a428, 0xff715b2c,
+    0xff372f2b, 0xff2e2a2b, 0xff322b2c, 0xff2f292e, 0xff2d292d, 0xff777576,
+    0xfffdfdfd, 0xfffefefe, 0xfffefefe, 0xfffefefe, 0xfffefefe, 0xfffefefe,
+    0xfffefefe, 0xfffefefe, 0xfffefefe, 0xfffefefe, 0xfffefefe, 0xfffdfdfd,
+    0xfffefefe, 0xfffefefe, 0xfffefefe, 0xffc9c8c8, 0xfe363236, 0xff322f32,
+    0x583d3a3c, 0xe23d3d3e, 0xff2b272a, 0xfe2c272b, 0xff2d282c, 0xff2e292d,
+    0xff2f2a2e, 0xff342f33, 0xfe332f33, 0xff757174, 0xb36c6d6d, 0x12060507,
+    0x0b030003, 0x0b020204, 0x07010103, 0x15121314, 0x221f2022, 0x211e1e20,
+    0x0f0b0b0d, 0x0e0b0c0d, 0x1a17181a, 0x0c090a0b, 0x03000001, 0x05000001,
+    0x06000002, 0x07000002, 0x07000002, 0x07000002, 0x06000001, 0x07000002,
+    0x05000001, 0x03010101, 0x04010000, 0x04000000, 0x835a686c, 0xb97d9698,
+    0x3822292d, 0xeb62a5c0, 0xff3797c1, 0xfe40a1ca, 0xff48a7d2, 0xff63b9d1,
+    0xff98c498, 0xffe7d445, 0xffffda27, 0xfffdd926, 0xffffd826, 0xfffcda38,
+    0xfffddd45, 0xfffdda25, 0xfffed826, 0xfffeda2b, 0xfffbc521, 0xffffd12e,
+    0xffcf9822, 0xff584a33, 0xff302c2a, 0xff2f2b2c, 0xff2d292d, 0xff2d292c,
+    0xff2e282c, 0xff747172, 0xfffbfbfb, 0xfffefefe, 0xfffefefe, 0xfffefefe,
+    0xfffefefe, 0xfffefefe, 0xfffefefe, 0xfffefefe, 0xfffefefe, 0xfffefefe,
+    0xfffefefe, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffc3c3c4,
+    0xff322e32, 0xff383738, 0x332a2929, 0x7d282829, 0xff312d30, 0xff2d282d,
+    0xff2d282c, 0xff2e292d, 0xff2f2a2e, 0xff312c30, 0xfe373135, 0xff6e686b,
+    0x8b545354, 0x0d030205, 0x0a030003, 0x0c060508, 0x110d0e10, 0x2a27282a,
+    0x1b18181a, 0x08030205, 0x05000001, 0x05000002, 0x0a060608, 0x05000001,
+    0x06000001, 0x07000002, 0x07000002, 0x06000001, 0x07000002, 0x06000001,
+    0x07000002, 0x06000001, 0x0a020304, 0x08020002, 0x09020002, 0x0a010002,
+    0x36222528, 0x150a0a0e, 0xc36994a4, 0xff3896be, 0xff3596c0, 0xfe41a3cd,
+    0xff51afd7, 0xff6db9c2, 0xffd5d156, 0xfffbd429, 0xfffed525, 0xfffdd926,
+    0xfffed627, 0xfffbca24, 0xffffd82c, 0xfffed925, 0xffffd928, 0xffffd729,
+    0xfff9bb1d, 0xfff8c628, 0xffb48320, 0xff594f3e, 0xff2e2929, 0xff2e292d,
+    0xff2d282d, 0xff2d292c, 0xff2d282c, 0xff6a686a, 0xfff6f6f6, 0xfffefefe,
+    0xfffefefe, 0xfffefefe, 0xfffefefe, 0xfffefefe, 0xfffefefe, 0xfffefefe,
+    0xfffefefe, 0xfffdfdfd, 0xfffefefe, 0xffebebec, 0xff999798, 0xff6e6d6d,
+    0xffc1c0c0, 0xffb6b4b6, 0xff312c30, 0xff484748, 0x07070707, 0x38171717,
+    0xe5333034, 0xff2d292a, 0xff2d282d, 0xff302b2f, 0xff302b2f, 0xff302b2f,
+    0xff363034, 0xff605d5f, 0x5c333334, 0x07010001, 0x06030102, 0x08070707,
+    0x0c0b0c0b, 0x0b08090a, 0x07000103, 0x06000001, 0x07000002, 0x06000002,
+    0x06000001, 0x07000002, 0x06000001, 0x07000002, 0x07000002, 0x07000002,
+    0x07000002, 0x07000002, 0x07000002, 0x09010203, 0x0b030204, 0x0a020002,
+    0x0a010002, 0x0a020003, 0x05000000, 0x7142575f, 0xff4ba2c8, 0xfe2a8bb9,
+    0xff409fca, 0xff49a6d2, 0xff4ba6c2, 0xffb6cb7b, 0xffe1d048, 0xfff3cc2c,
+    0xfffddb25, 0xffffd92a, 0xfffbc822, 0xfffcd026, 0xfffed927, 0xffffd827,
+    0xfffed929, 0xfffed328, 0xfff9b81a, 0xfff4bd23, 0xff956f25, 0xff4a4337,
+    0xff2d282c, 0xff2d292c, 0xff2d282d, 0xff2e292c, 0xff2d282c, 0xff474446,
+    0xffd3d2d3, 0xfffefefe, 0xfffdfdfd, 0xfffefefe, 0xfffefefe, 0xfffefefe,
+    0xfffefefe, 0xfffefefe, 0xfffdfdfd, 0xfffefefe, 0xfff4f3f4, 0xff908e91,
+    0xff716c71, 0xff575256, 0xff464145, 0xff5c585c, 0xff2e292d, 0xff454547,
+    0x24181616, 0x120e0e0d, 0xb73f3e40, 0xff322d31, 0xfe2c272b, 0xff302b2f,
+    0xff312c30, 0xff312c30, 0xff342f33, 0xe8434144, 0x3c1f1d1f, 0x09080508,
+    0x09090607, 0x05040404, 0x02000000, 0x06000001, 0x07000002, 0x07000002,
+    0x06000001, 0x05000002, 0x07000002, 0x06000001, 0x07000002, 0x07000002,
+    0x07000002, 0x07000002, 0x07000002, 0x06000001, 0x09010203, 0x10060608,
+    0x0a020002, 0x09010002, 0x09020002, 0x09030001, 0x52323e42, 0xfa5facca,
+    0xfe2789b4, 0xfe3a99c3, 0xff41a2cd, 0xff3e9dc0, 0xff90c3a3, 0xffb6c976,
+    0xffd2c551, 0xfffdd629, 0xfffedb2e, 0xfffcd42a, 0xfffdcb26, 0xfffed72a,
+    0xfffdd926, 0xfffed828, 0xfffeda29, 0xfffed52b, 0xfff8b91c, 0xffd9a621,
+    0xff5f4723, 0xff302a2b, 0xff2d292d, 0xff2e292c, 0xff2e282d, 0xff2d292c,
+    0xff2e292d, 0xff292428, 0xff575457, 0xffdcdbdc, 0xffffffff, 0xfffdfdfd,
+    0xfffdfdfd, 0xfffefefe, 0xfffefefe, 0xfffdfdfd, 0xfffdfdfd, 0xffffffff,
+    0xffbebdbf, 0xff757074, 0xff4e494d, 0xff2e292d, 0xff2d282c, 0xff322e32,
+    0xfe2d292d, 0xff2b282c, 0xe4464546, 0x52232222, 0x241e1d1d, 0xad373839,
+    0xff342f33, 0xff2e292d, 0xff302b2f, 0xff312c30, 0xff302c30, 0xce48484a,
+    0x231e1c1d, 0x08080506, 0x07050306, 0x08030406, 0x06000001, 0x06000002,
+    0x06000002, 0x07000002, 0x05000001, 0x05040403, 0x07000002, 0x07000002,
+    0x07000002, 0x07000002, 0x07000002, 0x07000002, 0x07000002, 0x05000001,
+    0x0f060608, 0x0b020003, 0x0a020003, 0x0a020002, 0x08000000, 0x3620262b,
+    0xde5b97ad, 0xff348fb7, 0xfe2c8bb7, 0xff3799c3, 0xff3b9ac3, 0xff6eb8c2,
+    0xff87bfad, 0xff9abb8e, 0xffe8d23d, 0xffffda31, 0xfffcd72b, 0xfffccc25,
+    0xfffdd227, 0xfffed829, 0xfffed927, 0xfffdd829, 0xfffed929, 0xfffed52a,
+    0xfff5b71c, 0xffb88b24, 0xff564425, 0xff2f2b2b, 0xff2e292e, 0xff2e292d,
+    0xff2e282d, 0xff2d292c, 0xff2e292d, 0xff2e282c, 0xff282327, 0xff534f53,
+    0xffdddcde, 0xffffffff, 0xfffdfdfd, 0xfffdfdfd, 0xfffefefe, 0xfffdfdfd,
+    0xffffffff, 0xffd5d4d6, 0xff767276, 0xff544e53, 0xff322c30, 0xff2e292d,
+    0xff2d292d, 0xff2d292c, 0xff2f2a2e, 0xfe2d282c, 0xff312c30, 0xf83a383a,
+    0x702f2f2f, 0x16161616, 0xff413f41, 0xff2c272b, 0xff2e292d, 0xfe2d282c,
+    0xff322f33, 0xb8545656, 0x0f0e0c0d, 0x05030002, 0x110a090b, 0x06000001,
+    0x06000001, 0x07000002, 0x06000001, 0x06000002, 0x03010101, 0x07060706,
+    0x07000002, 0x07000002, 0x07000002, 0x07000002, 0x07000002, 0x07000002,
+    0x06000001, 0x0e050608, 0x0d040305, 0x0a030002, 0x0b010003, 0x08010001,
+    0x170f0e13, 0xc1588798, 0xff338db5, 0xff3090ba, 0xff2b8db8, 0xff3091bc,
+    0xff50acce, 0xff6abac6, 0xff83b9ab, 0xffc7ca60, 0xfffdd32a, 0xfffcd329,
+    0xfffccb27, 0xfffbc620, 0xfffdd428, 0xfffcda28, 0xffffd82b, 0xfffed928,
+    0xfffdd928, 0xffffd72b, 0xfff4b81d, 0xffb98c1f, 0xff664d26, 0xff312c2a,
+    0xff2e2a2b, 0xff2d292b, 0xff2d282d, 0xff2e292c, 0xff2e292d, 0xff2e292d,
+    0xff2d282c, 0xff2e292d, 0xff4b484c, 0xffd6d6d6, 0xffffffff, 0xfffdfdfd,
+    0xfffdfdfd, 0xffffffff, 0xffefefef, 0xff736f73, 0xff443f43, 0xff363034,
+    0xff2d282c, 0xff2d282c, 0xff2d282c, 0xff2e292d, 0xff2e292d, 0xff2e292d,
+    0xfe2f2a2e, 0xff2f2a2e, 0xfa383637, 0x784d4e4f, 0xff3b393c, 0xff2d272b,
+    0xff2e292d, 0xfe2f292d, 0xf239373a, 0x34252124, 0x05020002, 0x09010002,
+    0x1209080a, 0x06000001, 0x06000001, 0x06000001, 0x07000002, 0x07000002,
+    0x05000001, 0x03020202, 0x07000002, 0x07000002, 0x07000002, 0x07000002,
+    0x06000001, 0x06000001, 0x09010203, 0x0f060507, 0x09010002, 0x0a020003,
+    0x08000002, 0x0e090505, 0xad608391, 0xff338ab2, 0xfe2b8cb5, 0xff2787b3,
+    0xff2d8eb8, 0xff3e9dc8, 0xff5fb8d8, 0xff6bb8c9, 0xffa6c286, 0xffe7c63c,
+    0xfff9ca24, 0xfffdca24, 0xfff9bd1c, 0xfffdd52a, 0xfffdd62a, 0xffffd62a,
+    0xfffad023, 0xffffd929, 0xfffdd829, 0xffffd92c, 0xfff7b81c, 0xffd39f1e,
+    0xff8a6a24, 0xff3c3429, 0xff2f2a2d, 0xff2d292e, 0xff2e282d, 0xff2e282c,
+    0xff2d292d, 0xff2e292d, 0xff2e292d, 0xff2d282c, 0xff2e292d, 0xff4c484c,
+    0xffd1d0d1, 0xfffefefe, 0xfffefefe, 0xffffffff, 0xff9a9899, 0xff4e494d,
+    0xff353034, 0xff2d282c, 0xff2d292c, 0xff2e282d, 0xff2e292d, 0xff2d282c,
+    0xff2d282c, 0xff2f2a2e, 0xff2f2a2e, 0xfe322e32, 0xff2c2529, 0xff545657,
+    0xfe3f3d40, 0xfe2b262a, 0xfe2d282c, 0xff302d2f, 0x99414342, 0x00000000,
+    0x09010002, 0x0a020003, 0x0f060708, 0x06000001, 0x07000002, 0x06000001,
+    0x06000001, 0x07000002, 0x06000002, 0x06020204, 0x07000002, 0x07000002,
+    0x07000002, 0x06000001, 0x06000001, 0x0a020304, 0x0c020205, 0x0a020002,
+    0x0a020002, 0x09010002, 0x06030001, 0x894f6772, 0xff4293b5, 0xfe2181ab,
+    0xfe2282ae, 0xff2b88b4, 0xff3695c1, 0xff54b3da, 0xff61b7d7, 0xff98be96,
+    0xffd2c251, 0xffefc32d, 0xfffcc41d, 0xfffabe20, 0xffffd428, 0xfffcd128,
+    0xfffcd62d, 0xfffbc622, 0xfffdd628, 0xfffed929, 0xfffdda2a, 0xfffdd42a,
+    0xfff9ba1b, 0xfffcc023, 0xffdca524, 0xff664f24, 0xff322b28, 0xff2d292c,
+    0xff2d292c, 0xff2e292c, 0xff2d292d, 0xff2e292d, 0xff2e292d, 0xff2d292d,
+    0xff2d282c, 0xff2a2529, 0xff484547, 0xffc1bfc1, 0xffffffff, 0xffe9e8e9,
+    0xff5f5c5e, 0xff312c30, 0xff2e292d, 0xff2d292c, 0xff2d292d, 0xff2e282d,
+    0xff2d282c, 0xff2d282c, 0xff2e292d, 0xff2f2a2e, 0xff302b2f, 0xff342f33,
+    0xfe2f2b2f, 0xff322d32, 0xfe363437, 0xfe2b262a, 0xff332f32, 0xdf494c4c,
+    0x1f161314, 0x06000000, 0x0a020003, 0x10070709, 0x06000001, 0x06000001,
+    0x06000001, 0x07000002, 0x07000002, 0x06000002, 0x05000002, 0x05000001,
+    0x07000002, 0x06000001, 0x06000001, 0x07000002, 0x0d040506, 0x0d040306,
+    0x0b030003, 0x0b020004, 0x09000000, 0x04000000, 0x5f36464b, 0xf64794b4,
+    0xff2282ab, 0xfe2383ab, 0xff2b88ae, 0xff3894be, 0xff4ba7d1, 0xff57b3d9,
+    0xff80bab0, 0xffbebd67, 0xffedc32e, 0xfffdc115, 0xfffbc01e, 0xfffed22a,
+    0xfffcd132, 0xfffccf2a, 0xfffbc525, 0xfffdd027, 0xfffdd829, 0xfffed82b,
+    0xfffeda30, 0xfffcc621, 0xfffdcb23, 0xfffcc624, 0xfffebf23, 0xffd3981c,
+    0xff7c6026, 0xff433728, 0xff2c292a, 0xff2d292c, 0xff2e292d, 0xff2d292d,
+    0xff2d292d, 0xff2e292d, 0xff2d282c, 0xff2d282c, 0xff2b252a, 0xff3d383c,
+    0xff959394, 0xffa5a4a6, 0xff373337, 0xff2d282c, 0xff2d282c, 0xff2d282c,
+    0xff2e282d, 0xff2d282c, 0xff2d282c, 0xff2d282c, 0xff2d282c, 0xff2e292d,
+    0xff2c272b, 0xff302b2f, 0xff312c30, 0xff3c363a, 0xfe2e2a2e, 0xff312c2f,
+    0xc7414343, 0x351c1a1d, 0x05000000, 0x09010002, 0x0d040305, 0x09010103,
+    0x05000001, 0x03000000, 0x05000001, 0x06000001, 0x05000001, 0x06000002,
+    0x06000002, 0x07000002, 0x06000001, 0x07000002, 0x09010203, 0x0f060507,
+    0x08010001, 0x06000000, 0x05000000, 0x08000002, 0x08020001, 0x3720262b,
+    0xdc4d8ba5, 0xff2a89b2, 0xfe2684af, 0xff3089aa, 0xff469ab2, 0xff49a6ca,
+    0xff53b1d8, 0xff73b6c4, 0xffa3b787, 0xffd9c34a, 0xffdeb836, 0xfffdc01b,
+    0xfffdd326, 0xfffbc933, 0xfffcd33a, 0xfffcc923, 0xfffac222, 0xfffdd528,
+    0xfffed829, 0xfffdd931, 0xfffdd02f, 0xfffbc922, 0xfffed92a, 0xfffdd132,
+    0xfffcc223, 0xfffebb21, 0xffcb951d, 0xff705725, 0xff322d2a, 0xff2d282c,
+    0xff2e292d, 0xff2e292d, 0xff2d282c, 0xff2d282c, 0xff2e292d, 0xff2e292d,
+    0xff2d282c, 0xff2d282c, 0xff332e31, 0xff373337, 0xff2f292d, 0xff2e282c,
+    0xff2f2a2e, 0xff2f2a2e, 0xff2d282c, 0xff2e292d, 0xff2e292d, 0xff353034,
+    0xff423d41, 0xff3d383c, 0xff423d41, 0xff343134, 0xff3d393d, 0xff3c393c,
+    0xff302c30, 0xde3d3d3e, 0x351b1b1d, 0x09010002, 0x0a020002, 0x0a020002,
+    0x0a030204, 0x04010102, 0x00000000, 0x01000000, 0x03000001, 0x04000001,
+    0x05000001, 0x05000001, 0x05000001, 0x05000001, 0x0d040406, 0x0d050406,
+    0x10060407, 0x1f0c0a0d, 0x411c1d21, 0x6134393b, 0x7f4c5356, 0x59343b3f,
+    0x3621282d, 0xdb6c9dad, 0xff358fb7, 0xfe2480aa, 0xff3b8faa, 0xff75a588,
+    0xff68a9a7, 0xff55aed3, 0xff6eb5c5, 0xff96b798, 0xffaec17c, 0xffc7b24b,
+    0xfffbbb16, 0xfffdd025, 0xfffbcc26, 0xfffbcc2f, 0xfffdcc25, 0xfffac31f,
+    0xfffbcb23, 0xfffdd727, 0xfffdd82b, 0xfffdd42e, 0xfff8c120, 0xfffcd629,
+    0xfffed92a, 0xfffdd62b, 0xfffdce2d, 0xfffac222, 0xfff7b61d, 0xff9f7a25,
+    0xff463b29, 0xff2d282c, 0xff2d292d, 0xff2d292d, 0xff2e292d, 0xff2e292d,
+    0xff2e292d, 0xff2d282c, 0xff2e292d, 0xff2f2a2e, 0xff2e292d, 0xff2c292c,
+    0xff2d292c, 0xff2e282d, 0xfe2c272b, 0xff2f2a2d, 0xff302b2f, 0xff383337,
+    0xff555054, 0xff777377, 0xff8a888b, 0xff767477, 0xfb555356, 0xf7555656,
+    0xfb636464, 0xff3b3a3b, 0xf4373638, 0x5d2f2f30, 0x02000000, 0x0a000002,
+    0x09010002, 0x08020103, 0x06030304, 0x02010101, 0x01000000, 0x01000000,
+    0x01000000, 0x01000000, 0x02000000, 0x03000000, 0x03000000, 0x02000000,
+    0x09030003, 0x3818191d, 0x972f3439, 0xec465358, 0xff4b6471, 0xff527686,
+    0xff548ba4, 0xff6daac3, 0xf961a3bf, 0xff529ebd, 0xfe2682ae, 0xfe2484b0,
+    0xff80ad8c, 0xff9db06d, 0xff66adbb, 0xff70b3bf, 0xffbab46a, 0xffa0b887,
+    0xffbdb964, 0xfff5bf22, 0xfffccb23, 0xfffcd227, 0xfffbc827, 0xfffbcc27,
+    0xfffdca24, 0xfffcc822, 0xfffcd026, 0xfffcd426, 0xfffdd529, 0xfffac021,
+    0xfffbcc29, 0xfffdd82b, 0xfffed727, 0xfffed92a, 0xfffdda32, 0xfffdd22c,
+    0xfff9c220, 0xffc59826, 0xff584828, 0xff2e292d, 0xff2e292c, 0xff2d292d,
+    0xff2d282d, 0xff2d282c, 0xff2e292d, 0xff2e292d, 0xff2d282c, 0xff2e292d,
+    0xff2f2a2e, 0xff2e292d, 0xff2d282c, 0xff2e292c, 0xff2f2a2e, 0xf2363335,
+    0xd9302e30, 0xdd363437, 0xd7373537, 0xc42c2a2b, 0xc2313031, 0xae3b3c3c,
+    0x82343435, 0x7d4d5050, 0xe0626767, 0xff444346, 0x83444445, 0x00000000,
+    0x09000002, 0x09020002, 0x08020103, 0x08050606, 0x05040504, 0x01000100,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x01000000, 0x00000000, 0x552c2b2c, 0xe23d454b, 0xff3c6577, 0xff6aa0bb,
+    0xff8ec4d8, 0xfe92c9e2, 0xfe70b7d4, 0xff7ebad4, 0xff8bbed6, 0xfe7ab7ce,
+    0xff1d7eaa, 0xff2b8dbb, 0xff99af78, 0xffa3b077, 0xff89b19d, 0xffc5b45c,
+    0xffecb72f, 0xffbaba65, 0xffecbf2e, 0xfffcca21, 0xfffbce23, 0xfffccc27,
+    0xfffbc723, 0xfffcce29, 0xfffac21f, 0xfffdcd25, 0xfffac61a, 0xfffcd125,
+    0xfffabc1e, 0xfffabf26, 0xfffcd333, 0xfffbca22, 0xfffdd426, 0xffffd82a,
+    0xfffdda2e, 0xfffdd62b, 0xfff5bc20, 0xffa58524, 0xff70572a, 0xff352d2a,
+    0xff2d2a2c, 0xff2e282e, 0xff2d292d, 0xff2e292c, 0xff2e292d, 0xff2e292d,
+    0xff2e292d, 0xff2d282c, 0xff2d282c, 0xff2e292d, 0xff2d282c, 0xff2d292c,
+    0xff312d31, 0xb63c3c3e, 0x26100e11, 0x301b191b, 0x2d202020, 0x1b171616,
+    0x13131112, 0x00000000, 0x00000000, 0x5d313031, 0xff616765, 0x6b2e2e32,
+    0x06020003, 0x09000001, 0x09010002, 0x07020001, 0x0b08090a, 0x06040505,
+    0x01000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xf144484c, 0xff457b94,
+    0xff5bb2d8, 0xfe8acae6, 0xff91cde8, 0xff79bfdc, 0xff4e9dc2, 0xff3d92b4,
+    0xff9cc3d8, 0xff97b8c5, 0xfe3881a3, 0xff3399c5, 0xfea2b68b, 0xffd3af45,
+    0xffecb12c, 0xfff8b821, 0xffd1bb4d, 0xffe5bb36, 0xfffdc722, 0xfff9c21d,
+    0xfffdd128, 0xfffbc624, 0xfffcce28, 0xfffabd1c, 0xfffecd25, 0xfffcca1e,
+    0xfffcca21, 0xfffabc1e, 0xfffbaf17, 0xfffdce2c, 0xfffabd1d, 0xfffdc722,
+    0xfffed82b, 0xfffdd72a, 0xffffd82c, 0xfff9c31e, 0xfffdca25, 0xffdaa421,
+    0xff5d4922, 0xff53422a, 0xff2d282c, 0xff2d292b, 0xff2d282d, 0xff2e282c,
+    0xff2d282d, 0xff2d282c, 0xff2d282c, 0xff2e292d, 0xff2d282c, 0xff2e292d,
+    0xff2e292d, 0xff2c292c, 0xff322b2f, 0xba373739, 0x110c0a0d, 0x0f0c0b0d,
+    0x1a18191a, 0x08070807, 0x05040304, 0x02010000, 0x0a060505, 0x94535755,
+    0x733c3d3d, 0x0f050306, 0x09010002, 0x09010002, 0x0a040205, 0x07060505,
+    0x03000001, 0x02000000, 0x01000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0xff4f7484, 0xfe5bb8e0, 0xff5db6dc, 0xff7fc8e6, 0xff68b6da, 0xff5fadca,
+    0xff348fb6, 0xff267da2, 0xfe96bccc, 0xfcb2887d, 0xff3b6577, 0xfe8aada6,
+    0xffd7af45, 0xfff8ad16, 0xfffcb516, 0xffe9b931, 0xffe8b72e, 0xfffebc1a,
+    0xfff9ba18, 0xfffdcd25, 0xfffbc520, 0xfffdca26, 0xfff9bd1e, 0xfffdc21e,
+    0xfff8b70f, 0xfffbc71c, 0xfffbc221, 0xfff9ae15, 0xfffcc522, 0xfffbc223,
+    0xfffbbc1b, 0xfffbca21, 0xfffccc24, 0xfffdd333, 0xfffbca24, 0xfffcc220,
+    0xfff7cf29, 0xfff8b921, 0xffa17520, 0xff463727, 0xff443b2a, 0xff302b2b,
+    0xff2e2a2d, 0xff2d292d, 0xff2e2a2c, 0xff2f2a2e, 0xff2e292d, 0xff2d282c,
+    0xff2d282c, 0xff2e292d, 0xff2f2a2e, 0xff2f2a2e, 0xff2f2a2d, 0xc6322f31,
+    0x1d111013, 0x110e0d10, 0x13131212, 0x0b0a0a0a, 0x01000101, 0x02010000,
+    0x07050303, 0x22171717, 0x02000000, 0x08020002, 0x0a020001, 0x0a040404,
+    0x05020302, 0x04000001, 0x03000001, 0x02000000, 0x01000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0xff5db1d6, 0xff50add4, 0xff65b7da, 0xff6abcde,
+    0xff6db2cf, 0xff469abf, 0xff1d7aa3, 0xff4a97b9, 0xffa37e73, 0xff9d5d35,
+    0xfe417280, 0xffdd9c26, 0xfef1a01e, 0xfffcb017, 0xfffaaf16, 0xfff9ae15,
+    0xfffbb114, 0xfff9b218, 0xfffcc11e, 0xfffbc622, 0xfffdc724, 0xfff9bd1d,
+    0xfffbb91d, 0xfffab313, 0xfffabe17, 0xfffdc61f, 0xfffcb418, 0xfffbbe1e,
+    0xfffdc422, 0xfffbbd1f, 0xfffcbe20, 0xfffabd1c, 0xfffabb22, 0xfffcce2b,
+    0xfffbc11e, 0xfffcc01c, 0xffeabc25, 0xffb98a1e, 0xffe4a424, 0xff6c5223,
+    0xff3f3327, 0xff3c332b, 0xff2e2a2a, 0xff2d282e, 0xff2e292d, 0xff2d282c,
+    0xff2d282c, 0xff2e292d, 0xff2e292d, 0xff2d282c, 0xff2e292d, 0xff2f2a2c,
+    0xff2e292c, 0xda373638, 0x331b1d1e, 0x09050607, 0x09080908, 0x00000000,
+    0x00000000, 0x05030203, 0x07030103, 0x06030102, 0x04010001, 0x08030304,
+    0x09040406, 0x0a060709, 0x06000102, 0x06000002, 0x05000001, 0x02000000,
+    0x01000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xff5fb6dd, 0xff419bc2,
+    0xff5ca8ca, 0xff51a3c6, 0xff3e92b9, 0xff2581a9, 0xff1c7ba4, 0xff428bad,
+    0xff7c401d, 0xffbc782c, 0xff61624e, 0xffbe6314, 0xfff4a525, 0xfffaac15,
+    0xfffaac1a, 0xfffba911, 0xfffab31d, 0xfffcba1a, 0xfffbc321, 0xfffcc122,
+    0xfffbb81c, 0xfffbb41a, 0xfffab518, 0xfffcbd1e, 0xfffaba19, 0xfffcbf1d,
+    0xfffbbb1e, 0xfffbbf1e, 0xfffcc023, 0xfffbc020, 0xfffbbb1d, 0xfffab51a,
+    0xfffcc722, 0xfffbc21e, 0xfffab61b, 0xfffbbb1d, 0xfff9b11a, 0xffbf8b1e,
+    0xffbd881f, 0xffd39520, 0xff684f22, 0xff403628, 0xff3b332a, 0xff2c292d,
+    0xff2d292d, 0xff2e282c, 0xff2e282d, 0xff2d282d, 0xff2d282c, 0xff2e282d,
+    0xff2e292d, 0xff2f2a2e, 0xff2c292d, 0xdc373333, 0x32191818, 0x07020204,
+    0x00000000, 0x00000000, 0x00000000, 0x07010203, 0x08010103, 0x0a030305,
+    0x02000000, 0x03020202, 0x04010203, 0x03000000, 0x05000001, 0x05000001,
+    0x05000001, 0x04000001, 0x02000000, 0x01000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0xff5bafd2, 0xff4496b8, 0xff4397bb, 0xff2a85b1, 0xff2083aa, 0xff1d79a1,
+    0xff2583ac, 0xff3b6e85, 0xff77331a, 0xff994a26, 0xff52443b, 0xffd98218,
+    0xfffcad17, 0xfffbab1f, 0xfffaac1c, 0xfffcb223, 0xfffcb215, 0xfffbb719,
+    0xfffbb61a, 0xfffaaf18, 0xfffbb116, 0xfffbb419, 0xfffcbb1d, 0xfffaad17,
+    0xfffab41e, 0xfffcb917, 0xfffcba1d, 0xfffbbf1d, 0xfffbc123, 0xfffab91c,
+    0xfffab51b, 0xfffcc01d, 0xfffed329, 0xfffbb81c, 0xfffcb51c, 0xfffab117,
+    0xfffbac19, 0xfff9ac17, 0xffad7b19, 0xffbb861e, 0xffdd9c1f, 0xff6e5220,
+    0xff564a2d, 0xff3a3027, 0xff282528, 0xff2c292e, 0xff2d2a2b, 0xff2e292c,
+    0xff2e292d, 0xff2d2a2c, 0xff2d282c, 0xff322c30, 0xff312c2a, 0xe54f412f,
+    0x43332b1f, 0x0e050707, 0x0a080709, 0x03000201, 0x04000000, 0x06000001,
+    0x05000001, 0x04000001, 0x02000000, 0x01000000, 0x02000000, 0x02000000,
+    0x03000000, 0x04000001, 0x05000001, 0x05000001, 0x03000000, 0x01000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x01000000, 0xff2c8ab1, 0xff3f94b8, 0xff2b86ad, 0xff2686ae,
+    0xff1e77a1, 0xff207ba3, 0xff3387ab, 0xff3a5363, 0xff903616, 0xff933920,
+    0xff64402e, 0xffdf8e19, 0xffde7f0e, 0xffd67a10, 0xfffab12a, 0xfffba919,
+    0xfffbab12, 0xfffbaf16, 0xfff9ac16, 0xfffaab14, 0xfffdb318, 0xfffbba1d,
+    0xfffaad16, 0xfffcae17, 0xfffab013, 0xfffcbd1e, 0xfffabb19, 0xfffabd20,
+    0xfffdb51e, 0xfffcb61a, 0xfffcb918, 0xfff8b012, 0xfff9b71a, 0xfff8ae17,
+    0xfffbad16, 0xfffaac16, 0xfffbab15, 0xfff9a816, 0xfffeac1c, 0xffba841c,
+    0xffb37d1a, 0xffdda624, 0xff8c6620, 0xff9a6f1f, 0xffb77f2a, 0xff473424,
+    0xff302a2d, 0xff2f2a30, 0xff2d292c, 0xff2e292e, 0xff2f2b2c, 0xff37322c,
+    0xfe5b4627, 0xffbc8b2a, 0x887c602b, 0x130a0808, 0x1b16171b, 0x02000000,
+    0x06000001, 0x05000001, 0x03000000, 0x02000000, 0x01000000, 0x00000000,
+    0x00000000, 0x00000000, 0x01000000, 0x02000000, 0x04000001, 0x05000001,
+    0x05000001, 0x03000000, 0x02000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x01000000, 0x01000100, 0x03020302, 0xff2f8bb2, 0xff2e89b0,
+    0xff2c89b2, 0xff1a799f, 0xff1f759f, 0xff2782a6, 0xff3b84a4, 0xff393c43,
+    0xff863519, 0xff7a2f18, 0xff6d301f, 0xffb24e1b, 0xffc65a10, 0xffe59528,
+    0xfffdb123, 0xfffba60d, 0xfffbaa12, 0xfffba913, 0xfffaab15, 0xfffcb319,
+    0xfffbb519, 0xfffaab16, 0xfffbac16, 0xfffab417, 0xfffcbc1c, 0xfffcb61a,
+    0xfff5ac1c, 0xfff8b01d, 0xfffbb419, 0xfffab217, 0xfffbb616, 0xfff19d0b,
+    0xfffbaa17, 0xfff9a713, 0xfffaa914, 0xfffbac16, 0xfffaab16, 0xfff6a216,
+    0xfff9a512, 0xffffae15, 0xffe7a71e, 0xffc98d19, 0xffda931e, 0xffd48e1b,
+    0xffdb8f17, 0xffb5791d, 0xff6e5123, 0xff53432d, 0xff433626, 0xff3c3228,
+    0xff564225, 0xff84622c, 0xfe9e7016, 0xffffb424, 0xdab29758, 0x5f3a3c3b,
+    0x09060709, 0x04020201, 0x04000001, 0x04000001, 0x02000000, 0x01000000,
+    0x01000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x01000000,
+    0x03000000, 0x05000001, 0x06000001, 0x04000001, 0x03000001, 0x01000000,
+    0x00000000, 0x00000000, 0x01000000, 0x03020302, 0x03020202, 0x08070807,
+    0xff348db4, 0xff2a88b0, 0xff257ba2, 0xff1b6e90, 0xff2280a8, 0xff3080a2,
+    0xff327692, 0xff42322e, 0xff612b1e, 0xff5f2a1b, 0xff693120, 0xffa3350f,
+    0xffc55a0d, 0xfff0a023, 0xfffba913, 0xfffca912, 0xfff8a911, 0xfffaaa13,
+    0xfff9aa13, 0xfffbae15, 0xfffba812, 0xfffaac16, 0xfffcb317, 0xfffab518,
+    0xfffcb51b, 0xfff3a419, 0xfff39d18, 0xfffbad17, 0xfff9ac17, 0xfffbb215,
+    0xfff7aa15, 0xfff8a816, 0xfff9a914, 0xfffaa815, 0xfffaab17, 0xfffbad17,
+    0xfff9aa18, 0xfff5a114, 0xfffba815, 0xfff6a714, 0xfffcaf19, 0xfff7a417,
+    0xffec9c19, 0xfff7a813, 0xfff8a516, 0xffe99412, 0xffda981d, 0xffcb8f22,
+    0xffc18d2a, 0xffb98321, 0xffda9719, 0xfffdb222, 0xffffb01a, 0xffe8a828,
+    0xf39c8b63, 0x924a4d4c, 0x05040405, 0x0a080809, 0x01000000, 0x02000000,
+    0x01000000, 0x01000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x01000000, 0x03000000, 0x05000001, 0x06000002, 0x06000001,
+    0x04000001, 0x02000001, 0x07050506, 0x0b0a0a0b, 0x07050606, 0x04030403,
+    0x0a090a09, 0x110f1110, 0xff2885ae, 0xff2586b1, 0xff1c6484, 0xff247393,
+    0xff24799c, 0xff25769a, 0xff2b596f, 0xff432e30, 0xff3d2925, 0xff5c2b1c,
+    0xff642718, 0xffaf410d, 0xffd16d0e, 0xfffdad1e, 0xfff7a20f, 0xfff8a911,
+    0xfffaa913, 0xfffaaa13, 0xfffbaa13, 0xfffca70f, 0xfffcac16, 0xfffcac16,
+    0xfffbb015, 0xfffbae16, 0xfff39e15, 0xfff29116, 0xfff9a617, 0xfffbab16,
+    0xfffbae14, 0xfff9b118, 0xffef9912, 0xfffaa915, 0xfffba915, 0xfff8a512,
+    0xfff6a013, 0xfffaac16, 0xfffaa214, 0xfff39e13, 0xfff7a713, 0xfff9a816,
+    0xfff69f15, 0xffef9714, 0xfff9a817, 0xfff8a613, 0xfff19910, 0xfffca817,
+    0xfffbae17, 0xfffaab15, 0xfffab126, 0xfffaaa14, 0xfffbab16, 0xfffaad13,
+    0xfff8ac16, 0xffaa7f2e, 0xff69695f, 0x9c424244, 0x0b0b0b0a, 0x0a070a09,
+    0x02000000, 0x03000001, 0x01000000, 0x01000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x01000000, 0x03000001, 0x05000001,
+    0x07000003, 0x06000002, 0x06000001, 0x0c0a0b0a, 0x1b1a1815, 0x2524211e,
+    0x0d0b0c0b, 0x0d0c0d0c, 0x15131514, 0x0f0e0e0e, 0xff2988b1, 0xff1d7ca5,
+    0xff1e5e79, 0xff216685, 0xff257696, 0xff1b6689, 0xff2c4451, 0xff362f34,
+    0xff342927, 0xff5e2b1f, 0xff602a18, 0xffa73705, 0xffed951d, 0xfff6a311,
+    0xfff9a412, 0xfff9a910, 0xfffaaa14, 0xfffaaa12, 0xfffaa911, 0xfffbab16,
+    0xfff49e15, 0xfffaa30f, 0xfffba813, 0xfff29211, 0xffe57d0b, 0xfff29b11,
+    0xfffcad16, 0xfffaac15, 0xfffba415, 0xffef9314, 0xfff59f12, 0xfff9a917,
+    0xfffbab16, 0xfff6a114, 0xfff6a014, 0xfff8a112, 0xffe37d12, 0xfff9a516,
+    0xfff9a816, 0xfff7a016, 0xffee930e, 0xfff8a216, 0xfff6a212, 0xfff39a12,
+    0xfff7a214, 0xfffbab16, 0xfffcab17, 0xfffba911, 0xfffbab14, 0xfffcab14,
+    0xfffaab13, 0xfffbad1b, 0xffba8726, 0xff2f393c, 0xff213f4c, 0xab394346,
+    0x12121112, 0x1f1c1c1e, 0x0d0a0b0c, 0x04000001, 0x03000000, 0x01000000,
+    0x01000000, 0x01000000, 0x01000000, 0x01000000, 0x01000000, 0x03000101,
+    0x04000102, 0x05000001, 0x07000100, 0x0a020507, 0x18151515, 0x211c1710,
+    0x2c21190d, 0x12100b03, 0x17131512, 0x17161616, 0x0c0b0b0b, 0x08070807,
+    0xff2382ac, 0xff19759a, 0xff21617e, 0xff24607c, 0xff176e91, 0xff1c5b79,
+    0xff2b3c47, 0xff302b2e, 0xff432922, 0xff5e2b1d, 0xff672816, 0xffbc5410,
+    0xffcc650b, 0xfff19b17, 0xfff5a011, 0xfffaa813, 0xfff8a714, 0xfff9aa11,
+    0xfffcab16, 0xfff6a119, 0xfff08e0d, 0xfff9a412, 0xfff1920d, 0xffed8b0e,
+    0xfff49813, 0xfffbad16, 0xfffaab15, 0xfffbad15, 0xfff69d11, 0xfff5a014,
+    0xfffaaa16, 0xfffbac15, 0xfff7a216, 0xffea8810, 0xfff39914, 0xffe78010,
+    0xfff29515, 0xfffbab15, 0xfff6a314, 0xffef8f15, 0xfff5a015, 0xfff7a417,
+    0xffe68913, 0xfff6a113, 0xfffaab16, 0xfffaab16, 0xfffbaa15, 0xfff7a411,
+    0xfffbaa14, 0xfffaaa13, 0xfffead15, 0xffbf871e, 0xff383228, 0xff284b5a,
+    0xff1e3a49, 0xfc5d747b, 0x87596c6f, 0x523b4346, 0x09060607, 0x05000001,
+    0x04000001, 0x04000001, 0x03000000, 0x02000000, 0x02000000, 0x02000000,
+    0x04020203, 0x12101111, 0x12101111, 0x0a060607, 0x03000001, 0x0c0a0908,
+    0x5a473d24, 0x6a50472a, 0x241e180e, 0x56433b29, 0x664e452d, 0x1814120e,
+    0x0d0b0b0b, 0x0a09090a, 0xff1f79a0, 0xff1c749c, 0xff22576e, 0xff235b76,
+    0xff106589, 0xff24536a, 0xff2c363d, 0xff31292c, 0xff5f2f22, 0xff58291c,
+    0xff6f2b12, 0xffb44f10, 0xffd97612, 0xffe48a17, 0xfff49a0f, 0xfff6a015,
+    0xfff9a813, 0xfffdaa17, 0xfff7a41a, 0xffef920e, 0xfff69e0f, 0xffef8e0c,
+    0xfff09011, 0xffdf7109, 0xffee8e10, 0xfffcab16, 0xfffbab17, 0xfffba816,
+    0xffee9010, 0xfff69e13, 0xfff19b12, 0xfff79e14, 0xffeb8c10, 0xfff09112,
+    0xffed8d11, 0xfff09916, 0xfffdac17, 0xfff49a12, 0xffe57913, 0xffed8c13,
+    0xfff8a815, 0xffe58513, 0xffe78914, 0xfff5a116, 0xfff7a615, 0xfffbac14,
+    0xfffaa915, 0xfff8a716, 0xfffbac18, 0xfffea814, 0xffcc8f23, 0xff443827,
+    0xff2b2e35, 0xff2c5363, 0xfe213341, 0xff3d5d6b, 0xeb95bac2, 0x60364348,
+    0x00000000, 0x06000002, 0x05000001, 0x05000001, 0x05000001, 0x05000001,
+    0x04000001, 0x07040504, 0x110f100f, 0x1a191717, 0x100e0e0f, 0x05040403,
+    0x1613100b, 0x93726743, 0xf3b5a057, 0xeeada148, 0xa67b7845, 0x82665d43,
+    0xb7897e50, 0x3e302b1e, 0x03030000, 0x4e3b3421, 0xff166c90, 0xff236f90,
+    0xff243b47, 0xff1a536c, 0xff1a6e93, 0xff2b4a5b, 0xff333139, 0xff393433,
+    0xff7b301a, 0xff55271d, 0xff7a321c, 0xffbf5912, 0xffd77510, 0xffda730b,
+    0xfff69d15, 0xfff6a211, 0xfff7a110, 0xfff39813, 0xffeb8d10, 0xfff39613,
+    0xffea7e05, 0xfff49312, 0xffda6706, 0xffd05303, 0xffed8910, 0xfffcaf16,
+    0xfffaa813, 0xffe8820e, 0xffe87e0f, 0xfff79e14, 0xffe47a0c, 0xffef8d13,
+    0xffec860b, 0xfff39a13, 0xfff6a213, 0xfffbab13, 0xfff1910f, 0xffdc6309,
+    0xffe88211, 0xfff8a314, 0xfff29619, 0xffda7714, 0xffda6e11, 0xffe38011,
+    0xfff9a715, 0xfffbaa16, 0xfff29d12, 0xfff2a016, 0xfffaa814, 0xffdf961f,
+    0xff574627, 0xff2b282c, 0xff2e4955, 0xff2a4d5f, 0xff253542, 0xfe2b4f61,
+    0xf68fb8c1, 0x59323f45, 0x00000000, 0x06000001, 0x06000001, 0x06000001,
+    0x06000001, 0x05000001, 0x0c080a0b, 0x14111313, 0x100e0f0f, 0x09080708,
+    0x08040603, 0x28221c14, 0xb0867748, 0xfeb8a84d, 0xe6aaa14e, 0xcc968d41,
+    0x44322e15, 0x2019170d, 0x48383021, 0x1e19140e, 0x73584d34, 0xf4b5a566,
+    0xff207ba1, 0xff275d75, 0xff272e35, 0xff1d4e65, 0xff287599, 0xff30414c,
+    0xff373539, 0xff533835, 0xff8e2e0d, 0xff4d2920, 0xff7d3118, 0xffc5600d,
+    0xffbd4c08, 0xffed9010, 0xfff19711, 0xfff79d12, 0xffec8a0d, 0xffec890f,
+    0xffeb8110, 0xffe57e08, 0xfff69414, 0xffd55f03, 0xffcf5104, 0xffe2740c,
+    0xfff59c10, 0xfff79d14, 0xffef8f11, 0xffe77a0a, 0xfff49d12, 0xffe8850f,
+    0xffea800f, 0xffe27708, 0xfff59e15, 0xfff8a813, 0xfff5a215, 0xffe7870f,
+    0xffd95a07, 0xffdf710b, 0xfff39614, 0xfff49d14, 0xffd86f0a, 0xffdd7b12,
+    0xffd06c0b, 0xffed9114, 0xfff3a114, 0xfff6a514, 0xfff29b14, 0xfff6a214,
+    0xffe19318, 0xff664d26, 0xff2a242b, 0xff253d4c, 0xff2b4251, 0xff234656,
+    0xff253845, 0xff2a4d5b, 0xf79cc3cb, 0x60374548, 0x01000000, 0x07000002,
+    0x07000002, 0x08010103, 0x0b040406, 0x17131211, 0x413c372f, 0x1f1b1713,
+    0x08070302, 0x09070301, 0x362c2317, 0xca978b4c, 0xfdb7a94a, 0xb2857e42,
+    0x34262009, 0x231b170d, 0x1c181009, 0xb78b7e52, 0xe5ab9b61, 0x7055492d,
+    0xe0a8975f, 0x7a605537, 0xff2a81a7, 0xff1e4d63, 0xff282d34, 0xff294c5f,
+    0xff185977, 0xff38373c, 0xff34353a, 0xff7e3623, 0xff7e2b11, 0xff4b2a23,
+    0xff904618, 0xffb34109, 0xffbf5008, 0xfff29313, 0xfff09511, 0xffe9830c,
+    0xffed8811, 0xffe4750d, 0xffe77705, 0xfff19118, 0xffda6807, 0xffcd5105,
+    0xffde6909, 0xffe1720b, 0xffe7840c, 0xffed8910, 0xffec830c, 0xffef8f0d,
+    0xfff19212, 0xffe77f0d, 0xffe77f0e, 0xffed8c11, 0xfff29813, 0xffeb8f10,
+    0xffda740f, 0xffc94f05, 0xffe3720d, 0xffe38110, 0xffe98914, 0xffde790e,
+    0xffe88b13, 0xffe38712, 0xffdd7e12, 0xffdb780d, 0xffed9713, 0xffdf8112,
+    0xffd66f0d, 0xffe5991b, 0xff6f4e23, 0xff272229, 0xff2b333a, 0xff234b5e,
+    0xff29363f, 0xff244556, 0xff263946, 0xff2d4753, 0xf57fabb7, 0x5229373c,
+    0x03000000, 0x09010204, 0x08000103, 0x0b060405, 0x372b251a, 0x67534934,
+    0x5a463b26, 0x2e252018, 0x0b080300, 0x624a4123, 0xe0a5964a, 0xd99e9240,
+    0x77595121, 0x392b2716, 0x27201c15, 0x3b2b2417, 0xc590814f, 0xefb29c5d,
+    0x9a73663a, 0x624b4027, 0x71564c30, 0xd48d834b, 0xff1f7194, 0xff1d4b60,
+    0xff283038, 0xff213f50, 0xff21495d, 0xff383234, 0xff343134, 0xff9a300f,
+    0xff652a19, 0xff563220, 0xff933e16, 0xffa92f04, 0xffce5e0a, 0xffef8d0f,
+    0xffe68411, 0xffea8511, 0xffe27008, 0xffe16505, 0xffe67407, 0xffd76308,
+    0xffcc4e04, 0xffda5e06, 0xffd55a07, 0xffcb5005, 0xffe1750a, 0xffe87a0d,
+    0xffe97d08, 0xffea810b, 0xffe9820f, 0xfff49815, 0xffeb890e, 0xffe87d0e,
+    0xffdf770c, 0xffc95d0e, 0xffbe4909, 0xffe17510, 0xffd67110, 0xffd0640d,
+    0xffe28211, 0xffea8d16, 0xffd77710, 0xffca610b, 0xffc15109, 0xffd57413,
+    0xffc14f0b, 0xffb03604, 0xffd57213, 0xff7f5c28, 0xff2f292b, 0xff2b2b2e,
+    0xff283f4d, 0xff273a46, 0xff2a2e36, 0xff244353, 0xff273a44, 0xff344448,
+    0xfa6b9aa1, 0x91546b70, 0x17100f0f, 0x04010002, 0x1912110d, 0x664d4731,
+    0x80635c41, 0x9e796c47, 0xd39d8c52, 0x98756845, 0x9d746934, 0xf4afa139,
+    0xfeb4a836, 0xaa817a44, 0x56433c26, 0x04030000, 0x5d403b24, 0xe891894a,
+    0xd98e8546, 0x85645734, 0x73564b2d, 0x79574e2e, 0xdb887f40, 0xc67b753f,
+    0xff126488, 0xff1e4c64, 0xff28313b, 0xff253f4c, 0xff2b363c, 0xff333137,
+    0xff412a26, 0xffa0300d, 0xff4f2a20, 0xff633820, 0xff94300e, 0xffa72c03,
+    0xffd86a0d, 0xffec9014, 0xffc25407, 0xffdb690a, 0xffe06911, 0xffe06706,
+    0xffdd680a, 0xffcb5005, 0xffd65a05, 0xffd05206, 0xffc84703, 0xffda6706,
+    0xffec8112, 0xffde6604, 0xffe77c09, 0xffe97f12, 0xffed8814, 0xffe2770a,
+    0xffe5760d, 0xffd66208, 0xffc7570d, 0xffb03705, 0xffcf5f0d, 0xffc95d09,
+    0xffcd6210, 0xffd7750f, 0xffe18213, 0xffba4c07, 0xffc2520b, 0xffac3506,
+    0xffbd4f0b, 0xffc2581c, 0xffa72c06, 0xff9c3b15, 0xff80562a, 0xff44392e,
+    0xff2d292c, 0xff2d2e34, 0xff293740, 0xff2a3033, 0xff292d35, 0xff234657,
+    0xff273d4a, 0xff363f3f, 0xff57888c, 0xea8cb0b4, 0x4d393e3d, 0x7c5e563a,
+    0xbe8f825b, 0x7f625a3f, 0x795e5640, 0xbc918258, 0xffbda75f, 0xfab7a444,
+    0xfdb5a630, 0xffb5a734, 0xecac9d46, 0xecae9d57, 0x5c463f28, 0x835a5639,
+    0xfa938e42, 0xc97c7940, 0x8e615c3e, 0x996b6745, 0x815f5737, 0xe188803e,
+    0x8f4e4818, 0x2714120a, 0xff15658a, 0xff224d63, 0xff26333d, 0xff2e353c,
+    0xff2f2d33, 0xff312f32, 0xff4c2921, 0xff9a2f10, 0xff382826, 0xff682f1c,
+    0xff9e2c08, 0xffa83201, 0xffec8c1a, 0xffcd660f, 0xffcb5309, 0xffdf6b12,
+    0xffe17009, 0xffea8010, 0xffcf5506, 0xffd75804, 0xffd15009, 0xffc54704,
+    0xffd96206, 0xfff08b11, 0xffdd6507, 0xffe06d07, 0xffe77b0c, 0xffe46e0b,
+    0xffdb6704, 0xffe7770d, 0xffcf5707, 0xffbd4506, 0xffae3203, 0xffb74207,
+    0xffc35009, 0xffcc5f0e, 0xffca600c, 0xffd46f0d, 0xffb43f04, 0xffb33c09,
+    0xffa62c01, 0xffac3406, 0xffb84714, 0xffae3d16, 0xffa62d05, 0xff623324,
+    0xff363134, 0xff302c2d, 0xff2e2a2d, 0xff2e2b2f, 0xff2b2b2f, 0xff2c2c30,
+    0xff292e35, 0xff1e4d62, 0xff26495c, 0xff36414a, 0xff508a95, 0xea7d988b,
+    0xb4817d5e, 0xbf8e8053, 0x5a483e28, 0x44312917, 0x221b150e, 0x382e2618,
+    0xf4b8a563, 0xffb8a83d, 0xfbb2a63a, 0x9a756732, 0x78584d25, 0xffafa354,
+    0xffa19a4f, 0xfe969145, 0xba787245, 0x8f5e5b36, 0xcc827f49, 0xa76f6b42,
+    0xf8948d45, 0x914f4819, 0x1a0b0900, 0x83565744, 0xff1c6c90, 0xff1e4254,
+    0xff27333c, 0xff2d2d31, 0xff2f2b2e, 0xff2f2a2e, 0xff5b2b1e, 0xff812d13,
+    0xff332727, 0xff6b2c1a, 0xff9f2906, 0xffc45a0f, 0xffe78c1d, 0xffbe4403,
+    0xffde6909, 0xffee8711, 0xfff18b13, 0xffd35d06, 0xffd65605, 0xffce4f06,
+    0xffc94f02, 0xffdd6307, 0xffed8510, 0xffe57407, 0xffde6a05, 0xffef880b,
+    0xffe16908, 0xffdd6707, 0xffe5720e, 0xffce5305, 0xffbb3e06, 0xffad3208,
+    0xffaa3206, 0xffbe4507, 0xffba4309, 0xffb13b03, 0xffd1670e, 0xffba4909,
+    0xffa82c04, 0xffa72b02, 0xffa82e06, 0xffae3608, 0xffb4431c, 0xffaa2f07,
+    0xff9e2f0b, 0xff3c2826, 0xff2e2a2e, 0xff2c2a2d, 0xff2f292e, 0xff2e292d,
+    0xff2c292e, 0xff2c2d31, 0xff293038, 0xff195973, 0xff235e79, 0xff314a55,
+    0xff48859f, 0xf981a193, 0xb07f7a58, 0x664f4732, 0x4836301f, 0x17140e0b,
+    0x45342e1d, 0xc897864d, 0xffbba84f, 0xebad9e45, 0xa1797339, 0xa8776f3b,
+    0xffafa356, 0xffaba352, 0xdd8c8644, 0x9a66623d, 0xa96b693a, 0xff9b9045,
+    0xff9a8f40, 0xfa938a3d, 0x82474117, 0x1b0d0a03, 0x86504f3d, 0xff7c7b4d,
+    0xff146388, 0xff213c4b, 0xff29323a, 0xff2f2c30, 0xff302b2e, 0xff2f282d,
+    0xff662c19, 0xff6c2b17, 0xff352728, 0xff712c16, 0xffac3708, 0xffe5851b,
+    0xffcc5609, 0xffdf730e, 0xfff9a116, 0xffe9840d, 0xffdd6806, 0xffdf6707,
+    0xffcf5205, 0xffd35902, 0xffe77008, 0xffe97d0b, 0xffe47408, 0xffe26f06,
+    0xfff7990f, 0xffe97b07, 0xffe36f0a, 0xffe06b09, 0xffcd4e04, 0xffbd4005,
+    0xffb03506, 0xffac3005, 0xffae3103, 0xffbb3d08, 0xffa82d03, 0xffbd4909,
+    0xffc15009, 0xffa92c04, 0xffa82c04, 0xffa72c02, 0xffa82e06, 0xffad360d,
+    0xffad3911, 0xffaa2d07, 0xff9e310c, 0xff342828, 0xff2d292d, 0xff2e292d,
+    0xff2d292c, 0xff2d292c, 0xff2c292d, 0xff2a2e33, 0xff2b3039, 0xff185d7c,
+    0xff206f92, 0xff325a6d, 0xff377e98, 0xf380abac, 0xe6a59f70, 0x83645838,
+    0x16151110, 0x84635a33, 0xecaf9d49, 0xffb7a63c, 0xdfa49a42, 0x96706a34,
+    0xdd9d954b, 0xffaea450, 0xffada352, 0xb9807a44, 0x8c65603c, 0xd68b8547,
+    0xff999043, 0xff978e3c, 0xf08d853b, 0x6a3a3613, 0x180b0903, 0x97565641,
+    0xf9747048, 0xff706a39, 0xff115f80, 0xff253642, 0xff292f36, 0xff2f2b2f,
+    0xff2e2a2c, 0xff31292a, 0xff732d16, 0xff552a1d, 0xff41292a, 0xff782d13,
+    0xffca5910, 0xffd15809, 0xffc7560a, 0xfffcac1e, 0xfff8a114, 0xffe57b0a,
+    0xffe8790a, 0xffd35b04, 0xffd96306, 0xfff18708, 0xffe77505, 0xffe16f07,
+    0xffe37106, 0xfffa9c0d, 0xffe87705, 0xffe47409, 0xffdf6908, 0xffce4e05,
+    0xffbd4003, 0xffb53904, 0xffaf3405, 0xffa92f07, 0xffb73907, 0xffb13704,
+    0xffb13904, 0xffbd4907, 0xffa82c05, 0xffa82c04, 0xffa72d05, 0xffa92b03,
+    0xffa92e06, 0xffac360e, 0xffaa3008, 0xffaa2e07, 0xff9e320e, 0xff302728,
+    0xff302b2f, 0xff2d2a2e, 0xff2e292d, 0xff2d282c, 0xff2e282c, 0xff2b2e35,
+    0xff293239, 0xff1d6280, 0xff227196, 0xff2a6a84, 0xff2e7792, 0xff84aba1,
+    0xbb858058, 0x7861573b, 0xbf8b823c, 0xfdb5a736, 0xffb5a734, 0xc28f893f,
+    0x916e6931, 0xf4aca23e, 0xffafa347, 0xffada659, 0xa5757247, 0x976e6a44,
+    0xed9d944b, 0xff9a9142, 0xff978e3b, 0xe988823b, 0x59332d13, 0x14080501,
+    0xb3676951, 0xff726f44, 0xff706a3a, 0xfe716a3a, 0xff145d7b, 0xff25323c,
+    0xff2d2d33, 0xff2f2b2e, 0xff2e282d, 0xff31282a, 0xff7c2d16, 0xff422923,
+    0xff542c21, 0xff8b4016, 0xffd45d0d, 0xffac3303, 0xfff19519, 0xfff8a311,
+    0xffea850c, 0xffec850f, 0xffe47407, 0xffe9810d, 0xfffc9b09, 0xfff18805,
+    0xffdf6906, 0xffe47208, 0xfff8950d, 0xffe17203, 0xffe3720a, 0xffe67409,
+    0xffcc4e05, 0xffbd3f02, 0xffbd3f07, 0xffb63906, 0xffb03507, 0xffab2f03,
+    0xffb33504, 0xffb53b05, 0xffb73f06, 0xffac3406, 0xffad3203, 0xffa92e04,
+    0xffa72c04, 0xffaa2d04, 0xffa92f06, 0xffab310a, 0xffa92e04, 0xffab3006,
+    0xff9b2f0d, 0xff2d2829, 0xff2f292d, 0xff2d292d, 0xff302a2e, 0xff2d282c,
+    0xff2e282c, 0xff2d2e34, 0xff2b353d, 0xff256888, 0xff207293, 0xff287494,
+    0xff2b7790, 0xe275a1a2, 0xab7d7d51, 0xf7b0a53c, 0xffb7a52f, 0xfbb4a637,
+    0xaa7d7635, 0x9d756f2e, 0xffb4a837, 0xffb2a535, 0xffafa654, 0x74534d26,
+    0xb07e794d, 0xffa59b4c, 0xff998f41, 0xff958d3a, 0xe0827b38, 0x4426220e,
+    0x1b0c0a07, 0xba65644a, 0xff6f6a3d, 0xfe6f6a39, 0xfe6e6a3b, 0xff6e6b39,
+    0xff195773, 0xff273138, 0xff2d2c31, 0xff2e292d, 0xff2d292d, 0xff362828,
+    0xff7e2d15, 0xff3a2728, 0xff62311c, 0xff9c501a, 0xffb84709, 0xffc75f1f,
+    0xfff79d13, 0xffed8c0f, 0xffeb850f, 0xffea7f0a, 0xffee870a, 0xffffa10c,
+    0xfff6920b, 0xffe26f05, 0xffeb7e0b, 0xfff08d10, 0xffe57506, 0xffe87605,
+    0xffe46f07, 0xffcd4e04, 0xffc04004, 0xffbf4307, 0xffbc400a, 0xffbf4309,
+    0xffaa3002, 0xffb53808, 0xffb33707, 0xffb94005, 0xffb23905, 0xffb13906,
+    0xffb53c05, 0xffa62c04, 0xffa82d03, 0xffa82e05, 0xffab3007, 0xffaa2e06,
+    0xffa92e06, 0xffac2f03, 0xff923010, 0xff2b282b, 0xff2f292c, 0xff2d292d,
+    0xff2f2a2e, 0xff2d282c, 0xff2d292c, 0xff2b3036, 0xff273641, 0xff256b8c,
+    0xff1d6e93, 0xff227292, 0xff368098, 0xfe88b398, 0xffb4a838, 0xffb3a72f,
+    0xeda99f3a, 0xa2787236, 0xb8867f31, 0xffb5a631, 0xfeb3a52b, 0xffb1a53e,
+    0xffaea453, 0xefa19952, 0xffa19646, 0xff958d3c, 0xfe938b3a, 0xff958d38,
+    0xc66f692d, 0x53282617, 0xbe5d5c41, 0xff706b3d, 0xfe7b7639, 0xff77713b,
+    0xff726b39, 0xff857c3c, 0xff1c5069, 0xff28343c, 0xff2d2d32, 0xff2d2a2d,
+    0xff30282d, 0xff3e2825, 0xff762b17, 0xff352925, 0xff6a3321, 0xff984112,
+    0xffb7440f, 0xffd56f18, 0xfff19010, 0xffee8b10, 0xffec850e, 0xfff08f0c,
+    0xfff4950d, 0xfff39110, 0xffeb7f0c, 0xffef8309, 0xffea800f, 0xffe47407,
+    0xffea7b09, 0xffe06805, 0xffd05307, 0xffc04304, 0xffc44604, 0xffc34709,
+    0xffc94f11, 0xffaf3404, 0xffaf3204, 0xffb83c07, 0xffaf3404, 0xffbf4507,
+    0xffb23804, 0xffbc4607, 0xffa82d04, 0xffaa2f04, 0xffaa2e05, 0xffaa2f06,
+    0xffa92e05, 0xffa82d04, 0xffa92f06, 0xffa82d03, 0xff923618, 0xff29272a,
+    0xff2d292c, 0xff2d292d, 0xff2d282c, 0xff2e282c, 0xff2d292b, 0xff2b343d,
+    0xff263945, 0xff1d698a, 0xff1d6e91, 0xff1d6e91, 0xfe408183, 0xff87ac85,
+    0xffb9a43a, 0xfdb7ac49, 0x96726c35, 0xd0958d35, 0xffb4a62e, 0xffb4a729,
+    0xfeb2a530, 0xfeafa346, 0xfea69c4a, 0xff9b9240, 0xff958d3b, 0xfe948b3a,
+    0xff958c3b, 0xff948c3a, 0xfc8c833a, 0xf56e6b3e, 0xff75703c, 0xff88803a,
+    0xff938c3c, 0xff8b843b, 0xff8c833b, 0xff958c3c, 0xff1b495e, 0xff293843,
+    0xff2f2c30, 0xff2e2a2e, 0xff30282c, 0xff4a2a22, 0xff6b2b19, 0xff362629,
+    0xff6f301c, 0xff98310a, 0xffc9540d, 0xffed9014, 0xfff19614, 0xffee8e14,
+    0xfff89a10, 0xffef930f, 0xfff79e16, 0xfff38e12, 0xffef8108, 0xffe57205,
+    0xffe37007, 0xffe87c0c, 0xffdf6706, 0xffd15808, 0xffbf4702, 0xffcd4d08,
+    0xffc44605, 0xffcf550a, 0xffb23504, 0xffb43906, 0xffbe4408, 0xffbc4105,
+    0xffbe4408, 0xffb74006, 0xffbd490a, 0xffab3304, 0xffa92f03, 0xffac3106,
+    0xffab3007, 0xffaa2f06, 0xffa72c03, 0xffa82e05, 0xffa92d04, 0xffad3109,
+    0xff8a3214, 0xff29282b, 0xff32282b, 0xff2e2a2f, 0xff2e292c, 0xff2d282c,
+    0xff2d282c, 0xff2c3741, 0xff233b48, 0xff1b688a, 0xff1d6e8f, 0xff196c8e,
+    0xff528b8b, 0xff8cab82, 0xffb8a632, 0xfdb4a833, 0xeeab9e38, 0xfdb3a632,
+    0xe1a29b40, 0xf3ada13b, 0xffb2a536, 0xfeada34d, 0xffa19949, 0xff968e3d,
+    0xff958d3c, 0xff958d3c, 0xfe978f3e, 0xff988f3d, 0xff958b3d, 0xff89823c,
+    0xff978d43, 0xfe9c913f, 0xff958d3b, 0xff948c3c, 0xff948c3b, 0xff958d3b,
+    0xff204253, 0xff293e4c, 0xff2e2b2f, 0xff2e292e, 0xff2f2a2b, 0xff582b1f,
+    0xff612f22, 0xff392728, 0xff6e2d18, 0xffa63a13, 0xffdc781a, 0xfff7a316,
+    0xfff79e17, 0xfffaa214, 0xfff19911, 0xfff49910, 0xfff59a13, 0xffeb7e0a,
+    0xffe77504, 0xffe57607, 0xffec7e0f, 0xffdc6205, 0xffd56005, 0xffca5001,
+    0xffd75f0a, 0xffc54905, 0xffd0540a, 0xffb53804, 0xffbd3f06, 0xffc74f08,
+    0xffc45006, 0xffc34b09, 0xffb93e02, 0xffcc5b10, 0xffb53f0d, 0xffa93002,
+    0xffb94008, 0xffac3204, 0xffac3105, 0xffa72c03, 0xffa92e05, 0xffa92d05,
+    0xffa82c03, 0xffb0350d, 0xff8d2f10, 0xff2c2729, 0xff3d2c2c, 0xff2f2c2e,
+    0xff332b2e, 0xff2d292c, 0xff2d292c, 0xff293640, 0xff223f4e, 0xff1d6c8e,
+    0xff206f91, 0xff186b8c, 0xff60938a, 0xfe91a96a, 0xffb5a728, 0xffb5a72b,
+    0xf7b0a134, 0xae7e7639, 0x5b43412d, 0xba868342, 0xffb2a538, 0xffb0a449,
+    0xffac9f4e, 0xff9c9443, 0xff968e3d, 0xff9e9544, 0xfeab9f4d, 0xec978d45,
+    0xe6918a43, 0xffac9f50, 0xffaa9e4c, 0xff9a9040, 0xff948b3c, 0xff958d3c,
+    0xff978d3d, 0xff998d3e, 0xff243847, 0xff254250, 0xff2e2b2e, 0xff2f292d,
+    0xff312929, 0xff623025, 0xff553027, 0xff412925, 0xff632b1a, 0xffb44117,
+    0xffdf7b12, 0xffed9012, 0xfff8a21b, 0xfff6a012, 0xffef8e0e, 0xfff09212,
+    0xfff08b10, 0xffe87402, 0xffe97905, 0xffe8790f, 0xffd96002, 0xffe06d05,
+    0xffd45c05, 0xffda6405, 0xffd05506, 0xffd2550b, 0xffbf4005, 0xffca4e08,
+    0xffc74c09, 0xffd66108, 0xffcc580b, 0xffba4004, 0xffce5906, 0xffc2500b,
+    0xffa92e01, 0xffc6530f, 0xffb63d06, 0xffb4390c, 0xffaa2f04, 0xffa82d04,
+    0xffa92e05, 0xffa92d04, 0xffab3006, 0xffad3507, 0xff9a300f, 0xff332728,
+    0xff46302d, 0xff302c2d, 0xff31282b, 0xff2d2a2d, 0xff2c292e, 0xff2a3944,
+    0xff224252, 0xff1e6e91, 0xff227094, 0xff186a8b, 0xfe6d9e89, 0xff9ba958,
+    0xffb6a72b, 0xeca99e3b, 0x8c636134, 0x543e3b2b, 0x8f666238, 0xeba49b4a,
+    0xffafa44a, 0xfeaea34b, 0xfeaea34e, 0xffaba04e, 0xffa69c4a, 0xfaaa9f4f,
+    0xbd7f773a, 0x482c260d, 0x5138341c, 0xf8aba55c, 0xffa59b4a, 0xff978e3e,
+    0xff958d3b, 0xff968c3d, 0xff9e9042, 0xffad9c50, 0xff243642, 0xff214150,
+    0xff2e2a2c, 0xff2d2a2e, 0xff31292a, 0xff56342d, 0xff452c25, 0xff422b28,
+    0xff642e1a, 0xffae3309, 0xffd76a09, 0xffec8e13, 0xfffaa318, 0xffee8d0e,
+    0xffec8a0f, 0xfff39111, 0xffe47208, 0xffe47306, 0xffe36c07, 0xffda5c04,
+    0xffeb7d07, 0xffda6504, 0xffe77606, 0xffe26e05, 0xffcf5a07, 0xffc44908,
+    0xffd35207, 0xffcd4e09, 0xffd05307, 0xffd96609, 0xffc34c08, 0xffcb5306,
+    0xffca5808, 0xffb33b04, 0xffc65009, 0xffca5c14, 0xffaf3507, 0xffb93f0c,
+    0xffa82c03, 0xffa92e05, 0xffa82d03, 0xffa92e04, 0xffb53707, 0xffa92f06,
+    0xff9c2e0a, 0xff352626, 0xff513631, 0xff303c44, 0xff30282a, 0xff2d2b2f,
+    0xff2c2a30, 0xff283e4b, 0xff214657, 0xff196c8e, 0xff1f7092, 0xff1c6c8a,
+    0xff78a892, 0xffa5a94d, 0xcd938a38, 0x5e444027, 0x58413f2d, 0xb7817e48,
+    0xf6a9a04f, 0xffb0a346, 0xfeb0a43d, 0xfeb2a631, 0xffb2a540, 0xffada450,
+    0xeda29851, 0x86595328, 0x29171506, 0x452f2c1e, 0xad757140, 0xfdaea356,
+    0xffaca14f, 0xff9e9443, 0xff99903f, 0xffa9994b, 0xffb9a459, 0xffbca65d,
+    0xff243946, 0xff224456, 0xff2d282b, 0xff2d292d, 0xff362f31, 0xff41312f,
+    0xff3c2b29, 0xff3c2b2a, 0xff732a15, 0xffbc4004, 0xffea880f, 0xfff2991b,
+    0xffec910f, 0xffea850e, 0xffe9820d, 0xffe26d09, 0xffdf690b, 0xffde650d,
+    0xffda610b, 0xffe8750a, 0xffdf6b05, 0xffeb7c09, 0xffed8109, 0xffcc5602,
+    0xffd76713, 0xffda5b05, 0xffd35606, 0xffcf5109, 0xffd55a07, 0xffce5b06,
+    0xffd46009, 0xffc75105, 0xffbc4805, 0xffc44b05, 0xffe0720e, 0xffae3803,
+    0xffc0450d, 0xffaa2e03, 0xffaa2f06, 0xffa92e05, 0xffa82d02, 0xffb13607,
+    0xffb33805, 0xffae3006, 0xff9a2e0c, 0xff332625, 0xff523731, 0xff2b4a5b,
+    0xff31282a, 0xff2b2c33, 0xff2b2c30, 0xff283f4c, 0xff1f495e, 0xff16678b,
+    0xff1a6e93, 0xfe2b7286, 0xfc77a9a2, 0xaf777740, 0x43312d1d, 0x563c3b25,
+    0xd2938c49, 0xfeb0a344, 0xffb3a538, 0xffb2a62f, 0xfeb3a52b, 0xffb7a736,
+    0xffb9a64e, 0xcc8e844b, 0x4e322e15, 0x25161309, 0x7d565436, 0xe29a9452,
+    0xfdada253, 0xffaea351, 0xfeada24a, 0xffb5a351, 0xffbba35b, 0xffbea75e,
+    0xffae9c56, 0xff847b43, 0xff233b48, 0xff21465b, 0xff2c282a, 0xff2e292b,
+    0xff332c31, 0xff332828, 0xff372a29, 0xff362829, 0xff882e11, 0xffce5f0b,
+    0xffe98d15, 0xfff29719, 0xffe8850d, 0xffea810f, 0xffe57511, 0xffde670e,
+    0xffdb620b, 0xffda6c1a, 0xffd8600c, 0xffe26c08, 0xffe97d08, 0xffed8a0d,
+    0xffd96702, 0xffe78612, 0xffe8790a, 0xffdc6208, 0xffd1570c, 0xffd45406,
+    0xffc95106, 0xffd9690c, 0xffca5406, 0xffc04a05, 0xffc74f04, 0xffdc6809,
+    0xffc85809, 0xffb13505, 0xffbf410a, 0xffa82c04, 0xffaa3006, 0xffa82c03,
+    0xffac3205, 0xffb63a05, 0xffb63906, 0xffae3103, 0xff952e0f, 0xff312727,
+    0xff4e3834, 0xff294556, 0xff302b2b, 0xff2a3138, 0xff2a2a30, 0xff284351,
+    0xff1c4d64, 0xff13678c, 0xff156d94, 0xff407e7d, 0xec6da6af, 0x291c1b12,
+    0x6c4c482e, 0xeea39d58, 0xffafa444, 0xfeb3a62d, 0xfeb4a62c, 0xffb8a637,
+    0xfebaa549, 0xf3b39f59, 0xa1756b44, 0x2c1c170c, 0x3a262212, 0xb0797546,
+    0xfca8a056, 0xff988f4b, 0xfe867e42, 0xffaa9e40, 0xffb5a730, 0xffb7a63c,
+    0xffb3a04b, 0xff938846, 0xff736e3d, 0xff706b3c, 0xff263844, 0xff1e475c,
+    0xff2d282c, 0xff2e292c, 0xff312a2e, 0xff30292b, 0xff382a2a, 0xff332728,
+    0xffa14017, 0xffe07713, 0xffe78d17, 0xffea890f, 0xffec8711, 0xffe0700a,
+    0xffe17015, 0xffd65d09, 0xffdc6b20, 0xffd1590e, 0xffde6307, 0xffe67809,
+    0xfff0900e, 0xffe57707, 0xffef8d10, 0xffef8b0a, 0xffe06a05, 0xffe1730e,
+    0xffd35d0e, 0xffce5204, 0xffe27b1a, 0xffce5d09, 0xffc24c03, 0xffcf5806,
+    0xffd55d05, 0xffdd740a, 0xffb43e06, 0xffbe4009, 0xffae3204, 0xffac3206,
+    0xffa92f04, 0xffaf3305, 0xffb23905, 0xffbc3e07, 0xffb93c06, 0xffac2f04,
+    0xff8e2e10, 0xff312627, 0xff513a37, 0xff293f4f, 0xff322b2e, 0xff2b2b2f,
+    0xff2a2b2f, 0xff24495a, 0xff1d4e68, 0xff136688, 0xff15678a, 0xff4f8b87,
+    0xeb7aa7a3, 0x956d6641, 0xf3a79d55, 0xffb0a454, 0xfeb4a450, 0xffb9a644,
+    0xffbca64c, 0xffbea75a, 0xf5b39f5c, 0x7b595136, 0x25141109, 0x683f3c1f,
+    0xd8908b4c, 0xffa0984f, 0xff867f42, 0xfe6f6b39, 0xff797137, 0xffa2972f,
+    0xffb7a92c, 0xffa3982e, 0xff7a7336, 0xff6d6937, 0xff7b743d, 0xff928a47,
+    0xff26323d, 0xff20465a, 0xff2d292c, 0xff2d2a2d, 0xff30292b, 0xff30292a,
+    0xff392928, 0xff3a2825, 0xffc05a17, 0xffde7b11, 0xffe07710, 0xffe98310,
+    0xffe27408, 0xffee8b19, 0xffd95f09, 0xffd15e1b, 0xffd05711, 0xffd95a08,
+    0xffe37106, 0xfff18e0d, 0xffe87b08, 0xffe67d0b, 0xfff49814, 0xffe47705,
+    0xffe57409, 0xffe97d07, 0xffd35b04, 0xffeb7c0e, 0xffdc7115, 0xffc24b02,
+    0xffd05d05, 0xffd35705, 0xffe67e0d, 0xffc95b08, 0xffb33805, 0xffb83c05,
+    0xffaf3404, 0xffb23707, 0xffa82c04, 0xffb94206, 0xffb63b05, 0xffc14409,
+    0xffb63b05, 0xffaa2f04, 0xff8d3214, 0xff362725, 0xff523d3b, 0xff283d49,
+    0xff302e32, 0xff2c292d, 0xff2b2d32, 0xff24495d, 0xff1e4b5d, 0xff185e80,
+    0xff185873, 0xff5e9799, 0xfa95a784, 0xfbbba658, 0xffb3a356, 0xff9d904e,
+    0xffb59f58, 0xffbfa75b, 0xffbca55c, 0xffad9b4e, 0xf89c9148, 0xa96d693f,
+    0xaf6a6635, 0xf38f8741, 0xff8b8344, 0xff78723d, 0xfe6f683a, 0xff877f36,
+    0xffa69a30, 0xffb5a42e, 0xffb6a82b, 0xffa39937, 0xff7c7437, 0xff938a35,
+    0xffab9e33, 0xffb0a54d, 0xff283038, 0xff234658, 0xff2d292b, 0xff2c2a2d,
+    0xff31292b, 0xff33282b, 0xff3b2827, 0xff442822, 0xffd46b15, 0xffd2670b,
+    0xffe0710c, 0xffde6b06, 0xfff59d1b, 0xffe2710f, 0xffcb520e, 0xffcf5513,
+    0xffd35406, 0xffe16c08, 0xffed8008, 0xffe67b08, 0xffe77708, 0xffef8910,
+    0xffe77f0a, 0xffe6780a, 0xffeb8310, 0xffe77707, 0xffe5730c, 0xffe57c12,
+    0xffc74f03, 0xffd05d05, 0xffd35804, 0xffe4770a, 0xffda7409, 0xffbf490a,
+    0xffb83a06, 0xffb23504, 0xffbb3f08, 0xffad3004, 0xffac3105, 0xffb23904,
+    0xffc84a07, 0xffbe4206, 0xffb43a08, 0xffab2c05, 0xff933818, 0xff372523,
+    0xff503f3f, 0xff283c47, 0xff2d3036, 0xff2d292c, 0xff2a2e34, 0xff24485b,
+    0xff1d495c, 0xff1d546d, 0xff21566d, 0xfe65a1a9, 0xffada971, 0xffb39d57,
+    0xfe877b44, 0xfe706b3a, 0xff8c7e46, 0xffc1a95f, 0xffb39f52, 0xfe998f3f,
+    0xfe958b3c, 0xfd958d3f, 0xfe8c833e, 0xff78713a, 0xff706b3b, 0xfe7a713f,
+    0xffa29353, 0xffb5a449, 0xffb6a72f, 0xffb3a72f, 0xffb2a43c, 0xffb2a537,
+    0xffafa32f, 0xffb4a72a, 0xffb4a739, 0xffa79c4c
+};
diff --git a/pixman/demos/parrot.jpg b/pixman/demos/parrot.jpg
new file mode 100644
index 000000000..e7727f3b4
--- /dev/null
+++ b/pixman/demos/parrot.jpg
diff --git a/pixman/demos/quad2quad.c b/pixman/demos/quad2quad.c
new file mode 100644
index 000000000..66b838fc0
--- /dev/null
+++ b/pixman/demos/quad2quad.c
@@ -0,0 +1,2183 @@
+#include <math.h>
+#include <stdio.h>
+#include <pixman.h>
+
+/* This code is basically the output of Maxima translated into C.
+ *
+ * See http://maxima.sourceforge.net/
+ */
+static void
+quad_to_quad (double x0, double y0,
+	      double x1, double y1,
+	      double x2, double y2,
+	      double x3, double y3,
+
+	      double px0, double py0,
+	      double px1, double py1,
+	      double px2, double py2,
+	      double px3, double py3,
+
+	      struct pixman_f_transform *trans)
+{
+    double
+	t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13, t14, t15, t16, t17, t18,
+	t19, t20, t21, t22, t23, t24, t25, t26, t27, t28, t29, t30, t31, t32, t33, t34,
+	t35, t36, t37, t38, t39, t40, t41, t42, t43, t44, t45, t46, t47, t48, t49, t50,
+	t51, t52, t53, t54, t55, t56, t57, t58, t59, t60, t61, t62, t63, t64, t65, t66,
+	t67, t68, t69, t70, t71, t72, t73, t74, t75, t76, t77, t78, t79, t80, t81, t82,
+	t83, t84, t85, t86, t87, t88, t89, t90, t91, t92, t93, t94, t95, t96, t97, t98,
+	t99, t100, t101, t102, t103, t104, t105, t106, t107, t108, t109, t110, t111,
+	t112, t113, t114, t115, t116, t117, t118, t119, t120, t121, t122, t123,
+	t124, t125, t126, t127, t128, t129, t130, t131, t132, t133, t134, t135,
+	t136, t137, t138, t139, t140, t141, t142, t143, t144, t145, t146, t147,
+	t148, t149, t150, t151, t152, t153, t154, t155, t156, t157, t158, t159,
+	t160, t161, t162, t163, t164, t165, t166, t167, t168, t169, t170, t171,
+	t172, t173, t174, t175, t176, t177, t178, t179, t180, t181, t182, t183,
+	t184, t185, t186, t187, t188, t189, t190, t191, t192, t193, t194, t195,
+	t196, t197, t198, t199, t200, t201, t202, t203, t204, t205, t206, t207,
+	t208, t209, t210, t211, t212, t213, t214, t215, t216, t217, t218, t219,
+	t220, t221, t222, t223, t224, t225, t226, t227, t228, t229, t230, t231,
+	t232, t233, t234, t235, t236, t237, t238, t239, t240, t241, t242, t243,
+	t244, t245, t246, t247, t248, t249, t250, t251, t252, t253, t254, t255,
+	t256, t257, t258, t259, t260, t261, t262, t263, t264, t265, t266, t267,
+	t268, t269, t270, t271, t272, t273, t274, t275, t276, t277, t278, t279,
+	t280, t281, t282, t283, t284, t285, t286, t287, t288, t289, t290, t291,
+	t292, t293, t294, t295, t296, t297, t298, t299, t300, t301, t302, t303,
+	t304, t305, t306, t307, t308, t309, t310, t311, t312, t313, t314, t315,
+	t316, t317, t318, t319, t320, t321, t322, t323, t324, t325, t326, t327,
+	t328, t329, t330, t331, t332, t333, t334, t335, t336, t337, t338, t339,
+	t340, t341, t342, t343, t344, t345, t346, t347, t348, t349, t350, t351,
+	t352, t353, t354, t355, t356, t357, t358, t359, t360, t361, t362, t363,
+	t364, t365, t366, t367, t368, t369, t370, t371, t372, t373, t374, t375,
+	t376, t377, t378, t379, t380, t381, t382, t383, t384, t385, t386, t387,
+	t388, t389, t390, t391, t392, t393, t394, t395, t396, t397, t398, t399,
+	t400, t401, t402, t403, t404, t405, t406, t407, t408, t409, t410, t411,
+	t412, t413, t414, t415, t416, t417, t418, t419, t420, t421, t422, t423,
+	t424, t425, t426, t427, t428, t429, t430, t431, t432, t433, t434, t435,
+	t436, t437, t438, t439, t440, t441, t442, t443, t444, t445, t446, t447,
+	t448, t449, t450, t451, t452, t453, t454, t455, t456, t457, t458, t459,
+	t460, t461, t462, t463, t464, t465, t466, t467, t468, t469, t470, t471,
+	t472, t473, t474, t475, t476, t477, t478, t479, t480, t481, t482, t483,
+	t484, t485, t486, t487, t488, t489, t490, t491, t492, t493, t494, t495,
+	t496, t497, t498, t499, t500, t501, t502, t503, t504, t505, t506, t507,
+	t508, t509, t510, t511, t512, t513, t514, t515, t516, t517, t518, t519,
+	t520, t521, t522, t523, t524, t525, t526, t527, t528, t529, t530, t531,
+	t532, t533, t534, t535, t536, t537, t538, t539, t540, t541, t542, t543,
+	t544, t545, t546, t547, t548, t549, t550, t551, t552, t553, t554, t555,
+	t556, t557, t558, t559, t560, t561, t562, t563, t564, t565, t566, t567,
+	t568, t569, t570, t571, t572, t573, t574, t575, t576, t577, t578, t579,
+	t580, t581, t582, t583, t584, t585, t586, t587, t588, t589, t590, t591,
+	t592, t593, t594, t595, t596, t597, t598, t599, t600, t601, t602, t603,
+	t604, t605, t606, t607, t608, t609, t610, t611, t612, t613, t614, t615,
+	t616, t617, t618, t619, t620, t621, t622, t623, t624, t625, t626, t627,
+	t628, t629, t630, t631, t632, t633, t634, t635, t636, t637, t638, t639,
+	t640, t641, t642, t643, t644, t645, t646, t647, t648, t649, t650, t651,
+	t652, t653, t654, t655, t656, t657, t658, t659, t660, t661, t662, t663,
+	t664, t665, t666, t667, t668, t669, t670, t671, t672, t673, t674, t675,
+	t676, t677, t678, t679, t680, t681, t682, t683, t684, t685, t686, t687,
+	t688, t689, t690, t691, t692, t693, t694, t695, t696, t697, t698, t699,
+	t700, t701, t702, t703, t704, t705, t706, t707, t708, t709, t710, t711,
+	t712, t713, t714, t715, t716, t717, t718, t719, t720, t721, t722, t723,
+	t724, t725, t726, t727, t728, t729, t730, t731, t732, t733, t734, t735,
+	t736, t737, t738, t739, t740, t741, t742, t743, t744, t745, t746, t747,
+	t748, t749, t750, t751, t752, t753, t754, t755, t756, t757, t758, t759,
+	t760, t761, t762, t763, t764, t765, t766, t767, t768, t769, t770, t771,
+	t772, t773, t774, t775, t776, t777, t778, t779, t780, t781, t782, t783,
+	t784, t785, t786, t787, t788, t789, t790, t791, t792, t793, t794, t795,
+	t796, t797, t798, t799, t800, t801, t802, t803, t804, t805, t806, t807,
+	t808, t809, t810, t811, t812, t813, t814, t815, t816, t817, t818, t819,
+	t820, t821, t822, t823, t824, t825, t826, t827, t828, t829, t830, t831,
+	t832, t833, t834, t835, t836, t837, t838, t839, t840, t841, t842, t843,
+	t844, t845, t846, t847, t848, t849, t850, t851, t852, t853, t854, t855,
+	t856, t857, t858, t859, t860, t861, t862, t863, t864, t865, t866, t867,
+	t868, t869, t870, t871, t872, t873, t874, t875, t876, t877, t878, t879,
+	t880, t881, t882, t883, t884, t885, t886, t887, t888, t889, t890, t891,
+	t892, t893, t894, t895, t896, t897, t898, t899, t900, t901, t902, t903,
+	t904, t905, t906, t907, t908, t909, t910, t911, t912, t913, t914, t915,
+	t916, t917, t918, t919, t920, t921, t922, t923, t924, t925, t926, t927,
+	t928, t929, t930, t931, t932, t933, t934, t935, t936, t937, t938, t939,
+	t940, t941, t942, t943, t944, t945, t946, t947, t948, t949, t950, t951,
+	t952, t953, t954, t955, t956, t957, t958, t959, t960, t961, t962, t963,
+	t964, t965, t966, t967, t968, t969, t970, t971, t972, t973, t974, t975,
+	t976, t977, t978, t979, t980, t981, t982, t983, t984, t985, t986, t987,
+	t988, t989, t990, t991, t992, t993, t994, t995, t996, t997, t998, t999,
+	t1000, t1001, t1002, t1003, t1004, t1005, t1006, t1007, t1008, t1009,
+	t1010, t1011, t1012, t1013, t1014, t1015, t1016, t1017, t1018, t1019,
+	t1020, t1021, t1022, t1023, t1024, t1025, t1026, t1027, t1028, t1029,
+	t1030, t1031, t1032, t1033, t1034, t1035, t1036, t1037, t1038, t1039,
+	t1040, t1041, t1042, t1043, t1044, t1045, t1046, t1047, t1048, t1049,
+	t1050, t1051, t1052, t1053, t1054, t1055, t1056, t1057, t1058, t1059,
+	t1060, t1061, t1062, t1063, t1064, t1065, t1066, t1067, t1068, t1069,
+	t1070, t1071, t1072, t1073;
+
+    t1 = y1 * y1;
+    t2 = x3 * x3;
+    t3 = px2 * px3 * t2;
+    t4 = (t3 - px2 * px3 * x2 * x3) * y2;
+    t5 = x2 * x2;
+    t6 = px2 * px3 * t5 * y3;
+
+    t7 = - px2 * px3 * x2 * x3 * y3;
+    t8 = py1 * (t7 + t6 + t4);
+    t9 = px3 * py2 * x2 * x3;
+
+    t10 = - px3 * py2 * t2;
+    t11 = (t10 + t9) * y2;
+    t12 = - px2 * py3 * t5 * y3;
+
+    t13 = px2 * py3 * x2 * x3 * y3;
+    t14 = y0 * y0;
+    t15 = - px3 * py2;
+    t16 = px2 * py3;
+
+    t17 = t16 + t15;
+    t18 = t17 * x2;
+    t19 = px3 * py2 * x3;
+    t20 = - px2 * py3 * x3;
+
+    t21 = t20 + t19 + t18;
+    t22 = px2 * px3 * t5;
+    t23 = - 2 * px2 * px3 * x2 * x3;
+
+    t24 = py1 * (t3 + t23 + t22);
+    t25 = - px2 * py3 * t5;
+    t26 = px2 * py3 * x3;
+
+    t27 = x2 * (t26 + t19);
+    t28 = t10 + t27 + t25;
+    t29 = x1 * x1;
+    t30 = px3 * py2;
+
+    t31 = - px2 * py3;
+    t32 = t31 + t30;
+    t33 = t32 * y2;
+    t34 = - px3 * py2 * y3;
+
+    t35 = px2 * py3 * y3;
+    t36 = t35 + t34 + t33;
+    t37 = - px2 * px3 * t2;
+
+    t38 = (t37 + px2 * px3 * x2 * x3) * y2;
+    t39 = - px2 * px3 * t5 * y3;
+
+    t40 = px2 * px3 * x2 * x3 * y3;
+    t41 = py1 * (t40 + t39 + t38);
+    t42 = - px2 * py3 * x2 * x3;
+
+    t43 = px3 * py2 * t2;
+    t44 = (t43 + t42) * y2;
+    t45 = px2 * py3 * t5 * y3;
+
+    t46 = - px3 * py2 * x2 * x3 * y3;
+    t47 = (px2 * px3 * x3 - px2 * px3 * x2) * y2;
+
+    t48 = px2 * px3 * x2 * y3;
+    t49 = - px2 * px3 * x3 * y3;
+    t50 = py1 * (t49 + t48 + t47);
+
+    t51 = px2 * py3 * x2;
+    t52 = - 2 * px3 * py2 * x3;
+    t53 = (t26 + t52 + t51) * y2;
+
+    t54 = px3 * py2 * x3 * y3;
+    t55 = px3 * py2 * y3;
+    t56 = - 2 * px2 * py3 * y3;
+    t57 = t56 + t55;
+
+    t58 = x2 * t57;
+    t59 = - px2 * px3 * t5;
+    t60 = 2 * px2 * px3 * x2 * x3;
+    t61 = - px2;
+
+    t62 = px3 + t61;
+    t63 = t62 * x2;
+    t64 = px2 * x3;
+    t65 = - px3 * x3;
+    t66 = t65 + t64 + t63;
+
+    t67 = px2 * t5;
+    t68 = - px2 * x3;
+    t69 = x2 * (t65 + t68);
+    t70 = px3 * t2;
+
+    t71 = t70 + t69 + t67;
+    t72 = - px3;
+    t73 = t72 + px2;
+    t74 = - px2 * y3;
+    t75 = px3 * y3;
+
+    t76 = t75 + t74 + t73 * y2;
+    t77 = px2 * x2 * x3;
+    t78 = - px3 * t2;
+    t79 = - px2 * t5 * y3;
+
+    t80 = px3 * x2 * x3 * y3;
+    t81 = t80 + t79 + (t78 + t77) * y2;
+
+    t82 = (px2 * px3 * x2 - px2 * px3 * x3) * y2;
+    t83 = - px2 * px3 * x2 * y3;
+
+    t84 = px2 * px3 * x3 * y3;
+    t85 = - px2 * x2;
+    t86 = 2 * px3 * x3;
+    t87 = - px3 * x3 * y3;
+
+    t88 = 2 * px2 * y3;
+    t89 = - px3 * y3;
+    t90 = t89 + t88;
+    t91 = x2 * t90;
+
+    t92 = t91 + t87 + (t86 + t68 + t85) * y2;
+    t93 = px2 * py3 * t5;
+    t94 = - px3 * py2 * x3;
+
+    t95 = x2 * (t20 + t94);
+    t96 = t32 * x2;
+    t97 = t73 * x2;
+    t98 = px3 * x3;
+
+    t99 = t98 + t68 + t97;
+    t100 = py1 * t99;
+    t101 = - px2 * t5;
+    t102 = x2 * (t98 + t64);
+
+    t103 = t78 + t102 + t101;
+    t104 = py1 * t103;
+    t105 = - py2;
+    t106 = py3 + t105;
+
+    t107 = py2 * y3;
+    t108 = - py3 * y3;
+    t109 = t108 + t107 + t106 * y2;
+    t110 = - px3 * x2 * x3;
+
+    t111 = px2 * t5 * y3;
+    t112 = - px2 * x2 * x3 * y3;
+    t113 = t112 + t111 + (t70 + t110) * y2;
+
+    t114 = - py2 * x3;
+    t115 = py3 * x3;
+    t116 = t115 + t114;
+    t117 = py2 * x3 * y3;
+
+    t118 = - py3 * x3 * y3;
+    t119 = t118 + t117;
+    t120 = x2 * t119;
+
+    t121 = px1 * (t120 + x2 * t116 * y2);
+    t122 = - px3 * py2 * x2;
+    t123 = (t19 + t122) * y2;
+
+    t124 = px2 * py3 * x2 * y3;
+    t125 = - px2 * py3 * x3 * y3;
+    t126 = px3 * x2;
+
+    t127 = - px2 * x2 * y3;
+    t128 = px2 * x3 * y3;
+    t129 = t128 + t127 + (t65 + t126) * y2;
+
+    t130 = - py3;
+    t131 = t130 + py2;
+    t132 = t131 * x2;
+    t133 = py2 * x3;
+    t134 = - py3 * x3;
+
+    t135 = - py2 * x3 * y3;
+    t136 = py3 * x3 * y3;
+    t137 = - py2 * y3;
+    t138 = py3 * y3;
+
+    t139 = t138 + t137;
+    t140 = x2 * t139;
+
+    t141 = px1 * (t140 + t136 + t135 + (t134 + t133 + t132) * y2);
+    t142 = y2 * y2;
+
+    t143 = - px3 * py2 * x3 * y3;
+    t144 = px2 * py3 * x3 * y3;
+    t145 = t144 + t143;
+
+    t146 = t142 * t145;
+    t147 = y3 * y3;
+    t148 = px3 * py2 * t147;
+    t149 = - px2 * py3 * t147;
+
+    t150 = t149 + t148;
+    t151 = x2 * y2 * t150;
+    t152 = t151 + t146;
+    t153 = - px2 * py3 * y3;
+
+    t154 = t153 + t55;
+    t155 = t142 * t154;
+    t156 = - px3 * py2 * t147;
+
+    t157 = px2 * py3 * t147;
+    t158 = t157 + t156;
+    t159 = y2 * t158;
+    t160 = t159 + t155;
+
+    t161 = x0 * x0;
+    t162 = py1 * t76;
+    t163 = px1 * t109;
+    t164 = px2 * y3;
+    t165 = t89 + t164;
+
+    t166 = - px2 * t147;
+    t167 = px3 * t147;
+    t168 = t167 + t166;
+
+    t169 = y2 * t168 + t142 * t165;
+    t170 = py1 * t169;
+    t171 = py2 * t147;
+
+    t172 = - py3 * t147;
+    t173 = t172 + t171;
+    t174 = y2 * t173 + t142 * t139;
+
+    t175 = px1 * t174;
+    t176 = t17 * t142;
+    t177 = px2 * t147;
+    t178 = - px3 * t147;
+
+    t179 = t178 + t177 + t62 * t142;
+    t180 = - py2 * t147;
+    t181 = py3 * t147;
+
+    t182 = t181 + t180 + t131 * t142;
+
+    t183 = y1 * (px1 * t182 + py1 * t179 + t149 + t148 + t176)
+	+ t175 + t170 + t159 + t1 * (t163 + t162 + t35 + t34 + t33) + t155;
+
+    t184 = - px2 * px3 * t2 * t142;
+    t185 = 2 * px2 * px3 * x2 * x3 * y2 * y3;
+
+    t186 = - px2 * px3 * t5 * t147;
+    t187 = py1 * (t186 + t185 + t184);
+
+    t188 = px3 * py2 * t2 * t142;
+    t189 = x2 * y2 * (t125 + t143);
+    t190 = px2 * py3 * t5 * t147;
+
+    t191 = t190 + t189 + t188;
+    t192 = px2 * px3 * x3 * t142;
+    t193 = y2 * (t49 + t83);
+
+    t194 = px2 * px3 * x2 * t147;
+    t195 = py1 * (t194 + t193 + t192);
+
+    t196 = - px3 * py2 * x3 * t142;
+    t197 = 2 * px3 * py2 * x3 * y3;
+    t198 = 2 * px2 * py3 * y3;
+
+    t199 = t198 + t34;
+    t200 = x2 * t199;
+    t201 = y2 * (t200 + t125 + t197);
+
+    t202 = - px2 * py3 * x2 * t147;
+    t203 = - px2 * x3 * y3;
+    t204 = px3 * x3 * y3;
+
+    t205 = t204 + t203;
+    t206 = t142 * t205;
+    t207 = t178 + t177;
+    t208 = x2 * y2 * t207;
+
+    t209 = t208 + t206;
+    t210 = px2 * px3 * t2 * t142;
+    t211 = - 2 * px2 * px3 * x2 * x3 * y2 * y3;
+
+    t212 = px2 * px3 * t5 * t147;
+    t213 = - px3 * t2 * t142;
+    t214 = x2 * y2 * (t204 + t128);
+
+    t215 = - px2 * t5 * t147;
+    t216 = t215 + t214 + t213;
+    t217 = - px2 * px3 * x3 * t142;
+
+    t218 = y2 * (t84 + t48);
+    t219 = - px2 * px3 * x2 * t147;
+    t220 = px3 * x3 * t142;
+
+    t221 = - 2 * px3 * x3 * y3;
+    t222 = - 2 * px2 * y3;
+    t223 = t75 + t222;
+    t224 = x2 * t223;
+
+    t225 = y2 * (t224 + t221 + t128);
+    t226 = px2 * x2 * t147;
+    t227 = t226 + t225 + t220;
+
+    t228 = t125 + t54;
+    t229 = t142 * t228;
+    t230 = x2 * y2 * t158;
+    t231 = t87 + t128;
+
+    t232 = t142 * t231;
+    t233 = x2 * y2 * t168;
+    t234 = t233 + t232;
+    t235 = py1 * t234;
+
+    t236 = - px3 * py2 * t2 * t142;
+    t237 = x2 * y2 * (t144 + t54);
+
+    t238 = - px2 * py3 * t5 * t147;
+    t239 = px3 * t2 * t142;
+    t240 = x2 * y2 * (t87 + t203);
+
+    t241 = px2 * t5 * t147;
+    t242 = t241 + t240 + t239;
+    t243 = py1 * t242;
+
+    t244 = px2 * py3 * x3 * t142;
+    t245 = - px2 * py3 * x2 * y3;
+    t246 = y2 * (t143 + t245);
+
+    t247 = px3 * py2 * x2 * t147;
+    t248 = - px2 * x3 * t142;
+    t249 = px2 * x2 * y3;
+
+    t250 = y2 * (t204 + t249);
+    t251 = - px3 * x2 * t147;
+    t252 = t251 + t250 + t248;
+
+    t253 = t134 + t133;
+    t254 = t253 * t142;
+    t255 = t108 + t107;
+    t256 = x2 * t255;
+
+    t257 = t256 + t136 + t135;
+    t258 = y2 * t257;
+    t259 = t181 + t180;
+    t260 = x2 * t259;
+
+    t261 = px1 * (t260 + t258 + t254);
+    t262 = py1 * (t37 + t60 + t59);
+
+    t263 = t43 + t95 + t93;
+    t264 = px1 * t263;
+    t265 = t26 + t94;
+    t266 = x2 * t265 * y2;
+
+    t267 = x2 * t228;
+    t268 = t267 + t266;
+    t269 = py1 * (t84 + t83 + t82);
+
+    t270 = - 2 * px2 * py3;
+    t271 = (t26 + (t270 + t30) * x2) * y2;
+    t272 = px3 * py2 * x2 * y3;
+
+    t273 = - 2 * px3 * py2 * x3 * y3;
+    t274 = t149 + t148 + t176;
+
+    t275 = py1 * (t212 + t211 + t210);
+    t276 = t238 + t237 + t236;
+    t277 = px1 * t276;
+
+    t278 = py1 * (t219 + t218 + t217);
+    t279 = 2 * px3 * py2 * x3;
+    t280 = t20 + t279;
+
+    t281 = t280 * t142;
+    t282 = - px3 * py2 * x2 * y3;
+    t283 = y2 * (t125 + t282);
+
+    t284 = 2 * px2 * py3 * t147;
+    t285 = x2 * (t284 + t156);
+    t286 = px1 * t103;
+
+    t287 = t98 + t68;
+    t288 = x2 * t287 * y2;
+    t289 = x2 * t231;
+    t290 = t289 + t288;
+
+    t291 = 2 * px2;
+    t292 = - px3 * x2 * y3;
+    t293 = 2 * px3 * x3 * y3;
+
+    t294 = t293 + t203 + t292 + (t68 + (t72 + t291) * x2) * y2;
+    t295 = px1 * t242;
+
+    t296 = - 2 * px3 * x3;
+    t297 = t296 + t64;
+    t298 = px3 * x2 * y3;
+    t299 = y2 * (t128 + t298);
+
+    t300 = - 2 * px2 * t147;
+    t301 = x2 * (t167 + t300) + t299 + t297 * t142;
+    t302 = py1 * t71;
+
+    t303 = py1 * t290;
+    t304 = 2 * py2 * x3;
+    t305 = - 2 * py3 * x3;
+    t306 = - 2 * py2 * x3 * y3;
+
+    t307 = 2 * py3 * x3 * y3;
+    t308 = t307 + t306;
+    t309 = - 2 * px2 * py3 * x3;
+
+    t310 = (t309 + t19 + t51) * y2;
+    t311 = - 2 * px3 * py2 * y3;
+    t312 = t35 + t311;
+
+    t313 = x2 * t312;
+    t314 = 2 * px2 * x3;
+    t315 = 2 * px3 * y3;
+    t316 = t315 + t74;
+
+    t317 = x2 * t316;
+    t318 = t317 + t87 + (t65 + t314 + t85) * y2;
+    t319 = t106 * x2;
+
+    t320 = px1 * (t256 + t118 + t117 + (t115 + t114 + t319) * y2);
+    t321 = py1 * t216;
+
+    t322 = 2 * px2 * py3 * x3 * y3;
+    t323 = 2 * px3 * py2 * y3;
+    t324 = t153 + t323;
+
+    t325 = x2 * t324;
+    t326 = y2 * (t325 + t322 + t143);
+    t327 = - 2 * px2 * x3 * y3;
+
+    t328 = - 2 * px3 * y3;
+    t329 = t328 + t164;
+    t330 = x2 * t329;
+
+    t331 = y2 * (t330 + t204 + t327);
+    t332 = t226 + t331 + t220;
+    t333 = t116 * t142;
+
+    t334 = t140 + t118 + t117;
+    t335 = y2 * t334;
+    t336 = x2 * t173;
+
+    t337 = px1 * (t336 + t335 + t333);
+    t338 = t26 + t94 + t96;
+    t339 = t17 * y2;
+
+    t340 = t153 + t55 + t339;
+    t341 = px2 * px3 * t142;
+    t342 = - 2 * px2 * px3 * y2 * y3;
+
+    t343 = px2 * px3 * t147;
+    t344 = py1 * (t343 + t342 + t341);
+    t345 = - px2 * py3 * t142;
+
+    t346 = y2 * (t35 + t55);
+    t347 = t156 + t346 + t345;
+    t348 = px1 * t347 + t344;
+
+    t349 = t89 + t164 + t62 * y2;
+    t350 = - px2 * px3 * t142;
+    t351 = 2 * px2 * px3 * y2 * y3;
+
+    t352 = - px2 * px3 * t147;
+    t353 = px2 * t142;
+    t354 = y2 * (t89 + t74);
+
+    t355 = t167 + t354 + t353;
+    t356 = px1 * t355 + t352 + t351 + t350;
+    t357 = py1 * t66;
+
+    t358 = py1 * t349;
+    t359 = 2 * py2;
+    t360 = - 2 * py3;
+    t361 = - 2 * py2 * y3;
+
+    t362 = 2 * py3 * y3;
+    t363 = px3 * py2 * t142;
+    t364 = y2 * (t153 + t34);
+
+    t365 = - px3 * t142;
+    t366 = y2 * (t75 + t164);
+    t367 = t166 + t366 + t365;
+
+    t368 = py1 * t367;
+    t369 = px1 * (t172 + t171 + t106 * t142);
+    t370 = t35 + t34;
+
+    t371 = t142 * t370;
+    t372 = y2 * t150;
+    t373 = t372 + t371;
+    t374 = t230 + t229;
+
+    t375 = py1 * (t352 + t351 + t350);
+    t376 = t157 + t364 + t363;
+    t377 = px1 * t376 + t375;
+
+    t378 = t75 + t74;
+    t379 = y2 * t207 + t142 * t378;
+    t380 = px1 * t367 + t343 + t342 + t341;
+
+    t381 = py1 * t209;
+    t382 = py1 * t355;
+    t383 = py1 * t379;
+    t384 = 2 * py2 * y3;
+
+    t385 = - 2 * py3 * y3;
+    t386 = t385 + t384;
+    t387 = - 2 * py2 * t147;
+    t388 = 2 * py3 * t147;
+
+    t389 = px2 * py3 * t2;
+    t390 = t389 + t10;
+    t391 = x2 * t390 * y2;
+    t392 = t5 * t228;
+
+    t393 = - px2 * t2;
+    t394 = t70 + t393;
+    t395 = x2 * t394 * y2;
+    t396 = t5 * t231;
+
+    t397 = t396 + t395;
+    t398 = py1 * t397;
+    t399 = py2 * t2;
+    t400 = - py3 * t2;
+
+    t401 = t400 + t399;
+    t402 = x2 * t401 * y2;
+    t403 = t136 + t135;
+    t404 = t5 * t403;
+
+    t405 = t404 + t402;
+    t406 = px1 * t405;
+    t407 = t1 * (t406 + t398 + t392 + t391);
+
+    t408 = t65 + t64;
+    t409 = t5 * t408;
+    t410 = x2 * t394;
+    t411 = t410 + t409;
+
+    t412 = py1 * t411;
+    t413 = t5 * t116;
+    t414 = x2 * t401;
+    t415 = t414 + t413;
+
+    t416 = px1 * t415;
+    t417 = py2 * t5;
+    t418 = x2 * (t134 + t114);
+    t419 = py3 * t2;
+
+    t420 = t419 + t418 + t417;
+    t421 = px1 * t420;
+    t422 = t265 * y2;
+    t423 = x2 * t154;
+
+    t424 = px2 * x2;
+    t425 = (t68 + t424) * y2;
+    t426 = - py2 * x2;
+    t427 = (t133 + t426) * y2;
+
+    t428 = py3 * x2 * y3;
+    t429 = t20 + t19;
+    t430 = x2 * t429;
+    t431 = - px2 * py3 * t2;
+
+    t432 = (t431 + t43 + t430) * y2;
+    t433 = t5 * t370;
+    t434 = x2 * t145;
+
+    t435 = - px2 * x2 * x3;
+    t436 = px2 * t2;
+    t437 = (t436 + t435) * y2;
+    t438 = px3 * t5 * y3;
+
+    t439 = - px3 * x2 * x3 * y3;
+    t440 = py2 * x2 * x3;
+    t441 = - py2 * t2;
+
+    t442 = (t441 + t440) * y2;
+    t443 = - py3 * t5 * y3;
+    t444 = py3 * x2 * x3 * y3;
+
+    t445 = t5 * t287;
+    t446 = t78 + t436;
+    t447 = x2 * t446;
+    t448 = - t2;
+
+    t449 = t448 + 2 * x2 * x3 - t5;
+    t450 = px1 * t449;
+    t451 = (t98 + t85) * y2;
+    t452 = - x2 * y3;
+
+    t453 = x3 * y3;
+    t454 = t453 + t452 + (x2 - x3) * y2;
+    t455 = px1 * t454;
+    t456 = t65 + t314;
+
+    t457 = x2 * t456;
+    t458 = (t78 + t457) * y2;
+    t459 = x2 * (t293 + t203);
+
+    t460 = - x2 * x3 * y3 + t5 * y3 + (t2 - x2 * x3) * y2;
+    t461 = px1 * t460;
+    t462 = t5 * t253;
+
+    t463 = t419 + t441;
+    t464 = x2 * t463;
+    t465 = - py2 * t5;
+    t466 = x2 * (t115 + t133);
+
+    t467 = t2 - 2 * x2 * x3 + t5;
+    t468 = py1 * t467;
+    t469 = py2 * x2;
+    t470 = (t134 + t469) * y2;
+
+    t471 = - py2 * x2 * y3;
+    t472 = x2 * y3;
+    t473 = - x3 * y3;
+    t474 = t473 + t472 + (x3 - x2) * y2;
+
+    t475 = py1 * t474;
+    t476 = - 2 * py2 * x3;
+    t477 = t115 + t476;
+    t478 = x2 * t477;
+
+    t479 = (t419 + t478) * y2;
+    t480 = py2 * t5 * y3;
+    t481 = - 2 * py3 * x3 * y3;
+
+    t482 = x2 * (t481 + t117);
+    t483 = x2 * x3 * y3 - t5 * y3 + (t448 + x2 * x3) * y2;
+
+    t484 = py1 * t483;
+    t485 = t431 + t43;
+    t486 = t485 * t142;
+    t487 = t5 * t158;
+
+    t488 = t446 * t142;
+    t489 = t5 * t168;
+    t490 = t489 + t488;
+    t491 = py1 * t490;
+
+    t492 = t463 * t142;
+    t493 = t5 * t173;
+    t494 = t493 + t492;
+    t495 = px1 * t494;
+
+    t496 = x1 * y1 * (t495 + t491 + t487 + t486);
+    t497 = t142 * t119;
+    t498 = x2 * y2 * t259;
+
+    t499 = t498 + t497;
+    t500 = px1 * t499;
+    t501 = t29 * (t500 + t381 + t151 + t146);
+
+    t502 = t429 * t142;
+    t503 = x2 * t370;
+    t504 = y2 * (t503 + t125 + t54);
+    t505 = x2 * t158;
+
+    t506 = - px3 * x3 * t142;
+    t507 = - px2 * x2 * t147;
+    t508 = py3 * x3 * t142;
+
+    t509 = y2 * (t118 + t471);
+    t510 = py2 * x2 * t147;
+    t511 = - py2 * t142;
+
+    t512 = y2 * (t138 + t107);
+    t513 = t172 + t512 + t511;
+    t514 = px1 * t513;
+
+    t515 = y2 * t259 + t142 * t255;
+    t516 = px1 * t515;
+    t517 = py1 * t454;
+
+    t518 = - py2 * x3 * t142;
+    t519 = t108 + t384;
+    t520 = x2 * t519;
+
+    t521 = y2 * (t520 + t307 + t135);
+    t522 = - py3 * x2 * t147;
+    t523 = py2 * t142;
+
+    t524 = y2 * (t108 + t137);
+    t525 = - t147 + 2 * y2 * y3 - t142;
+    t526 = py1 * t525;
+
+    t527 = x2 * t147 + y2 * (t473 + t452) + x3 * t142;
+    t528 = py1 * t527;
+    t529 = px1 * t474;
+
+    t530 = px2 * x3 * t142;
+    t531 = px3 * x2 * t147;
+
+    t532 = - x2 * t147 + y2 * (t453 + t472) - x3 * t142;
+    t533 = px1 * t532;
+
+    t534 = - px2 * t142;
+    t535 = t147 - 2 * y2 * y3 + t142;
+    t536 = px1 * t535;
+
+    t537 = t447 + t445;
+    t538 = py1 * t537;
+    t539 = t464 + t462;
+    t540 = px1 * t539;
+
+    t541 = 2 * px3 * py2 * t2;
+    t542 = - 2 * px2 * py3 * t2;
+    t543 = x2 * t446 * y2;
+
+    t544 = t5 * t205;
+    t545 = t544 + t543;
+    t546 = py1 * t545;
+    t547 = x2 * t463 * y2;
+
+    t548 = t5 * t119;
+    t549 = t548 + t547;
+    t550 = px1 * t549;
+    t551 = x2 * t265;
+
+    t552 = (t389 + t10 + t551) * y2;
+    t553 = t5 * t154;
+    t554 = 2 * px3 * t2;
+
+    t555 = (t554 + t393 + t110) * y2;
+    t556 = t5 * t90;
+    t557 = py3 * x2 * x3;
+
+    t558 = - 2 * py3 * t2;
+    t559 = (t558 + t399 + t557) * y2;
+    t560 = py2 * x2 * x3 * y3;
+
+    t561 = t138 + t361;
+    t562 = t5 * t561;
+    t563 = t390 * t142;
+    t564 = t5 * t150;
+
+    t565 = - px2 * t2 * t142;
+    t566 = - px3 * t5 * t147;
+    t567 = t566 + t214 + t565;
+
+    t568 = py1 * t567;
+    t569 = py2 * t2 * t142;
+    t570 = x2 * y2 * (t118 + t135);
+
+    t571 = py3 * t5 * t147;
+    t572 = t571 + t570 + t569;
+    t573 = px1 * t572;
+    t574 = t86 + t68;
+
+    t575 = x2 * t574;
+    t576 = (t78 + t575) * y2;
+    t577 = 2 * px2 * x3 * y3;
+
+    t578 = x2 * (t87 + t577);
+    t579 = px1 * t527;
+
+    t580 = - t5 * t147 + 2 * x2 * x3 * y2 * y3 - t2 * t142;
+    t581 = px1 * t580;
+    t582 = t305 + t133;
+
+    t583 = x2 * t582;
+    t584 = (t419 + t583) * y2;
+    t585 = x2 * (t136 + t306);
+
+    t586 = py1 * t532;
+    t587 = - py3 * t2 * t142;
+    t588 = x2 * y2 * (t136 + t117);
+
+    t589 = - py2 * t5 * t147;
+    t590 = t5 * t147 - 2 * x2 * x3 * y2 * y3 + t2 * t142;
+
+    t591 = py1 * t590;
+    t592 = t400 + t466 + t465;
+    t593 = px1 * t592;
+    t594 = t309 + t279;
+
+    t595 = t198 + t311;
+    t596 = x2 * t378;
+    t597 = t596 + t408 * y2;
+    t598 = py1 * t597;
+
+    t599 = t256 + t116 * y2;
+    t600 = px1 * t599;
+    t601 = t178 + t366 + t534;
+
+    t602 = py1 * t601;
+    t603 = t181 + t524 + t523;
+    t604 = px1 * t603;
+    t605 = t265 * t142;
+
+    t606 = t423 + t144 + t143;
+    t607 = y2 * t606;
+    t608 = x2 * t150;
+    t609 = 2 * py2 * x3 * y3;
+
+    t610 = t362 + t137;
+    t611 = x2 * t610;
+    t612 = y2 * (t611 + t118 + t609);
+
+    t613 = py1 * t449;
+    t614 = t419 + t613 + t418 + t417;
+    t615 = py1 * t460;
+
+    t616 = py1 * t535;
+    t617 = t616 + t172 + t512 + t511;
+    t618 = t134 + t304;
+
+    t619 = t618 * t142;
+    t620 = - py3 * x2 * y3;
+    t621 = y2 * (t135 + t620);
+
+    t622 = x2 * (t388 + t180);
+    t623 = px1 * t467;
+    t624 = t623 + t78 + t102 + t101;
+
+    t625 = px1 * t483;
+    t626 = px1 * t525;
+    t627 = t167 + t626 + t354 + t353;
+
+    t628 = - 2 * px2 * x3;
+    t629 = t98 + t628;
+    t630 = t629 * t142;
+    t631 = - 2 * px3 * t147;
+
+    t632 = x2 * (t631 + t177);
+    t633 = - 2 * px2 * py3 * x3 * y3;
+    t634 = t633 + t197;
+
+    t635 = - 2 * px3 * py2 * t147;
+    t636 = t142 * t403;
+    t637 = x2 * y2 * t173;
+
+    t638 = t637 + t636;
+    t639 = px1 * t638;
+    t640 = t589 + t588 + t587;
+    t641 = px1 * t640;
+
+    t642 = px1 * t590;
+    t643 = py1 * t580;
+
+    t644 = (x0 * (px0 * (y1 * (x1 * (t528 + t522 + t612 + t518)
+			       + t643 + t571 + t570 + t569)
+			 + t29 * t515 + x1 * t638 + t1 * (t615 + t444 + t443 + t442))
+		  + py0 * (y1 * (x1 * (t533 + t531 + t331 + t530)
+				 + t642 + t566 + t214 + t565)
+			   + x1 * t234 + t29 * t379 + t1 * (t625 + t439 + t438 + t437))
+		  + y1 * (x1 * (px1 * (t622 + t621 + t619) + py1 * (t632 + t299 + t630)
+				+ t608 + t607 + t605)
+			  + t641 + t243 + t564 + t563)
+		  + x1 * (t639 + t235 + x2 * y2 * (t284 + t635) + t142 * t634)
+		  + t29 * (t175 + t170)
+		  + t1 * (px1 * (t482 + t480 + t479) + py1 * (t459 + t79 + t458) + t434
+			  + t433 + t432))
+	    + y0 * (x0 * (py0 * (x1 * (t579 + t632 + t299 + t630)
+				 + t489 + t29 * t627
+				 + y1 * (x1 * t597 + t625 + t556 + t112 + t555) + t488
+				 + t624 * t1)
+			  + px0 * (x1 * (t586 + t622 + t621 + t619)
+				   + t29 * t617 + t493
+				   + y1 * (x1 * t599 + t615 + t562 + t560 + t559) + t492
+				   + t614 * t1)
+			  + x1 * (px1 * (t522 + t612 + t518) + py1 * (t531 + t331 + t530)
+				  + t608 + t607 + t605)
+			  + t29 * (t604 + t602) + t487
+			  + y1 * (x1 * (t600 + t598 + x2 * t595 + t594 * y2)
+				  + px1 * (t585 + t480 + t584) + py1 * (t578 + t79 + t576) + t267
+				  + t553 + t552) + t486 + (t593 + t302) * t1)
+		    + px0 * (x1 * (t591 + t589 + t588 + t587)
+			     + t29 * (t586 + t510 + t509 + t508)
+			     + y1 * (x1 * (t484 + t585 + t480 + t584) + t548 + t547) + t415 * t1)
+		    + py0 * (x1 * (t581 + t241 + t240 + t239)
+			     + t29 * (t579 + t507 + t250 + t506)
+			     + y1 * (x1 * (t461 + t578 + t79 + t576) + t544 + t543) + t411 * t1)
+		    + x1 * (t573 + t568 + t564 + t563)
+		    + t29 * (px1 * (t522 + t521 + t518) + py1 * (t531 + t225 + t530) + t505
+			     + t504 + t502)
+		    + y1 * (x1 * (px1 * (t562 + t560 + t559) + py1 * (t556 + t112 + t555)
+				  + t267 + t553 + t552)
+			    + t550 + t546 + t5 * (t322 + t273) + x2 * (t542 + t541) * y2)
+		    + (t540 + t538) * t1)
+	    + t161 * (py0 * (y1 * (x1 * (t536 + t178 + t366 + t534)
+				   + t533 + t531 + t225 + t530)
+			     + x1 * t169 + t208 + t1 * (t529 + t204 + t292 + t425) + t206)
+		      + px0 * (y1 * (t528 + x1 * (t181 + t526 + t524 + t523) + t522 + t521
+				     + t518)
+			       + x1 * t174 + t498 + t1 * (t517 + t118 + t428 + t427) + t497)
+		      + x1 * (t516 + t383)
+		      + y1 * (x1 * (t514 + t382) + px1 * (t510 + t509 + t508)
+			      + py1 * (t507 + t250 + t506) + t505 + t504
+			      + t502) + t151
+		      + t1 * (px1 * (t136 + t471 + t470) + py1 * (t87 + t249 + t451) + t423
+			      + t422) + t146) + t501 + t496
+	    + t14 * (px0 * (x1 * (t484 + t482 + t480 + t479)
+			    + t29 * (t475 + t136 + t471 + t470) + t404 + t402
+			    + (x1 * (t468 + t400 + t466 + t465) + t464 + t462) * y1)
+		     + py0 * (x1 * (t461 + t459 + t79 + t458)
+			      + t29 * (t455 + t87 + t249 + t451) + t396 + t395
+			      + (x1 * (t70 + t450 + t69 + t67) + t447 + t445) * y1)
+		     + x1 * (px1 * (t444 + t443 + t442) + py1 * (t439 + t438 + t437) + t434
+			     + t433 + t432)
+		     + t29 * (px1 * (t118 + t428 + t427) + py1 * (t204 + t292 + t425) + t423
+			      + t422) + t392 + t391
+		     + (x1 * (t421 + t104) + t416 + t412) * y1) + t407);
+    t645 = t5 * t265;
+
+    t646 = t115 + t114 + t132;
+    t647 = px1 * t646;
+    t648 = x2 * t485;
+    t649 = t32 * t5;
+
+    t650 = t70 + t393 + t73 * t5;
+    t651 = t400 + t399 + t106 * t5;
+
+    t652 = t540 + x1 * (px1 * t651 + py1 * t650 + t389 + t10 + t649) + t538 + t648
+	+ t29 * (t647 + t357 + t20 + t19 + t18) + t645;
+    t653 = t648 + t645;
+
+    t654 = t392 + t391;
+    t655 = px1 * t654;
+    t656 = t309 + t19;
+    t657 = x2 * t656;
+
+    t658 = (t389 + t657) * y2;
+    t659 = px3 * py2 * t5 * y3;
+    t660 = x2 * (t144 + t273);
+
+    t661 = - px3 * py2 * t5;
+    t662 = t431 + t27 + t661;
+    t663 = px1 * t662 + t24;
+
+    t664 = t5 * t429;
+    t665 = x2 * t390;
+    t666 = t665 + t664;
+    t667 = px3 * py2 * x2;
+
+    t668 = (t20 + t667) * y2;
+    t669 = x2 * t485 * y2;
+    t670 = t5 * t145;
+    t671 = t670 + t669;
+
+    t672 = px1 * t671;
+    t673 = t26 + t52;
+    t674 = x2 * t673;
+    t675 = (t389 + t674) * y2;
+
+    t676 = x2 * (t633 + t54);
+    t677 = px3 * t5;
+    t678 = t436 + t69 + t677;
+
+    t679 = px1 * t678 + t37 + t60 + t59;
+    t680 = - px3 * x2;
+
+    t681 = t203 + t298 + (t64 + t680) * y2;
+    t682 = px1 * t545;
+    t683 = - px3 * t5 * y3;
+
+    t684 = t578 + t683 + (t393 + t575) * y2;
+    t685 = 2 * py3 * x3;
+    t686 = t685 + t476;
+
+    t687 = 2 * py2 * t2;
+    t688 = px1 * (t419 + t441 + t131 * t5);
+    t689 = - px2 * py3 * x2;
+
+    t690 = 2 * px2 * py3 * x3;
+    t691 = (t690 + t94 + t689) * y2;
+
+    t692 = t330 + t204 + (t98 + t628 + t424) * y2;
+    t693 = t134 + t133 + t319;
+
+    t694 = px1 * (t140 + t118 + t117 + t693 * y2);
+    t695 = (t542 + t43 + t9) * y2;
+
+    t696 = t5 * t312;
+    t697 = 2 * px2 * t2;
+    t698 = t5 * t316 + t112 + (t78 + t697 + t110) * y2;
+
+    t699 = x2 * t253;
+    t700 = t5 * t255;
+    t701 = x2 * t403;
+
+    t702 = px1 * (t701 + t700 + (t419 + t441 + t699) * y2);
+    t703 = px2 * py3 * x2 * x3;
+
+    t704 = (t10 + t703) * y2;
+    t705 = px3 * py2 * x2 * x3 * y3;
+    t706 = (t20 + t279 + t689) * y2;
+
+    t707 = t439 + t111 + (t70 + t435) * y2;
+    t708 = t224 + t204 + (t296 + t64 + t424) * y2;
+
+    t709 = - 2 * py2;
+    t710 = 2 * py3;
+    t711 = py1 * t678;
+
+    t712 = t459 + t683 + (t393 + t457) * y2;
+    t713 = x2 * t116;
+    t714 = t5 * t139;
+
+    t715 = px1 * (t120 + t714 + (t400 + t399 + t713) * y2);
+    t716 = 2 * px2 * py3;
+
+    t717 = (t94 + (t716 + t15) * x2) * y2;
+    t718 = - 2 * px2;
+
+    t719 = t221 + t128 + t249 + (t98 + (px3 + t718) * x2) * y2;
+
+    t720 = px1 * (t256 + t136 + t135 + t646 * y2);
+    t721 = - px2 * py3 * t2 * t142;
+
+    t722 = - px3 * py2 * t5 * t147;
+    t723 = t722 + t237 + t721;
+    t724 = - px2 * py3 * x3 * t142;
+
+    t725 = y2 * (t54 + t124);
+    t726 = px1 * y2 * t257;
+    t727 = - px3 * py2 * x2 * t147;
+
+    t728 = y2 * (t87 + t127);
+    t729 = t531 + t728 + t530;
+    t730 = px2 * py3 * t2 * t142;
+
+    t731 = px3 * py2 * t5 * t147;
+    t732 = px1 * t397;
+    t733 = t251 + t299 + t248;
+
+    t734 = px2 * t2 * t142;
+    t735 = px3 * t5 * t147;
+    t736 = t735 + t240 + t734;
+
+    t737 = t389 + t10 + t649;
+    t738 = t731 + t189 + t730;
+    t739 = px1 * t738;
+
+    t740 = x2 * t165;
+    t741 = t740 + t204 + t203;
+    t742 = py1 * y2 * t741;
+    t743 = py1 * t736;
+
+    t744 = px2 * py3 * t142;
+    t745 = px1 * t567;
+    t746 = t148 + t364 + t744;
+
+    t747 = px3 * py2 * t5;
+    t748 = t389 + t95 + t747;
+    t749 = (t26 + t122) * y2;
+
+    t750 = x2 * t280;
+    t751 = (t431 + t750) * y2;
+    t752 = - px3 * py2 * t5 * y3;
+
+    t753 = x2 * (t322 + t143);
+    t754 = - px3 * t5;
+    t755 = t393 + t102 + t754;
+
+    t756 = t128 + t292 + (t68 + t126) * y2;
+    t757 = x2 * t297;
+    t758 = x2 * (t204 + t327);
+
+    t759 = t758 + t438 + (t436 + t757) * y2;
+    t760 = (t94 + t667) * y2;
+
+    t761 = t203 + t249 + (t98 + t680) * y2;
+    t762 = px1 * (t140 + t253 * y2);
+
+    t763 = - px3 * py2 * x2 * x3;
+    t764 = (t43 + t763) * y2;
+    t765 = - px2 * py3 * x2 * x3 * y3;
+
+    t766 = px3 * x2 * x3;
+    t767 = px2 * x2 * x3 * y3;
+    t768 = t767 + t79 + (t78 + t766) * y2;
+
+    t769 = px1 * (t120 + t700 + (t419 + t441 + t713) * y2);
+    t770 = t501 + t496 + t407;
+
+    t771 = px3 * py2 * x3 * t142;
+    t772 = y2 * (t313 + t633 + t54);
+
+    t773 = px2 * py3 * x2 * t147;
+    t774 = - px3 * py2 * t142;
+    t775 = t149 + t346 + t774;
+
+    t776 = y2 * (t317 + t87 + t577);
+    t777 = t507 + t776 + t506;
+    t778 = px3 * t142;
+
+    t779 = t177 + t354 + t778;
+    t780 = y2 * (t144 + t272);
+    t781 = y2 * (t203 + t292);
+
+    t782 = t531 + t781 + t530;
+    t783 = px1 * (t336 + t258 + t333);
+    t784 = t690 + t94;
+
+    t785 = x2 * t784;
+    t786 = (t431 + t785) * y2;
+    t787 = x2 * (t125 + t197);
+
+    t788 = x2 * t629;
+    t789 = x2 * (t221 + t128);
+    t790 = t789 + t438 + (t436 + t788) * y2;
+
+    t791 = - 2 * py2 * t2;
+    t792 = 2 * py3 * t2;
+    t793 = 2 * px2 * py3 * t2;
+
+    t794 = (t793 + t10 + t42) * y2;
+    t795 = t5 * t324;
+    t796 = - 2 * px2 * t2;
+
+    t797 = t5 * t329 + t80 + (t70 + t796 + t77) * y2;
+
+    t798 = px1 * (t701 + t714 + (t400 + t399 + t699) * y2);
+
+    t799 = px1 * (t5 * t259 + t401 * t142);
+    t800 = t429 * y2;
+    t801 = t503 + t800;
+
+    t802 = t487 + t486;
+    t803 = t673 * t142;
+    t804 = - 2 * px2 * py3 * t147;
+
+    t805 = x2 * (t804 + t148);
+    t806 = 2 * px2 * t147;
+
+    t807 = x2 * (t178 + t806) + t728 + t574 * t142;
+    t808 = py1 * t755;
+    t809 = py1 * t779;
+
+    t810 = y2 * (t58 + t144 + t273);
+    t811 = y2 * (t91 + t293 + t203);
+
+    t812 = t507 + t811 + t506;
+    t813 = px1 * (t260 + t335 + t254);
+    t814 = 2 * py2 * t147;
+
+    t815 = - 2 * py3 * t147;
+    t816 = (t389 + t42) * y2;
+    t817 = - py2 * py3 * t2;
+
+    t818 = (t817 + py2 * py3 * x2 * x3) * y2;
+    t819 = - py2 * py3 * t5 * y3;
+
+    t820 = py2 * py3 * x2 * x3 * y3;
+    t821 = px1 * (t820 + t819 + t818);
+    t822 = - py2 * py3 * t5;
+
+    t823 = 2 * py2 * py3 * x2 * x3;
+    t824 = px1 * (t817 + t823 + t822);
+    t825 = (t431 + t9) * y2;
+
+    t826 = py2 * py3 * t2;
+    t827 = (t826 - py2 * py3 * x2 * x3) * y2;
+    t828 = py2 * py3 * t5 * y3;
+
+    t829 = - py2 * py3 * x2 * x3 * y3;
+    t830 = px1 * (t829 + t828 + t827);
+
+    t831 = (py2 * py3 * x2 - py2 * py3 * x3) * y2;
+    t832 = - py2 * py3 * x2 * y3;
+
+    t833 = py2 * py3 * x3 * y3;
+    t834 = px1 * (t833 + t832 + t831);
+
+    t835 = (t690 + t94 + t122) * y2;
+    t836 = px1 * t693;
+    t837 = - py2 * t5 * y3;
+
+    t838 = t560 + t837 + (t400 + t557) * y2;
+    t839 = x2 * t205;
+
+    t840 = py1 * (t839 + x2 * t408 * y2);
+    t841 = (t20 + t51) * y2;
+    t842 = - py3 * x2;
+
+    t843 = py2 * x2 * y3;
+    t844 = t135 + t843 + (t115 + t842) * y2;
+
+    t845 = py1 * (t740 + t87 + t128 + (t98 + t68 + t63) * y2);
+    t846 = py2 * py3 * t5;
+
+    t847 = - 2 * py2 * py3 * x2 * x3;
+    t848 = - py2 * x2 * x3;
+    t849 = - py3 * x2 * x3 * y3;
+
+    t850 = t849 + t480 + (t419 + t848) * y2;
+    t851 = (py2 * py3 * x3 - py2 * py3 * x2) * y2;
+
+    t852 = py2 * py3 * x2 * y3;
+    t853 = - py2 * py3 * x3 * y3;
+    t854 = x2 * t561;
+
+    t855 = t854 + t136 + (t305 + t133 + t469) * y2;
+    t856 = py2 * py3 * t2 * t142;
+
+    t857 = - 2 * py2 * py3 * x2 * x3 * y2 * y3;
+    t858 = py2 * py3 * t5 * t147;
+
+    t859 = px1 * (t858 + t857 + t856);
+    t860 = - py2 * py3 * x3 * t142;
+
+    t861 = y2 * (t833 + t852);
+    t862 = - py2 * py3 * x2 * t147;
+
+    t863 = px1 * (t862 + t861 + t860);
+    t864 = - py2 * py3 * t2 * t142;
+
+    t865 = 2 * py2 * py3 * x2 * x3 * y2 * y3;
+    t866 = - py2 * py3 * t5 * t147;
+
+    t867 = py3 * t2 * t142;
+    t868 = py2 * t5 * t147;
+    t869 = t868 + t570 + t867;
+
+    t870 = py2 * py3 * x3 * t142;
+    t871 = y2 * (t853 + t832);
+    t872 = py2 * py3 * x2 * t147;
+
+    t873 = - py3 * x3 * t142;
+    t874 = - py2 * x2 * t147;
+    t875 = t874 + t521 + t873;
+
+    t876 = py2 * x3 * t142;
+    t877 = py3 * x2 * t147;
+    t878 = t877 + t509 + t876;
+
+    t879 = t287 * t142;
+    t880 = t596 + t87 + t128;
+    t881 = y2 * t880;
+    t882 = x2 * t207;
+
+    t883 = py1 * (t882 + t881 + t879);
+    t884 = py1 * t662;
+
+    t885 = px1 * (t826 + t847 + t846);
+    t886 = 2 * px3 * py2;
+
+    t887 = (t94 + (t31 + t886) * x2) * y2;
+    t888 = px1 * (t853 + t852 + t851);
+
+    t889 = py1 * t738;
+    t890 = px1 * (t866 + t865 + t864);
+
+    t891 = px1 * (t872 + t871 + t870);
+    t892 = t656 * t142;
+    t893 = x2 * (t157 + t635);
+
+    t894 = t221 + t577;
+    t895 = x2 * t253 * y2;
+    t896 = t701 + t895;
+    t897 = px1 * t896;
+
+    t898 = (t20 + t279 + t122) * y2;
+
+    t899 = py1 * (t596 + t204 + t203 + (t65 + t64 + t97) * y2);
+    t900 = t385 + t107;
+
+    t901 = x2 * t900;
+    t902 = t901 + t136 + (t115 + t476 + t469) * y2;
+    t903 = px1 * t869;
+
+    t904 = t874 + t612 + t873;
+    t905 = t408 * t142;
+    t906 = y2 * t741;
+    t907 = x2 * t168;
+
+    t908 = py1 * (t907 + t906 + t905);
+    t909 = - py2 * py3 * t142;
+
+    t910 = 2 * py2 * py3 * y2 * y3;
+    t911 = - py2 * py3 * t147;
+
+    t912 = px1 * (t911 + t910 + t909);
+    t913 = t912 + py1 * t376;
+
+    t914 = t481 + t117 + t428 + (t133 + (py3 + t709) * x2) * y2;
+    t915 = 2 * px3;
+
+    t916 = t138 + t137 + t131 * y2;
+    t917 = px1 * t916;
+
+    t918 = py1 * (t167 + t166 + t73 * t142);
+    t919 = py3 * t142;
+    t920 = t171 + t524 + t919;
+
+    t921 = px1 * t920;
+    t922 = py2 * py3 * t142;
+    t923 = - 2 * py2 * py3 * y2 * y3;
+
+    t924 = py2 * py3 * t147;
+    t925 = py1 * t513 + t924 + t923 + t922;
+    t926 = py1 * t420;
+
+    t927 = py1 * t640;
+    t928 = t685 + t114;
+    t929 = x2 * (t172 + t814) + t621 + t928 * t142;
+
+    t930 = px1 * (t924 + t923 + t922);
+    t931 = t930 + py1 * t347;
+
+    t932 = py1 * t920 + t911 + t910 + t909;
+    t933 = t315 + t222;
+    t934 = py1 * t654;
+
+    t935 = (t10 + t750) * y2;
+    t936 = t824 + py1 * t263;
+    t937 = py1 * t671;
+
+    t938 = (t19 + t689) * y2;
+    t939 = (t10 + t785) * y2;
+    t940 = t296 + t314;
+
+    t941 = py1 * (t78 + t436 + t62 * t5);
+    t942 = (t26 + t52 + t667) * y2;
+
+    t943 = py1 * (t740 + t204 + t203 + t99 * y2);
+
+    t944 = t611 + t118 + (t134 + t304 + t426) * y2;
+    t945 = (t431 + t541 + t42) * y2;
+
+    t946 = t5 * t199;
+    t947 = t5 * t900 + t560 + (t419 + t791 + t557) * y2;
+    t948 = x2 * t287;
+
+    t949 = t5 * t378;
+    t950 = py1 * (t289 + t949 + (t78 + t436 + t948) * y2);
+
+    t951 = - py3 * t5;
+    t952 = t441 + t466 + t951;
+    t953 = py1 * t952 + t826 + t847 + t846;
+
+    t954 = py3 * x2;
+    t955 = t117 + t620 + (t114 + t954) * y2;
+    t956 = py1 * t549;
+
+    t957 = py3 * t5 * y3;
+    t958 = t585 + t957 + (t399 + t583) * y2;
+    t959 = (t389 + t763) * y2;
+
+    t960 = (t309 + t19 + t667) * y2;
+    t961 = - 2 * px3;
+    t962 = px1 * t952;
+    t963 = x2 * t408;
+
+    t964 = t5 * t165;
+    t965 = py1 * (t839 + t964 + (t70 + t393 + t963) * y2);
+
+    t966 = t482 + t957 + (t399 + t478) * y2;
+    t967 = - 2 * px3 * py2;
+
+    t968 = (t26 + (t16 + t967) * x2) * y2;
+
+    t969 = t307 + t135 + t471 + (t134 + (t130 + t359) * x2) * y2;
+
+    t970 = py1 * (t596 + t87 + t128 + t66 * y2);
+    t971 = t444 + t837 + (t400 + t440) * y2;
+
+    t972 = t520 + t118 + (t685 + t114 + t426) * y2;
+    t973 = py1 * t405;
+
+    t974 = t877 + t621 + t876;
+    t975 = - py2 * t2 * t142;
+    t976 = - py3 * t5 * t147;
+
+    t977 = t976 + t588 + t975;
+    t978 = py1 * y2 * t880;
+    t979 = y2 * (t136 + t843);
+
+    t980 = t522 + t979 + t518;
+    t981 = py1 * t276;
+    t982 = py1 * t572;
+    t983 = px1 * y2 * t334;
+
+    t984 = px1 * t977;
+    t985 = (t94 + t51) * y2;
+    t986 = (t43 + t657) * y2;
+
+    t987 = (t26 + t689) * y2;
+    t988 = t117 + t471 + (t134 + t954) * y2;
+
+    t989 = py1 * (t740 + t287 * y2);
+    t990 = (t431 + t703) * y2;
+    t991 = - py3 * x2 * x3;
+
+    t992 = - py2 * x2 * x3 * y3;
+    t993 = t992 + t480 + (t419 + t991) * y2;
+
+    t994 = py1 * (t839 + t949 + (t78 + t436 + t963) * y2);
+    t995 = py3 * t5;
+
+    t996 = t399 + t418 + t995;
+    t997 = t135 + t428 + (t133 + t842) * y2;
+    t998 = x2 * t928;
+
+    t999 = x2 * (t118 + t609);
+    t1000 = t999 + t443 + (t441 + t998) * y2;
+
+    t1001 = y2 * (t901 + t136 + t306);
+    t1002 = t510 + t1001 + t508;
+    t1003 = - py3 * t142;
+
+    t1004 = t180 + t512 + t1003;
+    t1005 = y2 * (t117 + t428);
+    t1006 = t522 + t1005 + t518;
+
+    t1007 = py1 * (t907 + t881 + t905);
+    t1008 = y2 * (t854 + t481 + t117);
+
+    t1009 = t510 + t1008 + t508;
+    t1010 = 2 * px3 * t147;
+
+    t1011 = py1 * (t5 * t207 + t394 * t142);
+    t1012 = t784 * t142;
+
+    t1013 = 2 * px3 * py2 * t147;
+    t1014 = x2 * (t149 + t1013);
+
+    t1015 = py1 * (t882 + t906 + t879);
+    t1016 = x2 * (t181 + t387) + t979 + t582 * t142;
+
+    t1017 = (t43 + t674) * y2;
+    t1018 = x2 * t618;
+    t1019 = x2 * (t307 + t135);
+
+    t1020 = t1019 + t443 + (t441 + t1018) * y2;
+    t1021 = - 2 * px3 * t2;
+
+    t1022 = - 2 * px3 * py2 * t2;
+    t1023 = (t389 + t1022 + t9) * y2;
+    t1024 = t5 * t57;
+
+    t1025 = t5 * t610 + t849 + (t400 + t687 + t848) * y2;
+
+    t1026 = py1 * (t289 + t964 + (t70 + t393 + t948) * y2);
+    t1027 = px1 * t996;
+
+    t1028 = px1 * t1004;
+    t1029 = x2 * t429 * y2;
+    t1030 = (t436 + t110) * y2;
+
+    t1031 = (t441 + t557) * y2;
+    t1032 = (t393 + t77) * y2;
+    t1033 = (t399 + t848) * y2;
+
+    t1034 = (t26 + t94 + t18) * y2;
+    t1035 = (t64 + t85) * y2;
+    t1036 = (t114 + t469) * y2;
+
+    t1037 = (t98 + t628 + t126) * y2;
+    t1038 = (t134 + t304 + t842) * y2;
+
+    t1039 = (t20 + t19 + t96) * y2;
+    t1040 = (t296 + t64 + t126) * y2;
+
+    t1041 = (t685 + t114 + t842) * y2;
+    t1042 = (t98 + (t961 + px2) * x2) * y2;
+
+    t1043 = t456 * t142;
+    t1044 = x2 * (t1010 + t166);
+
+    t1045 = (t134 + (t710 + t105) * x2) * y2;
+    t1046 = t477 * t142;
+
+    t1047 = x2 * (t815 + t171);
+    t1048 = t32 * t142;
+    t1049 = t171 + t526 + t524 + t919;
+
+    t1050 = t536 + t166 + t366 + t365;
+    t1051 = (t389 + t10 + t430) * y2;
+
+    t1052 = (t393 + t766) * y2;
+    t1053 = (t399 + t991) * y2;
+    t1054 = t17 * t5;
+
+    t1055 = (t431 + t43 + t551) * y2;
+    t1056 = (t1021 + t436 + t77) * y2;
+    t1057 = t5 * t223;
+
+    t1058 = (t792 + t441 + t848) * y2;
+    t1059 = t5 * t519;
+    t1060 = t338 * y2;
+
+    t1061 = (t86 + t68 + t680) * y2;
+    t1062 = (t305 + t133 + t954) * y2;
+
+    t1063 = (t115 + t426) * y2;
+    t1064 = (t400 + t1018) * y2;
+    t1065 = (t65 + t424) * y2;
+
+    t1066 = (t70 + t788) * y2;
+    t1067 = (t70 + t757) * y2;
+    t1068 = (t400 + t998) * y2;
+
+    t1069 = t21 * y2;
+    t1070 = (t68 + (t915 + t61) * x2) * y2;
+
+    t1071 = (t133 + (t360 + py2) * x2) * y2;
+    t1072 = (t115 + t476 + t954) * y2;
+
+    t1073 = (t65 + t314 + t680) * y2;
+
+    trans->m[0][0]
+	= (x0 * (px0 * (x1 * (px1 * (y2 * (t388 + t387) + t142 * t386)
+			      + t383 + t372 + t371)
+			+ y1 * (x1 * (t369 + t382 + t156 + t346 + t345)
+				+ t337 + py1 * t301 + t285 + t283 + t281) + t381 + t151
+			+ t1 * (t141 + py1 * t92 + t58 + t54 + t53) + t146)
+		 + py0 * (y1 * (x1 * t380 + px1 * t332 + t219 + t218 + t217)
+			  + px1 * t234 + px1 * x1 * t379 + t1 * (px1 * t129 + t49 + t48 + t47))
+		 + y1 * (x1 * t377 + px1 * (t202 + t326 + t196) + t195) + px1 * t374
+		 + px1 * x1 * t373 + t1 * (px1 * (t125 + t124 + t123) + t269))
+	   + y0 * (x0 * (px0 * (t261 + x1 * (t369 + t368 + t157 + t364 + t363) + py1 * t227
+				+ t202
+				+ y1
+				* (x1
+				   * (px1 * (t362 + t361 + (t360 + t359) * y2)
+				      + t358 + t153 + t55 + t339)
+				   + t320 + py1 * t294 + t144 + t273 + t272 + t271)
+				+ t201 + t196 + (t357 + t20 + t19 + t18) * t1)
+			 + py0 * (x1 * t356 + px1 * t252 + t194
+				  + y1 * (px1 * t318 + px1 * x1 * t349 + t84 + t83 + t82)
+				  + t193 + t192 + px1 * t99 * t1) + x1 * t348
+			 + px1 * (t247 + t246 + t244) + t278
+			 + y1 * (px1 * (t313 + t54 + t310) + t50 + px1 * x1 * t340)
+			 + px1 * t338 * t1)
+		   + px0 * (x1 * (t337 + py1 * t332 + t202 + t326 + t196)
+			    + t321 + px1 * t29 * t182 + t190
+			    + y1 * (x1 * (t320 + py1 * t318 + t313 + t54 + t310)
+				    + px1 * (x2 * t308 + x2 * (t305 + t304) * y2) + t303 + t267
+				    + t266) + t189 + t188 + (t302 + t10 + t27 + t25) * t1)
+		   + py0 * (x1 * (px1 * t301 + t194 + t193 + t192)
+			    + t295 + px1 * t29 * t179 + t186
+			    + y1 * (x1 * (px1 * t294 + t49 + t48 + t47) + px1 * t290) + t185 + t184
+			    + (t286 + t3 + t23 + t22) * t1)
+		   + x1 * (px1 * (t285 + t283 + t281) + t278) + t277 + t275 + px1 * t29 * t274
+		   + y1 * (x1 * (px1 * (t144 + t273 + t272 + t271) + t269) + px1 * t268)
+		   + (t264 + t262) * t1)
+	   + px0 * (y1 * (x1 * (t261 + py1 * t252 + t247 + t246 + t244)
+			  + t243 + t238 + t237 + t236)
+		    + x1 * (t235 + t230 + t229) + px1 * t29 * t174
+		    + t1 * (t121 + py1 * t81 + t46 + t45 + t44))
+	   + py0 * (y1 * (x1 * (px1 * t227 + t219 + t218 + t217)
+			  + px1 * t216 + t212 + t211 + t210)
+		    + px1 * t29 * t169 + px1 * x1 * t209 + t1 * (px1 * t113 + t40 + t39 + t38))
+	   + y1 * (x1 * (px1 * (t202 + t201 + t196) + t195) + px1 * t191 + t187)
+	   + px0 * t161 * t183 + px1 * t29 * t160 + px1 * x1 * t152
+	   + t14 * (px0 * (x1 * (t141 + py1 * t129 + t125 + t124 + t123)
+			   + t121 + py1 * t113 + px1 * t29 * t109 + t13 + t12 + t11
+			   + (t104 + t43 + x1 * (t100 + t26 + t94 + t96) + t95 + t93) * y1)
+		    + py0 * (x1 * (px1 * t92 + t84 + t83 + t82)
+			     + px1 * t81 + px1 * t29 * t76 + t7 + t6 + t4
+			     + (px1 * t71 + t37 + px1 * x1 * t66 + t60 + t59) * y1)
+		    + x1 * (px1 * (t58 + t54 + t53) + t50) + px1 * (t46 + t45 + t44) + t41
+		    + px1 * t29 * t36 + (px1 * t28 + t24 + px1 * x1 * t21) * y1)
+	   + t1 * (px1 * (t13 + t12 + t11) + t8));
+
+    trans->m[0][1] =
+	(t161 * (px0 * (x1 * (t382 + t156 + t346 + t345)
+			+ py1 * t733 + t247
+			+ y1 * (t694 + x1 * (t358 + t153 + t55 + t339) + py1 * t681
+				+ t144 + t282 + t668) + t726 + t283 + t244
+			+ px1 * t646 * t1)
+		 + py0 * (x1 * (px1 * t601 + t343 + t342 + t341)
+			  + px1 * t729 + t219
+			  + y1 * (px1 * t692 + px1 * x1 * t76 + t49 + t48 + t47) + t218
+			  + t217 + px1 * t66 * t1) + x1 * (px1 * t746 + t375)
+		 + px1 * (t727 + t725 + t724) + t195
+		 + y1 * (px1 * (t325 + t143 + t691) + t269 + px1 * x1 * t36)
+		 + px1 * t21 * t1)
+	 + x0 * (py0 * (t29 * t356 + t745 + t212
+			+ y1
+			* (x1 * (px1 * t719 + t84 + t83 + t82)
+			   + px1 * t698 + t40 + t39 + t38) + px1 * x1 * y2 * t741
+			+ t211 + t210 + px1 * t650 * t1)
+		 + px0 * (t29 * (t602 + t148 + t364 + t744)
+			  + t743 + t722
+			  + y1 * (x1 * (t720 + py1 * t708 + t200 + t143 + t706)
+				  + t702 + py1 * t684 + t676 + t659 + t675)
+			  + x1 * (t607 + px1 * y2 * (x2 * (t362 + t361) + t481 + t609) + t742)
+			  + t237 + t721 + px1 * t651 * t1) + t29 * t348 + t739 + t187
+		 + y1 * (x1 * (px1 * (t125 + t197 + t245 + t717) + t50)
+			 + px1 * (t696 + t13 + t695) + t8) + px1 * x1 * y2 * t606
+		 + px1 * t737 * t1)
+	 + py0 * (x1 * (px1 * t736 + t186 + t185 + t184)
+		  + t29 * (px1 * t733 + t194 + t193 + t192)
+		  + y1 * (x1 * (px1 * t712 + t7 + t6 + t4) + t732) + px1 * t537 * t1)
+	 + px0 * (x1 * (t568 + t731 + t189 + t730)
+		  + t29 * (py1 * t729 + t727 + t726 + t725 + t724)
+		  + y1 * (x1 * (t715 + py1 * t707 + t705 + t12 + t704) + t546 + t670 + t669)
+		  + px1 * t539 * t1) + x1 * (px1 * t723 + t275)
+	 + t29 * (px1 * (t247 + t283 + t244) + t278)
+	 + y0 * (x0 * (px0 * (x1 * (t720 + py1 * t719 + t125 + t197 + t245 + t717)
+			      + t715 + py1 * t712 + t29 * (t162 + t35 + t34 + t33) + t660
+			      + t659 + t658
+			      + (t688 + t711 + t431
+				 + x1
+				 * (px1 * (t305 + t304 + (t710 + t709) * x2)
+				    + t100 + t26 + t94 + t96) + t27 + t661)
+			      * y1)
+		       + py0 * (x1 * (px1 * t708 + t49 + t48 + t47)
+				+ px1 * t707 + px1 * t29 * t349 + t40 + t39 + t38
+				+ (t286 + t3 + px1 * x1 * t99 + t23 + t22) * y1)
+		       + x1 * (px1 * (t200 + t143 + t706) + t269) + px1 * (t705 + t12 + t704)
+		       + t8 + px1 * t29 * t340 + (t264 + t262 + px1 * x1 * t338) * y1)
+		 + px0 * (x1 * (t702 + py1 * t698 + t696 + t13 + t695)
+			  + t29 * (t694 + py1 * t692 + t325 + t143 + t691) + t398 + t392 + t391
+			  + (x1 * (t688 + t104 + t43 + t95 + t93)
+			     + px1 * (x2 * (t558 + t687) + t5 * t686) + t412 + t665 + t664)
+			  * y1)
+		 + py0 * (x1 * (px1 * t684 + t7 + t6 + t4) + t682
+			  + t29 * (px1 * t681 + t84 + t83 + t82)
+			  + (px1 * t411 + x1 * t679) * y1)
+		 + x1 * (px1 * (t676 + t659 + t675) + t41) + t672
+		 + t29 * (px1 * (t144 + t282 + t668) + t50) + (px1 * t666 + x1 * t663) * y1)
+	 + y1 * (x1 * (px1 * (t660 + t659 + t658) + t41) + t655) + px1 * t653 * t1
+	 + px0 * t652 * t14)
+	;
+
+    trans->m[0][2] =
+	(x0 * (px0 * (y1 * (x1 * (t813 + py1 * t807 + t805 + t725 + t803)
+			    + t799 + t568 + t731 + t189 + t730)
+		      + x1 * (px1 * (x2 * y2 * (t815 + t814) + t142 * t308)
+			      + t235 + t230 + t229) + t29 * (t170 + t159 + t155)
+		      + t1 * (t769 + py1 * t759 + t753 + t752 + t751))
+	       + py0 * (y1 * (x1 * (px1 * t812 + t194 + t193 + t192)
+			      + t295 + t186 + t185 + t184)
+			+ px1 * x1 * t234 + px1 * t29 * t379
+			+ t1 * (px1 * t768 + t7 + t6 + t4))
+	       + y1 * (x1 * (px1 * (t773 + t810 + t771) + t278) + t277 + t275)
+	       + px1 * x1 * t374 + px1 * t29 * t373
+	       + t1 * (px1 * (t765 + t45 + t764) + t41))
+	 + y0 * (x0 * (px0 * (x1 * (t813 + py1 * t812 + t773 + t810 + t771)
+			      + t495 + t29 * (t809 + t149 + t346 + t774)
+			      + y1
+			      * (x1
+				 * (px1 * (x2 * t386 + t686 * y2)
+				    + t598 + t503 + t800)
+				 + t798 + py1 * t790 + t787 + t752 + t786)
+			      + (t808 + t389 + t95 + t747) * t1)
+		       + py0 * (x1 * (px1 * t807 + t219 + t218 + t217)
+				+ px1 * t490 + t29 * t380
+				+ y1 * (px1 * x1 * t597 + px1 * t797 + t7 + t6 + t4)
+				+ t679 * t1)
+		       + x1 * (px1 * (t805 + t725 + t803) + t195) + px1 * t802
+		       + t29 * t377
+		       + y1 * (px1 * x1 * t801 + px1 * (t795 + t46 + t794) + t41)
+		       + t663 * t1)
+		 + px0 * (x1 * (t799 + t243 + t238 + t237 + t236)
+			  + t29 * (t783 + py1 * t777 + t773 + t772 + t771)
+			  + y1 * (x1 * (t798 + py1 * t797 + t795 + t46 + t794)
+				  + px1 * (t5 * (t481 + t609) + x2 * (t792 + t791) * y2)
+				  + t546 + t670 + t669) + (t538 + t648 + t645) * t1)
+		 + py0 * (x1 * (t745 + t212 + t211 + t210)
+			  + t29 * (px1 * t782 + t219 + t218 + t217)
+			  + y1 * (x1 * (px1 * t790 + t40 + t39 + t38) + t682)
+			  + px1 * t411 * t1) + x1 * (t739 + t187)
+		 + t29 * (px1 * (t727 + t780 + t724) + t195)
+		 + y1 * (x1 * (px1 * (t787 + t752 + t786) + t8) + t672)
+		 + px1 * t666 * t1)
+	 + t161 * (px0 * (y1
+			  * (t783 + x1 * (t368 + t157 + t364 + t363) + py1 * t782
+			     + t727 + t780 + t724)
+			  + x1 * (t383 + t372 + t371) + t500
+			  + t1 * (t762 + py1 * t756 + t125 + t272 + t749))
+		   + py0 * (y1
+			    * (x1 * (px1 * t779 + t352 + t351 + t350)
+			       + px1 * t777 + t194 + t193 + t192)
+			    + px1 * x1 * t169 + px1 * t209
+			    + t1 * (px1 * t761 + t84 + t83 + t82))
+		   + y1 * (x1 * (px1 * t775 + t344) + px1 * (t773 + t772 + t771)
+			   + t278) + px1 * x1 * t160
+		   + px1 * t152 + t1 * (px1 * (t144 + t245 + t760) + t50))
+	 + px0 * t770
+	 + t14 * (px0 * (x1 * (t769 + py1 * t768 + t765 + t45 + t764)
+			 + t29 * (t762 + py1 * t761 + t144 + t245 + t760) + t406
+			 + (t412 + x1 * (t711 + t431 + t27 + t661) + t665 + t664)
+			 * y1)
+		  + py0 * (x1 * (px1 * t759 + t40 + t39 + t38)
+			   + t732 + t29 * (px1 * t756 + t49 + t48 + t47)
+			   + (px1 * t537 + x1 * (px1 * t755 + t3 + t23 + t22)) * y1)
+		  + x1 * (px1 * (t753 + t752 + t751) + t8) + t655
+		  + t29 * (px1 * (t125 + t272 + t749) + t269)
+		  + (x1 * (px1 * t748 + t262) + px1 * t653) * y1));
+
+    trans->m[1][0] = (x0 * (py0 * (x1 * (t516 + py1 * (y2 * (t631 + t806) + t142 * t933) + t372
+					 + t371)
+				   + y1 * (px1 * t929 + x1 * (t514 + t918 + t157 + t364 + t363) + t908
+					   + t893 + t725 + t892) + t500 + t151
+				   + t1 * (px1 * t855 + t845 + t325 + t125 + t835) + t146)
+			    + px0 * (y1 * (x1 * t932 + py1 * t904 + t872 + t871 + t870)
+				     + py1 * x1 * t515 + py1 * t638
+				     + t1 * (py1 * t844 + t833 + t832 + t831))
+			    + y1 * (x1 * t931 + t863 + py1 * (t247 + t810 + t244)) + py1 * t374
+			    + py1 * x1 * t373 + t1 * (t888 + py1 * (t54 + t282 + t841)))
+		      + y0 * (px0 * (x1 * (py1 * t929 + t862 + t861 + t860)
+				     + t927 + py1 * t29 * t182 + t858
+				     + y1 * (py1 * t896 + x1 * (py1 * t914 + t833 + t832 + t831)) + t857
+				     + t856 + (t926 + t817 + t823 + t822) * t1)
+			      + x0 * (px0 * (x1 * t925 + py1 * t878 + t862
+					     + y1
+					     * (py1 * t902 + py1 * x1 * t916 + t853 + t852
+						+ t851) + t861 + t860
+					     + py1 * t693 * t1)
+				      + py0 * (x1 * (t921 + t918 + t156 + t346 + t345)
+					       + t883 + px1 * t875 + t247
+					       + y1 * (x1 * (t917 + py1 * (t328 + t88 + (t915 + t718) * y2)
+							     + t153 + t55 + t339)
+						       + t899 + px1 * t914 + t322 + t143 + t245 + t887) + t772
+					       + t244 + (t647 + t20 + t19 + t18) * t1) + x1 * t913 + t891
+				      + py1 * (t202 + t780 + t196)
+				      + y1 * (py1 * (t200 + t125 + t898) + t834 + py1 * x1 * t340)
+				      + py1 * t338 * t1)
+			      + py0 * (x1 * (t908 + px1 * t904 + t247 + t810 + t244)
+				       + t903 + py1 * t29 * t179 + t722
+				       + y1 * (x1 * (px1 * t902 + t899 + t200 + t125 + t898)
+					       + t897 + py1 * (x2 * t894 + x2 * (t86 + t628) * y2) + t267
+					       + t266) + t237 + t721 + (t593 + t389 + t95 + t747) * t1)
+			      + x1 * (py1 * (t893 + t725 + t892) + t891) + t890 + t889 + py1 * t29 * t274
+			      + y1 * (x1 * (t888 + py1 * (t322 + t143 + t245 + t887)) + py1 * t268)
+			      + (t885 + t884) * t1)
+		      + py0 * (y1 * (x1 * (t883 + px1 * t878 + t202 + t780 + t196)
+				     + t641 + t731 + t189 + t730)
+			       + x1 * (t639 + t230 + t229) + py1 * t29 * t169
+			       + t1 * (t840 + px1 * t850 + t13 + t752 + t825))
+		      + px0 * (y1 * (x1 * (py1 * t875 + t872 + t871 + t870)
+				     + py1 * t869 + t866 + t865 + t864)
+			       + py1 * x1 * t499 + py1 * t29 * t174
+			       + t1 * (py1 * t838 + t829 + t828 + t827))
+		      + y1 * (x1 * (t863 + py1 * (t247 + t772 + t244)) + t859 + py1 * t723)
+		      + py0 * t161 * t183 + py1 * t29 * t160 + py1 * x1 * t152
+		      + t14 * (px0 * (x1 * (py1 * t855 + t853 + t852 + t851)
+				      + py1 * t850 + py1 * t29 * t109 + t820 + t819 + t818
+				      + (py1 * t592 + t826 + py1 * x1 * t646 + t847 + t846) * y1)
+			       + py0 * (x1 * (t845 + px1 * t844 + t54 + t282 + t841)
+					+ t840 + px1 * t838 + py1 * t29 * t76 + t46 + t659 + t816
+					+ (t421 + t431 + x1 * (t836 + t26 + t94 + t96) + t27 + t661) * y1)
+			       + x1 * (py1 * (t325 + t125 + t835) + t834) + t830
+			       + py1 * (t13 + t752 + t825) + py1 * t29 * t36
+			       + (t824 + py1 * t748 + py1 * x1 * t21) * y1)
+		      + t1 * (t821 + py1 * (t46 + t659 + t816)))
+	;
+
+    trans->m[1][1] = (t161 * (px0 * (x1 * (py1 * t603 + t911 + t910 + t909)
+				     + py1 * t980 + t872
+				     + y1 * (py1 * t944 + py1 * x1 * t109 + t833 + t832 + t831) + t871
+				     + t870 + py1 * t646 * t1)
+			      + py0 * (x1 * (t514 + t157 + t364 + t363)
+				       + px1 * t974 + t202
+				       + y1 * (x1 * (t917 + t153 + t55 + t339)
+					       + t943 + px1 * t955 + t143 + t124 + t938) + t978 + t725
+				       + t196 + py1 * t66 * t1) + x1 * (t930 + py1 * t775) + t863
+			      + py1 * (t773 + t283 + t771)
+			      + y1 * (py1 * (t58 + t144 + t942) + t888 + py1 * x1 * t36)
+			      + py1 * t21 * t1)
+		      + x0 * (py0 * (t29 * (t604 + t149 + t346 + t774)
+				     + t984 + t190
+				     + y1 * (x1 * (px1 * t972 + t970 + t313 + t144 + t960)
+					     + px1 * t958 + t950 + t787 + t12 + t939)
+				     + x1 * (t607 + t983 + py1 * y2 * (x2 * (t328 + t88) + t293 + t327))
+				     + t189 + t188 + py1 * t650 * t1)
+			      + px0 * (t29 * t925 + t982 + t866
+				       + y1
+				       * (x1 * (py1 * t969 + t853 + t852 + t851)
+					  + py1 * t947 + t829 + t828 + t827)
+				       + py1 * x1 * y2 * t334 + t865 + t864 + py1 * t651 * t1)
+			      + t29 * t913 + t859 + t981
+			      + y1 * (x1 * (t834 + py1 * (t633 + t54 + t272 + t968))
+				      + py1 * (t946 + t46 + t945) + t821) + py1 * x1 * y2 * t606
+			      + py1 * t737 * t1)
+		      + py0 * (x1 * (t573 + t238 + t237 + t236)
+			       + t29 * (px1 * t980 + t773 + t978 + t283 + t771)
+			       + y1 * (x1 * (t965 + px1 * t971 + t765 + t659 + t959) + t550 + t670 + t669)
+			       + py1 * t537 * t1)
+		      + px0 * (x1 * (py1 * t977 + t858 + t857 + t856)
+			       + t29 * (py1 * t974 + t862 + t861 + t860)
+			       + y1 * (x1 * (py1 * t966 + t820 + t819 + t818) + t973) + py1 * t539 * t1)
+		      + x1 * (t890 + py1 * t191) + t29 * (t891 + py1 * (t202 + t725 + t196))
+		      + y0 * (x0 * (px0 * (x1 * (py1 * t972 + t833 + t832 + t831)
+					   + py1 * t971 + py1 * t29 * t916 + t829 + t828 + t827
+					   + (t926 + t817 + py1 * x1 * t693 + t823 + t822) * y1)
+				    + py0 * (x1 * (t970 + px1 * t969 + t633 + t54 + t272 + t968)
+					     + px1 * t966 + t965 + t29 * (t163 + t35 + t34 + t33) + t753 + t12
+					     + t935
+					     + (t962 + t941 + t43
+						+ x1
+						* (t836 + py1 * (t86 + t628 + (t961 + t291) * x2)
+						   + t26 + t94 + t96) + t95 + t93)
+					     * y1) + x1 * (py1 * (t313 + t144 + t960) + t888) + t821
+				    + py1 * (t765 + t659 + t959) + py1 * t29 * t340
+				    + (t885 + t884 + py1 * x1 * t338) * y1)
+			      + px0 * (x1 * (py1 * t958 + t820 + t819 + t818)
+				       + t956 + t29 * (py1 * t955 + t853 + t852 + t851)
+				       + (py1 * t415 + x1 * t953) * y1)
+			      + py0 * (x1 * (t950 + px1 * t947 + t946 + t46 + t945)
+				       + t29 * (px1 * t944 + t943 + t58 + t144 + t942) + t406 + t392 + t391
+				       + (x1 * (t421 + t941 + t431 + t27 + t661)
+					  + t416 + py1 * (x2 * (t554 + t796) + t5 * t940) + t665 + t664)
+				       * y1) + x1 * (py1 * (t787 + t12 + t939) + t830)
+			      + t29 * (t834 + py1 * (t143 + t124 + t938)) + t937
+			      + (x1 * t936 + py1 * t666) * y1)
+		      + y1 * (x1 * (py1 * (t753 + t12 + t935) + t830) + t934) + py1 * t653 * t1
+		      + py0 * t652 * t14)
+	;
+
+    trans->m[1][2] = (y0 * (x0 * (px0 * (x1 * (py1 * t1016 + t872 + t871 + t870)
+					 + py1 * t494 + t29 * t932
+					 + y1
+					 * (py1 * t1025 + py1 * x1 * t599 + t820 + t819
+					    + t818) + t953 * t1)
+				  + py0 * (x1 * (t1015 + px1 * t1009 + t727 + t326 + t724)
+					   + t29 * (t1028 + t148 + t364 + t744) + t491
+					   + y1
+					   * (x1
+					      * (t600 + py1 * (x2 * t933 + t940 * y2) + t503
+						 + t800)
+					      + px1 * t1020 + t1026 + t676 + t45 + t1017)
+					   + (t1027 + t10 + t27 + t25) * t1)
+				  + x1 * (py1 * (t1014 + t283 + t1012) + t863) + t29 * t931
+				  + py1 * t802
+				  + y1 * (py1 * x1 * t801 + py1 * (t1024 + t13 + t1023) + t830)
+				  + t936 * t1)
+			    + py0 * (t29 * (t1007 + px1 * t1002 + t727 + t201 + t724)
+				     + x1 * (t1011 + t641 + t731 + t189 + t730)
+				     + y1 * (x1 * (t1026 + px1 * t1025 + t1024 + t13 + t1023)
+					     + t550
+					     + py1
+					     * (t5 * (t293 + t327) + x2 * (t1021 + t697) * y2)
+					     + t670 + t669) + (t540 + t648 + t645) * t1)
+			    + px0 * (x1 * (t982 + t866 + t865 + t864)
+				     + t29 * (py1 * t1006 + t872 + t871 + t870)
+				     + y1 * (x1 * (py1 * t1020 + t829 + t828 + t827) + t956)
+				     + py1 * t415 * t1) + x1 * (t859 + t981)
+			    + t29 * (t863 + py1 * (t773 + t246 + t771))
+			    + y1 * (x1 * (py1 * (t676 + t45 + t1017) + t821) + t937)
+			    + py1 * t666 * t1)
+		      + x0 * (py0 * (y1 * (x1 * (px1 * t1016 + t1015 + t1014 + t283 + t1012)
+					   + t1011 + t573 + t238 + t237 + t236)
+				     + x1 * (t639
+					     + py1 * (x2 * y2 * (t1010 + t300) + t142 * t894)
+					     + t230 + t229) + t29 * (t175 + t159 + t155)
+				     + t1 * (px1 * t1000 + t994 + t660 + t45 + t986))
+			      + px0 * (y1 * (x1 * (py1 * t1009 + t862 + t861 + t860)
+					     + t927 + t858 + t857 + t856)
+				       + py1 * t29 * t515 + py1 * x1 * t638
+				       + t1 * (py1 * t993 + t820 + t819 + t818))
+			      + y1 * (x1 * (t891 + py1 * (t727 + t326 + t724)) + t890 + t889)
+			      + py1 * x1 * t374 + py1 * t29 * t373
+			      + t1 * (t830 + py1 * (t705 + t752 + t990)))
+		      + t161 * (py0 * (x1 * (t516 + t372 + t371)
+				       + y1
+				       * (x1 * (t921 + t156 + t346 + t345)
+					  + t1007 + px1 * t1006 + t773 + t246 + t771) + t381
+				       + t1 * (t989 + px1 * t997 + t54 + t245 + t985))
+				+ px0 * (y1
+					 * (x1 * (py1 * t1004 + t924 + t923 + t922)
+					    + py1 * t1002 + t862 + t861 + t860)
+					 + py1 * t499 + py1 * x1 * t174
+					 + t1 * (py1 * t988 + t853 + t852 + t851))
+				+ y1 * (x1 * (t912 + py1 * t746) + t891
+					+ py1 * (t727 + t201 + t724))
+				+ py1 * x1 * t160 + py1 * t152
+				+ t1 * (t834 + py1 * (t143 + t272 + t987))) + py0 * t770
+		      + t14 * (px0 * (x1 * (py1 * t1000 + t829 + t828 + t827)
+				      + t973 + t29 * (py1 * t997 + t833 + t832 + t831)
+				      + (py1 * t539 + x1 * (py1 * t996 + t817 + t823 + t822))
+				      * y1)
+			       + py0 * (x1 * (t994 + px1 * t993 + t705 + t752 + t990)
+					+ t29 * (t989 + px1 * t988 + t143 + t272 + t987) + t398
+					+ (t416 + x1 * (t962 + t43 + t95 + t93) + t665 + t664)
+					* y1) + x1 * (py1 * (t660 + t45 + t986) + t821)
+			       + t29 * (t888 + py1 * (t54 + t245 + t985)) + t934
+			       + (x1 * (t885 + py1 * t28) + py1 * t653) * y1));
+
+    trans->m[2][0] = (x0 * (px0 * (y1 * (x1 * t617 + t586 + t877 + t1008 + t876)
+				   + x1 * t515 + t637 + t1 * (t475 + t136 + t620 + t1036) + t636)
+			    + py0 * (y1 * (t579 + x1 * t627 + t251 + t811 + t248)
+				     + x1 * t379 + t233 + t1 * (t455 + t87 + t298 + t1035) + t232)
+			    + x1 * (t516 + t383 + y2 * (t804 + t1013) + t142 * t595)
+			    + y1 * (px1 * (t1047 + t979 + t1046)
+				    + x1 * (t921 + t368 + t157 + t156 + t1048)
+				    + py1 * (t1044 + t728 + t1043) + t505 + t607 + t502) + t500 + t381
+			    + t1 * (px1 * (t611 + t135 + t1038) + py1 * (t330 + t128 + t1037) + t423
+				    + t125 + t54 + t1034))
+		      + y0 * (x0 * (py0 * (x1 * t1050 + t533 + t226
+					   + y1 * (t529 + t224 + x1 * t349 + t128 + t1040)
+					   + t781 + t220 + t99 * t1)
+				    + px0 * (t528 + x1 * t1049 + t874
+					     + y1 * (t517 + x1 * t916 + t520 + t135 + t1041) + t1005
+					     + t873 + t693 * t1)
+				    + x1 * (t514 + t382 + t157 + t156 + t1048)
+				    + px1 * (t877 + t1001 + t876) + py1 * (t251 + t776 + t248) + t608
+				    + y1 * (x1 * (t917 + t358 + t56 + t323 + (t716 + t967) * y2)
+					    + px1 * (t118 + t609 + t471 + t1045)
+					    + py1 * (t204 + t327 + t249 + t1042) + t503 + t144 + t143
+					    + t1039) + t504 + t605 + (t647 + t357) * t1)
+			      + px0 * (x1 * (t528 + t1047 + t979 + t1046)
+				       + t643 + t29 * t182 + t571
+				       + y1 * (x1 * (t475 + t118 + t609 + t471 + t1045) + t701 + t895)
+				       + t570 + t569 + (t468 + t441 + t466 + t951) * t1)
+			      + py0 * (x1 * (t533 + t1044 + t728 + t1043)
+				       + t642 + t29 * t179 + t566
+				       + y1 * (x1 * (t455 + t204 + t327 + t249 + t1042) + t289 + t288)
+				       + t214 + t565 + (t436 + t450 + t69 + t677) * t1)
+			      + x1 * (px1 * (t877 + t1008 + t876) + py1 * (t251 + t811 + t248) + t505
+				      + t607 + t502) + t984 + t743
+			      + t29 * t274
+			      + y1 * (x1 * (px1 * (t520 + t135 + t1041) + py1 * (t224 + t128 + t1040)
+					    + t503 + t144 + t143 + t1039)
+				      + t897 + t303 + x2 * t634 + x2 * (t690 + t52) * y2)
+			      + (t1027 + t808) * t1)
+		      + py0 * (y1 * (x1 * (t579 + t251 + t776 + t248) + t581 + t735 + t240 + t734)
+			       + t29 * t169 + x1 * t209 + t1 * (t461 + t80 + t683 + t1032))
+		      + px0 * (y1 * (x1 * (t586 + t877 + t1001 + t876) + t591 + t976 + t588 + t975)
+			       + x1 * t499 + t29 * t174 + t1 * (t484 + t849 + t957 + t1033))
+		      + y1 * (x1 * (px1 * (t874 + t1005 + t873) + py1 * (t226 + t781 + t220) + t608
+				    + t504 + t605)
+			      + t573 + t568) + t161 * t183 + x1 * (t639 + t235) + t29 * t160
+		      + t14 * (px0 * (x1 * (t517 + t611 + t135 + t1038)
+				      + t615 + t29 * t109 + t560 + t443 + t1031
+				      + (t399 + t613 + x1 * t646 + t418 + t995) * y1)
+			       + py0 * (x1 * (t529 + t330 + t128 + t1037)
+					+ t625 + t29 * t76 + t112 + t438 + t1030
+					+ (t623 + t393 + t102 + x1 * t66 + t754) * y1)
+			       + x1 * (px1 * (t136 + t620 + t1036) + py1 * (t87 + t298 + t1035) + t423
+				       + t125 + t54 + t1034)
+			       + px1 * (t849 + t957 + t1033) + py1 * (t80 + t683 + t1032) + t434
+			       + t29 * t36 + t1029 + (t962 + t711 + x1 * (t836 + t100)) * y1)
+		      + t1 * (px1 * (t560 + t443 + t1031) + py1 * (t112 + t438 + t1030) + t434
+			      + t1029))
+	;
+
+    trans->m[2][1] = (t161 * (px0 * (x1 * (t616 + t180 + t512 + t1003)
+				     + t586 + t510 + y1 * (t475 + t854 + x1 * t109 + t117 + t1062)
+				     + t621 + t508 + t646 * t1)
+			      + py0 * (t579 + x1 * (t177 + t626 + t354 + t778) + t507
+				       + y1 * (t455 + x1 * t76 + t91 + t203 + t1061) + t299
+				       + t506 + t66 * t1) + x1 * (t921 + t368)
+			      + px1 * (t874 + t979 + t873) + py1 * (t226 + t728 + t220)
+			      + y1 * (x1 * (t917 + t358) + px1 * (t118 + t843 + t1063)
+				      + py1 * (t204 + t127 + t1065) + t423 + t144
+				      + t143 + t1060) + t504 + t21 * t1)
+		      + x0 * (py0 * (t29 * t1050 + t581 + t241
+				     + y1
+				     * (x1 * (t529 + t87 + t577 + t292 + t1070)
+					+ t461 + t1057 + t80 + t1056) + x1 * y2 * t741 + t240
+				     + t239 + t650 * t1)
+			      + px0 * (t591 + t29 * t1049 + t589
+				       + y1 * (x1 * (t517 + t136 + t306 + t428 + t1071)
+					       + t484 + t1059 + t849 + t1058) + x1 * y2 * t334 + t588
+				       + t587 + t651 * t1) + t29 * (t1028 + t809) + t903 + t321
+			      + y1 * (x1 * (px1 * (t901 + t117 + t1072) + py1 * (t317 + t203 + t1073)
+					    + t503 + t125 + t54 + t1069)
+				      + px1 * (t1019 + t837 + t1064) + py1 * (t789 + t111 + t1066) + t267
+				      + t433 + t1055)
+			      + x1 * (y2 * (x2 * (t56 + t323) + t322 + t273) + t983 + t742) + t737 * t1)
+		      + py0 * (x1 * (t642 + t215 + t214 + t213) + t29 * (t533 + t226 + t728 + t220)
+			       + y1
+			       * (x1 * (t625 + t758 + t111 + t1067)
+				  + t396 + t395) + t537 * t1)
+		      + px0 * (x1 * (t643 + t868 + t570 + t867) + t29 * (t528 + t874 + t979 + t873)
+			       + y1
+			       * (x1 * (t615 + t999 + t837 + t1068)
+				  + t404 + t402) + t539 * t1)
+		      + x1 * (t641 + t243)
+		      + t29 * (px1 * (t510 + t621 + t508) + py1 * (t507 + t299 + t506) + t504)
+		      + y0 * (x0 * (py0 * (x1 * (t455 + t317 + t203 + t1073)
+					   + t461 + t29 * t349 + t767 + t683 + t1052
+					   + (t436 + t450 + x1 * t99 + t69 + t677) * y1)
+				    + px0 * (x1 * (t475 + t901 + t117 + t1072)
+					     + t484 + t29 * t916 + t992 + t957 + t1053
+					     + (t468 + t441 + t466 + x1 * t693 + t951) * y1)
+				    + x1 * (px1 * (t136 + t306 + t428 + t1071)
+					    + py1 * (t87 + t577 + t292 + t1070) + t503 + t125 + t54 + t1069)
+				    + px1 * (t999 + t837 + t1068) + py1 * (t758 + t111 + t1067)
+				    + t29 * (t163 + t162) + t434 + t553 + t1051
+				    + (t421 + t104 + t431 + t43
+				       + x1 * (t836 + t100 + t690 + t52 + (t270 + t886) * x2)
+				       + t1054)
+				    * y1)
+			      + py0 * (x1 * (t625 + t789 + t111 + t1066)
+				       + t29 * (t529 + t204 + t127 + t1065) + t544 + t543
+				       + (x1 * t624 + t410 + t409) * y1)
+			      + px0 * (x1 * (t615 + t1019 + t837 + t1064)
+				       + t29 * (t517 + t118 + t843 + t1063) + t548 + t547
+				       + (x1 * t614 + t414 + t413) * y1)
+			      + t29 * (px1 * (t854 + t117 + t1062) + py1 * (t91 + t203 + t1061) + t423
+				       + t144 + t143 + t1060)
+			      + x1 * (px1 * (t1059 + t849 + t1058) + py1 * (t1057 + t80 + t1056) + t267
+				      + t433 + t1055) + t406 + t398
+			      + (t416 + x1 * (t962 + t711 + t431 + t43 + t1054) + t412
+				 + x2 * (t793 + t1022) + t5 * t594)
+			      * y1)
+		      + y1 * (x1 * (px1 * (t992 + t957 + t1053) + py1 * (t767 + t683 + t1052) + t434
+				    + t553 + t1051)
+			      + t550 + t546) + t653 * t1 + t652 * t14)
+	;
+    trans->m[2][2] = t644;
+}
+
+static void
+print_trans (const char *header, struct pixman_f_transform *trans)
+{
+    int i, j;
+    double max;
+
+    max = 0;
+
+    printf ("%s\n", header);
+
+    for (i = 0; i < 3; ++i)
+    {
+	for (j = 0; j < 3; ++j)
+	{
+	    double a = fabs (trans->m[i][j]);
+
+	    if (a > max)
+		max = a;
+	}
+    }
+
+    if (max == 0.0)
+	max = 1.0;
+
+    for (i = 0; i < 3; ++i)
+    {
+	printf ("{ ");
+	for (j = 0; j < 3; ++j)
+	{
+	    printf ("D2F (%.5f)%s", 16384 * (trans->m[i][j] / max), j == 2 ? "" : ", ");
+	}
+
+	printf ("},\n");
+    }
+}
+
+int
+main ()
+{
+    struct pixman_f_transform t;
+
+#if 0
+    quad_to_quad (75, 200,
+		  325, 200,
+		  450, 335,
+		  -50, 335,
+
+		  0, 0,
+		  400, 0,
+		  400, 400,
+		  0, 400,
+
+		  &t);
+#endif
+    quad_to_quad (
+	1, 0,
+	1, 2,
+	2, 2,
+	2, 0,
+
+	1, 0,
+	1, 112,
+	2, 2,
+	2, 0,
+
+	&t);
+
+    print_trans ("0->0", &t);
+
+    return 0;
+}
diff --git a/pixman/demos/radial-test.c b/pixman/demos/radial-test.c
new file mode 100644
index 000000000..08a367cd2
--- /dev/null
+++ b/pixman/demos/radial-test.c
@@ -0,0 +1,208 @@
+#include "../test/utils.h"
+#include "gtk-utils.h"
+
+#define NUM_GRADIENTS 9
+#define NUM_STOPS 3
+#define NUM_REPEAT 4
+#define SIZE 128
+#define WIDTH (SIZE * NUM_GRADIENTS)
+#define HEIGHT (SIZE * NUM_REPEAT)
+
+/*
+ * We want to test all the possible relative positions of the start
+ * and end circle:
+ *
+ *  - The start circle can be smaller/equal/bigger than the end
+ *    circle. A radial gradient can be classified in one of these
+ *    three cases depending on the sign of dr.
+ *
+ *  - The smaller circle can be completely inside/internally
+ *    tangent/outside (at least in part) of the bigger circle. This
+ *    classification is the same as the one which can be computed by
+ *    examining the sign of a = (dx^2 + dy^2 - dr^2).
+ *
+ *  - If the two circles have the same size, neither can be inside or
+ *    internally tangent
+ *
+ * This test draws radial gradients whose circles always have the same
+ * centers (0, 0) and (1, 0), but with different radiuses. From left
+ * to right:
+ *
+ * - Degenerate start circle completely inside the end circle
+ *     0.00 -> 1.75; dr = 1.75 > 0; a = 1 - 1.75^2 < 0
+ *
+ * - Small start circle completely inside the end circle
+ *     0.25 -> 1.75; dr =  1.5 > 0; a = 1 - 1.50^2 < 0
+ *
+ * - Small start circle internally tangent to the end circle
+ *     0.50 -> 1.50; dr =  1.0 > 0; a = 1 - 1.00^2 = 0
+ *
+ * - Small start circle outside of the end circle
+ *     0.50 -> 1.00; dr =  0.5 > 0; a = 1 - 0.50^2 > 0
+ *
+ * - Start circle with the same size as the end circle
+ *     1.00 -> 1.00; dr =  0.0 = 0; a = 1 - 0.00^2 > 0
+ *
+ * - Small end circle outside of the start circle
+ *     1.00 -> 0.50; dr = -0.5 > 0; a = 1 - 0.50^2 > 0
+ *
+ * - Small end circle internally tangent to the start circle
+ *     1.50 -> 0.50; dr = -1.0 > 0; a = 1 - 1.00^2 = 0
+ *
+ * - Small end circle completely inside the start circle
+ *     1.75 -> 0.25; dr = -1.5 > 0; a = 1 - 1.50^2 < 0
+ *
+ * - Degenerate end circle completely inside the start circle
+ *     0.00 -> 1.75; dr = 1.75 > 0; a = 1 - 1.75^2 < 0
+ *
+ */
+
+const static double radiuses[NUM_GRADIENTS] = {
+    0.00,
+    0.25,
+    0.50,
+    0.50,
+    1.00,
+    1.00,
+    1.50,
+    1.75,
+    1.75
+};
+
+#define double_to_color(x)					\
+    (((uint32_t) ((x)*65536)) - (((uint32_t) ((x)*65536)) >> 16))
+
+#define PIXMAN_STOP(offset,r,g,b,a)		\
+    { pixman_double_to_fixed (offset),		\
+	{					\
+	double_to_color (r),			\
+	double_to_color (g),			\
+	double_to_color (b),			\
+	double_to_color (a)			\
+	}					\
+    }
+
+static const pixman_gradient_stop_t stops[NUM_STOPS] = {
+    PIXMAN_STOP (0.0,        1, 0, 0, 0.75),
+    PIXMAN_STOP (0.70710678, 0, 1, 0, 0),
+    PIXMAN_STOP (1.0,        0, 0, 1, 1)
+};
+
+static pixman_image_t *
+create_radial (int index)
+{
+    pixman_point_fixed_t p0, p1;
+    pixman_fixed_t r0, r1;
+    double x0, x1, radius0, radius1, left, right, center;
+
+    x0 = 0;
+    x1 = 1;
+    radius0 = radiuses[index];
+    radius1 = radiuses[NUM_GRADIENTS - index - 1];
+
+    /* center the gradient */
+    left = MIN (x0 - radius0, x1 - radius1);
+    right = MAX (x0 + radius0, x1 + radius1);
+    center = (left + right) * 0.5;
+    x0 -= center;
+    x1 -= center;
+
+    /* scale to make it fit within a 1x1 rect centered in (0,0) */
+    x0 *= 0.25;
+    x1 *= 0.25;
+    radius0 *= 0.25;
+    radius1 *= 0.25;
+
+    p0.x = pixman_double_to_fixed (x0);
+    p0.y = pixman_double_to_fixed (0);
+
+    p1.x = pixman_double_to_fixed (x1);
+    p1.y = pixman_double_to_fixed (0);
+
+    r0 = pixman_double_to_fixed (radius0);
+    r1 = pixman_double_to_fixed (radius1);
+
+    return pixman_image_create_radial_gradient (&p0, &p1,
+						r0, r1,
+						stops, NUM_STOPS);
+}
+
+static const pixman_repeat_t repeat[NUM_REPEAT] = {
+    PIXMAN_REPEAT_NONE,
+    PIXMAN_REPEAT_NORMAL,
+    PIXMAN_REPEAT_REFLECT,
+    PIXMAN_REPEAT_PAD
+};
+
+int
+main (int argc, char **argv)
+{
+    pixman_transform_t transform;
+    pixman_image_t *src_img, *dest_img;
+    int i, j;
+
+    enable_divbyzero_exceptions ();
+
+    dest_img = pixman_image_create_bits (PIXMAN_a8r8g8b8,
+					 WIDTH, HEIGHT,
+					 NULL, 0);
+
+    draw_checkerboard (dest_img, 25, 0xffaaaaaa, 0xffbbbbbb);
+    
+    pixman_transform_init_identity (&transform);
+
+    /*
+     * The create_radial() function returns gradients centered in the
+     * origin and whose interesting part fits a 1x1 square. We want to
+     * paint these gradients on a SIZExSIZE square and to make things
+     * easier we want the origin in the top-left corner of the square
+     * we want to see.
+     */
+    pixman_transform_translate (NULL, &transform,
+				pixman_double_to_fixed (0.5),
+				pixman_double_to_fixed (0.5));
+
+    pixman_transform_scale (NULL, &transform,
+			    pixman_double_to_fixed (SIZE),
+			    pixman_double_to_fixed (SIZE));
+
+    /*
+     * Gradients are evaluated at the center of each pixel, so we need
+     * to translate by half a pixel to trigger some interesting
+     * cornercases. In particular, the original implementation of PDF
+     * radial gradients tried to divide by 0 when using this transform
+     * on the "tangent circles" cases.
+     */
+    pixman_transform_translate (NULL, &transform,
+				pixman_double_to_fixed (0.5),
+				pixman_double_to_fixed (0.5));
+
+    for (i = 0; i < NUM_GRADIENTS; i++)
+    {
+	src_img = create_radial (i);
+	pixman_image_set_transform (src_img, &transform);
+
+	for (j = 0; j < NUM_REPEAT; j++)
+	{
+	    pixman_image_set_repeat (src_img, repeat[j]);
+
+	    pixman_image_composite32 (PIXMAN_OP_OVER,
+				      src_img,
+				      NULL,
+				      dest_img,
+				      0, 0,
+				      0, 0,
+				      i * SIZE, j * SIZE,
+				      SIZE, SIZE);
+
+	}
+
+	pixman_image_unref (src_img);
+    }
+
+    show_image (dest_img);
+
+    pixman_image_unref (dest_img);
+
+    return 0;
+}
diff --git a/pixman/demos/scale.c b/pixman/demos/scale.c
new file mode 100644
index 000000000..d00307e44
--- /dev/null
+++ b/pixman/demos/scale.c
@@ -0,0 +1,436 @@
+/*
+ * Copyright 2012, Red Hat, Inc.
+ * Copyright 2012, Soren Sandmann
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Author: Soren Sandmann <soren.sandmann@gmail.com>
+ */
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <math.h>
+#include <gtk/gtk.h>
+#include <pixman.h>
+#include <stdlib.h>
+#include "gtk-utils.h"
+
+typedef struct
+{
+    GtkBuilder *        builder;
+    pixman_image_t *	original;
+    GtkAdjustment *     scale_x_adjustment;
+    GtkAdjustment *     scale_y_adjustment;
+    GtkAdjustment *     rotate_adjustment;
+    GtkAdjustment *	subsample_adjustment;
+    int                 scaled_width;
+    int                 scaled_height;
+} app_t;
+
+static GtkWidget *
+get_widget (app_t *app, const char *name)
+{
+    GtkWidget *widget = GTK_WIDGET (gtk_builder_get_object (app->builder, name));
+
+    if (!widget)
+        g_error ("Widget %s not found\n", name);
+
+    return widget;
+}
+
+static double
+min4 (double a, double b, double c, double d)
+{
+    double m1, m2;
+
+    m1 = MIN (a, b);
+    m2 = MIN (c, d);
+    return MIN (m1, m2);
+}
+
+static double
+max4 (double a, double b, double c, double d)
+{
+    double m1, m2;
+
+    m1 = MAX (a, b);
+    m2 = MAX (c, d);
+    return MAX (m1, m2);
+}
+
+static void
+compute_extents (pixman_f_transform_t *trans, double *sx, double *sy)
+{
+    double min_x, max_x, min_y, max_y;
+    pixman_f_vector_t v[4] =
+    {
+	{ { 1, 1, 1 } },
+	{ { -1, 1, 1 } },
+	{ { -1, -1, 1 } },
+	{ { 1, -1, 1 } },
+    };
+
+    pixman_f_transform_point (trans, &v[0]);
+    pixman_f_transform_point (trans, &v[1]);
+    pixman_f_transform_point (trans, &v[2]);
+    pixman_f_transform_point (trans, &v[3]);
+
+    min_x = min4 (v[0].v[0], v[1].v[0], v[2].v[0], v[3].v[0]);
+    max_x = max4 (v[0].v[0], v[1].v[0], v[2].v[0], v[3].v[0]);
+    min_y = min4 (v[0].v[1], v[1].v[1], v[2].v[1], v[3].v[1]);
+    max_y = max4 (v[0].v[1], v[1].v[1], v[2].v[1], v[3].v[1]);
+
+    *sx = (max_x - min_x) / 2.0;
+    *sy = (max_y - min_y) / 2.0;
+}
+
+typedef struct
+{
+    char	name [20];
+    int		value;
+} named_int_t;
+
+static const named_int_t filters[] =
+{
+    { "Box",			PIXMAN_KERNEL_BOX },
+    { "Impulse",		PIXMAN_KERNEL_IMPULSE },
+    { "Linear",			PIXMAN_KERNEL_LINEAR },
+    { "Cubic",			PIXMAN_KERNEL_CUBIC },
+    { "Lanczos2",		PIXMAN_KERNEL_LANCZOS2 },
+    { "Lanczos3",		PIXMAN_KERNEL_LANCZOS3 },
+    { "Lanczos3 Stretched",	PIXMAN_KERNEL_LANCZOS3_STRETCHED },
+    { "Gaussian",		PIXMAN_KERNEL_GAUSSIAN },
+};
+
+static const named_int_t repeats[] =
+{
+    { "None",                   PIXMAN_REPEAT_NONE },
+    { "Normal",                 PIXMAN_REPEAT_NORMAL },
+    { "Reflect",                PIXMAN_REPEAT_REFLECT },
+    { "Pad",                    PIXMAN_REPEAT_PAD },
+};
+
+static int
+get_value (app_t *app, const named_int_t table[], const char *box_name)
+{
+    GtkComboBox *box = GTK_COMBO_BOX (get_widget (app, box_name));
+
+    return table[gtk_combo_box_get_active (box)].value;
+}
+
+static void
+copy_to_counterpart (app_t *app, GObject *object)
+{
+    static const char *xy_map[] =
+    {
+	"reconstruct_x_combo_box", "reconstruct_y_combo_box",
+	"sample_x_combo_box",      "sample_y_combo_box",
+	"scale_x_adjustment",      "scale_y_adjustment",
+    };
+    GObject *counterpart = NULL;
+    int i;
+
+    for (i = 0; i < G_N_ELEMENTS (xy_map); i += 2)
+    {
+	GObject *x = gtk_builder_get_object (app->builder, xy_map[i]);
+	GObject *y = gtk_builder_get_object (app->builder, xy_map[i + 1]);
+
+	if (object == x)
+	    counterpart = y;
+	if (object == y)
+	    counterpart = x;
+    }
+
+    if (!counterpart)
+	return;
+    
+    if (GTK_IS_COMBO_BOX (counterpart))
+    {
+	gtk_combo_box_set_active (
+	    GTK_COMBO_BOX (counterpart),
+	    gtk_combo_box_get_active (
+		GTK_COMBO_BOX (object)));
+    }
+    else if (GTK_IS_ADJUSTMENT (counterpart))
+    {
+	gtk_adjustment_set_value (
+	    GTK_ADJUSTMENT (counterpart),
+	    gtk_adjustment_get_value (
+		GTK_ADJUSTMENT (object)));
+    }
+}
+
+static double
+to_scale (double v)
+{
+    return pow (1.15, v);
+}
+
+static void
+rescale (GtkWidget *may_be_null, app_t *app)
+{
+    pixman_f_transform_t ftransform;
+    pixman_transform_t transform;
+    double new_width, new_height;
+    double fscale_x, fscale_y;
+    double rotation;
+    pixman_fixed_t *params;
+    int n_params;
+    double sx, sy;
+
+    pixman_f_transform_init_identity (&ftransform);
+
+    if (may_be_null && gtk_toggle_button_get_active (
+	    GTK_TOGGLE_BUTTON (get_widget (app, "lock_checkbutton"))))
+    {
+	copy_to_counterpart (app, G_OBJECT (may_be_null));
+    }
+    
+    fscale_x = gtk_adjustment_get_value (app->scale_x_adjustment);
+    fscale_y = gtk_adjustment_get_value (app->scale_y_adjustment);
+    rotation = gtk_adjustment_get_value (app->rotate_adjustment);
+
+    fscale_x = to_scale (fscale_x);
+    fscale_y = to_scale (fscale_y);
+    
+    new_width = pixman_image_get_width (app->original) * fscale_x;
+    new_height = pixman_image_get_height (app->original) * fscale_y;
+
+    pixman_f_transform_scale (&ftransform, NULL, fscale_x, fscale_y);
+
+    pixman_f_transform_translate (&ftransform, NULL, - new_width / 2.0, - new_height / 2.0);
+
+    rotation = (rotation / 360.0) * 2 * M_PI;
+    pixman_f_transform_rotate (&ftransform, NULL, cos (rotation), sin (rotation));
+
+    pixman_f_transform_translate (&ftransform, NULL, new_width / 2.0, new_height / 2.0);
+
+    pixman_f_transform_invert (&ftransform, &ftransform);
+
+    compute_extents (&ftransform, &sx, &sy);
+    
+    pixman_transform_from_pixman_f_transform (&transform, &ftransform);
+    pixman_image_set_transform (app->original, &transform);
+
+    params = pixman_filter_create_separable_convolution (
+        &n_params,
+        sx * 65536.0 + 0.5,
+	sy * 65536.0 + 0.5,
+	get_value (app, filters, "reconstruct_x_combo_box"),
+	get_value (app, filters, "reconstruct_y_combo_box"),
+	get_value (app, filters, "sample_x_combo_box"),
+	get_value (app, filters, "sample_y_combo_box"),
+	gtk_adjustment_get_value (app->subsample_adjustment),
+	gtk_adjustment_get_value (app->subsample_adjustment));
+
+    pixman_image_set_filter (app->original, PIXMAN_FILTER_SEPARABLE_CONVOLUTION, params, n_params);
+
+    pixman_image_set_repeat (
+        app->original, get_value (app, repeats, "repeat_combo_box"));
+    
+    free (params);
+
+    app->scaled_width = ceil (new_width);
+    app->scaled_height = ceil (new_height);
+    
+    gtk_widget_set_size_request (
+        get_widget (app, "drawing_area"), new_width + 0.5, new_height + 0.5);
+
+    gtk_widget_queue_draw (
+        get_widget (app, "drawing_area"));
+}
+
+static gboolean
+on_expose (GtkWidget *da, GdkEvent *event, gpointer data)
+{
+    app_t *app = data;
+    GdkRectangle *area = &event->expose.area;
+    cairo_surface_t *surface;
+    pixman_image_t *tmp;
+    cairo_t *cr;
+    uint32_t *pixels;
+
+    pixels = calloc (1, area->width * area->height * 4);
+    tmp = pixman_image_create_bits (
+        PIXMAN_a8r8g8b8, area->width, area->height, pixels, area->width * 4);
+
+    if (area->x < app->scaled_width && area->y < app->scaled_height)
+    {
+        pixman_image_composite (
+            PIXMAN_OP_SRC,
+            app->original, NULL, tmp,
+            area->x, area->y, 0, 0, 0, 0,
+            app->scaled_width - area->x, app->scaled_height - area->y);
+    }
+
+    surface = cairo_image_surface_create_for_data (
+        (uint8_t *)pixels, CAIRO_FORMAT_ARGB32,
+        area->width, area->height, area->width * 4);
+
+    cr = gdk_cairo_create (da->window);
+
+    cairo_set_source_surface (cr, surface, area->x, area->y);
+
+    cairo_paint (cr);
+
+    cairo_destroy (cr);
+    cairo_surface_destroy (surface);
+    free (pixels);
+    pixman_image_unref (tmp);
+
+    return TRUE;
+}
+
+static void
+set_up_combo_box (app_t *app, const char *box_name,
+                  int n_entries, const named_int_t table[])
+{
+    GtkWidget *widget = get_widget (app, box_name);
+    GtkListStore *model;
+    GtkCellRenderer *cell;
+    int i;
+
+    model = gtk_list_store_new (1, G_TYPE_STRING);
+    
+    cell = gtk_cell_renderer_text_new ();
+    gtk_cell_layout_pack_start (GTK_CELL_LAYOUT (widget), cell, TRUE);
+    gtk_cell_layout_set_attributes (GTK_CELL_LAYOUT (widget), cell,
+				    "text", 0,
+				    NULL);
+
+    gtk_combo_box_set_model (GTK_COMBO_BOX (widget), GTK_TREE_MODEL (model));
+    
+    for (i = 0; i < n_entries; ++i)
+    {
+	const named_int_t *info = &(table[i]);
+	GtkTreeIter iter;
+
+	gtk_list_store_append (model, &iter);
+	gtk_list_store_set (model, &iter, 0, info->name, -1);
+    }
+
+    gtk_combo_box_set_active (GTK_COMBO_BOX (widget), 0);
+
+    g_signal_connect (widget, "changed", G_CALLBACK (rescale), app);
+}
+
+static void
+set_up_filter_box (app_t *app, const char *box_name)
+{
+    set_up_combo_box (app, box_name, G_N_ELEMENTS (filters), filters);
+}
+
+static char *
+format_value (GtkWidget *widget, double value)
+{
+    return g_strdup_printf ("%.4f", to_scale (value));
+}
+
+static app_t *
+app_new (pixman_image_t *original)
+{
+    GtkWidget *widget;
+    app_t *app = g_malloc (sizeof *app);
+    GError *err = NULL;
+
+    app->builder = gtk_builder_new ();
+    app->original = original;
+
+    if (!gtk_builder_add_from_file (app->builder, "scale.ui", &err))
+	g_error ("Could not read file scale.ui: %s", err->message);
+
+    app->scale_x_adjustment =
+        GTK_ADJUSTMENT (gtk_builder_get_object (app->builder, "scale_x_adjustment"));
+    app->scale_y_adjustment =
+        GTK_ADJUSTMENT (gtk_builder_get_object (app->builder, "scale_y_adjustment"));
+    app->rotate_adjustment =
+        GTK_ADJUSTMENT (gtk_builder_get_object (app->builder, "rotate_adjustment"));
+    app->subsample_adjustment =
+	GTK_ADJUSTMENT (gtk_builder_get_object (app->builder, "subsample_adjustment"));
+
+    g_signal_connect (app->scale_x_adjustment, "value_changed", G_CALLBACK (rescale), app);
+    g_signal_connect (app->scale_y_adjustment, "value_changed", G_CALLBACK (rescale), app);
+    g_signal_connect (app->rotate_adjustment, "value_changed", G_CALLBACK (rescale), app);
+    g_signal_connect (app->subsample_adjustment, "value_changed", G_CALLBACK (rescale), app);
+    
+    widget = get_widget (app, "scale_x_scale");
+    gtk_scale_add_mark (GTK_SCALE (widget), 0.0, GTK_POS_LEFT, NULL);
+    g_signal_connect (widget, "format_value", G_CALLBACK (format_value), app);
+    widget = get_widget (app, "scale_y_scale");
+    gtk_scale_add_mark (GTK_SCALE (widget), 0.0, GTK_POS_LEFT, NULL);
+    g_signal_connect (widget, "format_value", G_CALLBACK (format_value), app);
+    widget = get_widget (app, "rotate_scale");
+    gtk_scale_add_mark (GTK_SCALE (widget), 0.0, GTK_POS_LEFT, NULL);
+
+    widget = get_widget (app, "drawing_area");
+    g_signal_connect (widget, "expose_event", G_CALLBACK (on_expose), app);
+
+    set_up_filter_box (app, "reconstruct_x_combo_box");
+    set_up_filter_box (app, "reconstruct_y_combo_box");
+    set_up_filter_box (app, "sample_x_combo_box");
+    set_up_filter_box (app, "sample_y_combo_box");
+
+    set_up_combo_box (
+        app, "repeat_combo_box", G_N_ELEMENTS (repeats), repeats);
+
+    g_signal_connect (
+	gtk_builder_get_object (app->builder, "lock_checkbutton"),
+	"toggled", G_CALLBACK (rescale), app);
+    
+    rescale (NULL, app);
+    
+    return app;
+}
+
+int
+main (int argc, char **argv)
+{
+    GtkWidget *window;
+    pixman_image_t *image;
+    app_t *app;
+    
+    gtk_init (&argc, &argv);
+
+    if (argc < 2)
+    {
+	printf ("%s <image file>\n", argv[0]);
+	return -1;
+    }
+
+    if (!(image = pixman_image_from_file (argv[1], PIXMAN_a8r8g8b8)))
+    {
+	printf ("Could not load image \"%s\"\n", argv[1]);
+	return -1;
+    }
+
+    app = app_new (image);
+    
+    window = get_widget (app, "main");
+
+    g_signal_connect (window, "delete_event", G_CALLBACK (gtk_main_quit), NULL);
+    
+    gtk_window_set_default_size (GTK_WINDOW (window), 1024, 768);
+    
+    gtk_widget_show_all (window);
+    
+    gtk_main ();
+
+    return 0;
+}
diff --git a/pixman/demos/scale.ui b/pixman/demos/scale.ui
new file mode 100644
index 000000000..ee985dd1c
--- /dev/null
+++ b/pixman/demos/scale.ui
@@ -0,0 +1,332 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<interface>
+  <!-- interface-requires gtk+ 2.12 -->
+  <!-- interface-naming-policy toplevel-contextual -->
+  <object class="GtkAdjustment" id="rotate_adjustment">
+    <property name="lower">-180</property>
+    <property name="upper">190</property>
+    <property name="step_increment">1</property>
+    <property name="page_increment">10</property>
+    <property name="page_size">10</property>
+  </object>
+  <object class="GtkAdjustment" id="scale_y_adjustment">
+    <property name="lower">-32</property>
+    <property name="upper">42</property>
+    <property name="step_increment">1</property>
+    <property name="page_increment">10</property>
+    <property name="page_size">10</property>
+  </object>
+  <object class="GtkAdjustment" id="scale_x_adjustment">
+    <property name="lower">-32</property>
+    <property name="upper">42</property>
+    <property name="step_increment">1</property>
+    <property name="page_increment">10</property>
+    <property name="page_size">10</property>
+  </object>
+  <object class="GtkAdjustment" id="subsample_adjustment">
+    <property name="lower">0</property>
+    <property name="upper">12</property>
+    <property name="step_increment">1</property>
+    <property name="page_increment">1</property>
+    <property name="page_size">0</property>
+    <property name="value">4</property>
+  </object>
+  <object class="GtkWindow" id="main">
+    <child>
+      <object class="GtkHBox" id="u">
+        <property name="visible">True</property>
+        <property name="spacing">12</property>
+        <child>
+          <object class="GtkScrolledWindow" id="scrolledwindow1">
+            <property name="visible">True</property>
+            <property name="can_focus">True</property>
+            <property name="shadow_type">in</property>
+            <child>
+              <object class="GtkViewport" id="viewport1">
+                <property name="visible">True</property>
+                <child>
+                  <object class="GtkDrawingArea" id="drawing_area">
+                    <property name="visible">True</property>
+                  </object>
+                </child>
+              </object>
+            </child>
+          </object>
+          <packing>
+            <property name="position">0</property>
+          </packing>
+        </child>
+        <child>
+          <object class="GtkVBox" id="box1">
+            <property name="visible">True</property>
+	    <property name="spacing">12</property>
+            <child>
+              <object class="GtkHBox" id="box2">
+                <property name="visible">True</property>
+                <property name="homogeneous">True</property>
+                <child>
+                  <object class="GtkVBox" id="box3">
+                    <property name="visible">True</property>
+                    <property name="spacing">6</property>
+                    <child>
+                      <object class="GtkLabel" id="label1">
+                        <property name="visible">True</property>
+                        <property name="label" translatable="yes">&lt;b&gt;Scale X&lt;/b&gt;</property>
+                        <property name="use_markup">True</property>
+                      </object>
+                      <packing>
+                        <property name="expand">False</property>
+                        <property name="position">0</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkVScale" id="scale_x_scale">
+                        <property name="visible">True</property>
+                        <property name="can_focus">True</property>
+                        <property name="adjustment">scale_x_adjustment</property>
+                        <property name="fill_level">32</property>
+                        <property name="value_pos">right</property>
+                      </object>
+                      <packing>
+                        <property name="position">1</property>
+                      </packing>
+                    </child>
+                  </object>
+                  <packing>
+                    <property name="expand">False</property>
+                    <property name="position">0</property>
+                  </packing>
+                </child>
+                <child>
+                  <object class="GtkVBox" id="box4">
+                    <property name="visible">True</property>
+                    <property name="spacing">6</property>
+                    <child>
+                      <object class="GtkLabel" id="label2">
+                        <property name="visible">True</property>
+                        <property name="label" translatable="yes">&lt;b&gt;Scale Y&lt;/b&gt;</property>
+                        <property name="use_markup">True</property>
+                      </object>
+                      <packing>
+                        <property name="expand">False</property>
+                        <property name="position">0</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkVScale" id="scale_y_scale">
+                        <property name="visible">True</property>
+                        <property name="can_focus">True</property>
+                        <property name="adjustment">scale_y_adjustment</property>
+                        <property name="fill_level">32</property>
+                        <property name="value_pos">right</property>
+                      </object>
+                      <packing>
+                        <property name="position">1</property>
+                      </packing>
+                    </child>
+                  </object>
+                  <packing>
+                    <property name="expand">False</property>
+                    <property name="position">1</property>
+                  </packing>
+                </child>
+                <child>
+                  <object class="GtkVBox" id="box5">
+                    <property name="visible">True</property>
+                    <property name="spacing">6</property>
+                    <child>
+                      <object class="GtkLabel" id="label3">
+                        <property name="visible">True</property>
+                        <property name="label" translatable="yes">&lt;b&gt;Rotate&lt;/b&gt;</property>
+                        <property name="use_markup">True</property>
+                      </object>
+                      <packing>
+                        <property name="expand">False</property>
+                        <property name="position">0</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkVScale" id="rotate_scale">
+                        <property name="visible">True</property>
+                        <property name="can_focus">True</property>
+                        <property name="adjustment">rotate_adjustment</property>
+                        <property name="fill_level">180</property>
+                        <property name="value_pos">right</property>
+                      </object>
+                      <packing>
+                        <property name="position">1</property>
+                      </packing>
+                    </child>
+                  </object>
+                  <packing>
+                    <property name="expand">False</property>
+                    <property name="position">2</property>
+                  </packing>
+                </child>
+              </object>
+              <packing>
+                <property name="padding">6</property>
+                <property name="position">0</property>
+              </packing>
+            </child>
+            <child>
+              <object class="GtkVBox" id="box6">
+                <property name="visible">True</property>
+		<child>
+		  <object class="GtkCheckButton"
+			  id="lock_checkbutton">
+		    <property name="label" translatable="yes">Lock X and Y Dimensions</property>
+		    <property name="xalign">0.0</property>
+		  </object>
+                  <packing>
+                    <property name="expand">False</property>
+                    <property name="fill">False</property>
+                    <property name="padding">6</property>
+                    <property name="position">1</property>
+                  </packing>
+		</child>
+                <child>
+                  <object class="GtkTable" id="grid1">
+                    <property name="visible">True</property>
+                    <property name="column_spacing">8</property>
+                    <property name="row_spacing">6</property>
+                    <child>
+                      <object class="GtkLabel" id="label4">
+                        <property name="visible">True</property>
+                        <property name="xalign">1</property>
+                        <property name="label" translatable="yes">&lt;b&gt;Reconstruct X:&lt;/b&gt;</property>
+                        <property name="use_markup">True</property>
+                      </object>
+                    </child>
+                    <child>
+                      <object class="GtkLabel" id="label5">
+                        <property name="visible">True</property>
+                        <property name="xalign">1</property>
+                        <property name="label" translatable="yes">&lt;b&gt;Reconstruct Y:&lt;/b&gt;</property>
+                        <property name="use_markup">True</property>
+                      </object>
+                      <packing>
+                        <property name="top_attach">1</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkLabel" id="label6">
+                        <property name="visible">True</property>
+                        <property name="xalign">1</property>
+                        <property name="label" translatable="yes">&lt;b&gt;Sample X:&lt;/b&gt;</property>
+                        <property name="use_markup">True</property>
+                      </object>
+                      <packing>
+                        <property name="top_attach">2</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkLabel" id="label7">
+                        <property name="visible">True</property>
+                        <property name="xalign">1</property>
+                        <property name="label" translatable="yes">&lt;b&gt;Sample Y:&lt;/b&gt;</property>
+                        <property name="use_markup">True</property>
+                      </object>
+                      <packing>
+                        <property name="top_attach">3</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkLabel" id="label8">
+                        <property name="visible">True</property>
+                        <property name="xalign">1</property>
+                        <property name="label" translatable="yes">&lt;b&gt;Repeat:&lt;/b&gt;</property>
+                        <property name="use_markup">True</property>
+                      </object>
+                      <packing>
+                        <property name="top_attach">4</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkLabel" id="label9">
+                        <property name="visible">True</property>
+                        <property name="xalign">1</property>
+                        <property name="label" translatable="yes">&lt;b&gt;Subsample:&lt;/b&gt;</property>
+                        <property name="use_markup">True</property>
+                      </object>
+                      <packing>
+                        <property name="top_attach">5</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkComboBox" id="reconstruct_x_combo_box">
+                        <property name="visible">True</property>
+                      </object>
+                      <packing>
+                        <property name="left_attach">1</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkComboBox" id="reconstruct_y_combo_box">
+                        <property name="visible">True</property>
+                      </object>
+                      <packing>
+                        <property name="left_attach">1</property>
+                        <property name="top_attach">1</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkComboBox" id="sample_x_combo_box">
+                        <property name="visible">True</property>
+                      </object>
+                      <packing>
+                        <property name="left_attach">1</property>
+                        <property name="top_attach">2</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkComboBox" id="sample_y_combo_box">
+                        <property name="visible">True</property>
+                      </object>
+                      <packing>
+                        <property name="left_attach">1</property>
+                        <property name="top_attach">3</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkComboBox" id="repeat_combo_box">
+                        <property name="visible">True</property>
+                      </object>
+                      <packing>
+                        <property name="left_attach">1</property>
+                        <property name="top_attach">4</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkSpinButton" id="subsample_spin_button">
+                        <property name="visible">True</property>
+			<property name="adjustment">subsample_adjustment</property>
+                      </object>
+                      <packing>
+                        <property name="left_attach">1</property>
+                        <property name="top_attach">5</property>
+                      </packing>
+                    </child>
+                  </object>
+                  <packing>
+                    <property name="expand">False</property>
+                    <property name="padding">6</property>
+                    <property name="position">1</property>
+                  </packing>
+                </child>
+              </object>
+              <packing>
+                <property name="expand">False</property>
+                <property name="position">0</property>
+              </packing>
+            </child>
+          </object>
+          <packing>
+            <property name="expand">False</property>
+            <property name="position">1</property>
+          </packing>
+        </child>
+      </object>
+    </child>
+  </object>
+</interface>
diff --git a/pixman/test/screen-test.c b/pixman/demos/screen-test.c
index e69dba3de..e69dba3de 100644
--- a/pixman/test/screen-test.c
+++ b/pixman/demos/screen-test.c
diff --git a/pixman/demos/srgb-test.c b/pixman/demos/srgb-test.c
new file mode 100644
index 000000000..681d52181
--- /dev/null
+++ b/pixman/demos/srgb-test.c
@@ -0,0 +1,87 @@
+#include <math.h>
+
+#include "pixman.h"
+#include "gtk-utils.h"
+
+static uint32_t
+linear_argb_to_premult_argb (float a,
+			     float r,
+			     float g,
+			     float b)
+{
+    r *= a;
+    g *= a;
+    b *= a;
+    return (uint32_t) (a * 255.0f + 0.5f) << 24
+	 | (uint32_t) (r * 255.0f + 0.5f) << 16
+	 | (uint32_t) (g * 255.0f + 0.5f) <<  8
+	 | (uint32_t) (b * 255.0f + 0.5f) <<  0;
+}
+
+static float
+lin2srgb (float linear)
+{
+    if (linear < 0.0031308f)
+	return linear * 12.92f;
+    else
+	return 1.055f * powf (linear, 1.0f/2.4f) - 0.055f;
+}
+
+static uint32_t
+linear_argb_to_premult_srgb_argb (float a,
+				  float r,
+				  float g,
+				  float b)
+{
+    r = lin2srgb (r * a);
+    g = lin2srgb (g * a);
+    b = lin2srgb (b * a);
+    return (uint32_t) (a * 255.0f + 0.5f) << 24
+	 | (uint32_t) (r * 255.0f + 0.5f) << 16
+	 | (uint32_t) (g * 255.0f + 0.5f) <<  8
+	 | (uint32_t) (b * 255.0f + 0.5f) <<  0;
+}
+
+int
+main (int argc, char **argv)
+{
+#define WIDTH 400
+#define HEIGHT 200
+    int y, x, p;
+    float alpha;
+ 
+    uint32_t *dest = malloc (WIDTH * HEIGHT * 4);
+    uint32_t *src1 = malloc (WIDTH * HEIGHT * 4);
+    pixman_image_t *dest_img, *src1_img;
+   
+    dest_img = pixman_image_create_bits (PIXMAN_a8r8g8b8_sRGB,
+					 WIDTH, HEIGHT,
+					 dest,
+					 WIDTH * 4);
+    src1_img = pixman_image_create_bits (PIXMAN_a8r8g8b8,
+					 WIDTH, HEIGHT,
+					 src1,
+					 WIDTH * 4);
+
+    for (y = 0; y < HEIGHT; y ++)
+    {
+	p = WIDTH * y;
+	for (x = 0; x < WIDTH; x ++)
+	{
+	     alpha = (float) x / WIDTH;
+	     src1[p + x] = linear_argb_to_premult_argb (alpha, 1, 0, 1);
+	     dest[p + x] = linear_argb_to_premult_srgb_argb (1-alpha, 0, 1, 0);
+	}
+    }
+    
+    pixman_image_composite (PIXMAN_OP_ADD, src1_img, NULL, dest_img,
+			    0, 0, 0, 0, 0, 0, WIDTH, HEIGHT);
+    pixman_image_unref (src1_img);
+    free (src1);
+
+    show_image (dest_img);
+    pixman_image_unref (dest_img);
+    free (dest);
+    
+    return 0;
+}
diff --git a/pixman/demos/srgb-trap-test.c b/pixman/demos/srgb-trap-test.c
new file mode 100644
index 000000000..d5ae16a06
--- /dev/null
+++ b/pixman/demos/srgb-trap-test.c
@@ -0,0 +1,119 @@
+#include <math.h>
+#include "pixman.h"
+#include "gtk-utils.h"
+
+#define F(x)								\
+    pixman_double_to_fixed (x)
+
+#define WIDTH 600
+#define HEIGHT 300
+
+static uint16_t
+convert_to_srgb (uint16_t in)
+{
+    double d = in * (1/65535.0);
+    double a = 0.055;
+
+    if (d < 0.0031308)
+	d = 12.92 * d;
+    else
+	d = (1 + a) * pow (d, 1 / 2.4) - a;
+
+    return (d * 65535.0) + 0.5;
+}
+
+static void
+convert_color (pixman_color_t *dest_srgb, pixman_color_t *linear)
+{
+    dest_srgb->alpha = convert_to_srgb (linear->alpha);
+    dest_srgb->red = convert_to_srgb (linear->red);
+    dest_srgb->green = convert_to_srgb (linear->green);
+    dest_srgb->blue = convert_to_srgb (linear->blue);
+}
+
+int
+main (int argc, char **argv)
+{
+    static const pixman_trapezoid_t traps[] =
+    {
+	{ F(10.10), F(280.0),
+	  { { F(20.0), F(10.10) },
+	    { F(5.3), F(280.0) } },
+	  { { F(20.3), F(10.10) },
+	    { F(5.6), F(280.0) } }
+	},
+	{ F(10.10), F(280.0),
+	  { { F(40.0), F(10.10) },
+	    { F(15.3), F(280.0) } },
+	  { { F(41.0), F(10.10) },
+	    { F(16.3), F(280.0) } }
+	},
+	{ F(10.10), F(280.0),
+	  { { F(120.0), F(10.10) },
+	    { F(5.3), F(280.0) } },
+	  { { F(128.3), F(10.10) },
+	    { F(6.6), F(280.0) } }
+	},
+	{ F(10.10), F(280.0),
+	  { { F(60.0), F(10.10) },
+	    { F(25.3), F(280.0) } },
+	  { { F(61.0), F(10.10) },
+	    { F(26.3), F(280.0) } }
+	},
+	{ F(10.10), F(280.0),
+	  { { F(90.0), F(10.10) },
+	    { F(55.3), F(280.0) } },
+	  { { F(93.0), F(10.10) },
+	    { F(58.3), F(280.0) } }
+	},
+	{ F(130.10), F(150.0),
+	  { { F(100.0), F(130.10) },
+	    { F(250.3), F(150.0) } },
+	  { { F(110.0), F(130.10) },
+	    { F(260.3), F(150.0) } }
+	},
+	{ F(170.10), F(240.0),
+	  { { F(100.0), F(170.10) },
+	    { F(120.3), F(240.0) } },
+	  { { F(250.0), F(170.10) },
+	    { F(250.3), F(240.0) } }
+	},
+    };
+
+    pixman_image_t *src, *dest_srgb, *dest_linear;
+    pixman_color_t bg = { 0x0000, 0x0000, 0x0000, 0xffff };
+    pixman_color_t fg = { 0xffff, 0xffff, 0xffff, 0xffff };
+    pixman_color_t fg_srgb;
+    uint32_t *d;
+
+    d = malloc (WIDTH * HEIGHT * 4);
+    
+    dest_srgb = pixman_image_create_bits (
+	PIXMAN_a8r8g8b8_sRGB, WIDTH, HEIGHT, d, WIDTH * 4);
+    dest_linear = pixman_image_create_bits (
+	PIXMAN_a8r8g8b8, WIDTH, HEIGHT, d, WIDTH * 4);
+    
+    src = pixman_image_create_solid_fill (&bg);
+    pixman_image_composite32 (PIXMAN_OP_SRC,
+			      src, NULL, dest_srgb,
+			      0, 0, 0, 0, 0, 0, WIDTH, HEIGHT);
+    
+    src = pixman_image_create_solid_fill (&fg);
+    
+    pixman_composite_trapezoids (PIXMAN_OP_OVER,
+				 src, dest_srgb, PIXMAN_a8,
+				 0, 0, 10, 10, G_N_ELEMENTS (traps), traps);
+
+    convert_color (&fg_srgb, &fg);
+    src = pixman_image_create_solid_fill (&fg_srgb);
+    
+    pixman_composite_trapezoids (PIXMAN_OP_OVER,
+				 src, dest_linear, PIXMAN_a8,
+				 0, 0, 310, 10, G_N_ELEMENTS (traps), traps);
+
+    show_image (dest_linear);
+    pixman_image_unref(dest_linear);
+    free(d);
+    
+    return 0;
+}
diff --git a/pixman/test/trap-test.c b/pixman/demos/trap-test.c
index 19295e7a5..19295e7a5 100644
--- a/pixman/test/trap-test.c
+++ b/pixman/demos/trap-test.c
diff --git a/pixman/demos/tri-test.c b/pixman/demos/tri-test.c
new file mode 100644
index 000000000..a71869a6a
--- /dev/null
+++ b/pixman/demos/tri-test.c
@@ -0,0 +1,48 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "../test/utils.h"
+#include "gtk-utils.h"
+
+int
+main (int argc, char **argv)
+{
+#define WIDTH 200
+#define HEIGHT 200
+
+#define POINT(x,y)							\
+    { pixman_double_to_fixed ((x)), pixman_double_to_fixed ((y)) }
+    
+    pixman_image_t *src_img, *dest_img;
+    pixman_triangle_t tris[4] =
+    {
+	{ POINT (100, 100), POINT (10, 50), POINT (110, 10) },
+	{ POINT (100, 100), POINT (150, 10), POINT (200, 50) },
+	{ POINT (100, 100), POINT (10, 170), POINT (90, 175) },
+	{ POINT (100, 100), POINT (170, 150), POINT (120, 190) },
+    };
+    pixman_color_t color = { 0x4444, 0x4444, 0xffff, 0xffff };
+    uint32_t *bits = malloc (WIDTH * HEIGHT * 4);
+    int i;
+
+    for (i = 0; i < WIDTH * HEIGHT; ++i)
+	bits[i] = (i / HEIGHT) * 0x01010000;
+    
+    src_img = pixman_image_create_solid_fill (&color);
+    dest_img = pixman_image_create_bits (PIXMAN_a8r8g8b8, WIDTH, HEIGHT, bits, WIDTH * 4);
+    
+    pixman_composite_triangles (PIXMAN_OP_ATOP_REVERSE,
+				src_img,
+				dest_img,
+				PIXMAN_a8,
+				200, 200,
+				-5, 5,
+				ARRAY_LENGTH (tris), tris);
+    show_image (dest_img);
+    
+    pixman_image_unref (src_img);
+    pixman_image_unref (dest_img);
+    free (bits);
+    
+    return 0;
+}
diff --git a/pixman/demos/zone_plate.png b/pixman/demos/zone_plate.png
new file mode 100644
index 000000000..519291d6d
--- /dev/null
+++ b/pixman/demos/zone_plate.png
diff --git a/pixman/pixman-1.pc.in b/pixman/pixman-1.pc.in
index 936d95db0..e3b9711ae 100644
--- a/pixman/pixman-1.pc.in
+++ b/pixman/pixman-1.pc.in
@@ -6,6 +6,6 @@ includedir=@includedir@
 Name: Pixman
 Description: The pixman library (version 1)
 Version: @PACKAGE_VERSION@
-Cflags: -I${includedir}/pixman-1 @DEP_CFLAGS@
-Libs: -L${libdir} -lpixman-1 @DEP_LIBS@
+Cflags: -I${includedir}/pixman-1
+Libs: -L${libdir} -lpixman-1
 
diff --git a/pixman/pixman/Makefile.am b/pixman/pixman/Makefile.am
index 66ad7f005..b376d9aeb 100644
--- a/pixman/pixman/Makefile.am
+++ b/pixman/pixman/Makefile.am
@@ -1,66 +1,27 @@
+include $(top_srcdir)/pixman/Makefile.sources
+
 lib_LTLIBRARIES = libpixman-1.la
+
 libpixman_1_la_LDFLAGS = -version-info $(LT_VERSION_INFO) -no-undefined @PTHREAD_LDFLAGS@ 
-libpixman_1_la_LIBADD = @DEP_LIBS@ -lm
-libpixman_1_la_SOURCES =			\
-	pixman.h				\
-	pixman-accessor.h			\
-	pixman-access.c				\
-	pixman-access-accessors.c		\
-	pixman-cpu.c				\
-	pixman-gradient-walker.c		\
-	pixman-region16.c			\
-	pixman-region32.c			\
-	pixman-compiler.h			\
-	pixman-private.h			\
-	pixman-image.c				\
-	pixman-implementation.c			\
-	pixman-combine32.c			\
-	pixman-combine32.h			\
-	pixman-combine64.c			\
-	pixman-combine64.h			\
-	pixman-general.c			\
-	pixman.c				\
-	pixman-fast-path.c			\
-	pixman-solid-fill.c			\
-	pixman-conical-gradient.c		\
-	pixman-linear-gradient.c		\
-	pixman-radial-gradient.c		\
-	pixman-bits-image.c			\
-	pixman-utils.c				\
-	pixman-edge.c				\
-	pixman-edge-accessors.c			\
-	pixman-edge-imp.h			\
-	pixman-trap.c				\
-	pixman-timer.c				\
-	pixman-matrix.c
-
-libpixmanincludedir = $(includedir)/pixman-1/
+libpixman_1_la_LIBADD = @PTHREAD_LIBS@ -lm
+libpixman_1_la_SOURCES = $(libpixman_sources) $(libpixman_headers)
+
+libpixmanincludedir = $(includedir)/pixman-1
 libpixmaninclude_HEADERS = pixman.h pixman-version.h
 noinst_LTLIBRARIES = 
 
-BUILT_SOURCES = pixman-combine32.h pixman-combine32.c pixman-combine64.h pixman-combine64.c
-
-pixman-combine32.c : pixman-combine.c.template pixman-combine32.h make-combine.pl
-	$(PERL) $(srcdir)/make-combine.pl 8 < $(srcdir)/pixman-combine.c.template > $@ || ($(RM) $@; exit 1)
-pixman-combine32.h : pixman-combine.h.template make-combine.pl
-	$(PERL) $(srcdir)/make-combine.pl 8 < $(srcdir)/pixman-combine.h.template > $@ || ($(RM) $@; exit 1)
-
-pixman-combine64.c : pixman-combine.c.template pixman-combine64.h make-combine.pl
-	$(PERL) $(srcdir)/make-combine.pl 16 < $(srcdir)/pixman-combine.c.template > $@ || ($(RM) $@; exit 1)
-pixman-combine64.h : pixman-combine.h.template make-combine.pl
-	$(PERL) $(srcdir)/make-combine.pl 16 < $(srcdir)/pixman-combine.h.template > $@ || ($(RM) $@; exit 1)
-
-EXTRA_DIST = Makefile.win32 pixman-combine.c.template make-combine.pl pixman-region.c \
-	pixman-combine.h.template solaris-hwcap.mapfile pixman-x64-mmx-emulation.h
-CLEANFILES = pixman-combine32.c pixman-combine64.c pixman-combine32.h pixman-combine64.h
+EXTRA_DIST =				\
+	Makefile.win32			\
+	pixman-region.c			\
+	solaris-hwcap.mapfile		\
+	$(NULL)
 
 # mmx code
-if USE_MMX
+if USE_X86_MMX
 noinst_LTLIBRARIES += libpixman-mmx.la
 libpixman_mmx_la_SOURCES = \
 	pixman-mmx.c
-libpixman_mmx_la_CFLAGS = $(DEP_CFLAGS) $(MMX_CFLAGS)
-libpixman_mmx_la_LIBADD = $(DEP_LIBS)
+libpixman_mmx_la_CFLAGS = $(MMX_CFLAGS)
 libpixman_1_la_LDFLAGS += $(MMX_LDFLAGS)
 libpixman_1_la_LIBADD += libpixman-mmx.la
 
@@ -73,8 +34,7 @@ noinst_LTLIBRARIES += libpixman-vmx.la
 libpixman_vmx_la_SOURCES = \
 	pixman-vmx.c \
 	pixman-combine32.h
-libpixman_vmx_la_CFLAGS = $(DEP_CFLAGS) $(VMX_CFLAGS)
-libpixman_vmx_la_LIBADD = $(DEP_LIBS)
+libpixman_vmx_la_CFLAGS = $(VMX_CFLAGS)
 libpixman_1_la_LIBADD += libpixman-vmx.la
 
 ASM_CFLAGS_vmx=$(VMX_CFLAGS)
@@ -85,23 +45,34 @@ if USE_SSE2
 noinst_LTLIBRARIES += libpixman-sse2.la
 libpixman_sse2_la_SOURCES = \
 	pixman-sse2.c
-libpixman_sse2_la_CFLAGS = $(DEP_CFLAGS) $(SSE2_CFLAGS)
-libpixman_sse2_la_LIBADD = $(DEP_LIBS)
+libpixman_sse2_la_CFLAGS = $(SSE2_CFLAGS)
 libpixman_1_la_LDFLAGS += $(SSE2_LDFLAGS)
 libpixman_1_la_LIBADD += libpixman-sse2.la
 
 ASM_CFLAGS_sse2=$(SSE2_CFLAGS)
 endif
 
+# ssse3 code
+if USE_SSSE3
+noinst_LTLIBRARIES += libpixman-ssse3.la
+libpixman_ssse3_la_SOURCES = \
+	pixman-ssse3.c
+libpixman_ssse3_la_CFLAGS = $(SSSE3_CFLAGS)
+libpixman_1_la_LDFLAGS += $(SSSE3_LDFLAGS)
+libpixman_1_la_LIBADD += libpixman-ssse3.la
+
+ASM_CFLAGS_ssse3=$(SSSE3_CFLAGS)
+endif
+
 # arm simd code
 if USE_ARM_SIMD
 noinst_LTLIBRARIES += libpixman-arm-simd.la
 libpixman_arm_simd_la_SOURCES = \
 	pixman-arm-simd.c	\
 	pixman-arm-common.h	\
-	pixman-arm-simd-asm.S
-libpixman_arm_simd_la_CFLAGS = $(DEP_CFLAGS)
-libpixman_arm_simd_la_LIBADD = $(DEP_LIBS)
+	pixman-arm-simd-asm.S   \
+	pixman-arm-simd-asm-scaled.S \
+	pixman-arm-simd-asm.h
 libpixman_1_la_LIBADD += libpixman-arm-simd.la
 
 ASM_CFLAGS_arm_simd=
@@ -114,13 +85,55 @@ libpixman_arm_neon_la_SOURCES = \
         pixman-arm-neon.c	\
         pixman-arm-common.h	\
         pixman-arm-neon-asm.S	\
+		pixman-arm-neon-asm-bilinear.S \
         pixman-arm-neon-asm.h
-libpixman_arm_neon_la_CFLAGS = $(DEP_CFLAGS)
-libpixman_arm_neon_la_LIBADD = $(DEP_LIBS)
 libpixman_1_la_LIBADD += libpixman-arm-neon.la
 
 ASM_CFLAGS_arm_neon=
 endif
 
-.c.s : $(libpixmaninclude_HEADERS) $(BUILT_SOURCES)
+# iwmmxt code
+if USE_ARM_IWMMXT
+libpixman_iwmmxt_la_SOURCES = pixman-mmx.c
+noinst_LTLIBRARIES += libpixman-iwmmxt.la
+libpixman_1_la_LIBADD += libpixman-iwmmxt.la
+
+libpixman_iwmmxt_la-pixman-mmx.lo: pixman-mmx.c
+	$(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(AM_CPPFLAGS) $(AM_CPPFLAGS) $(CPPFLAGS) $(CFLAGS) $(IWMMXT_CFLAGS) -MT libpixman_iwmmxt_la-pixman-mmx.lo -MD -MP -MF $(DEPDIR)/libpixman_iwmmxt_la-pixman-mmx.Tpo -c -o libpixman_iwmmxt_la-pixman-mmx.lo `test -f 'pixman-mmx.c' || echo '$(srcdir)/'`pixman-mmx.c
+	$(AM_V_at)$(am__mv) $(DEPDIR)/libpixman_iwmmxt_la-pixman-mmx.Tpo $(DEPDIR)/libpixman_iwmmxt_la-pixman-mmx.Plo
+
+libpixman_iwmmxt_la_DEPENDENCIES = $(am__DEPENDENCIES_1)
+libpixman_iwmmxt_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \
+        $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \
+	$(CFLAGS) $(IWMMXT_CFLAGS) $(AM_LDFLAGS) \
+	$(LDFLAGS) -o $@
+
+libpixman-iwmmxt.la: libpixman_iwmmxt_la-pixman-mmx.lo $(libpixman_iwmmxt_la_DEPENDENCIES) 
+	$(AM_V_CCLD)$(libpixman_iwmmxt_la_LINK) libpixman_iwmmxt_la-pixman-mmx.lo $(libpixman_iwmmxt_la_LIBADD) $(LIBS)
+endif
+
+# mips dspr2 code
+if USE_MIPS_DSPR2
+noinst_LTLIBRARIES += libpixman-mips-dspr2.la
+libpixman_mips_dspr2_la_SOURCES = \
+        pixman-mips-dspr2.c \
+        pixman-mips-dspr2.h \
+        pixman-mips-dspr2-asm.S \
+        pixman-mips-dspr2-asm.h \
+        pixman-mips-memcpy-asm.S
+libpixman_1_la_LIBADD += libpixman-mips-dspr2.la
+
+ASM_CFLAGS_mips_dspr2=
+endif
+
+# loongson code
+if USE_LOONGSON_MMI
+noinst_LTLIBRARIES += libpixman-loongson-mmi.la
+libpixman_loongson_mmi_la_SOURCES = pixman-mmx.c loongson-mmintrin.h
+libpixman_loongson_mmi_la_CFLAGS = $(LS_CFLAGS)
+libpixman_1_la_LDFLAGS += $(LS_LDFLAGS)
+libpixman_1_la_LIBADD += libpixman-loongson-mmi.la
+endif
+
+.c.s : $(libpixmaninclude_HEADERS)
 	$(CC) $(CFLAGS) $(ASM_CFLAGS_$(@:pixman-%.s=%)) $(ASM_CFLAGS_$(@:pixman-arm-%.s=arm_%)) -DHAVE_CONFIG_H -I$(srcdir) -I$(builddir) -I$(top_builddir) -S -o $@ $<
diff --git a/pixman/pixman/Makefile.sources b/pixman/pixman/Makefile.sources
new file mode 100644
index 000000000..c624eb9a8
--- /dev/null
+++ b/pixman/pixman/Makefile.sources
@@ -0,0 +1,42 @@
+libpixman_sources =			\
+	pixman.c			\
+	pixman-access.c			\
+	pixman-access-accessors.c	\
+	pixman-bits-image.c		\
+	pixman-combine32.c		\
+	pixman-combine-float.c		\
+	pixman-conical-gradient.c	\
+	pixman-filter.c			\
+	pixman-x86.c			\
+	pixman-mips.c			\
+	pixman-arm.c			\
+	pixman-ppc.c			\
+	pixman-edge.c			\
+	pixman-edge-accessors.c		\
+	pixman-fast-path.c		\
+	pixman-glyph.c			\
+	pixman-general.c		\
+	pixman-gradient-walker.c	\
+	pixman-image.c			\
+	pixman-implementation.c		\
+	pixman-linear-gradient.c	\
+	pixman-matrix.c			\
+	pixman-noop.c			\
+	pixman-radial-gradient.c	\
+	pixman-region16.c		\
+	pixman-region32.c		\
+	pixman-solid-fill.c		\
+	pixman-timer.c			\
+	pixman-trap.c			\
+	pixman-utils.c			\
+	$(NULL)
+
+libpixman_headers =			\
+	pixman.h			\
+	pixman-accessor.h		\
+	pixman-combine32.h		\
+	pixman-compiler.h		\
+	pixman-edge-imp.h		\
+	pixman-inlines.h		\
+	pixman-private.h		\
+	$(NULL)
diff --git a/pixman/pixman/Makefile.win32 b/pixman/pixman/Makefile.win32
index 388bee23a..7b64033bc 100644
--- a/pixman/pixman/Makefile.win32
+++ b/pixman/pixman/Makefile.win32
@@ -1,12 +1,8 @@
-LIBRARY     = pixman-1
+default: all
 
-CC   = cl
-LINK = link
-
-CFG_VAR = $(CFG)
-ifeq ($(CFG_VAR),)
-CFG_VAR=release
-endif
+top_srcdir = ..
+include $(top_srcdir)/pixman/Makefile.sources
+include $(top_srcdir)/Makefile.win32.common
 
 MMX_VAR = $(MMX)
 ifeq ($(MMX_VAR),)
@@ -18,79 +14,37 @@ ifeq ($(SSE2_VAR),)
 SSE2_VAR=on
 endif
 
-CFLAGS     = -MD -nologo -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE -I../pixman/src -I. -DPACKAGE=$(LIBRARY) -DPACKAGE_VERSION="" -DPACKAGE_BUGREPORT=""
-MMX_CFLAGS = -DUSE_MMX -w14710 -w14714
-SSE2_CFLAGS = -DUSE_SSE2
-
-# optimization flags
-ifeq ($(CFG_VAR),debug)
-CFLAGS += -Od -Zi
-else
-CFLAGS += -O2
+SSSE3_VAR = $(SSSE3)
+ifeq ($(SSSE3_VAR),)
+SSSE3_VAR=on
 endif
 
-SOURCES =				\
-	pixman-image.c			\
-	pixman-access.c			\
-	pixman-access-accessors.c	\
-	pixman-region16.c		\
-	pixman-region32.c		\
-	pixman-combine32.c		\
-	pixman-combine64.c		\
-	pixman-utils.c			\
-	pixman-edge.c			\
-	pixman-edge-accessors.c		\
-	pixman-trap.c			\
-	pixman-timer.c			\
-	pixman-matrix.c			\
-	pixman-gradient-walker.c	\
-	pixman-linear-gradient.c	\
-	pixman-radial-gradient.c	\
-	pixman-bits-image.c		\
-	pixman.c			\
-	pixman-cpu.c			\
-	pixman-fast-path.c		\
-	pixman-implementation.c		\
-	pixman-solid-fill.c		\
-	pixman-general.c		\
-	$(NULL)
+MMX_CFLAGS = -DUSE_X86_MMX -w14710 -w14714
+SSE2_CFLAGS = -DUSE_SSE2
+SSSE3_CFLAGS = -DUSE_SSSE3
 
 # MMX compilation flags
 ifeq ($(MMX_VAR),on)
-CFLAGS += $(MMX_CFLAGS)
-SOURCES += pixman-mmx.c
+PIXMAN_CFLAGS += $(MMX_CFLAGS)
+libpixman_sources += pixman-mmx.c
 endif
 
 # SSE2 compilation flags
 ifeq ($(SSE2_VAR),on)
-CFLAGS += $(SSE2_CFLAGS)
-SOURCES += pixman-sse2.c
+PIXMAN_CFLAGS += $(SSE2_CFLAGS)
+libpixman_sources += pixman-sse2.c
 endif
 
-OBJECTS     = $(patsubst %.c, $(CFG_VAR)/%.obj, $(SOURCES))
+# SSSE3 compilation flags
+ifeq ($(SSSE3_VAR),on)
+PIXMAN_CFLAGS += $(SSSE3_CFLAGS)
+libpixman_sources += pixman-ssse3.c
+endif
+
+OBJECTS = $(patsubst %.c, $(CFG_VAR)/%.obj, $(libpixman_sources))
 
 # targets
-all: inform informMMX informSSE2 $(CFG_VAR)/$(LIBRARY).lib
-	@exit 0
-clean: inform clean_r 
-	@exit 0
-pixman: inform informMMX informSSE2 $(CFG_VAR)/$(LIBRARY).lib 
-	@exit 0
-
-inform:
-ifneq ($(CFG),release)
-ifneq ($(CFG),debug)
-ifneq ($(CFG),)
-	@echo "Invalid specified configuration option : "$(CFG)"."
-	@echo
-	@echo -n "Possible choices for configuration are "
-	@echo "'release' and 'debug'"
-	@echo ""
-	@exit 1
-endif
-	@echo "Using default RELEASE configuration... (use CFG=release or CFG=debug)"
-endif
-endif
+all: inform informMMX informSSE2 informSSSE3 $(CFG_VAR)/$(LIBRARY).lib
 
 informMMX:
 ifneq ($(MMX),off)
@@ -98,8 +52,7 @@ ifneq ($(MMX),on)
 ifneq ($(MMX),)
 	@echo "Invalid specified MMX option : "$(MMX_VAR)"."
 	@echo
-	@echo -n "Possible choices for MMX are 'on' or 'off'"
-	@echo ""
+	@echo "Possible choices for MMX are 'on' or 'off'"
 	@exit 1
 endif
 	@echo "Setting MMX flag to default value 'on'... (use MMX=on or MMX=off)"
@@ -112,32 +65,29 @@ ifneq ($(SSE2),on)
 ifneq ($(SSE2),)
 	@echo "Invalid specified SSE option : "$(SSE2)"."
 	@echo
-	@echo -n "Possible choices for SSE2 are 'on' or 'off'"
-	@echo ""
+	@echo "Possible choices for SSE2 are 'on' or 'off'"
 	@exit 1
 endif
 	@echo "Setting SSE2 flag to default value 'on'... (use SSE2=on or SSE2=off)"
 endif
 endif
 
-# pixman compilation and linking
-$(CFG_VAR)/%.obj: %.c
-	@mkdir -p $(CFG_VAR)
-	@$(CC) -c $(CFLAGS) -Fo"$@" $<
-
-$(CFG_VAR)/$(LIBRARY).lib: $(OBJECTS)
-	lib -NOLOGO -OUT:$@ $(OBJECTS) || exit 0
+informSSSE3:
+ifneq ($(SSSE3),off)
+ifneq ($(SSSE3),on)
+ifneq ($(SSSE3),)
+	@echo "Invalid specified SSE option : "$(SSSE3)"."
+	@echo
+	@echo "Possible choices for SSSE3 are 'on' or 'off'"
+	@exit 1
+endif
+	@echo "Setting SSSE3 flag to default value 'on'... (use SSSE3=on or SSSE3=off)"
+endif
+endif
 
-pixman-combine32.c: pixman-combine.c.template pixman-combine32.h make-combine.pl
-	perl ./make-combine.pl 8 < $< > $@ || ($(RM) $@; exit 1)
-pixman-combine32.h: pixman-combine.h.template make-combine.pl
-	perl ./make-combine.pl 8 < $< > $@ || ($(RM) $@; exit 1)
 
-pixman-combine64.c: pixman-combine.c.template pixman-combine64.h make-combine.pl
-	perl ./make-combine.pl 16 < $< > $@ || ($(RM) $@; exit 1)
-pixman-combine64.h: pixman-combine.h.template make-combine.pl
-	perl ./make-combine.pl 16 < $< > $@ || ($(RM) $@; exit 1)
+# pixman linking
+$(CFG_VAR)/$(LIBRARY).lib: $(OBJECTS)
+	@$(AR) $(PIXMAN_ARFLAGS) -OUT:$@ $^
 
-clean_r:
-	@rm -f $(CFG_VAR)/*.obj $(CFG_VAR)/*.lib $(CFG_VAR)/*.pdb $(CFG)/*.ilk || exit 0
-	@rm -f $(CFG)/*.obj $(CFG)/*.lib $(CFG)/*.pdb $(CFG)/*.ilk pixman-combine32.c pixman-combine64.c pixman-combine64.c pixman-combine64.h || exit 0
+.PHONY: all informMMX informSSE2 informSSSE3
diff --git a/pixman/pixman/loongson-mmintrin.h b/pixman/pixman/loongson-mmintrin.h
new file mode 100644
index 000000000..086c6e0f1
--- /dev/null
+++ b/pixman/pixman/loongson-mmintrin.h
@@ -0,0 +1,410 @@
+/* The gcc-provided loongson intrinsic functions are way too fucking broken
+ * to be of any use, otherwise I'd use them.
+ *
+ * - The hardware instructions are very similar to MMX or iwMMXt. Certainly
+ *   close enough that they could have implemented the _mm_*-style intrinsic
+ *   interface and had a ton of optimized code available to them. Instead they
+ *   implemented something much, much worse.
+ *
+ * - pshuf takes a dead first argument, causing extra instructions to be
+ *   generated.
+ *
+ * - There are no 64-bit shift or logical intrinsics, which means you have
+ *   to implement them with inline assembly, but this is a nightmare because
+ *   gcc doesn't understand that the integer vector datatypes are actually in
+ *   floating-point registers, so you end up with braindead code like
+ *
+ *	punpcklwd	$f9,$f9,$f5
+ *	    dmtc1	v0,$f8
+ *	punpcklwd	$f19,$f19,$f5
+ *	    dmfc1	t9,$f9
+ *	    dmtc1	v0,$f9
+ *	    dmtc1	t9,$f20
+ *	    dmfc1	s0,$f19
+ *	punpcklbh	$f20,$f20,$f2
+ *
+ *   where crap just gets copied back and forth between integer and floating-
+ *   point registers ad nauseum.
+ *
+ * Instead of trying to workaround the problems from these crap intrinsics, I
+ * just implement the _mm_* intrinsics needed for pixman-mmx.c using inline
+ * assembly.
+ */
+
+#include <stdint.h>
+
+/* vectors are stored in 64-bit floating-point registers */
+typedef double __m64;
+/* having a 32-bit datatype allows us to use 32-bit loads in places like load8888 */
+typedef float  __m32;
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_setzero_si64 (void)
+{
+	return 0.0;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_add_pi16 (__m64 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("paddh %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_add_pi32 (__m64 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("paddw %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_adds_pu16 (__m64 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("paddush %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_adds_pu8 (__m64 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("paddusb %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_and_si64 (__m64 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("and %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpeq_pi32 (__m64 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("pcmpeqw %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_empty (void)
+{
+
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_madd_pi16 (__m64 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("pmaddhw %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_mulhi_pu16 (__m64 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("pmulhuh %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_mullo_pi16 (__m64 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("pmullh %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_or_si64 (__m64 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("or %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_packs_pu16 (__m64 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("packushb %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_packs_pi32 (__m64 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("packsswh %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+#define _MM_SHUFFLE(fp3,fp2,fp1,fp0) \
+ (((fp3) << 6) | ((fp2) << 4) | ((fp1) << 2) | (fp0))
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_set_pi16 (uint16_t __w3, uint16_t __w2, uint16_t __w1, uint16_t __w0)
+{
+	if (__builtin_constant_p (__w3) &&
+	    __builtin_constant_p (__w2) &&
+	    __builtin_constant_p (__w1) &&
+	    __builtin_constant_p (__w0))
+	{
+		uint64_t val = ((uint64_t)__w3 << 48)
+			     | ((uint64_t)__w2 << 32)
+			     | ((uint64_t)__w1 << 16)
+			     | ((uint64_t)__w0 <<  0);
+		return *(__m64 *)&val;
+	}
+	else if (__w3 == __w2 && __w2 == __w1 && __w1 == __w0)
+	{
+		/* TODO: handle other cases */
+		uint64_t val = __w3;
+		uint64_t imm = _MM_SHUFFLE (0, 0, 0, 0);
+		__m64 ret;
+		asm("pshufh %0, %1, %2\n\t"
+		    : "=f" (ret)
+		    : "f" (*(__m64 *)&val), "f" (*(__m64 *)&imm)
+		);
+		return ret;
+	}
+	uint64_t val = ((uint64_t)__w3 << 48)
+		     | ((uint64_t)__w2 << 32)
+		     | ((uint64_t)__w1 << 16)
+		     | ((uint64_t)__w0 <<  0);
+	return *(__m64 *)&val;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_set_pi32 (unsigned __i1, unsigned __i0)
+{
+	if (__builtin_constant_p (__i1) &&
+	    __builtin_constant_p (__i0))
+	{
+		uint64_t val = ((uint64_t)__i1 << 32)
+			     | ((uint64_t)__i0 <<  0);
+		return *(__m64 *)&val;
+	}
+	else if (__i1 == __i0)
+	{
+		uint64_t imm = _MM_SHUFFLE (1, 0, 1, 0);
+		__m64 ret;
+		asm("pshufh %0, %1, %2\n\t"
+		    : "=f" (ret)
+		    : "f" (*(__m32 *)&__i1), "f" (*(__m64 *)&imm)
+		);
+		return ret;
+	}
+	uint64_t val = ((uint64_t)__i1 << 32)
+		     | ((uint64_t)__i0 <<  0);
+	return *(__m64 *)&val;
+}
+#undef _MM_SHUFFLE
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_shuffle_pi16 (__m64 __m, int64_t __n)
+{
+	__m64 ret;
+	asm("pshufh %0, %1, %2\n\t"
+	    : "=f" (ret)
+	    : "f" (__m), "f" (*(__m64 *)&__n)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_slli_pi16 (__m64 __m, int64_t __count)
+{
+	__m64 ret;
+	asm("psllh  %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m), "f" (*(__m64 *)&__count)
+	);
+	return ret;
+}
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_slli_si64 (__m64 __m, int64_t __count)
+{
+	__m64 ret;
+	asm("dsll  %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m), "f" (*(__m64 *)&__count)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_srli_pi16 (__m64 __m, int64_t __count)
+{
+	__m64 ret;
+	asm("psrlh %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m), "f" (*(__m64 *)&__count)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_srli_pi32 (__m64 __m, int64_t __count)
+{
+	__m64 ret;
+	asm("psrlw %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m), "f" (*(__m64 *)&__count)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_srli_si64 (__m64 __m, int64_t __count)
+{
+	__m64 ret;
+	asm("dsrl  %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m), "f" (*(__m64 *)&__count)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_sub_pi16 (__m64 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("psubh %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_unpackhi_pi8 (__m64 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("punpckhbh %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_unpackhi_pi16 (__m64 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("punpckhhw %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_unpacklo_pi8 (__m64 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("punpcklbh %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+/* Since punpcklbh doesn't care about the high 32-bits, we use the __m32 datatype which
+ * allows load8888 to use 32-bit loads */
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_unpacklo_pi8_f (__m32 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("punpcklbh %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_unpacklo_pi16 (__m64 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("punpcklhw %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_xor_si64 (__m64 __m1, __m64 __m2)
+{
+	__m64 ret;
+	asm("xor %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+loongson_extract_pi16 (__m64 __m, int64_t __pos)
+{
+	__m64 ret;
+	asm("pextrh %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m), "f" (*(__m64 *)&__pos)
+	);
+	return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+loongson_insert_pi16 (__m64 __m1, __m64 __m2, int64_t __pos)
+{
+	__m64 ret;
+	asm("pinsrh_%3 %0, %1, %2\n\t"
+	   : "=f" (ret)
+	   : "f" (__m1), "f" (__m2), "i" (__pos)
+	);
+	return ret;
+}
diff --git a/pixman/pixman/make-combine.pl b/pixman/pixman/make-combine.pl
deleted file mode 100644
index 210a5da12..000000000
--- a/pixman/pixman/make-combine.pl
+++ /dev/null
@@ -1,86 +0,0 @@
-$usage = "Usage: combine.pl { 8 | 16 } < pixman-combine.c.template";
-
-$#ARGV == 0 or die $usage;
-
-# Get the component size.
-$size = int($ARGV[0]);
-$size == 8 or $size == 16 or die $usage;
-
-$pixel_size = $size * 4;
-$half_pixel_size = $size * 2;
-
-sub mask {
-    my $str = shift;
-    my $suffix;
-    $suffix = "ULL" if $size > 8;
-
-    return "0x" . $str . $suffix;
-}
-
-# Generate mask strings.
-$nibbles = $size / 4;
-$mask = "f" x $nibbles;
-$zero_mask = "0" x $nibbles;
-$one_half = "8" . "0" x ($nibbles - 1);
-
-print "/* WARNING: This file is generated by combine.pl from combine.inc.\n";
-print "   Please edit one of those files rather than this one. */\n";
-print "\n";
-
-print "#line 1 \"pixman-combine.c.template\"\n";
-
-$mask_ = mask($mask);
-$one_half_ = mask($one_half);
-$g_mask = mask($mask . $zero_mask);
-$b_mask = mask($mask . $zero_mask x 2);
-$a_mask = mask($mask . $zero_mask x 3);
-$rb_mask = mask($mask . $zero_mask . $mask);
-$ag_mask = mask($mask . $zero_mask . $mask . $zero_mask);
-$rb_one_half = mask($one_half . $zero_mask . $one_half);
-$rb_mask_plus_one = mask("1" . $zero_mask x 2 . "1" .  $zero_mask);
-
-while (<STDIN>) {
-    # Mask and 1/2 value for a single component.
-    s/#define COMPONENT_SIZE\b/$& $size/;
-    s/#define MASK\b/$& $mask_/;
-    s/#define ONE_HALF\b/$& $one_half_/;
-
-    # Shifts and masks for green, blue, and alpha.
-    s/#define G_SHIFT\b/$& $size/;
-    s/#define R_SHIFT\b/$& $size * 2/;
-    s/#define A_SHIFT\b/$& $size * 3/;
-    s/#define G_MASK\b/$& $g_mask/;
-    s/#define R_MASK\b/$& $b_mask/;
-    s/#define A_MASK\b/$& $a_mask/;
-
-    # Special values for dealing with red + blue at the same time.
-    s/#define RB_MASK\b/$& $rb_mask/;
-    s/#define AG_MASK\b/$& $ag_mask/;
-    s/#define RB_ONE_HALF\b/$& $rb_one_half/;
-    s/#define RB_MASK_PLUS_ONE\b/$& $rb_mask_plus_one/;
-
-    # Add 32/64 suffix to combining function types.
-    s/\bCombineFunc\b/CombineFunc$pixel_size/;
-    s/\bFbComposeFunctions\b/FbComposeFunctions$pixel_size/;
-    s/combine_width/combine_$pixel_size/;
-    s/_pixman_setup_combiner_functions_width/_pixman_setup_combiner_functions_$pixel_size/;
-    s/UNc/UN$size/g;
-    s/ALPHA_c/ALPHA_$size/g;
-    s/RED_c/RED_$size/g;
-    s/GREEN_c/GREEN_$size/g;
-    s/BLUE_c/BLUE_$size/g;
-
-    # Convert comp*_t values into the appropriate real types.
-    s/comp1_t/uint${size}_t/g;
-    s/comp2_t/uint${half_pixel_size}_t/g;
-    s/comp4_t/uint${pixel_size}_t/g;
-
-    # Change the function table name for the 64-bit version.
-    s/pixman_composeFunctions/pixman_composeFunctions64/ if $size == 16;
-
-    # Change the header for the 64-bit version
-    s/pixman-combine.h/pixman-combine64.h/ if $size == 16;
-    s/pixman-combine.h/pixman-combine32.h/ if $size == 8;
-
-    print;
-}
diff --git a/pixman/pixman/make-srgb.pl b/pixman/pixman/make-srgb.pl
new file mode 100644
index 000000000..cdaa80ba5
--- /dev/null
+++ b/pixman/pixman/make-srgb.pl
@@ -0,0 +1,115 @@
+#!/usr/bin/perl -w
+
+use strict;
+
+sub linear_to_srgb
+{
+    my ($c) = @_;
+
+    if ($c < 0.0031308)
+    {
+	return $c * 12.92;
+    }
+    else
+    {
+	return 1.055 * $c ** (1.0/2.4) - 0.055;
+    }
+}
+
+sub srgb_to_linear
+{
+    my ($c) = @_;
+
+    if ($c < 0.04045)
+    {
+	return $c / 12.92;
+    }
+    else
+    {
+	return (($c + 0.055) / 1.055) ** 2.4
+    }
+}
+
+my @linear_to_srgb;
+for my $linear (0 .. 4095)
+{
+    my $srgb = int(linear_to_srgb($linear / 4095.0) * 255.0 + 0.5);
+    push @linear_to_srgb, $srgb;
+}
+
+my @srgb_to_linear;
+for my $srgb (0 .. 255)
+{
+    my $linear = int(srgb_to_linear($srgb / 255.0) * 65535.0 + 0.5);
+    push @srgb_to_linear, $linear;
+}
+
+# Ensure that we have a lossless sRGB and back conversion loop.
+# some of the darkest shades need a little bias -- maximum is just
+# 5 increments out of 16. This gives us useful property with
+# least amount of error in the sRGB-to-linear table, and keeps the actual
+# table lookup in the other direction as simple as possible.
+for my $srgb (0 .. $#srgb_to_linear)
+{
+    my $add = 0;
+    while (1)
+    {
+	my $linear = $srgb_to_linear[$srgb];
+	my $srgb_lossy = $linear_to_srgb[$linear >> 4];
+	last if $srgb == $srgb_lossy;
+
+	# Add slight bias to this component until it rounds correctly
+	$srgb_to_linear[$srgb] ++;
+	$add ++;
+    }
+    die "Too many adds at $srgb" if $add > 5;
+}
+
+print <<"PROLOG";
+/* WARNING: This file is generated by $0.
+ * Please edit that file instead of this one.
+ */
+
+#include <stdint.h>
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "pixman-private.h"
+
+PROLOG
+
+print "const uint8_t linear_to_srgb[" . @linear_to_srgb . "] =\n";
+print "{\n";
+for my $linear (0 .. $#linear_to_srgb)
+{
+    if (($linear % 10) == 0)
+    {
+	print "\t";
+    }
+    print sprintf("%d, ", $linear_to_srgb[$linear]);
+    if (($linear % 10) == 9)
+    {
+	print "\n";
+    }
+}
+print "\n};\n";
+print "\n";
+
+print "const uint16_t srgb_to_linear[" . @srgb_to_linear . "] =\n";
+print "{\n";
+for my $srgb (0 .. $#srgb_to_linear)
+{
+    if (($srgb % 10) == 0)
+    {
+	print "\t";
+    }
+    print sprintf("%d, ", $srgb_to_linear[$srgb]);
+    if (($srgb % 10) == 9)
+    {
+	print "\n";
+    }
+}
+print "\n};\n";
+
diff --git a/pixman/pixman/pixman-access.c b/pixman/pixman/pixman-access.c
index b65ef661d..4f0642d77 100644
--- a/pixman/pixman/pixman-access.c
+++ b/pixman/pixman/pixman-access.c
@@ -31,9 +31,10 @@
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
+#include <math.h>
 
-#include "pixman-private.h"
 #include "pixman-accessor.h"
+#include "pixman-private.h"
 
 #define CONVERT_RGB24_TO_Y15(s)						\
     (((((s) >> 16) & 0xff) * 153 +					\
@@ -45,14 +46,119 @@
      (((s) >> 6) & 0x03e0) |                                            \
      (((s) >> 9) & 0x7c00))
 
-#define RGB15_TO_ENTRY(mif,rgb15)					\
-    ((mif)->ent[rgb15])
+/* Fetch macros */
+
+#ifdef WORDS_BIGENDIAN
+#define FETCH_1(img,l,o)						\
+    (((READ ((img), ((uint32_t *)(l)) + ((o) >> 5))) >> (0x1f - ((o) & 0x1f))) & 0x1)
+#else
+#define FETCH_1(img,l,o)						\
+    ((((READ ((img), ((uint32_t *)(l)) + ((o) >> 5))) >> ((o) & 0x1f))) & 0x1)
+#endif
+
+#define FETCH_8(img,l,o)    (READ (img, (((uint8_t *)(l)) + ((o) >> 3))))
+
+#ifdef WORDS_BIGENDIAN
+#define FETCH_4(img,l,o)						\
+    (((4 * (o)) & 4) ? (FETCH_8 (img,l, 4 * (o)) & 0xf) : (FETCH_8 (img,l,(4 * (o))) >> 4))
+#else
+#define FETCH_4(img,l,o)						\
+    (((4 * (o)) & 4) ? (FETCH_8 (img, l, 4 * (o)) >> 4) : (FETCH_8 (img, l, (4 * (o))) & 0xf))
+#endif
+
+#ifdef WORDS_BIGENDIAN
+#define FETCH_24(img,l,o)                                              \
+    ((READ (img, (((uint8_t *)(l)) + ((o) * 3) + 0)) << 16)    |       \
+     (READ (img, (((uint8_t *)(l)) + ((o) * 3) + 1)) << 8)     |       \
+     (READ (img, (((uint8_t *)(l)) + ((o) * 3) + 2)) << 0))
+#else
+#define FETCH_24(img,l,o)						\
+    ((READ (img, (((uint8_t *)(l)) + ((o) * 3) + 0)) << 0)	|	\
+     (READ (img, (((uint8_t *)(l)) + ((o) * 3) + 1)) << 8)	|	\
+     (READ (img, (((uint8_t *)(l)) + ((o) * 3) + 2)) << 16))
+#endif
+
+/* Store macros */
 
-#define RGB24_TO_ENTRY(mif,rgb24)					\
-    RGB15_TO_ENTRY (mif,CONVERT_RGB24_TO_RGB15 (rgb24))
+#ifdef WORDS_BIGENDIAN
+#define STORE_1(img,l,o,v)						\
+    do									\
+    {									\
+	uint32_t  *__d = ((uint32_t *)(l)) + ((o) >> 5);		\
+	uint32_t __m, __v;						\
+									\
+	__m = 1 << (0x1f - ((o) & 0x1f));				\
+	__v = (v)? __m : 0;						\
+									\
+	WRITE((img), __d, (READ((img), __d) & ~__m) | __v);		\
+    }									\
+    while (0)
+#else
+#define STORE_1(img,l,o,v)						\
+    do									\
+    {									\
+	uint32_t  *__d = ((uint32_t *)(l)) + ((o) >> 5);		\
+	uint32_t __m, __v;						\
+									\
+	__m = 1 << ((o) & 0x1f);					\
+	__v = (v)? __m : 0;						\
+									\
+	WRITE((img), __d, (READ((img), __d) & ~__m) | __v);		\
+    }									\
+    while (0)
+#endif
 
-#define RGB24_TO_ENTRY_Y(mif,rgb24)					\
-    ((mif)->ent[CONVERT_RGB24_TO_Y15 (rgb24)])
+#define STORE_8(img,l,o,v)  (WRITE (img, (uint8_t *)(l) + ((o) >> 3), (v)))
+
+#ifdef WORDS_BIGENDIAN
+#define STORE_4(img,l,o,v)						\
+    do									\
+    {									\
+	int bo = 4 * (o);						\
+	int v4 = (v) & 0x0f;						\
+									\
+	STORE_8 (img, l, bo, (						\
+		     bo & 4 ?						\
+		     (FETCH_8 (img, l, bo) & 0xf0) | (v4) :		\
+		     (FETCH_8 (img, l, bo) & 0x0f) | (v4 << 4)));	\
+    } while (0)
+#else
+#define STORE_4(img,l,o,v)						\
+    do									\
+    {									\
+	int bo = 4 * (o);						\
+	int v4 = (v) & 0x0f;						\
+									\
+	STORE_8 (img, l, bo, (						\
+		     bo & 4 ?						\
+		     (FETCH_8 (img, l, bo) & 0x0f) | (v4 << 4) :	\
+		     (FETCH_8 (img, l, bo) & 0xf0) | (v4)));		\
+    } while (0)
+#endif
+
+#ifdef WORDS_BIGENDIAN
+#define STORE_24(img,l,o,v)                                            \
+    do                                                                 \
+    {                                                                  \
+	uint8_t *__tmp = (l) + 3 * (o);				       \
+        							       \
+	WRITE ((img), __tmp++, ((v) & 0x00ff0000) >> 16);	       \
+	WRITE ((img), __tmp++, ((v) & 0x0000ff00) >>  8);	       \
+	WRITE ((img), __tmp++, ((v) & 0x000000ff) >>  0);	       \
+    }                                                                  \
+    while (0)
+#else
+#define STORE_24(img,l,o,v)                                            \
+    do                                                                 \
+    {                                                                  \
+	uint8_t *__tmp = (l) + 3 * (o);				       \
+        							       \
+	WRITE ((img), __tmp++, ((v) & 0x000000ff) >>  0);	       \
+	WRITE ((img), __tmp++, ((v) & 0x0000ff00) >>  8);	       \
+	WRITE ((img), __tmp++, ((v) & 0x00ff0000) >> 16);	       \
+    }								       \
+    while (0)
+#endif
 
 /*
  * YV12 setup and access macros
@@ -86,997 +192,555 @@
     ((uint8_t *) ((bits) + offset0 +                                    \
                   ((stride) >> 1) * ((line) >> 1)))
 
-/********************************** Fetch ************************************/
+/* Misc. helpers */
+
+static force_inline void
+get_shifts (pixman_format_code_t  format,
+	    int			 *a,
+	    int			 *r,
+	    int                  *g,
+	    int                  *b)
+{
+    switch (PIXMAN_FORMAT_TYPE (format))
+    {
+    case PIXMAN_TYPE_A:
+	*b = 0;
+	*g = 0;
+	*r = 0;
+	*a = 0;
+	break;
+
+    case PIXMAN_TYPE_ARGB:
+    case PIXMAN_TYPE_ARGB_SRGB:
+	*b = 0;
+	*g = *b + PIXMAN_FORMAT_B (format);
+	*r = *g + PIXMAN_FORMAT_G (format);
+	*a = *r + PIXMAN_FORMAT_R (format);
+	break;
+
+    case PIXMAN_TYPE_ABGR:
+	*r = 0;
+	*g = *r + PIXMAN_FORMAT_R (format);
+	*b = *g + PIXMAN_FORMAT_G (format);
+	*a = *b + PIXMAN_FORMAT_B (format);
+	break;
+
+    case PIXMAN_TYPE_BGRA:
+	/* With BGRA formats we start counting at the high end of the pixel */
+	*b = PIXMAN_FORMAT_BPP (format) - PIXMAN_FORMAT_B (format);
+	*g = *b - PIXMAN_FORMAT_B (format);
+	*r = *g - PIXMAN_FORMAT_G (format);
+	*a = *r - PIXMAN_FORMAT_R (format);
+	break;
+
+    case PIXMAN_TYPE_RGBA:
+	/* With BGRA formats we start counting at the high end of the pixel */
+	*r = PIXMAN_FORMAT_BPP (format) - PIXMAN_FORMAT_R (format);
+	*g = *r - PIXMAN_FORMAT_R (format);
+	*b = *g - PIXMAN_FORMAT_G (format);
+	*a = *b - PIXMAN_FORMAT_B (format);
+	break;
+
+    default:
+	assert (0);
+	break;
+    }
+}
+
+static force_inline uint32_t
+convert_channel (uint32_t pixel, uint32_t def_value,
+		 int n_from_bits, int from_shift,
+		 int n_to_bits, int to_shift)
+{
+    uint32_t v;
+
+    if (n_from_bits && n_to_bits)
+	v  = unorm_to_unorm (pixel >> from_shift, n_from_bits, n_to_bits);
+    else if (n_to_bits)
+	v = def_value;
+    else
+	v = 0;
 
-static void
-fetch_scanline_a8r8g8b8 (pixman_image_t *image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         uint32_t *      buffer,
-                         const uint32_t *mask,
-                         uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    
-    MEMCPY_WRAPPED (image,
-                    buffer, (const uint32_t *)bits + x,
-                    width * sizeof(uint32_t));
+    return (v & ((1 << n_to_bits) - 1)) << to_shift;
 }
 
-static void
-fetch_scanline_x8r8g8b8 (pixman_image_t *image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         uint32_t *      buffer,
-                         const uint32_t *mask,
-                         uint32_t        mask_bits)
+static force_inline uint32_t
+convert_pixel (pixman_format_code_t from, pixman_format_code_t to, uint32_t pixel)
 {
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint32_t *pixel = (const uint32_t *)bits + x;
-    const uint32_t *end = pixel + width;
-    
-    while (pixel < end)
-	*buffer++ = READ (image, pixel++) | 0xff000000;
-}
+    int a_from_shift, r_from_shift, g_from_shift, b_from_shift;
+    int a_to_shift, r_to_shift, g_to_shift, b_to_shift;
+    uint32_t a, r, g, b;
 
-static void
-fetch_scanline_a8b8g8r8 (pixman_image_t *image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         uint32_t *      buffer,
-                         const uint32_t *mask,
-                         uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint32_t *pixel = (uint32_t *)bits + x;
-    const uint32_t *end = pixel + width;
-    
-    while (pixel < end)
-    {
-	uint32_t p = READ (image, pixel++);
-	
-	*buffer++ = (p & 0xff00ff00)	|
-	    ((p >> 16) & 0xff)		|
-	    ((p & 0xff) << 16);
-    }
-}
+    get_shifts (from, &a_from_shift, &r_from_shift, &g_from_shift, &b_from_shift);
+    get_shifts (to, &a_to_shift, &r_to_shift, &g_to_shift, &b_to_shift);
 
-static void
-fetch_scanline_x8b8g8r8 (pixman_image_t *image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         uint32_t *      buffer,
-                         const uint32_t *mask,
-                         uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint32_t *pixel = (uint32_t *)bits + x;
-    const uint32_t *end = pixel + width;
-    
-    while (pixel < end)
-    {
-	uint32_t p = READ (image, pixel++);
-	
-	*buffer++ = 0xff000000		|
-	    (p & 0x0000ff00)		|
-	    ((p >> 16) & 0xff)		|
-	    ((p & 0xff) << 16);
-    }
-}
+    a = convert_channel (pixel, ~0,
+			 PIXMAN_FORMAT_A (from), a_from_shift,
+			 PIXMAN_FORMAT_A (to), a_to_shift);
 
-static void
-fetch_scanline_b8g8r8a8 (pixman_image_t *image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         uint32_t *      buffer,
-                         const uint32_t *mask,
-                         uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint32_t *pixel = (uint32_t *)bits + x;
-    const uint32_t *end = pixel + width;
+    r = convert_channel (pixel, 0,
+			 PIXMAN_FORMAT_R (from), r_from_shift,
+			 PIXMAN_FORMAT_R (to), r_to_shift);
 
-    while (pixel < end)
-    {
-	uint32_t p = READ (image, pixel++);
+    g = convert_channel (pixel, 0,
+			 PIXMAN_FORMAT_G (from), g_from_shift,
+			 PIXMAN_FORMAT_G (to), g_to_shift);
 
-	*buffer++ = (((p & 0xff000000) >> 24)	|
-	             ((p & 0x00ff0000) >> 8)	|
-	             ((p & 0x0000ff00) << 8)	|
-	             ((p & 0x000000ff) << 24));
-    }
-}
+    b = convert_channel (pixel, 0,
+			 PIXMAN_FORMAT_B (from), b_from_shift,
+			 PIXMAN_FORMAT_B (to), b_to_shift);
 
-static void
-fetch_scanline_b8g8r8x8 (pixman_image_t *image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         uint32_t *      buffer,
-                         const uint32_t *mask,
-                         uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint32_t *pixel = (uint32_t *)bits + x;
-    const uint32_t *end = pixel + width;
-    
-    while (pixel < end)
-    {
-	uint32_t p = READ (image, pixel++);
-	
-	*buffer++ = (0xff000000 |
-	             ((p & 0xff000000) >> 24)	|
-	             ((p & 0x00ff0000) >> 8)	|
-	             ((p & 0x0000ff00) << 8));
-    }
+    return a | r | g | b;
 }
 
-/* Expects a uint64_t buffer */
-static void
-fetch_scanline_a2r10g10b10 (pixman_image_t *image,
-                            int             x,
-                            int             y,
-                            int             width,
-                            uint32_t *      b,
-                            const uint32_t *mask,
-                            uint32_t        mask_bits)
+static force_inline uint32_t
+convert_pixel_to_a8r8g8b8 (bits_image_t *image,
+			   pixman_format_code_t format,
+			   uint32_t pixel)
 {
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint32_t *pixel = bits + x;
-    const uint32_t *end = pixel + width;
-    uint64_t *buffer = (uint64_t *)b;
-
-    while (pixel < end)
+    if (PIXMAN_FORMAT_TYPE (format) == PIXMAN_TYPE_GRAY		||
+	PIXMAN_FORMAT_TYPE (format) == PIXMAN_TYPE_COLOR)
     {
-	uint32_t p = READ (image, pixel++);
-	uint64_t a = p >> 30;
-	uint64_t r = (p >> 20) & 0x3ff;
-	uint64_t g = (p >> 10) & 0x3ff;
-	uint64_t b = p & 0x3ff;
-
-	r = r << 6 | r >> 4;
-	g = g << 6 | g >> 4;
-	b = b << 6 | b >> 4;
-
-	a <<= 14;
-	a |= a >> 2;
-	a |= a >> 4;
-	a |= a >> 8;
-
-	*buffer++ = a << 48 | r << 32 | g << 16 | b;
+	return image->indexed->rgba[pixel];
     }
-}
-
-/* Expects a uint64_t buffer */
-static void
-fetch_scanline_x2r10g10b10 (pixman_image_t *image,
-                            int             x,
-                            int             y,
-                            int             width,
-                            uint32_t *      b,
-                            const uint32_t *mask,
-                            uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint32_t *pixel = (uint32_t *)bits + x;
-    const uint32_t *end = pixel + width;
-    uint64_t *buffer = (uint64_t *)b;
-    
-    while (pixel < end)
+    else
     {
-	uint32_t p = READ (image, pixel++);
-	uint64_t r = (p >> 20) & 0x3ff;
-	uint64_t g = (p >> 10) & 0x3ff;
-	uint64_t b = p & 0x3ff;
-	
-	r = r << 6 | r >> 4;
-	g = g << 6 | g >> 4;
-	b = b << 6 | b >> 4;
-	
-	*buffer++ = 0xffffULL << 48 | r << 32 | g << 16 | b;
+	return convert_pixel (format, PIXMAN_a8r8g8b8, pixel);
     }
 }
 
-/* Expects a uint64_t buffer */
-static void
-fetch_scanline_a2b10g10r10 (pixman_image_t *image,
-                            int             x,
-                            int             y,
-                            int             width,
-                            uint32_t *      b,
-                            const uint32_t *mask,
-                            uint32_t        mask_bits)
+static force_inline uint32_t
+convert_pixel_from_a8r8g8b8 (pixman_image_t *image,
+			     pixman_format_code_t format, uint32_t pixel)
 {
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint32_t *pixel = bits + x;
-    const uint32_t *end = pixel + width;
-    uint64_t *buffer = (uint64_t *)b;
-    
-    while (pixel < end)
+    if (PIXMAN_FORMAT_TYPE (format) == PIXMAN_TYPE_GRAY)
     {
-	uint32_t p = READ (image, pixel++);
-	uint64_t a = p >> 30;
-	uint64_t b = (p >> 20) & 0x3ff;
-	uint64_t g = (p >> 10) & 0x3ff;
-	uint64_t r = p & 0x3ff;
-	
-	r = r << 6 | r >> 4;
-	g = g << 6 | g >> 4;
-	b = b << 6 | b >> 4;
-	
-	a <<= 14;
-	a |= a >> 2;
-	a |= a >> 4;
-	a |= a >> 8;
+	pixel = CONVERT_RGB24_TO_Y15 (pixel);
 
-	*buffer++ = a << 48 | r << 32 | g << 16 | b;
+	return image->bits.indexed->ent[pixel & 0x7fff];
     }
-}
-
-/* Expects a uint64_t buffer */
-static void
-fetch_scanline_x2b10g10r10 (pixman_image_t *image,
-                            int             x,
-                            int             y,
-                            int             width,
-                            uint32_t *      b,
-                            const uint32_t *mask,
-                            uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint32_t *pixel = (uint32_t *)bits + x;
-    const uint32_t *end = pixel + width;
-    uint64_t *buffer = (uint64_t *)b;
-    
-    while (pixel < end)
+    else if (PIXMAN_FORMAT_TYPE (format) == PIXMAN_TYPE_COLOR)
     {
-	uint32_t p = READ (image, pixel++);
-	uint64_t b = (p >> 20) & 0x3ff;
-	uint64_t g = (p >> 10) & 0x3ff;
-	uint64_t r = p & 0x3ff;
-	
-	r = r << 6 | r >> 4;
-	g = g << 6 | g >> 4;
-	b = b << 6 | b >> 4;
-	
-	*buffer++ = 0xffffULL << 48 | r << 32 | g << 16 | b;
-    }
-}
+	pixel = convert_pixel (PIXMAN_a8r8g8b8, PIXMAN_x1r5g5b5, pixel);
 
-static void
-fetch_scanline_r8g8b8 (pixman_image_t *image,
-                       int             x,
-                       int             y,
-                       int             width,
-                       uint32_t *      buffer,
-                       const uint32_t *mask,
-                       uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint8_t *pixel = (const uint8_t *)bits + 3 * x;
-    const uint8_t *end = pixel + 3 * width;
-    
-    while (pixel < end)
-    {
-	uint32_t b = 0xff000000;
-	
-#ifdef WORDS_BIGENDIAN
-	b |= (READ (image, pixel++) << 16);
-	b |= (READ (image, pixel++) << 8);
-	b |= (READ (image, pixel++));
-#else
-	b |= (READ (image, pixel++));
-	b |= (READ (image, pixel++) << 8);
-	b |= (READ (image, pixel++) << 16);
-#endif
-	
-	*buffer++ = b;
+	return image->bits.indexed->ent[pixel & 0x7fff];
     }
-}
-
-static void
-fetch_scanline_b8g8r8 (pixman_image_t *image,
-                       int             x,
-                       int             y,
-                       int             width,
-                       uint32_t *      buffer,
-                       const uint32_t *mask,
-                       uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint8_t *pixel = (const uint8_t *)bits + 3 * x;
-    const uint8_t *end = pixel + 3 * width;
-    
-    while (pixel < end)
+    else
     {
-	uint32_t b = 0xff000000;
-#ifdef WORDS_BIGENDIAN
-	b |= (READ (image, pixel++));
-	b |= (READ (image, pixel++) << 8);
-	b |= (READ (image, pixel++) << 16);
-#else
-	b |= (READ (image, pixel++) << 16);
-	b |= (READ (image, pixel++) << 8);
-	b |= (READ (image, pixel++));
-#endif
-	*buffer++ = b;
+	return convert_pixel (PIXMAN_a8r8g8b8, format, pixel);
     }
 }
 
-static void
-fetch_scanline_r5g6b5 (pixman_image_t *image,
-                       int             x,
-                       int             y,
-                       int             width,
-                       uint32_t *      buffer,
-                       const uint32_t *mask,
-                       uint32_t        mask_bits)
+static force_inline uint32_t
+fetch_and_convert_pixel (bits_image_t *		image,
+			 const uint8_t *	bits,
+			 int			offset,
+			 pixman_format_code_t	format)
 {
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint16_t *pixel = (const uint16_t *)bits + x;
-    const uint16_t *end = pixel + width;
-    
-    while (pixel < end)
-    {
-	uint32_t p = READ (image, pixel++);
-	uint32_t r = (((p) << 3) & 0xf8) |
-	    (((p) << 5) & 0xfc00) |
-	    (((p) << 8) & 0xf80000);
-	
-	r |= (r >> 5) & 0x70007;
-	r |= (r >> 6) & 0x300;
-	
-	*buffer++ = 0xff000000 | r;
-    }
-}
+    uint32_t pixel;
 
-static void
-fetch_scanline_b5g6r5 (pixman_image_t *image,
-                       int             x,
-                       int             y,
-                       int             width,
-                       uint32_t *      buffer,
-                       const uint32_t *mask,
-                       uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint16_t *pixel = (const uint16_t *)bits + x;
-    const uint16_t *end = pixel + width;
-    
-    while (pixel < end)
+    switch (PIXMAN_FORMAT_BPP (format))
     {
-	uint32_t p = READ (image, pixel++);
-	uint32_t r, g, b;
-	
-	b = ((p & 0xf800) | ((p & 0xe000) >> 5)) >> 8;
-	g = ((p & 0x07e0) | ((p & 0x0600) >> 6)) << 5;
-	r = ((p & 0x001c) | ((p & 0x001f) << 5)) << 14;
-	
-	*buffer++ = 0xff000000 | r | g | b;
-    }
-}
+    case 1:
+	pixel = FETCH_1 (image, bits, offset);
+	break;
 
-static void
-fetch_scanline_a1r5g5b5 (pixman_image_t *image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         uint32_t *      buffer,
-                         const uint32_t *mask,
-                         uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint16_t *pixel = (const uint16_t *)bits + x;
-    const uint16_t *end = pixel + width;
-    
-    while (pixel < end)
-    {
-	uint32_t p = READ (image, pixel++);
-	uint32_t r, g, b, a;
-	
-	a = (uint32_t) ((uint8_t) (0 - ((p & 0x8000) >> 15))) << 24;
-	r = ((p & 0x7c00) | ((p & 0x7000) >> 5)) << 9;
-	g = ((p & 0x03e0) | ((p & 0x0380) >> 5)) << 6;
-	b = ((p & 0x001c) | ((p & 0x001f) << 5)) >> 2;
-	
-	*buffer++ = a | r | g | b;
-    }
-}
+    case 4:
+	pixel = FETCH_4 (image, bits, offset);
+	break;
 
-static void
-fetch_scanline_x1r5g5b5 (pixman_image_t *image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         uint32_t *      buffer,
-                         const uint32_t *mask,
-                         uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint16_t *pixel = (const uint16_t *)bits + x;
-    const uint16_t *end = pixel + width;
-    
-    while (pixel < end)
-    {
-	uint32_t p = READ (image, pixel++);
-	uint32_t r, g, b;
-	
-	r = ((p & 0x7c00) | ((p & 0x7000) >> 5)) << 9;
-	g = ((p & 0x03e0) | ((p & 0x0380) >> 5)) << 6;
-	b = ((p & 0x001c) | ((p & 0x001f) << 5)) >> 2;
-	
-	*buffer++ = 0xff000000 | r | g | b;
-    }
-}
+    case 8:
+	pixel = READ (image, bits + offset);
+	break;
 
-static void
-fetch_scanline_a1b5g5r5 (pixman_image_t *image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         uint32_t *      buffer,
-                         const uint32_t *mask,
-                         uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint16_t *pixel = (const uint16_t *)bits + x;
-    const uint16_t *end = pixel + width;
-    uint32_t r, g, b, a;
-    
-    while (pixel < end)
-    {
-	uint32_t p = READ (image, pixel++);
-	
-	a = (uint32_t) ((uint8_t) (0 - ((p & 0x8000) >> 15))) << 24;
-	b = ((p & 0x7c00) | ((p & 0x7000) >> 5)) >> 7;
-	g = ((p & 0x03e0) | ((p & 0x0380) >> 5)) << 6;
-	r = ((p & 0x001c) | ((p & 0x001f) << 5)) << 14;
-	
-	*buffer++ = a | r | g | b;
-    }
-}
+    case 16:
+	pixel = READ (image, ((uint16_t *)bits + offset));
+	break;
 
-static void
-fetch_scanline_x1b5g5r5 (pixman_image_t *image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         uint32_t *      buffer,
-                         const uint32_t *mask,
-                         uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint16_t *pixel = (const uint16_t *)bits + x;
-    const uint16_t *end = pixel + width;
-    
-    while (pixel < end)
-    {
-	uint32_t p = READ (image, pixel++);
-	uint32_t r, g, b;
-	
-	b = ((p & 0x7c00) | ((p & 0x7000) >> 5)) >> 7;
-	g = ((p & 0x03e0) | ((p & 0x0380) >> 5)) << 6;
-	r = ((p & 0x001c) | ((p & 0x001f) << 5)) << 14;
-	
-	*buffer++ = 0xff000000 | r | g | b;
-    }
-}
+    case 24:
+	pixel = FETCH_24 (image, bits, offset);
+	break;
 
-static void
-fetch_scanline_a4r4g4b4 (pixman_image_t *image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         uint32_t *      buffer,
-                         const uint32_t *mask,
-                         uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint16_t *pixel = (const uint16_t *)bits + x;
-    const uint16_t *end = pixel + width;
-    
-    while (pixel < end)
-    {
-	uint32_t p = READ (image, pixel++);
-	uint32_t r, g, b, a;
-	
-	a = ((p & 0xf000) | ((p & 0xf000) >> 4)) << 16;
-	r = ((p & 0x0f00) | ((p & 0x0f00) >> 4)) << 12;
-	g = ((p & 0x00f0) | ((p & 0x00f0) >> 4)) << 8;
-	b = ((p & 0x000f) | ((p & 0x000f) << 4));
-	
-	*buffer++ = a | r | g | b;
-    }
-}
+    case 32:
+	pixel = READ (image, ((uint32_t *)bits + offset));
+	break;
 
-static void
-fetch_scanline_x4r4g4b4 (pixman_image_t *image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         uint32_t *      buffer,
-                         const uint32_t *mask,
-                         uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint16_t *pixel = (const uint16_t *)bits + x;
-    const uint16_t *end = pixel + width;
-    
-    while (pixel < end)
-    {
-	uint32_t p = READ (image, pixel++);
-	uint32_t r, g, b;
-	
-	r = ((p & 0x0f00) | ((p & 0x0f00) >> 4)) << 12;
-	g = ((p & 0x00f0) | ((p & 0x00f0) >> 4)) << 8;
-	b = ((p & 0x000f) | ((p & 0x000f) << 4));
-	
-	*buffer++ = 0xff000000 | r | g | b;
+    default:
+	pixel = 0xffff00ff; /* As ugly as possible to detect the bug */
+	break;
     }
-}
 
-static void
-fetch_scanline_a4b4g4r4 (pixman_image_t *image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         uint32_t *      buffer,
-                         const uint32_t *mask,
-                         uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint16_t *pixel = (const uint16_t *)bits + x;
-    const uint16_t *end = pixel + width;
-    
-    while (pixel < end)
-    {
-	uint32_t p = READ (image, pixel++);
-	uint32_t r, g, b, a;
-	
-	a = ((p & 0xf000) | ((p & 0xf000) >> 4)) << 16;
-	b = ((p & 0x0f00) | ((p & 0x0f00) >> 4)) >> 4;
-	g = ((p & 0x00f0) | ((p & 0x00f0) >> 4)) << 8;
-	r = ((p & 0x000f) | ((p & 0x000f) << 4)) << 16;
-	
-	*buffer++ = a | r | g | b;
-    }
+    return convert_pixel_to_a8r8g8b8 (image, format, pixel);
 }
 
-static void
-fetch_scanline_x4b4g4r4 (pixman_image_t *image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         uint32_t *      buffer,
-                         const uint32_t *mask,
-                         uint32_t        mask_bits)
+static force_inline void
+convert_and_store_pixel (bits_image_t *		image,
+			 uint8_t *		dest,
+			 int                    offset,
+			 pixman_format_code_t	format,
+			 uint32_t		pixel)
 {
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint16_t *pixel = (const uint16_t *)bits + x;
-    const uint16_t *end = pixel + width;
-    
-    while (pixel < end)
-    {
-	uint32_t p = READ (image, pixel++);
-	uint32_t r, g, b;
-	
-	b = ((p & 0x0f00) | ((p & 0x0f00) >> 4)) >> 4;
-	g = ((p & 0x00f0) | ((p & 0x00f0) >> 4)) << 8;
-	r = ((p & 0x000f) | ((p & 0x000f) << 4)) << 16;
-	
-	*buffer++ = 0xff000000 | r | g | b;
-    }
-}
-
-static void
-fetch_scanline_a8 (pixman_image_t *image,
-                   int             x,
-                   int             y,
-                   int             width,
-                   uint32_t *      buffer,
-                   const uint32_t *mask,
-                   uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint8_t *pixel = (const uint8_t *)bits + x;
-    const uint8_t *end = pixel + width;
-    
-    while (pixel < end)
-	*buffer++ = READ (image, pixel++) << 24;
-}
+    uint32_t converted = convert_pixel_from_a8r8g8b8 (
+	(pixman_image_t *)image, format, pixel);
 
-static void
-fetch_scanline_r3g3b2 (pixman_image_t *image,
-                       int             x,
-                       int             y,
-                       int             width,
-                       uint32_t *      buffer,
-                       const uint32_t *mask,
-                       uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint8_t *pixel = (const uint8_t *)bits + x;
-    const uint8_t *end = pixel + width;
-    
-    while (pixel < end)
+    switch (PIXMAN_FORMAT_BPP (format))
     {
-	uint32_t p = READ (image, pixel++);
-	uint32_t r, g, b;
-	
-	r = ((p & 0xe0) | ((p & 0xe0) >> 3) | ((p & 0xc0) >> 6)) << 16;
-	g = ((p & 0x1c) | ((p & 0x18) >> 3) | ((p & 0x1c) << 3)) << 8;
-	b = (((p & 0x03)     ) |
-	     ((p & 0x03) << 2) |
-	     ((p & 0x03) << 4) |
-	     ((p & 0x03) << 6));
-	
-	*buffer++ = 0xff000000 | r | g | b;
-    }
-}
+    case 1:
+	STORE_1 (image, dest, offset, converted & 0x01);
+	break;
 
-static void
-fetch_scanline_b2g3r3 (pixman_image_t *image,
-                       int             x,
-                       int             y,
-                       int             width,
-                       uint32_t *      buffer,
-                       const uint32_t *mask,
-                       uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint8_t *pixel = (const uint8_t *)bits + x;
-    const uint8_t *end = pixel + width;
+    case 4:
+	STORE_4 (image, dest, offset, converted & 0xf);
+	break;
 
-    while (pixel < end)
-    {
-	uint32_t p = READ (image, pixel++);
-	uint32_t r, g, b;
+    case 8:
+	WRITE (image, (dest + offset), converted & 0xff);
+	break;
 
-	b  = p & 0xc0;
-	b |= b >> 2;
-	b |= b >> 4;
-	b &= 0xff;
+    case 16:
+	WRITE (image, ((uint16_t *)dest + offset), converted & 0xffff);
+	break;
 
-	g  = (p & 0x38) << 10;
-	g |= g >> 3;
-	g |= g >> 6;
-	g &= 0xff00;
+    case 24:
+	STORE_24 (image, dest, offset, converted);
+	break;
 
-	r  = (p & 0x7) << 21;
-	r |= r >> 3;
-	r |= r >> 6;
-	r &= 0xff0000;
+    case 32:
+	WRITE (image, ((uint32_t *)dest + offset), converted);
+	break;
 
-	*buffer++ = 0xff000000 | r | g | b;
+    default:
+	*dest = 0x0;
+	break;
     }
 }
 
-static void
-fetch_scanline_a2r2g2b2 (pixman_image_t *image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         uint32_t *      buffer,
-                         const uint32_t *mask,
-                         uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint8_t *pixel = (const uint8_t *)bits + x;
-    const uint8_t *end = pixel + width;
-    
-    while (pixel < end)
-    {
-	uint32_t p = READ (image, pixel++);
-	uint32_t a, r, g, b;
-	
-	a = ((p & 0xc0) * 0x55) << 18;
-	r = ((p & 0x30) * 0x55) << 12;
-	g = ((p & 0x0c) * 0x55) << 6;
-	b = ((p & 0x03) * 0x55);
-	
-	*buffer++ = a | r | g | b;
-    }
-}
+#define MAKE_ACCESSORS(format)						\
+    static void								\
+    fetch_scanline_ ## format (bits_image_t *image,			\
+			       int	       x,			\
+			       int             y,			\
+			       int             width,			\
+			       uint32_t *      buffer,			\
+			       const uint32_t *mask)			\
+    {									\
+	uint8_t *bits =							\
+	    (uint8_t *)(image->bits + y * image->rowstride);		\
+	int i;								\
+									\
+	for (i = 0; i < width; ++i)					\
+	{								\
+	    *buffer++ =							\
+		fetch_and_convert_pixel (image, bits, x + i, PIXMAN_ ## format); \
+	}								\
+    }									\
+									\
+    static void								\
+    store_scanline_ ## format (bits_image_t *  image,			\
+			       int             x,			\
+			       int             y,			\
+			       int             width,			\
+			       const uint32_t *values)			\
+    {									\
+	uint8_t *dest =							\
+	    (uint8_t *)(image->bits + y * image->rowstride);		\
+	int i;								\
+									\
+	for (i = 0; i < width; ++i)					\
+	{								\
+	    convert_and_store_pixel (					\
+		image, dest, i + x, PIXMAN_ ## format, values[i]);	\
+	}								\
+    }									\
+									\
+    static uint32_t							\
+    fetch_pixel_ ## format (bits_image_t *image,			\
+			    int		offset,				\
+			    int		line)				\
+    {									\
+	uint8_t *bits =							\
+	    (uint8_t *)(image->bits + line * image->rowstride);		\
+									\
+	return fetch_and_convert_pixel (				\
+	    image, bits, offset, PIXMAN_ ## format);			\
+    }									\
+									\
+    static const void *const __dummy__ ## format
+
+MAKE_ACCESSORS(a8r8g8b8);
+MAKE_ACCESSORS(x8r8g8b8);
+MAKE_ACCESSORS(a8b8g8r8);
+MAKE_ACCESSORS(x8b8g8r8);
+MAKE_ACCESSORS(x14r6g6b6);
+MAKE_ACCESSORS(b8g8r8a8);
+MAKE_ACCESSORS(b8g8r8x8);
+MAKE_ACCESSORS(r8g8b8x8);
+MAKE_ACCESSORS(r8g8b8a8);
+MAKE_ACCESSORS(r8g8b8);
+MAKE_ACCESSORS(b8g8r8);
+MAKE_ACCESSORS(r5g6b5);
+MAKE_ACCESSORS(b5g6r5);
+MAKE_ACCESSORS(a1r5g5b5);
+MAKE_ACCESSORS(x1r5g5b5);
+MAKE_ACCESSORS(a1b5g5r5);
+MAKE_ACCESSORS(x1b5g5r5);
+MAKE_ACCESSORS(a4r4g4b4);
+MAKE_ACCESSORS(x4r4g4b4);
+MAKE_ACCESSORS(a4b4g4r4);
+MAKE_ACCESSORS(x4b4g4r4);
+MAKE_ACCESSORS(a8);
+MAKE_ACCESSORS(c8);
+MAKE_ACCESSORS(g8);
+MAKE_ACCESSORS(r3g3b2);
+MAKE_ACCESSORS(b2g3r3);
+MAKE_ACCESSORS(a2r2g2b2);
+MAKE_ACCESSORS(a2b2g2r2);
+MAKE_ACCESSORS(x4a4);
+MAKE_ACCESSORS(a4);
+MAKE_ACCESSORS(g4);
+MAKE_ACCESSORS(c4);
+MAKE_ACCESSORS(r1g2b1);
+MAKE_ACCESSORS(b1g2r1);
+MAKE_ACCESSORS(a1r1g1b1);
+MAKE_ACCESSORS(a1b1g1r1);
+MAKE_ACCESSORS(a1);
+MAKE_ACCESSORS(g1);
 
-static void
-fetch_scanline_a2b2g2r2 (pixman_image_t *image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         uint32_t *      buffer,
-                         const uint32_t *mask,
-                         uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint8_t *pixel = (const uint8_t *)bits + x;
-    const uint8_t *end = pixel + width;
-    
-    while (pixel < end)
-    {
-	uint32_t p = READ (image, pixel++);
-	uint32_t a, r, g, b;
-	
-	a = ((p & 0xc0) * 0x55) << 18;
-	b = ((p & 0x30) * 0x55) >> 4;
-	g = ((p & 0x0c) * 0x55) << 6;
-	r = ((p & 0x03) * 0x55) << 16;
-	
-	*buffer++ = a | r | g | b;
-    }
-}
+/********************************** Fetch ************************************/
+/* Table mapping sRGB-encoded 8 bit numbers to linearly encoded
+ * floating point numbers. We assume that single precision
+ * floating point follows the IEEE 754 format.
+ */
+static const uint32_t to_linear_u[256] =
+{
+    0x00000000, 0x399f22b4, 0x3a1f22b4, 0x3a6eb40e, 0x3a9f22b4, 0x3ac6eb61,
+    0x3aeeb40e, 0x3b0b3e5d, 0x3b1f22b4, 0x3b33070b, 0x3b46eb61, 0x3b5b518a,
+    0x3b70f18a, 0x3b83e1c5, 0x3b8fe614, 0x3b9c87fb, 0x3ba9c9b5, 0x3bb7ad6d,
+    0x3bc63547, 0x3bd5635f, 0x3be539bd, 0x3bf5ba70, 0x3c0373b5, 0x3c0c6152,
+    0x3c15a703, 0x3c1f45bc, 0x3c293e68, 0x3c3391f4, 0x3c3e4149, 0x3c494d43,
+    0x3c54b6c7, 0x3c607eb1, 0x3c6ca5df, 0x3c792d22, 0x3c830aa8, 0x3c89af9e,
+    0x3c9085db, 0x3c978dc5, 0x3c9ec7c0, 0x3ca63432, 0x3cadd37d, 0x3cb5a601,
+    0x3cbdac20, 0x3cc5e639, 0x3cce54ab, 0x3cd6f7d2, 0x3cdfd00e, 0x3ce8ddb9,
+    0x3cf2212c, 0x3cfb9ac1, 0x3d02a569, 0x3d0798dc, 0x3d0ca7e4, 0x3d11d2ae,
+    0x3d171963, 0x3d1c7c2e, 0x3d21fb3a, 0x3d2796af, 0x3d2d4ebb, 0x3d332380,
+    0x3d39152b, 0x3d3f23e3, 0x3d454fd0, 0x3d4b991c, 0x3d51ffeb, 0x3d588466,
+    0x3d5f26b7, 0x3d65e6fe, 0x3d6cc564, 0x3d73c210, 0x3d7add25, 0x3d810b65,
+    0x3d84b793, 0x3d88732e, 0x3d8c3e48, 0x3d9018f4, 0x3d940343, 0x3d97fd48,
+    0x3d9c0714, 0x3da020b9, 0x3da44a48, 0x3da883d6, 0x3daccd70, 0x3db12728,
+    0x3db59110, 0x3dba0b38, 0x3dbe95b2, 0x3dc3308f, 0x3dc7dbe0, 0x3dcc97b4,
+    0x3dd1641c, 0x3dd6412a, 0x3ddb2eec, 0x3de02d75, 0x3de53cd3, 0x3dea5d16,
+    0x3def8e52, 0x3df4d091, 0x3dfa23e5, 0x3dff885e, 0x3e027f06, 0x3e05427f,
+    0x3e080ea2, 0x3e0ae376, 0x3e0dc104, 0x3e10a752, 0x3e139669, 0x3e168e50,
+    0x3e198f0e, 0x3e1c98ab, 0x3e1fab2e, 0x3e22c6a0, 0x3e25eb08, 0x3e29186a,
+    0x3e2c4ed0, 0x3e2f8e42, 0x3e32d6c4, 0x3e362861, 0x3e39831e, 0x3e3ce702,
+    0x3e405416, 0x3e43ca5e, 0x3e4749e4, 0x3e4ad2ae, 0x3e4e64c2, 0x3e520027,
+    0x3e55a4e6, 0x3e595303, 0x3e5d0a8a, 0x3e60cb7c, 0x3e6495e0, 0x3e6869bf,
+    0x3e6c4720, 0x3e702e08, 0x3e741e7f, 0x3e78188c, 0x3e7c1c34, 0x3e8014c0,
+    0x3e822039, 0x3e84308b, 0x3e8645b8, 0x3e885fc3, 0x3e8a7eb0, 0x3e8ca281,
+    0x3e8ecb3a, 0x3e90f8df, 0x3e932b72, 0x3e9562f6, 0x3e979f6f, 0x3e99e0e0,
+    0x3e9c274e, 0x3e9e72b8, 0x3ea0c322, 0x3ea31892, 0x3ea57308, 0x3ea7d28a,
+    0x3eaa3718, 0x3eaca0b7, 0x3eaf0f69, 0x3eb18332, 0x3eb3fc16, 0x3eb67a15,
+    0x3eb8fd34, 0x3ebb8576, 0x3ebe12de, 0x3ec0a56e, 0x3ec33d2a, 0x3ec5da14,
+    0x3ec87c30, 0x3ecb2380, 0x3ecdd008, 0x3ed081ca, 0x3ed338c9, 0x3ed5f508,
+    0x3ed8b68a, 0x3edb7d52, 0x3ede4962, 0x3ee11abe, 0x3ee3f168, 0x3ee6cd64,
+    0x3ee9aeb6, 0x3eec955d, 0x3eef815d, 0x3ef272ba, 0x3ef56976, 0x3ef86594,
+    0x3efb6717, 0x3efe6e02, 0x3f00bd2b, 0x3f02460c, 0x3f03d1a5, 0x3f055ff8,
+    0x3f06f105, 0x3f0884ce, 0x3f0a1b54, 0x3f0bb499, 0x3f0d509f, 0x3f0eef65,
+    0x3f1090ef, 0x3f12353c, 0x3f13dc50, 0x3f15862a, 0x3f1732cc, 0x3f18e237,
+    0x3f1a946d, 0x3f1c4970, 0x3f1e013f, 0x3f1fbbde, 0x3f21794c, 0x3f23398c,
+    0x3f24fca0, 0x3f26c286, 0x3f288b42, 0x3f2a56d3, 0x3f2c253d, 0x3f2df680,
+    0x3f2fca9d, 0x3f31a195, 0x3f337b6a, 0x3f35581e, 0x3f3737b1, 0x3f391a24,
+    0x3f3aff7a, 0x3f3ce7b2, 0x3f3ed2d0, 0x3f40c0d2, 0x3f42b1bc, 0x3f44a58e,
+    0x3f469c49, 0x3f4895ee, 0x3f4a9280, 0x3f4c91ff, 0x3f4e946c, 0x3f5099c8,
+    0x3f52a216, 0x3f54ad55, 0x3f56bb88, 0x3f58ccae, 0x3f5ae0cb, 0x3f5cf7de,
+    0x3f5f11ec, 0x3f612ef0, 0x3f634eef, 0x3f6571ea, 0x3f6797e1, 0x3f69c0d6,
+    0x3f6beccb, 0x3f6e1bc0, 0x3f704db6, 0x3f7282af, 0x3f74baac, 0x3f76f5ae,
+    0x3f7933b6, 0x3f7b74c6, 0x3f7db8de, 0x3f800000
+};
 
-static void
-fetch_scanline_c8 (pixman_image_t *image,
-                   int             x,
-                   int             y,
-                   int             width,
-                   uint32_t *      buffer,
-                   const uint32_t *mask,
-                   uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const pixman_indexed_t * indexed = image->bits.indexed;
-    const uint8_t *pixel = (const uint8_t *)bits + x;
-    const uint8_t *end = pixel + width;
-    
-    while (pixel < end)
-    {
-	uint32_t p = READ (image, pixel++);
-	
-	*buffer++ = indexed->rgba[p];
-    }
-}
+static const float * const to_linear = (const float *)to_linear_u;
 
-static void
-fetch_scanline_x4a4 (pixman_image_t *image,
-                     int             x,
-                     int             y,
-                     int             width,
-                     uint32_t *      buffer,
-                     const uint32_t *mask,
-                     uint32_t        mask_bits)
+static uint8_t
+to_srgb (float f)
 {
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const uint8_t *pixel = (const uint8_t *)bits + x;
-    const uint8_t *end = pixel + width;
-   
-    while (pixel < end)
+    uint8_t low = 0;
+    uint8_t high = 255;
+
+    while (high - low > 1)
     {
-	uint8_t p = READ (image, pixel++) & 0xf;
+	uint8_t mid = (low + high) / 2;
 
-	*buffer++ = (p | (p << 4)) << 24;
+	if (to_linear[mid] > f)
+	    high = mid;
+	else
+	    low = mid;
     }
-}
 
-#define FETCH_8(img,l,o)    (READ (img, (((uint8_t *)(l)) + ((o) >> 3))))
-#ifdef WORDS_BIGENDIAN
-#define FETCH_4(img,l,o)						\
-    (((4 * (o)) & 4) ? (FETCH_8 (img,l, 4 * (o)) & 0xf) : (FETCH_8 (img,l,(4 * (o))) >> 4))
-#else
-#define FETCH_4(img,l,o)						\
-    (((4 * (o)) & 4) ? (FETCH_8 (img, l, 4 * (o)) >> 4) : (FETCH_8 (img, l, (4 * (o))) & 0xf))
-#endif
+    if (to_linear[high] - f < f - to_linear[low])
+	return high;
+    else
+	return low;
+}
 
 static void
-fetch_scanline_a4 (pixman_image_t *image,
-                   int             x,
-                   int             y,
-                   int             width,
-                   uint32_t *      buffer,
-                   const uint32_t *mask,
-                   uint32_t        mask_bits)
+fetch_scanline_a8r8g8b8_sRGB_float (bits_image_t *  image,
+				    int             x,
+				    int             y,
+				    int             width,
+				    uint32_t *      b,
+				    const uint32_t *mask)
 {
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    int i;
+    const uint32_t *bits = image->bits + y * image->rowstride;
+    const uint32_t *pixel = bits + x;
+    const uint32_t *end = pixel + width;
+    argb_t *buffer = (argb_t *)b;
 
-    for (i = 0; i < width; ++i)
+    while (pixel < end)
     {
-	uint32_t p = FETCH_4 (image, bits, i + x);
+	uint32_t p = READ (image, pixel++);
+	argb_t *argb = buffer;
 
-	p |= p << 4;
+	argb->a = pixman_unorm_to_float ((p >> 24) & 0xff, 8);
 
-	*buffer++ = p << 24;
-    }
-}
+	argb->r = to_linear [(p >> 16) & 0xff];
+	argb->g = to_linear [(p >>  8) & 0xff];
+	argb->b = to_linear [(p >>  0) & 0xff];
 
-static void
-fetch_scanline_r1g2b1 (pixman_image_t *image,
-                       int             x,
-                       int             y,
-                       int             width,
-                       uint32_t *      buffer,
-                       const uint32_t *mask,
-                       uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	uint32_t p = FETCH_4 (image, bits, i + x);
-	uint32_t r, g, b;
-	
-	r = ((p & 0x8) * 0xff) << 13;
-	g = ((p & 0x6) * 0x55) << 7;
-	b = ((p & 0x1) * 0xff);
-	
-	*buffer++ = 0xff000000 | r | g | b;
+	buffer++;
     }
 }
 
+/* Expects a float buffer */
 static void
-fetch_scanline_b1g2r1 (pixman_image_t *image,
-                       int             x,
-                       int             y,
-                       int             width,
-                       uint32_t *      buffer,
-                       const uint32_t *mask,
-                       uint32_t        mask_bits)
+fetch_scanline_a2r10g10b10_float (bits_image_t *  image,
+				  int             x,
+				  int             y,
+				  int             width,
+				  uint32_t *      b,
+				  const uint32_t *mask)
 {
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    int i;
-    
-    for (i = 0; i < width; ++i)
+    const uint32_t *bits = image->bits + y * image->rowstride;
+    const uint32_t *pixel = bits + x;
+    const uint32_t *end = pixel + width;
+    argb_t *buffer = (argb_t *)b;
+
+    while (pixel < end)
     {
-	uint32_t p = FETCH_4 (image, bits, i + x);
-	uint32_t r, g, b;
-	
-	b = ((p & 0x8) * 0xff) >> 3;
-	g = ((p & 0x6) * 0x55) << 7;
-	r = ((p & 0x1) * 0xff) << 16;
+	uint32_t p = READ (image, pixel++);
+	uint64_t a = p >> 30;
+	uint64_t r = (p >> 20) & 0x3ff;
+	uint64_t g = (p >> 10) & 0x3ff;
+	uint64_t b = p & 0x3ff;
+
+	buffer->a = pixman_unorm_to_float (a, 2);
+	buffer->r = pixman_unorm_to_float (r, 10);
+	buffer->g = pixman_unorm_to_float (g, 10);
+	buffer->b = pixman_unorm_to_float (b, 10);
 
-	*buffer++ = 0xff000000 | r | g | b;
+	buffer++;
     }
 }
 
+/* Expects a float buffer */
 static void
-fetch_scanline_a1r1g1b1 (pixman_image_t *image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         uint32_t *      buffer,
-                         const uint32_t *mask,
-                         uint32_t        mask_bits)
+fetch_scanline_x2r10g10b10_float (bits_image_t   *image,
+				  int             x,
+				  int             y,
+				  int             width,
+				  uint32_t *      b,
+				  const uint32_t *mask)
 {
-    uint32_t a, r, g, b;
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    int i;
+    const uint32_t *bits = image->bits + y * image->rowstride;
+    const uint32_t *pixel = (uint32_t *)bits + x;
+    const uint32_t *end = pixel + width;
+    argb_t *buffer = (argb_t *)b;
 
-    for (i = 0; i < width; ++i)
+    while (pixel < end)
     {
-	uint32_t p = FETCH_4 (image, bits, i + x);
+	uint32_t p = READ (image, pixel++);
+	uint64_t r = (p >> 20) & 0x3ff;
+	uint64_t g = (p >> 10) & 0x3ff;
+	uint64_t b = p & 0x3ff;
 
-	a = ((p & 0x8) * 0xff) << 21;
-	r = ((p & 0x4) * 0xff) << 14;
-	g = ((p & 0x2) * 0xff) << 7;
-	b = ((p & 0x1) * 0xff);
+	buffer->a = 1.0;
+	buffer->r = pixman_unorm_to_float (r, 10);
+	buffer->g = pixman_unorm_to_float (g, 10);
+	buffer->b = pixman_unorm_to_float (b, 10);
 
-	*buffer++ = a | r | g | b;
+	buffer++;
     }
 }
 
+/* Expects a float buffer */
 static void
-fetch_scanline_a1b1g1r1 (pixman_image_t *image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         uint32_t *      buffer,
-                         const uint32_t *mask,
-                         uint32_t        mask_bits)
+fetch_scanline_a2b10g10r10_float (bits_image_t   *image,
+				  int             x,
+				  int             y,
+				  int             width,
+				  uint32_t *      b,
+				  const uint32_t *mask)
 {
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    int i;
+    const uint32_t *bits = image->bits + y * image->rowstride;
+    const uint32_t *pixel = bits + x;
+    const uint32_t *end = pixel + width;
+    argb_t *buffer = (argb_t *)b;
 
-    for (i = 0; i < width; ++i)
+    while (pixel < end)
     {
-	uint32_t p = FETCH_4 (image, bits, i + x);
-	uint32_t a, r, g, b;
+	uint32_t p = READ (image, pixel++);
+	uint64_t a = p >> 30;
+	uint64_t b = (p >> 20) & 0x3ff;
+	uint64_t g = (p >> 10) & 0x3ff;
+	uint64_t r = p & 0x3ff;
 
-	a = ((p & 0x8) * 0xff) << 21;
-	b = ((p & 0x4) * 0xff) >> 2;
-	g = ((p & 0x2) * 0xff) << 7;
-	r = ((p & 0x1) * 0xff) << 16;
+	buffer->a = pixman_unorm_to_float (a, 2);
+	buffer->r = pixman_unorm_to_float (r, 10);
+	buffer->g = pixman_unorm_to_float (g, 10);
+	buffer->b = pixman_unorm_to_float (b, 10);
 
-	*buffer++ = a | r | g | b;
+	buffer++;
     }
 }
 
+/* Expects a float buffer */
 static void
-fetch_scanline_c4 (pixman_image_t *image,
-                   int             x,
-                   int             y,
-                   int             width,
-                   uint32_t *      buffer,
-                   const uint32_t *mask,
-                   uint32_t        mask_bits)
+fetch_scanline_x2b10g10r10_float (bits_image_t   *image,
+				  int             x,
+				  int             y,
+				  int             width,
+				  uint32_t *      b,
+				  const uint32_t *mask)
 {
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const pixman_indexed_t * indexed = image->bits.indexed;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	uint32_t p = FETCH_4 (image, bits, i + x);
-	
-	*buffer++ = indexed->rgba[p];
-    }
-}
+    const uint32_t *bits = image->bits + y * image->rowstride;
+    const uint32_t *pixel = (uint32_t *)bits + x;
+    const uint32_t *end = pixel + width;
+    argb_t *buffer = (argb_t *)b;
 
-static void
-fetch_scanline_a1 (pixman_image_t *image,
-                   int             x,
-                   int             y,
-                   int             width,
-                   uint32_t *      buffer,
-                   const uint32_t *mask,
-                   uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    int i;
-    
-    for (i = 0; i < width; ++i)
+    while (pixel < end)
     {
-	uint32_t p = READ (image, bits + ((i + x) >> 5));
-	uint32_t a;
-	
-#ifdef WORDS_BIGENDIAN
-	a = p >> (0x1f - ((i + x) & 0x1f));
-#else
-	a = p >> ((i + x) & 0x1f);
-#endif
-	a = a & 1;
-	a |= a << 1;
-	a |= a << 2;
-	a |= a << 4;
-	
-	*buffer++ = a << 24;
-    }
-}
+	uint32_t p = READ (image, pixel++);
+	uint64_t b = (p >> 20) & 0x3ff;
+	uint64_t g = (p >> 10) & 0x3ff;
+	uint64_t r = p & 0x3ff;
 
-static void
-fetch_scanline_g1 (pixman_image_t *image,
-                   int             x,
-                   int             y,
-                   int             width,
-                   uint32_t *      buffer,
-                   const uint32_t *mask,
-                   uint32_t        mask_bits)
-{
-    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
-    const pixman_indexed_t * indexed = image->bits.indexed;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	uint32_t p = READ (image, bits + ((i + x) >> 5));
-	uint32_t a;
-	
-#ifdef WORDS_BIGENDIAN
-	a = p >> (0x1f - ((i + x) & 0x1f));
-#else
-	a = p >> ((i + x) & 0x1f);
-#endif
-	a = a & 1;
-	
-	*buffer++ = indexed->rgba[a];
+	buffer->a = 1.0;
+	buffer->r = pixman_unorm_to_float (r, 10);
+	buffer->g = pixman_unorm_to_float (g, 10);
+	buffer->b = pixman_unorm_to_float (b, 10);
+
+	buffer++;
     }
 }
 
 static void
-fetch_scanline_yuy2 (pixman_image_t *image,
+fetch_scanline_yuy2 (bits_image_t   *image,
                      int             x,
                      int             line,
                      int             width,
                      uint32_t *      buffer,
-                     const uint32_t *mask,
-                     uint32_t        mask_bits)
+                     const uint32_t *mask)
 {
-    const uint32_t *bits = image->bits.bits + image->bits.rowstride * line;
+    const uint32_t *bits = image->bits + image->rowstride * line;
     int i;
     
     for (i = 0; i < width; i++)
@@ -1103,13 +767,12 @@ fetch_scanline_yuy2 (pixman_image_t *image,
 }
 
 static void
-fetch_scanline_yv12 (pixman_image_t *image,
+fetch_scanline_yv12 (bits_image_t   *image,
                      int             x,
                      int             line,
                      int             width,
                      uint32_t *      buffer,
-                     const uint32_t *mask,
-                     uint32_t        mask_bits)
+                     const uint32_t *mask)
 {
     YV12_SETUP (image);
     uint8_t *y_line = YV12_Y (line);
@@ -1142,55 +805,51 @@ fetch_scanline_yv12 (pixman_image_t *image,
 
 /**************************** Pixel wise fetching *****************************/
 
-/* Despite the type, expects a uint64_t buffer */
-static uint64_t
-fetch_pixel_a2r10g10b10 (bits_image_t *image,
-			 int		  offset,
-			 int           line)
+static argb_t
+fetch_pixel_x2r10g10b10_float (bits_image_t *image,
+			       int	   offset,
+			       int           line)
 {
     uint32_t *bits = image->bits + line * image->rowstride;
     uint32_t p = READ (image, bits + offset);
-    uint64_t a = p >> 30;
     uint64_t r = (p >> 20) & 0x3ff;
     uint64_t g = (p >> 10) & 0x3ff;
     uint64_t b = p & 0x3ff;
+    argb_t argb;
 
-    r = r << 6 | r >> 4;
-    g = g << 6 | g >> 4;
-    b = b << 6 | b >> 4;
-
-    a <<= 14;
-    a |= a >> 2;
-    a |= a >> 4;
-    a |= a >> 8;
+    argb.a = 1.0;
+    argb.r = pixman_unorm_to_float (r, 10);
+    argb.g = pixman_unorm_to_float (g, 10);
+    argb.b = pixman_unorm_to_float (b, 10);
 
-    return a << 48 | r << 32 | g << 16 | b;
+    return argb;
 }
 
-/* Despite the type, this function expects a uint64_t buffer */
-static uint64_t
-fetch_pixel_x2r10g10b10 (bits_image_t *image,
-			 int	   offset,
-			 int           line)
+static argb_t
+fetch_pixel_a2r10g10b10_float (bits_image_t *image,
+			       int	     offset,
+			       int           line)
 {
     uint32_t *bits = image->bits + line * image->rowstride;
     uint32_t p = READ (image, bits + offset);
+    uint64_t a = p >> 30;
     uint64_t r = (p >> 20) & 0x3ff;
     uint64_t g = (p >> 10) & 0x3ff;
     uint64_t b = p & 0x3ff;
-    
-    r = r << 6 | r >> 4;
-    g = g << 6 | g >> 4;
-    b = b << 6 | b >> 4;
-    
-    return 0xffffULL << 48 | r << 32 | g << 16 | b;
+    argb_t argb;
+
+    argb.a = pixman_unorm_to_float (a, 2);
+    argb.r = pixman_unorm_to_float (r, 10);
+    argb.g = pixman_unorm_to_float (g, 10);
+    argb.b = pixman_unorm_to_float (b, 10);
+
+    return argb;
 }
 
-/* Despite the type, expects a uint64_t buffer */
-static uint64_t
-fetch_pixel_a2b10g10r10 (bits_image_t *image,
-			 int           offset,
-			 int           line)
+static argb_t
+fetch_pixel_a2b10g10r10_float (bits_image_t *image,
+			       int           offset,
+			       int           line)
 {
     uint32_t *bits = image->bits + line * image->rowstride;
     uint32_t p = READ (image, bits + offset);
@@ -1198,568 +857,52 @@ fetch_pixel_a2b10g10r10 (bits_image_t *image,
     uint64_t b = (p >> 20) & 0x3ff;
     uint64_t g = (p >> 10) & 0x3ff;
     uint64_t r = p & 0x3ff;
-    
-    r = r << 6 | r >> 4;
-    g = g << 6 | g >> 4;
-    b = b << 6 | b >> 4;
-    
-    a <<= 14;
-    a |= a >> 2;
-    a |= a >> 4;
-    a |= a >> 8;
-    
-    return a << 48 | r << 32 | g << 16 | b;
+    argb_t argb;
+
+    argb.a = pixman_unorm_to_float (a, 2);
+    argb.r = pixman_unorm_to_float (r, 10);
+    argb.g = pixman_unorm_to_float (g, 10);
+    argb.b = pixman_unorm_to_float (b, 10);
+
+    return argb;
 }
 
-/* Despite the type, this function expects a uint64_t buffer */
-static uint64_t
-fetch_pixel_x2b10g10r10 (bits_image_t *image,
-			 int           offset,
-			 int           line)
+static argb_t
+fetch_pixel_x2b10g10r10_float (bits_image_t *image,
+			       int           offset,
+			       int           line)
 {
     uint32_t *bits = image->bits + line * image->rowstride;
     uint32_t p = READ (image, bits + offset);
     uint64_t b = (p >> 20) & 0x3ff;
     uint64_t g = (p >> 10) & 0x3ff;
     uint64_t r = p & 0x3ff;
-    
-    r = r << 6 | r >> 4;
-    g = g << 6 | g >> 4;
-    b = b << 6 | b >> 4;
-    
-    return 0xffffULL << 48 | r << 32 | g << 16 | b;
-}
-
-static uint32_t
-fetch_pixel_a8r8g8b8 (bits_image_t *image,
-		      int           offset,
-		      int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    return READ (image, (uint32_t *)bits + offset);
-}
-
-static uint32_t
-fetch_pixel_x8r8g8b8 (bits_image_t *image,
-		      int           offset,
-		      int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-
-    return READ (image, (uint32_t *)bits + offset) | 0xff000000;
-}
-
-static uint32_t
-fetch_pixel_a8b8g8r8 (bits_image_t *image,
-		      int           offset,
-		      int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint32_t *)bits + offset);
-    
-    return ((pixel & 0xff000000) |
-	    ((pixel >> 16) & 0xff) |
-	    (pixel & 0x0000ff00) |
-	    ((pixel & 0xff) << 16));
-}
-
-static uint32_t
-fetch_pixel_x8b8g8r8 (bits_image_t *image,
-		      int           offset,
-		      int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint32_t *)bits + offset);
-    
-    return ((0xff000000) |
-	    ((pixel >> 16) & 0xff) |
-	    (pixel & 0x0000ff00) |
-	    ((pixel & 0xff) << 16));
-}
-
-static uint32_t
-fetch_pixel_b8g8r8a8 (bits_image_t *image,
-		      int           offset,
-		      int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint32_t *)bits + offset);
-    
-    return ((pixel & 0xff000000) >> 24 |
-	    (pixel & 0x00ff0000) >> 8 |
-	    (pixel & 0x0000ff00) << 8 |
-	    (pixel & 0x000000ff) << 24);
-}
-
-static uint32_t
-fetch_pixel_b8g8r8x8 (bits_image_t *image,
-		      int           offset,
-		      int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint32_t *)bits + offset);
-    
-    return ((0xff000000) |
-	    (pixel & 0xff000000) >> 24 |
-	    (pixel & 0x00ff0000) >> 8 |
-	    (pixel & 0x0000ff00) << 8);
-}
-
-static uint32_t
-fetch_pixel_r8g8b8 (bits_image_t *image,
-		    int           offset,
-		    int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint8_t   *pixel = ((uint8_t *) bits) + (offset * 3);
-    
-#ifdef WORDS_BIGENDIAN
-    return (0xff000000 |
-	    (READ (image, pixel + 0) << 16) |
-	    (READ (image, pixel + 1) << 8) |
-	    (READ (image, pixel + 2)));
-#else
-    return (0xff000000 |
-	    (READ (image, pixel + 2) << 16) |
-	    (READ (image, pixel + 1) << 8) |
-	    (READ (image, pixel + 0)));
-#endif
-}
-
-static uint32_t
-fetch_pixel_b8g8r8 (bits_image_t *image,
-		    int           offset,
-		    int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint8_t   *pixel = ((uint8_t *) bits) + (offset * 3);
-#ifdef WORDS_BIGENDIAN
-    return (0xff000000 |
-	    (READ (image, pixel + 2) << 16) |
-	    (READ (image, pixel + 1) << 8) |
-	    (READ (image, pixel + 0)));
-#else
-    return (0xff000000 |
-	    (READ (image, pixel + 0) << 16) |
-	    (READ (image, pixel + 1) << 8) |
-	    (READ (image, pixel + 2)));
-#endif
-}
-
-static uint32_t
-fetch_pixel_r5g6b5 (bits_image_t *image,
-		    int           offset,
-		    int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
-    uint32_t r, g, b;
-    
-    r = ((pixel & 0xf800) | ((pixel & 0xe000) >> 5)) << 8;
-    g = ((pixel & 0x07e0) | ((pixel & 0x0600) >> 6)) << 5;
-    b = ((pixel & 0x001c) | ((pixel & 0x001f) << 5)) >> 2;
-    
-    return (0xff000000 | r | g | b);
-}
-
-static uint32_t
-fetch_pixel_b5g6r5 (bits_image_t *image,
-		    int           offset,
-		    int           line)
-{
-    uint32_t r, g, b;
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
-    
-    b = ((pixel & 0xf800) | ((pixel & 0xe000) >> 5)) >> 8;
-    g = ((pixel & 0x07e0) | ((pixel & 0x0600) >> 6)) << 5;
-    r = ((pixel & 0x001c) | ((pixel & 0x001f) << 5)) << 14;
-    
-    return (0xff000000 | r | g | b);
-}
-
-static uint32_t
-fetch_pixel_a1r5g5b5 (bits_image_t *image,
-		      int           offset,
-		      int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
-    uint32_t a, r, g, b;
-    
-    a = (uint32_t) ((uint8_t) (0 - ((pixel & 0x8000) >> 15))) << 24;
-    r = ((pixel & 0x7c00) | ((pixel & 0x7000) >> 5)) << 9;
-    g = ((pixel & 0x03e0) | ((pixel & 0x0380) >> 5)) << 6;
-    b = ((pixel & 0x001c) | ((pixel & 0x001f) << 5)) >> 2;
-    
-    return (a | r | g | b);
-}
-
-static uint32_t
-fetch_pixel_x1r5g5b5 (bits_image_t *image,
-		      int           offset,
-		      int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
-    uint32_t r, g, b;
-    
-    r = ((pixel & 0x7c00) | ((pixel & 0x7000) >> 5)) << 9;
-    g = ((pixel & 0x03e0) | ((pixel & 0x0380) >> 5)) << 6;
-    b = ((pixel & 0x001c) | ((pixel & 0x001f) << 5)) >> 2;
-    
-    return (0xff000000 | r | g | b);
-}
-
-static uint32_t
-fetch_pixel_a1b5g5r5 (bits_image_t *image,
-		      int           offset,
-		      int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
-    uint32_t a, r, g, b;
-    
-    a = (uint32_t) ((uint8_t) (0 - ((pixel & 0x8000) >> 15))) << 24;
-    b = ((pixel & 0x7c00) | ((pixel & 0x7000) >> 5)) >> 7;
-    g = ((pixel & 0x03e0) | ((pixel & 0x0380) >> 5)) << 6;
-    r = ((pixel & 0x001c) | ((pixel & 0x001f) << 5)) << 14;
-    
-    return (a | r | g | b);
-}
-
-static uint32_t
-fetch_pixel_x1b5g5r5 (bits_image_t *image,
-		      int           offset,
-		      int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
-    uint32_t r, g, b;
-    
-    b = ((pixel & 0x7c00) | ((pixel & 0x7000) >> 5)) >> 7;
-    g = ((pixel & 0x03e0) | ((pixel & 0x0380) >> 5)) << 6;
-    r = ((pixel & 0x001c) | ((pixel & 0x001f) << 5)) << 14;
-    
-    return (0xff000000 | r | g | b);
-}
-
-static uint32_t
-fetch_pixel_a4r4g4b4 (bits_image_t *image,
-		      int           offset,
-		      int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
-    uint32_t a, r, g, b;
-    
-    a = ((pixel & 0xf000) | ((pixel & 0xf000) >> 4)) << 16;
-    r = ((pixel & 0x0f00) | ((pixel & 0x0f00) >> 4)) << 12;
-    g = ((pixel & 0x00f0) | ((pixel & 0x00f0) >> 4)) << 8;
-    b = ((pixel & 0x000f) | ((pixel & 0x000f) << 4));
-    
-    return (a | r | g | b);
-}
-
-static uint32_t
-fetch_pixel_x4r4g4b4 (bits_image_t *image,
-		      int           offset,
-		      int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
-    uint32_t r, g, b;
-    
-    r = ((pixel & 0x0f00) | ((pixel & 0x0f00) >> 4)) << 12;
-    g = ((pixel & 0x00f0) | ((pixel & 0x00f0) >> 4)) << 8;
-    b = ((pixel & 0x000f) | ((pixel & 0x000f) << 4));
-    
-    return (0xff000000 | r | g | b);
-}
+    argb_t argb;
 
-static uint32_t
-fetch_pixel_a4b4g4r4 (bits_image_t *image,
-		      int           offset,
-		      int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
-    uint32_t a, r, g, b;
-    
-    a = ((pixel & 0xf000) | ((pixel & 0xf000) >> 4)) << 16;
-    b = ((pixel & 0x0f00) | ((pixel & 0x0f00) >> 4)) >> 4;
-    g = ((pixel & 0x00f0) | ((pixel & 0x00f0) >> 4)) << 8;
-    r = ((pixel & 0x000f) | ((pixel & 0x000f) << 4)) << 16;
-    
-    return (a | r | g | b);
-}
+    argb.a = 1.0;
+    argb.r = pixman_unorm_to_float (r, 10);
+    argb.g = pixman_unorm_to_float (g, 10);
+    argb.b = pixman_unorm_to_float (b, 10);
 
-static uint32_t
-fetch_pixel_x4b4g4r4 (bits_image_t *image,
-		      int           offset,
-		      int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
-    uint32_t r, g, b;
-    
-    b = ((pixel & 0x0f00) | ((pixel & 0x0f00) >> 4)) >> 4;
-    g = ((pixel & 0x00f0) | ((pixel & 0x00f0) >> 4)) << 8;
-    r = ((pixel & 0x000f) | ((pixel & 0x000f) << 4)) << 16;
-    
-    return (0xff000000 | r | g | b);
+    return argb;
 }
 
-static uint32_t
-fetch_pixel_a8 (bits_image_t *image,
-		int           offset,
-		int           line)
+static argb_t
+fetch_pixel_a8r8g8b8_sRGB_float (bits_image_t *image,
+				 int	       offset,
+				 int           line)
 {
     uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint8_t *) bits + offset);
-    
-    return pixel << 24;
-}
-
-static uint32_t
-fetch_pixel_r3g3b2 (bits_image_t *image,
-		    int           offset,
-		    int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint8_t *) bits + offset);
-    uint32_t r, g, b;
-    
-    r = ((pixel & 0xe0) |
-	 ((pixel & 0xe0) >> 3) |
-	 ((pixel & 0xc0) >> 6)) << 16;
-    
-    g = ((pixel & 0x1c) |
-	 ((pixel & 0x18) >> 3) |
-	 ((pixel & 0x1c) << 3)) << 8;
-    
-    b = (((pixel & 0x03)     ) |
-	 ((pixel & 0x03) << 2) |
-	 ((pixel & 0x03) << 4) |
-	 ((pixel & 0x03) << 6));
-    
-    return (0xff000000 | r | g | b);
-}
-
-static uint32_t
-fetch_pixel_b2g3r3 (bits_image_t *image,
-		    int           offset,
-		    int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t p = READ (image, (uint8_t *) bits + offset);
-    uint32_t r, g, b;
-
-    b  = p & 0xc0;
-    b |= b >> 2;
-    b |= b >> 4;
-    b &= 0xff;
-
-    g  = (p & 0x38) << 10;
-    g |= g >> 3;
-    g |= g >> 6;
-    g &= 0xff00;
-
-    r  = (p & 0x7) << 21;
-    r |= r >> 3;
-    r |= r >> 6;
-    r &= 0xff0000;
-
-    return 0xff000000 | r | g | b;
-}
-
-static uint32_t
-fetch_pixel_a2r2g2b2 (bits_image_t *image,
-		      int           offset,
-		      int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint8_t *) bits + offset);
-    uint32_t a, r, g, b;
-    
-    a = ((pixel & 0xc0) * 0x55) << 18;
-    r = ((pixel & 0x30) * 0x55) << 12;
-    g = ((pixel & 0x0c) * 0x55) << 6;
-    b = ((pixel & 0x03) * 0x55);
-    
-    return a | r | g | b;
-}
-
-static uint32_t
-fetch_pixel_a2b2g2r2 (bits_image_t *image,
-		      int           offset,
-		      int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint8_t *) bits + offset);
-    uint32_t a, r, g, b;
-    
-    a = ((pixel & 0xc0) * 0x55) << 18;
-    b = ((pixel & 0x30) * 0x55) >> 4;
-    g = ((pixel & 0x0c) * 0x55) << 6;
-    r = ((pixel & 0x03) * 0x55) << 16;
-    
-    return a | r | g | b;
-}
-
-static uint32_t
-fetch_pixel_c8 (bits_image_t *image,
-		int           offset,
-		int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint8_t *) bits + offset);
-    const pixman_indexed_t * indexed = image->indexed;
-    
-    return indexed->rgba[pixel];
-}
-
-static uint32_t
-fetch_pixel_x4a4 (bits_image_t *image,
-		  int           offset,
-		  int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, (uint8_t *) bits + offset);
-    
-    return ((pixel & 0xf) | ((pixel & 0xf) << 4)) << 24;
-}
-
-static uint32_t
-fetch_pixel_a4 (bits_image_t *image,
-		int           offset,
-		int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = FETCH_4 (image, bits, offset);
-    
-    pixel |= pixel << 4;
-    return pixel << 24;
-}
-
-static uint32_t
-fetch_pixel_r1g2b1 (bits_image_t *image,
-		    int           offset,
-		    int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = FETCH_4 (image, bits, offset);
-    uint32_t r, g, b;
-    
-    r = ((pixel & 0x8) * 0xff) << 13;
-    g = ((pixel & 0x6) * 0x55) << 7;
-    b = ((pixel & 0x1) * 0xff);
-    
-    return 0xff000000 | r | g | b;
-}
-
-static uint32_t
-fetch_pixel_b1g2r1 (bits_image_t *image,
-		    int           offset,
-		    int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = FETCH_4 (image, bits, offset);
-    uint32_t r, g, b;
-    
-    b = ((pixel & 0x8) * 0xff) >> 3;
-    g = ((pixel & 0x6) * 0x55) << 7;
-    r = ((pixel & 0x1) * 0xff) << 16;
-    
-    return 0xff000000 | r | g | b;
-}
-
-static uint32_t
-fetch_pixel_a1r1g1b1 (bits_image_t *image,
-		      int           offset,
-		      int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = FETCH_4 (image, bits, offset);
-    uint32_t a, r, g, b;
-
-    a = ((pixel & 0x8) * 0xff) << 21;
-    r = ((pixel & 0x4) * 0xff) << 14;
-    g = ((pixel & 0x2) * 0xff) << 7;
-    b = ((pixel & 0x1) * 0xff);
-
-    return a | r | g | b;
-}
+    uint32_t p = READ (image, bits + offset);
+    argb_t argb;
 
-static uint32_t
-fetch_pixel_a1b1g1r1 (bits_image_t *image,
-		      int           offset,
-		      int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = FETCH_4 (image, bits, offset);
-    uint32_t a, r, g, b;
+    argb.a = pixman_unorm_to_float ((p >> 24) & 0xff, 8);
 
-    a = ((pixel & 0x8) * 0xff) << 21;
-    b = ((pixel & 0x4) * 0xff) >> 2;
-    g = ((pixel & 0x2) * 0xff) << 7;
-    r = ((pixel & 0x1) * 0xff) << 16;
+    argb.r = to_linear [(p >> 16) & 0xff];
+    argb.g = to_linear [(p >>  8) & 0xff];
+    argb.b = to_linear [(p >>  0) & 0xff];
 
-    return a | r | g | b;
-}
-
-static uint32_t
-fetch_pixel_c4 (bits_image_t *image,
-		int           offset,
-		int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = FETCH_4 (image, bits, offset);
-    const pixman_indexed_t * indexed = image->indexed;
-
-    return indexed->rgba[pixel];
-}
-
-static uint32_t
-fetch_pixel_a1 (bits_image_t *image,
-		int           offset,
-		int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, bits + (offset >> 5));
-    uint32_t a;
-    
-#ifdef WORDS_BIGENDIAN
-    a = pixel >> (0x1f - (offset & 0x1f));
-#else
-    a = pixel >> (offset & 0x1f);
-#endif
-    a = a & 1;
-    a |= a << 1;
-    a |= a << 2;
-    a |= a << 4;
-    
-    return a << 24;
-}
-
-static uint32_t
-fetch_pixel_g1 (bits_image_t *image,
-		int           offset,
-		int           line)
-{
-    uint32_t *bits = image->bits + line * image->rowstride;
-    uint32_t pixel = READ (image, bits + (offset >> 5));
-    const pixman_indexed_t * indexed = image->indexed;
-    uint32_t a;
-    
-#ifdef WORDS_BIGENDIAN
-    a = pixel >> (0x1f - (offset & 0x1f));
-#else
-    a = pixel >> (offset & 0x1f);
-#endif
-    a = a & 1;
-    
-    return indexed->rgba[a];
+    return argb;
 }
 
 static uint32_t
@@ -1819,894 +962,276 @@ fetch_pixel_yv12 (bits_image_t *image,
 
 /*********************************** Store ************************************/
 
-#define SPLIT_A(v)              \
-    uint32_t a = ((v) >> 24),   \
-	r = ((v) >> 16) & 0xff, \
-	g = ((v) >> 8) & 0xff,  \
-	b = (v) & 0xff
-
-#define SPLIT(v)                     \
-    uint32_t r = ((v) >> 16) & 0xff, \
-	g = ((v) >> 8) & 0xff,       \
-	b = (v) & 0xff
-
 static void
-store_scanline_a2r10g10b10 (bits_image_t *  image,
-                            int             x,
-                            int             y,
-                            int             width,
-                            const uint32_t *v)
+store_scanline_a2r10g10b10_float (bits_image_t *  image,
+				  int             x,
+				  int             y,
+				  int             width,
+				  const uint32_t *v)
 {
     uint32_t *bits = image->bits + image->rowstride * y;
     uint32_t *pixel = bits + x;
-    uint64_t *values = (uint64_t *)v;
+    argb_t *values = (argb_t *)v;
     int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	WRITE (image, pixel++,
-	       ((values[i] >> 32) & 0xc0000000) |
-	       ((values[i] >> 18) & 0x3ff00000) |
-	       ((values[i] >> 12) & 0xffc00) | 
-	       ((values[i] >> 6) & 0x3ff));    
-    }
-}
 
-static void
-store_scanline_x2r10g10b10 (bits_image_t *  image,
-                            int             x,
-                            int             y,
-                            int             width,
-                            const uint32_t *v)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint64_t *values = (uint64_t *)v;
-    uint32_t *pixel = bits + x;
-    int i;
-    
     for (i = 0; i < width; ++i)
     {
-	WRITE (image, pixel++,
-	       ((values[i] >> 18) & 0x3ff00000) | 
-	       ((values[i] >> 12) & 0xffc00) |
-	       ((values[i] >> 6) & 0x3ff));
-    }
-}
+	uint16_t a, r, g, b;
 
-static void
-store_scanline_a2b10g10r10 (bits_image_t *  image,
-                            int             x,
-                            int             y,
-                            int             width,
-                            const uint32_t *v)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint32_t *pixel = bits + x;
-    uint64_t *values = (uint64_t *)v;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	WRITE (image, pixel++,
-	       ((values[i] >> 32) & 0xc0000000) |
-	       ((values[i] >> 38) & 0x3ff) |
-	       ((values[i] >> 12) & 0xffc00) |
-	       ((values[i] << 14) & 0x3ff00000));
-    }
-}
+	a = pixman_float_to_unorm (values[i].a, 2);
+	r = pixman_float_to_unorm (values[i].r, 10);
+	g = pixman_float_to_unorm (values[i].g, 10);
+	b = pixman_float_to_unorm (values[i].b, 10);
 
-static void
-store_scanline_x2b10g10r10 (bits_image_t *  image,
-                            int             x,
-                            int             y,
-                            int             width,
-                            const uint32_t *v)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint64_t *values = (uint64_t *)v;
-    uint32_t *pixel = bits + x;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
 	WRITE (image, pixel++,
-	       ((values[i] >> 38) & 0x3ff) |
-	       ((values[i] >> 12) & 0xffc00) |
-	       ((values[i] << 14) & 0x3ff00000));
+	       (a << 30) | (r << 20) | (g << 10) | b);
     }
 }
 
 static void
-store_scanline_a8r8g8b8 (bits_image_t *  image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         const uint32_t *values)
+store_scanline_x2r10g10b10_float (bits_image_t *  image,
+				  int             x,
+				  int             y,
+				  int             width,
+				  const uint32_t *v)
 {
     uint32_t *bits = image->bits + image->rowstride * y;
-    
-    MEMCPY_WRAPPED (image, ((uint32_t *)bits) + x, values,
-                    width * sizeof(uint32_t));
-}
-
-static void
-store_scanline_x8r8g8b8 (bits_image_t *  image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint32_t *pixel = (uint32_t *)bits + x;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-	WRITE (image, pixel++, values[i] & 0xffffff);
-}
-
-static void
-store_scanline_a8b8g8r8 (bits_image_t *  image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint32_t *pixel = (uint32_t *)bits + x;
+    uint32_t *pixel = bits + x;
+    argb_t *values = (argb_t *)v;
     int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	WRITE (image, pixel++,
-	       (values[i] & 0xff00ff00)         |
-	       ((values[i] >> 16) & 0xff)       |
-	       ((values[i] & 0xff) << 16));
-    }
-}
 
-static void
-store_scanline_x8b8g8r8 (bits_image_t *  image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint32_t *pixel = (uint32_t *)bits + x;
-    int i;
-    
     for (i = 0; i < width; ++i)
     {
-	WRITE (image, pixel++,
-	       (values[i] & 0x0000ff00)         |
-	       ((values[i] >> 16) & 0xff)       |
-	       ((values[i] & 0xff) << 16));
-    }
-}
+	uint16_t r, g, b;
 
-static void
-store_scanline_b8g8r8a8 (bits_image_t *  image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint32_t *pixel = (uint32_t *)bits + x;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	WRITE (image, pixel++,
-	       ((values[i] >> 24) & 0x000000ff) |
-	       ((values[i] >>  8) & 0x0000ff00) |
-	       ((values[i] <<  8) & 0x00ff0000) |
-	       ((values[i] << 24) & 0xff000000));
-    }
-}
+	r = pixman_float_to_unorm (values[i].r, 10);
+	g = pixman_float_to_unorm (values[i].g, 10);
+	b = pixman_float_to_unorm (values[i].b, 10);
 
-static void
-store_scanline_b8g8r8x8 (bits_image_t *  image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint32_t *pixel = (uint32_t *)bits + x;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
 	WRITE (image, pixel++,
-	       ((values[i] >>  8) & 0x0000ff00) |
-	       ((values[i] <<  8) & 0x00ff0000) |
-	       ((values[i] << 24) & 0xff000000));
+	       (r << 20) | (g << 10) | b);
     }
 }
 
 static void
-store_scanline_r8g8b8 (bits_image_t *  image,
-                       int             x,
-                       int             y,
-                       int             width,
-                       const uint32_t *values)
+store_scanline_a2b10g10r10_float (bits_image_t *  image,
+				  int             x,
+				  int             y,
+				  int             width,
+				  const uint32_t *v)
 {
     uint32_t *bits = image->bits + image->rowstride * y;
-    uint8_t *pixel = ((uint8_t *) bits) + 3 * x;
+    uint32_t *pixel = bits + x;
+    argb_t *values = (argb_t *)v;
     int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	uint32_t val = values[i];
-	
-#ifdef WORDS_BIGENDIAN
-	WRITE (image, pixel++, (val & 0x00ff0000) >> 16);
-	WRITE (image, pixel++, (val & 0x0000ff00) >>  8);
-	WRITE (image, pixel++, (val & 0x000000ff) >>  0);
-#else
-	WRITE (image, pixel++, (val & 0x000000ff) >>  0);
-	WRITE (image, pixel++, (val & 0x0000ff00) >>  8);
-	WRITE (image, pixel++, (val & 0x00ff0000) >> 16);
-#endif
-    }
-}
 
-static void
-store_scanline_b8g8r8 (bits_image_t *  image,
-                       int             x,
-                       int             y,
-                       int             width,
-                       const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint8_t *pixel = ((uint8_t *) bits) + 3 * x;
-    int i;
-    
     for (i = 0; i < width; ++i)
     {
-	uint32_t val = values[i];
-	
-#ifdef WORDS_BIGENDIAN
-	WRITE (image, pixel++, (val & 0x000000ff) >>  0);
-	WRITE (image, pixel++, (val & 0x0000ff00) >>  8);
-	WRITE (image, pixel++, (val & 0x00ff0000) >> 16);
-#else
-	WRITE (image, pixel++, (val & 0x00ff0000) >> 16);
-	WRITE (image, pixel++, (val & 0x0000ff00) >>  8);
-	WRITE (image, pixel++, (val & 0x000000ff) >>  0);
-#endif
-    }
-}
+	uint16_t a, r, g, b;
 
-static void
-store_scanline_r5g6b5 (bits_image_t *  image,
-                       int             x,
-                       int             y,
-                       int             width,
-                       const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint16_t *pixel = ((uint16_t *) bits) + x;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	uint32_t s = values[i];
-	
-	WRITE (image, pixel++,
-	       ((s >> 3) & 0x001f) |
-	       ((s >> 5) & 0x07e0) |
-	       ((s >> 8) & 0xf800));
-    }
-}
+	a = pixman_float_to_unorm (values[i].a, 2);
+	r = pixman_float_to_unorm (values[i].r, 10);
+	g = pixman_float_to_unorm (values[i].g, 10);
+	b = pixman_float_to_unorm (values[i].b, 10);
 
-static void
-store_scanline_b5g6r5 (bits_image_t *  image,
-                       int             x,
-                       int             y,
-                       int             width,
-                       const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint16_t  *pixel = ((uint16_t *) bits) + x;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	SPLIT (values[i]);
-	
 	WRITE (image, pixel++,
-	       ((b << 8) & 0xf800) |
-	       ((g << 3) & 0x07e0) |
-	       ((r >> 3)         ));
+	       (a << 30) | (b << 20) | (g << 10) | r);
     }
 }
 
 static void
-store_scanline_a1r5g5b5 (bits_image_t *  image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         const uint32_t *values)
+store_scanline_x2b10g10r10_float (bits_image_t *  image,
+				  int             x,
+				  int             y,
+				  int             width,
+				  const uint32_t *v)
 {
     uint32_t *bits = image->bits + image->rowstride * y;
-    uint16_t  *pixel = ((uint16_t *) bits) + x;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	SPLIT_A (values[i]);
-	
-	WRITE (image, pixel++,
-	       ((a << 8) & 0x8000) |
-	       ((r << 7) & 0x7c00) |
-	       ((g << 2) & 0x03e0) |
-	       ((b >> 3)         ));
-    }
-}
-
-static void
-store_scanline_x1r5g5b5 (bits_image_t *  image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint16_t  *pixel = ((uint16_t *) bits) + x;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	SPLIT (values[i]);
-	
-	WRITE (image, pixel++,
-	       ((r << 7) & 0x7c00) |
-	       ((g << 2) & 0x03e0) |
-	       ((b >> 3)         ));
-    }
-}
-
-static void
-store_scanline_a1b5g5r5 (bits_image_t *  image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint16_t  *pixel = ((uint16_t *) bits) + x;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	SPLIT_A (values[i]);
-	
-	WRITE (image, pixel++,
-	       ((a << 8) & 0x8000) |
-	       ((b << 7) & 0x7c00) |
-	       ((g << 2) & 0x03e0) |
-	       ((r >> 3)         ));
-    }
-}
-
-static void
-store_scanline_x1b5g5r5 (bits_image_t *  image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint16_t  *pixel = ((uint16_t *) bits) + x;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	SPLIT (values[i]);
-	
-	WRITE (image, pixel++, ((b << 7) & 0x7c00) |
-	       ((g << 2) & 0x03e0) |
-	       ((r >> 3)         ));
-    }
-}
-
-static void
-store_scanline_a4r4g4b4 (bits_image_t *  image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint16_t  *pixel = ((uint16_t *) bits) + x;
+    uint32_t *pixel = bits + x;
+    argb_t *values = (argb_t *)v;
     int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	SPLIT_A (values[i]);
-	
-	WRITE (image, pixel++,
-	       ((a << 8) & 0xf000) |
-	       ((r << 4) & 0x0f00) |
-	       ((g     ) & 0x00f0) |
-	       ((b >> 4)         ));
-    }
-}
 
-static void
-store_scanline_x4r4g4b4 (bits_image_t *  image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint16_t  *pixel = ((uint16_t *) bits) + x;
-    int i;
-    
     for (i = 0; i < width; ++i)
     {
-	SPLIT (values[i]);
-	
-	WRITE (image, pixel++,
-	       ((r << 4) & 0x0f00) |
-	       ((g     ) & 0x00f0) |
-	       ((b >> 4)         ));
-    }
-}
+	uint16_t r, g, b;
 
-static void
-store_scanline_a4b4g4r4 (bits_image_t *  image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint16_t  *pixel = ((uint16_t *) bits) + x;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	SPLIT_A (values[i]);
-	WRITE (image, pixel++, ((a << 8) & 0xf000) |
-	       ((b << 4) & 0x0f00) |
-	       ((g     ) & 0x00f0) |
-	       ((r >> 4)         ));
-    }
-}
+	r = pixman_float_to_unorm (values[i].r, 10);
+	g = pixman_float_to_unorm (values[i].g, 10);
+	b = pixman_float_to_unorm (values[i].b, 10);
 
-static void
-store_scanline_x4b4g4r4 (bits_image_t *  image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint16_t  *pixel = ((uint16_t *) bits) + x;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	SPLIT (values[i]);
-	
 	WRITE (image, pixel++,
-	       ((b << 4) & 0x0f00) |
-	       ((g     ) & 0x00f0) |
-	       ((r >> 4)         ));
+	       (b << 20) | (g << 10) | r);
     }
 }
 
 static void
-store_scanline_a8 (bits_image_t *  image,
-                   int             x,
-                   int             y,
-                   int             width,
-                   const uint32_t *values)
+store_scanline_a8r8g8b8_sRGB_float (bits_image_t *  image,
+				    int             x,
+				    int             y,
+				    int             width,
+				    const uint32_t *v)
 {
     uint32_t *bits = image->bits + image->rowstride * y;
-    uint8_t   *pixel = ((uint8_t *) bits) + x;
+    uint32_t *pixel = bits + x;
+    argb_t *values = (argb_t *)v;
     int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	WRITE (image, pixel++, values[i] >> 24);
-    }
-}
 
-static void
-store_scanline_r3g3b2 (bits_image_t *  image,
-                       int             x,
-                       int             y,
-                       int             width,
-                       const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint8_t   *pixel = ((uint8_t *) bits) + x;
-    int i;
-    
     for (i = 0; i < width; ++i)
     {
-	SPLIT (values[i]);
-	
-	WRITE (image, pixel++,
-	       ((r     ) & 0xe0) |
-	       ((g >> 3) & 0x1c) |
-	       ((b >> 6)       ));
-    }
-}
+	uint8_t a, r, g, b;
 
-static void
-store_scanline_b2g3r3 (bits_image_t *  image,
-                       int             x,
-                       int             y,
-                       int             width,
-                       const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint8_t   *pixel = ((uint8_t *) bits) + x;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	SPLIT (values[i]);
-	
-	WRITE (image, pixel++,
-	       ((b     ) & 0xc0) |
-	       ((g >> 2) & 0x38) |
-	       ((r >> 5)       ));
-    }
-}
+	a = pixman_float_to_unorm (values[i].a, 8);
+	r = to_srgb (values[i].r);
+	g = to_srgb (values[i].g);
+	b = to_srgb (values[i].b);
 
-static void
-store_scanline_a2r2g2b2 (bits_image_t *  image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint8_t   *pixel = ((uint8_t *) bits) + x;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	SPLIT_A (values[i]);
-	
 	WRITE (image, pixel++,
-	       ((a     ) & 0xc0) |
-	       ((r >> 2) & 0x30) |
-	       ((g >> 4) & 0x0c) |
-	       ((b >> 6)       ));
+	       (a << 24) | (r << 16) | (g << 8) | b);
     }
 }
 
+/*
+ * Contracts a floating point image to 32bpp and then stores it using a
+ * regular 32-bit store proc. Despite the type, this function expects an
+ * argb_t buffer.
+ */
 static void
-store_scanline_a2b2g2r2 (bits_image_t *  image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint8_t   *pixel = ((uint8_t *) bits) + x;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	SPLIT_A (values[i]);
-	
-	*(pixel++) =
-	    ((a     ) & 0xc0) |
-	    ((b >> 2) & 0x30) |
-	    ((g >> 4) & 0x0c) |
-	    ((r >> 6)       );
-    }
-}
-
-static void
-store_scanline_c8 (bits_image_t *  image,
-                   int             x,
-                   int             y,
-                   int             width,
-                   const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint8_t *pixel = ((uint8_t *) bits) + x;
-    const pixman_indexed_t *indexed = image->indexed;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-	WRITE (image, pixel++, RGB24_TO_ENTRY (indexed,values[i]));
-}
-
-static void
-store_scanline_x4a4 (bits_image_t *  image,
-                     int             x,
-                     int             y,
-                     int             width,
-                     const uint32_t *values)
+store_scanline_generic_float (bits_image_t *  image,
+			      int             x,
+			      int             y,
+			      int             width,
+			      const uint32_t *values)
 {
-    uint32_t *bits = image->bits + image->rowstride * y;
-    uint8_t   *pixel = ((uint8_t *) bits) + x;
-    int i;
-
-    for (i = 0; i < width; ++i)
-	WRITE (image, pixel++, values[i] >> 28);
-}
+    uint32_t *argb8_pixels;
 
-#define STORE_8(img,l,o,v)  (WRITE (img, (uint8_t *)(l) + ((o) >> 3), (v)))
-#ifdef WORDS_BIGENDIAN
+    assert (image->common.type == BITS);
 
-#define STORE_4(img,l,o,v)						\
-    do									\
-    {									\
-	int bo = 4 * (o);						\
-	int v4 = (v) & 0x0f;						\
-									\
-	STORE_8 (img, l, bo, (						\
-		     bo & 4 ?						\
-		     (FETCH_8 (img, l, bo) & 0xf0) | (v4) :		\
-		     (FETCH_8 (img, l, bo) & 0x0f) | (v4 << 4)));	\
-    } while (0)
-#else
+    argb8_pixels = pixman_malloc_ab (width, sizeof(uint32_t));
+    if (!argb8_pixels)
+	return;
 
-#define STORE_4(img,l,o,v)						\
-    do									\
-    {									\
-	int bo = 4 * (o);						\
-	int v4 = (v) & 0x0f;						\
-									\
-	STORE_8 (img, l, bo, (						\
-		     bo & 4 ?						\
-		     (FETCH_8 (img, l, bo) & 0x0f) | (v4 << 4) :	\
-		     (FETCH_8 (img, l, bo) & 0xf0) | (v4)));		\
-    } while (0)
-#endif
+    /* Contract the scanline.  We could do this in place if values weren't
+     * const.
+     */
+    pixman_contract_from_float (argb8_pixels, (argb_t *)values, width);
 
-static void
-store_scanline_a4 (bits_image_t *  image,
-                   int             x,
-                   int             y,
-                   int             width,
-                   const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    int i;
+    image->store_scanline_32 (image, x, y, width, argb8_pixels);
 
-    for (i = 0; i < width; ++i)
-	STORE_4 (image, bits, i + x, values[i] >> 28);
+    free (argb8_pixels);
 }
 
 static void
-store_scanline_r1g2b1 (bits_image_t *  image,
-                       int             x,
-                       int             y,
-                       int             width,
-                       const uint32_t *values)
+fetch_scanline_generic_float (bits_image_t *  image,
+			      int	      x,
+			      int	      y,
+			      int	      width,
+			      uint32_t *      buffer,
+			      const uint32_t *mask)
 {
-    uint32_t *bits = image->bits + image->rowstride * y;
-    int i;
+    image->fetch_scanline_32 (image, x, y, width, buffer, NULL);
 
-    for (i = 0; i < width; ++i)
-    {
-	uint32_t pixel;
-
-	SPLIT (values[i]);
-	pixel = (((r >> 4) & 0x8) |
-	         ((g >> 5) & 0x6) |
-	         ((b >> 7)      ));
-	STORE_4 (image, bits, i + x, pixel);
-    }
+    pixman_expand_to_float ((argb_t *)buffer, buffer, image->format, width);
 }
 
+/* The 32_sRGB paths should be deleted after narrow processing
+ * is no longer invoked for formats that are considered wide.
+ * (Also see fetch_pixel_generic_lossy_32) */
 static void
-store_scanline_b1g2r1 (bits_image_t *  image,
-                       int             x,
-                       int             y,
-                       int             width,
-                       const uint32_t *values)
+fetch_scanline_a8r8g8b8_32_sRGB (bits_image_t   *image,
+                                 int             x,
+                                 int             y,
+                                 int             width,
+                                 uint32_t       *buffer,
+                                 const uint32_t *mask)
 {
-    uint32_t *bits = image->bits + image->rowstride * y;
-    int i;
-
-    for (i = 0; i < width; ++i)
+    const uint32_t *bits = image->bits + y * image->rowstride;
+    const uint32_t *pixel = (uint32_t *)bits + x;
+    const uint32_t *end = pixel + width;
+    uint32_t tmp;
+    
+    while (pixel < end)
     {
-	uint32_t pixel;
+	uint8_t a, r, g, b;
 
-	SPLIT (values[i]);
-	pixel = (((b >> 4) & 0x8) |
-	         ((g >> 5) & 0x6) |
-	         ((r >> 7)      ));
-	STORE_4 (image, bits, i + x, pixel);
-    }
-}
+	tmp = READ (image, pixel++);
 
-static void
-store_scanline_a1r1g1b1 (bits_image_t *  image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    int i;
+	a = (tmp >> 24) & 0xff;
+	r = (tmp >> 16) & 0xff;
+	g = (tmp >> 8) & 0xff;
+	b = (tmp >> 0) & 0xff;
 
-    for (i = 0; i < width; ++i)
-    {
-	uint32_t pixel;
-
-	SPLIT_A (values[i]);
-	pixel = (((a >> 4) & 0x8) |
-	         ((r >> 5) & 0x4) |
-	         ((g >> 6) & 0x2) |
-	         ((b >> 7)      ));
+	r = to_linear[r] * 255.0f + 0.5f;
+	g = to_linear[g] * 255.0f + 0.5f;
+	b = to_linear[b] * 255.0f + 0.5f;
 
-	STORE_4 (image, bits, i + x, pixel);
+	*buffer++ = (a << 24) | (r << 16) | (g << 8) | (b << 0);
     }
 }
 
-static void
-store_scanline_a1b1g1r1 (bits_image_t *  image,
-                         int             x,
-                         int             y,
-                         int             width,
-                         const uint32_t *values)
+static uint32_t
+fetch_pixel_a8r8g8b8_32_sRGB (bits_image_t *image,
+			      int           offset,
+			      int           line)
 {
-    uint32_t *bits = image->bits + image->rowstride * y;
-    int i;
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t tmp = READ (image, bits + offset);
+    uint8_t a, r, g, b;
 
-    for (i = 0; i < width; ++i)
-    {
-	uint32_t pixel;
+    a = (tmp >> 24) & 0xff;
+    r = (tmp >> 16) & 0xff;
+    g = (tmp >> 8) & 0xff;
+    b = (tmp >> 0) & 0xff;
 
-	SPLIT_A (values[i]);
-	pixel = (((a >> 4) & 0x8) |
-	         ((b >> 5) & 0x4) |
-	         ((g >> 6) & 0x2) |
-	         ((r >> 7)      ));
+    r = to_linear[r] * 255.0f + 0.5f;
+    g = to_linear[g] * 255.0f + 0.5f;
+    b = to_linear[b] * 255.0f + 0.5f;
 
-	STORE_4 (image, bits, i + x, pixel);
-    }
+    return (a << 24) | (r << 16) | (g << 8) | (b << 0);
 }
 
 static void
-store_scanline_c4 (bits_image_t *  image,
-                   int             x,
-                   int             y,
-                   int             width,
-                   const uint32_t *values)
+store_scanline_a8r8g8b8_32_sRGB (bits_image_t   *image,
+                                 int             x,
+                                 int             y,
+                                 int             width,
+                                 const uint32_t *v)
 {
     uint32_t *bits = image->bits + image->rowstride * y;
-    const pixman_indexed_t *indexed = image->indexed;
+    uint64_t *values = (uint64_t *)v;
+    uint32_t *pixel = bits + x;
+    uint64_t tmp;
     int i;
     
     for (i = 0; i < width; ++i)
     {
-	uint32_t pixel;
-	
-	pixel = RGB24_TO_ENTRY (indexed, values[i]);
-	STORE_4 (image, bits, i + x, pixel);
-    }
-}
+	uint8_t a, r, g, b;
 
-static void
-store_scanline_a1 (bits_image_t *  image,
-                   int             x,
-                   int             y,
-                   int             width,
-                   const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	uint32_t  *pixel = ((uint32_t *) bits) + ((i + x) >> 5);
-	uint32_t mask, v;
-	
-#ifdef WORDS_BIGENDIAN
-	mask = 1 << (0x1f - ((i + x) & 0x1f));
-#else
-	mask = 1 << ((i + x) & 0x1f);
-#endif
-	v = values[i] & 0x80000000 ? mask : 0;
-	
-	WRITE (image, pixel, (READ (image, pixel) & ~mask) | v);
-    }
-}
+	tmp = values[i];
 
-static void
-store_scanline_g1 (bits_image_t *  image,
-                   int             x,
-                   int             y,
-                   int             width,
-                   const uint32_t *values)
-{
-    uint32_t *bits = image->bits + image->rowstride * y;
-    const pixman_indexed_t *indexed = image->indexed;
-    int i;
-    
-    for (i = 0; i < width; ++i)
-    {
-	uint32_t  *pixel = ((uint32_t *) bits) + ((i + x) >> 5);
-	uint32_t mask, v;
-	
-#ifdef WORDS_BIGENDIAN
-	mask = 1 << (0x1f - ((i + x) & 0x1f));
-#else
-	mask = 1 << ((i + x) & 0x1f);
-#endif
-	v = RGB24_TO_ENTRY_Y (indexed, values[i]) & 0x1 ? mask : 0;
+	a = (tmp >> 24) & 0xff;
+	r = (tmp >> 16) & 0xff;
+	g = (tmp >> 8) & 0xff;
+	b = (tmp >> 0) & 0xff;
+
+	r = to_srgb (r * (1/255.0f));
+	g = to_srgb (g * (1/255.0f));
+	b = to_srgb (b * (1/255.0f));
 	
-	WRITE (image, pixel, (READ (image, pixel) & ~mask) | v);
+	WRITE (image, pixel++, a | (r << 16) | (g << 8) | (b << 0));
     }
 }
 
-/*
- * Contracts a 64bpp image to 32bpp and then stores it using a regular 32-bit
- * store proc. Despite the type, this function expects a uint64_t buffer.
- */
-static void
-store_scanline_generic_64 (bits_image_t *  image,
-                           int             x,
-                           int             y,
-                           int             width,
-                           const uint32_t *values)
-{
-    uint32_t *argb8_pixels;
-    
-    assert (image->common.type == BITS);
-    
-    argb8_pixels = pixman_malloc_ab (width, sizeof(uint32_t));
-    if (!argb8_pixels)
-	return;
-    
-    /* Contract the scanline.  We could do this in place if values weren't
-     * const.
-     */
-    pixman_contract (argb8_pixels, (uint64_t *)values, width);
-    
-    image->store_scanline_raw_32 (image, x, y, width, argb8_pixels);
-    
-    free (argb8_pixels);
-}
-
-/* Despite the type, this function expects both buffer
- * and mask to be uint64_t
- */
-static void
-fetch_scanline_generic_64 (pixman_image_t *image,
-                           int             x,
-                           int             y,
-                           int             width,
-                           uint32_t *      buffer,
-                           const uint32_t *mask,
-                           uint32_t        mask_bits)
+static argb_t
+fetch_pixel_generic_float (bits_image_t *image,
+			   int		 offset,
+			   int           line)
 {
-    /* Fetch the pixels into the first half of buffer and then expand them in
-     * place.
-     */
-    image->bits.fetch_scanline_raw_32 (image, x, y, width, buffer, NULL, 0);
-    
-    pixman_expand ((uint64_t *)buffer, buffer, image->bits.format, width);
-}
+    uint32_t pixel32 = image->fetch_pixel_32 (image, offset, line);
+    argb_t f;
 
-/* Despite the type, this function expects a uint64_t *buffer */
-static uint64_t
-fetch_pixel_generic_64 (bits_image_t *image,
-			int	      offset,
-			int           line)
-{
-    uint32_t pixel32 = image->fetch_pixel_raw_32 (image, offset, line);
-    uint64_t result;
-    
-    pixman_expand ((uint64_t *)&result, &pixel32, image->format, 1);
+    pixman_expand_to_float (&f, &pixel32, image->format, 1);
 
-    return result;
+    return f;
 }
 
 /*
@@ -2720,10 +1245,10 @@ fetch_pixel_generic_lossy_32 (bits_image_t *image,
 			      int           offset,
 			      int           line)
 {
-    uint64_t pixel64 = image->fetch_pixel_raw_64 (image, offset, line);
+    argb_t pixel64 = image->fetch_pixel_float (image, offset, line);
     uint32_t result;
-    
-    pixman_contract (&result, &pixel64, 1);
+
+    pixman_contract_from_float (&result, &pixel64, 1);
 
     return result;
 }
@@ -2731,21 +1256,23 @@ fetch_pixel_generic_lossy_32 (bits_image_t *image,
 typedef struct
 {
     pixman_format_code_t	format;
-    fetch_scanline_t		fetch_scanline_raw_32;
-    fetch_scanline_t		fetch_scanline_raw_64;
-    fetch_pixel_32_t		fetch_pixel_raw_32;
-    fetch_pixel_64_t		fetch_pixel_raw_64;
-    store_scanline_t		store_scanline_raw_32;
-    store_scanline_t		store_scanline_raw_64;
+    fetch_scanline_t		fetch_scanline_32;
+    fetch_scanline_t		fetch_scanline_float;
+    fetch_pixel_32_t		fetch_pixel_32;
+    fetch_pixel_float_t		fetch_pixel_float;
+    store_scanline_t		store_scanline_32;
+    store_scanline_t		store_scanline_float;
 } format_info_t;
 
 #define FORMAT_INFO(format) 						\
     {									\
 	PIXMAN_ ## format,						\
 	    fetch_scanline_ ## format,					\
-	    fetch_scanline_generic_64,					\
-	    fetch_pixel_ ## format, fetch_pixel_generic_64,		\
-	    store_scanline_ ## format, store_scanline_generic_64	\
+	    fetch_scanline_generic_float,				\
+	    fetch_pixel_ ## format,					\
+	    fetch_pixel_generic_float,					\
+	    store_scanline_ ## format,					\
+	    store_scanline_generic_float				\
     }
 
 static const format_info_t accessors[] =
@@ -2757,7 +1284,17 @@ static const format_info_t accessors[] =
     FORMAT_INFO (x8b8g8r8),
     FORMAT_INFO (b8g8r8a8),
     FORMAT_INFO (b8g8r8x8),
-    
+    FORMAT_INFO (r8g8b8a8),
+    FORMAT_INFO (r8g8b8x8),
+    FORMAT_INFO (x14r6g6b6),
+
+/* sRGB formats */
+  { PIXMAN_a8r8g8b8_sRGB,
+    fetch_scanline_a8r8g8b8_32_sRGB, fetch_scanline_a8r8g8b8_sRGB_float,
+    fetch_pixel_a8r8g8b8_32_sRGB, fetch_pixel_a8r8g8b8_sRGB_float,
+    store_scanline_a8r8g8b8_32_sRGB, store_scanline_a8r8g8b8_sRGB_float,
+  },
+
 /* 24bpp formats */
     FORMAT_INFO (r8g8b8),
     FORMAT_INFO (b8g8r8),
@@ -2784,9 +1321,6 @@ static const format_info_t accessors[] =
     
     FORMAT_INFO (c8),
     
-#define fetch_scanline_g8 fetch_scanline_c8
-#define fetch_pixel_g8 fetch_pixel_c8
-#define store_scanline_g8 store_scanline_c8
     FORMAT_INFO (g8),
     
 #define fetch_scanline_x4c4 fetch_scanline_c8
@@ -2794,9 +1328,9 @@ static const format_info_t accessors[] =
 #define store_scanline_x4c4 store_scanline_c8
     FORMAT_INFO (x4c4),
     
-#define fetch_scanline_x4g4 fetch_scanline_c8
-#define fetch_pixel_x4g4 fetch_pixel_c8
-#define store_scanline_x4g4 store_scanline_c8
+#define fetch_scanline_x4g4 fetch_scanline_g8
+#define fetch_pixel_x4g4 fetch_pixel_g8
+#define store_scanline_x4g4 store_scanline_g8
     FORMAT_INFO (x4g4),
     
     FORMAT_INFO (x4a4),
@@ -2810,9 +1344,6 @@ static const format_info_t accessors[] =
     
     FORMAT_INFO (c4),
     
-#define fetch_scanline_g4 fetch_scanline_c4
-#define fetch_pixel_g4 fetch_pixel_c4
-#define store_scanline_g4 store_scanline_c4
     FORMAT_INFO (g4),
     
 /* 1bpp formats */
@@ -2822,34 +1353,34 @@ static const format_info_t accessors[] =
 /* Wide formats */
     
     { PIXMAN_a2r10g10b10,
-      NULL, fetch_scanline_a2r10g10b10,
-      fetch_pixel_generic_lossy_32, fetch_pixel_a2r10g10b10,
-      NULL, store_scanline_a2r10g10b10 },
-    
+      NULL, fetch_scanline_a2r10g10b10_float,
+      fetch_pixel_generic_lossy_32, fetch_pixel_a2r10g10b10_float,
+      NULL, store_scanline_a2r10g10b10_float },
+
     { PIXMAN_x2r10g10b10,
-      NULL, fetch_scanline_x2r10g10b10,
-      fetch_pixel_generic_lossy_32, fetch_pixel_x2r10g10b10,
-      NULL, store_scanline_x2r10g10b10 },
-    
+      NULL, fetch_scanline_x2r10g10b10_float,
+      fetch_pixel_generic_lossy_32, fetch_pixel_x2r10g10b10_float,
+      NULL, store_scanline_x2r10g10b10_float },
+
     { PIXMAN_a2b10g10r10,
-      NULL, fetch_scanline_a2b10g10r10,
-      fetch_pixel_generic_lossy_32, fetch_pixel_a2b10g10r10,
-      NULL, store_scanline_a2b10g10r10 },
-    
+      NULL, fetch_scanline_a2b10g10r10_float,
+      fetch_pixel_generic_lossy_32, fetch_pixel_a2b10g10r10_float,
+      NULL, store_scanline_a2b10g10r10_float },
+
     { PIXMAN_x2b10g10r10,
-      NULL, fetch_scanline_x2b10g10r10,
-      fetch_pixel_generic_lossy_32, fetch_pixel_x2b10g10r10,
-      NULL, store_scanline_x2b10g10r10 },
-    
+      NULL, fetch_scanline_x2b10g10r10_float,
+      fetch_pixel_generic_lossy_32, fetch_pixel_x2b10g10r10_float,
+      NULL, store_scanline_x2b10g10r10_float },
+
 /* YUV formats */
     { PIXMAN_yuy2,
-      fetch_scanline_yuy2, fetch_scanline_generic_64,
-      fetch_pixel_yuy2, fetch_pixel_generic_64,
+      fetch_scanline_yuy2, fetch_scanline_generic_float,
+      fetch_pixel_yuy2, fetch_pixel_generic_float,
       NULL, NULL },
-    
+
     { PIXMAN_yv12,
-      fetch_scanline_yv12, fetch_scanline_generic_64,
-      fetch_pixel_yv12, fetch_pixel_generic_64,
+      fetch_scanline_yv12, fetch_scanline_generic_float,
+      fetch_pixel_yv12, fetch_pixel_generic_float,
       NULL, NULL },
     
     { PIXMAN_null },
@@ -2864,12 +1395,12 @@ setup_accessors (bits_image_t *image)
     {
 	if (info->format == image->format)
 	{
-	    image->fetch_scanline_raw_32 = info->fetch_scanline_raw_32;
-	    image->fetch_scanline_raw_64 = info->fetch_scanline_raw_64;
-	    image->fetch_pixel_raw_32 = info->fetch_pixel_raw_32;
-	    image->fetch_pixel_raw_64 = info->fetch_pixel_raw_64;
-	    image->store_scanline_raw_32 = info->store_scanline_raw_32;
-	    image->store_scanline_raw_64 = info->store_scanline_raw_64;
+	    image->fetch_scanline_32 = info->fetch_scanline_32;
+	    image->fetch_scanline_float = info->fetch_scanline_float;
+	    image->fetch_pixel_32 = info->fetch_pixel_32;
+	    image->fetch_pixel_float = info->fetch_pixel_float;
+	    image->store_scanline_32 = info->store_scanline_32;
+	    image->store_scanline_float = info->store_scanline_float;
 	    
 	    return;
 	}
@@ -2880,13 +1411,13 @@ setup_accessors (bits_image_t *image)
 
 #ifndef PIXMAN_FB_ACCESSORS
 void
-_pixman_bits_image_setup_raw_accessors_accessors (bits_image_t *image);
+_pixman_bits_image_setup_accessors_accessors (bits_image_t *image);
 
 void
-_pixman_bits_image_setup_raw_accessors (bits_image_t *image)
+_pixman_bits_image_setup_accessors (bits_image_t *image)
 {
     if (image->read_func || image->write_func)
-	_pixman_bits_image_setup_raw_accessors_accessors (image);
+	_pixman_bits_image_setup_accessors_accessors (image);
     else
 	setup_accessors (image);
 }
@@ -2894,7 +1425,7 @@ _pixman_bits_image_setup_raw_accessors (bits_image_t *image)
 #else
 
 void
-_pixman_bits_image_setup_raw_accessors_accessors (bits_image_t *image)
+_pixman_bits_image_setup_accessors_accessors (bits_image_t *image)
 {
     setup_accessors (image);
 }
diff --git a/pixman/pixman/pixman-accessor.h b/pixman/pixman/pixman-accessor.h
index 90c8ea7b7..8e0b03621 100644
--- a/pixman/pixman/pixman-accessor.h
+++ b/pixman/pixman/pixman-accessor.h
@@ -1,21 +1,10 @@
 #ifdef PIXMAN_FB_ACCESSORS
 
-#define ACCESS(sym) sym##_accessors
-
 #define READ(img, ptr)							\
     (((bits_image_t *)(img))->read_func ((ptr), sizeof(*(ptr))))
 #define WRITE(img, ptr,val)						\
     (((bits_image_t *)(img))->write_func ((ptr), (val), sizeof (*(ptr))))
 
-#define MEMCPY_WRAPPED(img, dst, src, size)				\
-    do {								\
-	size_t _i;							\
-	uint8_t *_dst = (uint8_t*)(dst), *_src = (uint8_t*)(src);	\
-	for(_i = 0; _i < size; _i++) {					\
-	    WRITE((img), _dst +_i, READ((img), _src + _i));		\
-	}								\
-    } while (0)
-
 #define MEMSET_WRAPPED(img, dst, val, size)				\
     do {								\
 	size_t _i;							\
@@ -27,12 +16,8 @@
 
 #else
 
-#define ACCESS(sym) sym
-
 #define READ(img, ptr)		(*(ptr))
 #define WRITE(img, ptr, val)	(*(ptr) = (val))
-#define MEMCPY_WRAPPED(img, dst, src, size)				\
-    memcpy(dst, src, size)
 #define MEMSET_WRAPPED(img, dst, val, size)				\
     memset(dst, val, size)
 
diff --git a/pixman/pixman/pixman-arm-common.h b/pixman/pixman/pixman-arm-common.h
index 8d432b1de..3a7cb2bef 100644
--- a/pixman/pixman/pixman-arm-common.h
+++ b/pixman/pixman/pixman-arm-common.h
@@ -26,6 +26,8 @@
 #ifndef PIXMAN_ARM_COMMON_H
 #define PIXMAN_ARM_COMMON_H
 
+#include "pixman-inlines.h"
+
 /* Define some macros which can expand into proxy functions between
  * ARM assembly optimized functions and the rest of pixman fast path API.
  *
@@ -45,6 +47,9 @@
  * or mask), the corresponding stride argument is unused.
  */
 
+#define SKIP_ZERO_SRC  1
+#define SKIP_ZERO_MASK 2
+
 #define PIXMAN_ARM_BIND_FAST_PATH_SRC_DST(cputype, name,                \
                                           src_type, src_cnt,            \
                                           dst_type, dst_cnt)            \
@@ -58,26 +63,16 @@ pixman_composite_##name##_asm_##cputype (int32_t   w,                   \
                                                                         \
 static void                                                             \
 cputype##_composite_##name (pixman_implementation_t *imp,               \
-                            pixman_op_t              op,                \
-                            pixman_image_t *         src_image,         \
-                            pixman_image_t *         mask_image,        \
-                            pixman_image_t *         dst_image,         \
-                            int32_t                  src_x,             \
-                            int32_t                  src_y,             \
-                            int32_t                  mask_x,            \
-                            int32_t                  mask_y,            \
-                            int32_t                  dest_x,            \
-                            int32_t                  dest_y,            \
-                            int32_t                  width,             \
-                            int32_t                  height)            \
+                            pixman_composite_info_t *info)              \
 {                                                                       \
-    dst_type *dst_line;                                                 \
+    PIXMAN_COMPOSITE_ARGS (info);                                       \
+    dst_type *dst_line;							\
     src_type *src_line;                                                 \
     int32_t dst_stride, src_stride;                                     \
                                                                         \
     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, src_type,           \
                            src_stride, src_line, src_cnt);              \
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, dst_type,         \
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, dst_type,        \
                            dst_stride, dst_line, dst_cnt);              \
                                                                         \
     pixman_composite_##name##_asm_##cputype (width, height,             \
@@ -85,7 +80,7 @@ cputype##_composite_##name (pixman_implementation_t *imp,               \
                                              src_line, src_stride);     \
 }
 
-#define PIXMAN_ARM_BIND_FAST_PATH_N_DST(cputype, name,                  \
+#define PIXMAN_ARM_BIND_FAST_PATH_N_DST(flags, cputype, name,           \
                                         dst_type, dst_cnt)              \
 void                                                                    \
 pixman_composite_##name##_asm_##cputype (int32_t    w,                  \
@@ -96,29 +91,20 @@ pixman_composite_##name##_asm_##cputype (int32_t    w,                  \
                                                                         \
 static void                                                             \
 cputype##_composite_##name (pixman_implementation_t *imp,               \
-                            pixman_op_t              op,                \
-                            pixman_image_t *         src_image,         \
-                            pixman_image_t *         mask_image,        \
-                            pixman_image_t *         dst_image,         \
-                            int32_t                  src_x,             \
-                            int32_t                  src_y,             \
-                            int32_t                  mask_x,            \
-                            int32_t                  mask_y,            \
-                            int32_t                  dest_x,            \
-                            int32_t                  dest_y,            \
-                            int32_t                  width,             \
-                            int32_t                  height)            \
+			    pixman_composite_info_t *info)              \
 {                                                                       \
+    PIXMAN_COMPOSITE_ARGS (info);					\
     dst_type  *dst_line;                                                \
     int32_t    dst_stride;                                              \
     uint32_t   src;                                                     \
                                                                         \
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);  \
+    src = _pixman_image_get_solid (					\
+	imp, src_image, dest_image->bits.format);			\
                                                                         \
-    if (src == 0)                                                       \
+    if ((flags & SKIP_ZERO_SRC) && src == 0)                            \
 	return;                                                         \
                                                                         \
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, dst_type,         \
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, dst_type,        \
                            dst_stride, dst_line, dst_cnt);              \
                                                                         \
     pixman_composite_##name##_asm_##cputype (width, height,             \
@@ -126,7 +112,7 @@ cputype##_composite_##name (pixman_implementation_t *imp,               \
                                              src);                      \
 }
 
-#define PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST(cputype, name,             \
+#define PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST(flags, cputype, name,      \
                                              mask_type, mask_cnt,       \
                                              dst_type, dst_cnt)         \
 void                                                                    \
@@ -141,30 +127,21 @@ pixman_composite_##name##_asm_##cputype (int32_t    w,                  \
                                                                         \
 static void                                                             \
 cputype##_composite_##name (pixman_implementation_t *imp,               \
-                            pixman_op_t              op,                \
-                            pixman_image_t *         src_image,         \
-                            pixman_image_t *         mask_image,        \
-                            pixman_image_t *         dst_image,         \
-                            int32_t                  src_x,             \
-                            int32_t                  src_y,             \
-                            int32_t                  mask_x,            \
-                            int32_t                  mask_y,            \
-                            int32_t                  dest_x,            \
-                            int32_t                  dest_y,            \
-                            int32_t                  width,             \
-                            int32_t                  height)            \
+                            pixman_composite_info_t *info)              \
 {                                                                       \
-    dst_type  *dst_line;                                                \
+    PIXMAN_COMPOSITE_ARGS (info);                                       \
+    dst_type  *dst_line;						\
     mask_type *mask_line;                                               \
     int32_t    dst_stride, mask_stride;                                 \
     uint32_t   src;                                                     \
                                                                         \
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);  \
+    src = _pixman_image_get_solid (					\
+	imp, src_image, dest_image->bits.format);			\
                                                                         \
-    if (src == 0)                                                       \
+    if ((flags & SKIP_ZERO_SRC) && src == 0)                            \
 	return;                                                         \
                                                                         \
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, dst_type,         \
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, dst_type,        \
                            dst_stride, dst_line, dst_cnt);              \
     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, mask_type,       \
                            mask_stride, mask_line, mask_cnt);           \
@@ -175,7 +152,7 @@ cputype##_composite_##name (pixman_implementation_t *imp,               \
                                              mask_line, mask_stride);   \
 }
 
-#define PIXMAN_ARM_BIND_FAST_PATH_SRC_N_DST(cputype, name,              \
+#define PIXMAN_ARM_BIND_FAST_PATH_SRC_N_DST(flags, cputype, name,       \
                                             src_type, src_cnt,          \
                                             dst_type, dst_cnt)          \
 void                                                                    \
@@ -189,30 +166,21 @@ pixman_composite_##name##_asm_##cputype (int32_t    w,                  \
                                                                         \
 static void                                                             \
 cputype##_composite_##name (pixman_implementation_t *imp,               \
-                            pixman_op_t              op,                \
-                            pixman_image_t *         src_image,         \
-                            pixman_image_t *         mask_image,        \
-                            pixman_image_t *         dst_image,         \
-                            int32_t                  src_x,             \
-                            int32_t                  src_y,             \
-                            int32_t                  mask_x,            \
-                            int32_t                  mask_y,            \
-                            int32_t                  dest_x,            \
-                            int32_t                  dest_y,            \
-                            int32_t                  width,             \
-                            int32_t                  height)            \
+                            pixman_composite_info_t *info)              \
 {                                                                       \
-    dst_type  *dst_line;                                                \
+    PIXMAN_COMPOSITE_ARGS (info);                                       \
+    dst_type  *dst_line;						\
     src_type  *src_line;                                                \
     int32_t    dst_stride, src_stride;                                  \
     uint32_t   mask;                                                    \
                                                                         \
-    mask = _pixman_image_get_solid (mask_image, dst_image->bits.format);\
+    mask = _pixman_image_get_solid (					\
+	imp, mask_image, dest_image->bits.format);			\
                                                                         \
-    if (mask == 0)                                                      \
+    if ((flags & SKIP_ZERO_MASK) && mask == 0)                          \
 	return;                                                         \
                                                                         \
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, dst_type,         \
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, dst_type,        \
                            dst_stride, dst_line, dst_cnt);              \
     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, src_type,           \
                            src_stride, src_line, src_cnt);              \
@@ -239,25 +207,15 @@ pixman_composite_##name##_asm_##cputype (int32_t    w,                  \
                                                                         \
 static void                                                             \
 cputype##_composite_##name (pixman_implementation_t *imp,               \
-                            pixman_op_t              op,                \
-                            pixman_image_t *         src_image,         \
-                            pixman_image_t *         mask_image,        \
-                            pixman_image_t *         dst_image,         \
-                            int32_t                  src_x,             \
-                            int32_t                  src_y,             \
-                            int32_t                  mask_x,            \
-                            int32_t                  mask_y,            \
-                            int32_t                  dest_x,            \
-                            int32_t                  dest_y,            \
-                            int32_t                  width,             \
-                            int32_t                  height)            \
+                            pixman_composite_info_t *info)              \
 {                                                                       \
-    dst_type  *dst_line;                                                \
+    PIXMAN_COMPOSITE_ARGS (info);                                       \
+    dst_type  *dst_line;						\
     src_type  *src_line;                                                \
     mask_type *mask_line;                                               \
     int32_t    dst_stride, src_stride, mask_stride;                     \
                                                                         \
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, dst_type,         \
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, dst_type,        \
                            dst_stride, dst_line, dst_cnt);              \
     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, src_type,           \
                            src_stride, src_line, src_cnt);              \
@@ -270,4 +228,201 @@ cputype##_composite_##name (pixman_implementation_t *imp,               \
                                              mask_line, mask_stride);   \
 }
 
+#define PIXMAN_ARM_BIND_SCALED_NEAREST_SRC_DST(cputype, name, op,             \
+                                               src_type, dst_type)            \
+void                                                                          \
+pixman_scaled_nearest_scanline_##name##_##op##_asm_##cputype (                \
+                                                   int32_t          w,        \
+                                                   dst_type *       dst,      \
+                                                   const src_type * src,      \
+                                                   pixman_fixed_t   vx,       \
+                                                   pixman_fixed_t   unit_x,   \
+                                                   pixman_fixed_t   max_vx);  \
+                                                                              \
+static force_inline void                                                      \
+scaled_nearest_scanline_##cputype##_##name##_##op (dst_type *       pd,       \
+                                                   const src_type * ps,       \
+                                                   int32_t          w,        \
+                                                   pixman_fixed_t   vx,       \
+                                                   pixman_fixed_t   unit_x,   \
+                                                   pixman_fixed_t   max_vx,   \
+                                                   pixman_bool_t    zero_src) \
+{                                                                             \
+    pixman_scaled_nearest_scanline_##name##_##op##_asm_##cputype (w, pd, ps,  \
+                                                                  vx, unit_x, \
+                                                                  max_vx);    \
+}                                                                             \
+                                                                              \
+FAST_NEAREST_MAINLOOP (cputype##_##name##_cover_##op,                         \
+                       scaled_nearest_scanline_##cputype##_##name##_##op,     \
+                       src_type, dst_type, COVER)                             \
+FAST_NEAREST_MAINLOOP (cputype##_##name##_none_##op,                          \
+                       scaled_nearest_scanline_##cputype##_##name##_##op,     \
+                       src_type, dst_type, NONE)                              \
+FAST_NEAREST_MAINLOOP (cputype##_##name##_pad_##op,                           \
+                       scaled_nearest_scanline_##cputype##_##name##_##op,     \
+                       src_type, dst_type, PAD)                               \
+FAST_NEAREST_MAINLOOP (cputype##_##name##_normal_##op,                        \
+                       scaled_nearest_scanline_##cputype##_##name##_##op,     \
+                       src_type, dst_type, NORMAL)
+
+/* Provide entries for the fast path table */
+#define PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH(op,s,d,func)                      \
+    SIMPLE_NEAREST_FAST_PATH_COVER (op,s,d,func),                             \
+    SIMPLE_NEAREST_FAST_PATH_NONE (op,s,d,func),                              \
+    SIMPLE_NEAREST_FAST_PATH_PAD (op,s,d,func),                               \
+    SIMPLE_NEAREST_FAST_PATH_NORMAL (op,s,d,func)
+
+#define PIXMAN_ARM_BIND_SCALED_NEAREST_SRC_A8_DST(flags, cputype, name, op,   \
+                                                  src_type, dst_type)         \
+void                                                                          \
+pixman_scaled_nearest_scanline_##name##_##op##_asm_##cputype (                \
+                                                   int32_t          w,        \
+                                                   dst_type *       dst,      \
+                                                   const src_type * src,      \
+                                                   pixman_fixed_t   vx,       \
+                                                   pixman_fixed_t   unit_x,   \
+                                                   pixman_fixed_t   max_vx,   \
+                                                   const uint8_t *  mask);    \
+                                                                              \
+static force_inline void                                                      \
+scaled_nearest_scanline_##cputype##_##name##_##op (const uint8_t *  mask,     \
+                                                   dst_type *       pd,       \
+                                                   const src_type * ps,       \
+                                                   int32_t          w,        \
+                                                   pixman_fixed_t   vx,       \
+                                                   pixman_fixed_t   unit_x,   \
+                                                   pixman_fixed_t   max_vx,   \
+                                                   pixman_bool_t    zero_src) \
+{                                                                             \
+    if ((flags & SKIP_ZERO_SRC) && zero_src)                                  \
+	return;                                                               \
+    pixman_scaled_nearest_scanline_##name##_##op##_asm_##cputype (w, pd, ps,  \
+                                                                  vx, unit_x, \
+                                                                  max_vx,     \
+                                                                  mask);      \
+}                                                                             \
+                                                                              \
+FAST_NEAREST_MAINLOOP_COMMON (cputype##_##name##_cover_##op,                  \
+                              scaled_nearest_scanline_##cputype##_##name##_##op,\
+                              src_type, uint8_t, dst_type, COVER, TRUE, FALSE)\
+FAST_NEAREST_MAINLOOP_COMMON (cputype##_##name##_none_##op,                   \
+                              scaled_nearest_scanline_##cputype##_##name##_##op,\
+                              src_type, uint8_t, dst_type, NONE, TRUE, FALSE) \
+FAST_NEAREST_MAINLOOP_COMMON (cputype##_##name##_pad_##op,                    \
+                              scaled_nearest_scanline_##cputype##_##name##_##op,\
+                              src_type, uint8_t, dst_type, PAD, TRUE, FALSE)  \
+FAST_NEAREST_MAINLOOP_COMMON (cputype##_##name##_normal_##op,                 \
+                              scaled_nearest_scanline_##cputype##_##name##_##op,\
+                              src_type, uint8_t, dst_type, NORMAL, TRUE, FALSE)
+
+/* Provide entries for the fast path table */
+#define PIXMAN_ARM_SIMPLE_NEAREST_A8_MASK_FAST_PATH(op,s,d,func)              \
+    SIMPLE_NEAREST_A8_MASK_FAST_PATH_COVER (op,s,d,func),                     \
+    SIMPLE_NEAREST_A8_MASK_FAST_PATH_NONE (op,s,d,func),                      \
+    SIMPLE_NEAREST_A8_MASK_FAST_PATH_PAD (op,s,d,func),                       \
+    SIMPLE_NEAREST_A8_MASK_FAST_PATH_NORMAL (op,s,d,func)
+
+/*****************************************************************************/
+
+#define PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_DST(flags, cputype, name, op,     \
+                                                src_type, dst_type)           \
+void                                                                          \
+pixman_scaled_bilinear_scanline_##name##_##op##_asm_##cputype (               \
+                                                dst_type *       dst,         \
+                                                const src_type * top,         \
+                                                const src_type * bottom,      \
+                                                int              wt,          \
+                                                int              wb,          \
+                                                pixman_fixed_t   x,           \
+                                                pixman_fixed_t   ux,          \
+                                                int              width);      \
+                                                                              \
+static force_inline void                                                      \
+scaled_bilinear_scanline_##cputype##_##name##_##op (                          \
+                                                dst_type *       dst,         \
+                                                const uint32_t * mask,        \
+                                                const src_type * src_top,     \
+                                                const src_type * src_bottom,  \
+                                                int32_t          w,           \
+                                                int              wt,          \
+                                                int              wb,          \
+                                                pixman_fixed_t   vx,          \
+                                                pixman_fixed_t   unit_x,      \
+                                                pixman_fixed_t   max_vx,      \
+                                                pixman_bool_t    zero_src)    \
+{                                                                             \
+    if ((flags & SKIP_ZERO_SRC) && zero_src)                                  \
+	return;                                                               \
+    pixman_scaled_bilinear_scanline_##name##_##op##_asm_##cputype (           \
+                            dst, src_top, src_bottom, wt, wb, vx, unit_x, w); \
+}                                                                             \
+                                                                              \
+FAST_BILINEAR_MAINLOOP_COMMON (cputype##_##name##_cover_##op,                 \
+                       scaled_bilinear_scanline_##cputype##_##name##_##op,    \
+                       src_type, uint32_t, dst_type, COVER, FLAG_NONE)        \
+FAST_BILINEAR_MAINLOOP_COMMON (cputype##_##name##_none_##op,                  \
+                       scaled_bilinear_scanline_##cputype##_##name##_##op,    \
+                       src_type, uint32_t, dst_type, NONE, FLAG_NONE)         \
+FAST_BILINEAR_MAINLOOP_COMMON (cputype##_##name##_pad_##op,                   \
+                       scaled_bilinear_scanline_##cputype##_##name##_##op,    \
+                       src_type, uint32_t, dst_type, PAD, FLAG_NONE)          \
+FAST_BILINEAR_MAINLOOP_COMMON (cputype##_##name##_normal_##op,                \
+                       scaled_bilinear_scanline_##cputype##_##name##_##op,    \
+                       src_type, uint32_t, dst_type, NORMAL,                  \
+                       FLAG_NONE)
+
+
+#define PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_A8_DST(flags, cputype, name, op,  \
+                                                src_type, dst_type)           \
+void                                                                          \
+pixman_scaled_bilinear_scanline_##name##_##op##_asm_##cputype (               \
+                                                dst_type *       dst,         \
+                                                const uint8_t *  mask,        \
+                                                const src_type * top,         \
+                                                const src_type * bottom,      \
+                                                int              wt,          \
+                                                int              wb,          \
+                                                pixman_fixed_t   x,           \
+                                                pixman_fixed_t   ux,          \
+                                                int              width);      \
+                                                                              \
+static force_inline void                                                      \
+scaled_bilinear_scanline_##cputype##_##name##_##op (                          \
+                                                dst_type *       dst,         \
+                                                const uint8_t *  mask,        \
+                                                const src_type * src_top,     \
+                                                const src_type * src_bottom,  \
+                                                int32_t          w,           \
+                                                int              wt,          \
+                                                int              wb,          \
+                                                pixman_fixed_t   vx,          \
+                                                pixman_fixed_t   unit_x,      \
+                                                pixman_fixed_t   max_vx,      \
+                                                pixman_bool_t    zero_src)    \
+{                                                                             \
+    if ((flags & SKIP_ZERO_SRC) && zero_src)                                  \
+	return;                                                                   \
+    pixman_scaled_bilinear_scanline_##name##_##op##_asm_##cputype (           \
+                      dst, mask, src_top, src_bottom, wt, wb, vx, unit_x, w); \
+}                                                                             \
+                                                                              \
+FAST_BILINEAR_MAINLOOP_COMMON (cputype##_##name##_cover_##op,                 \
+                       scaled_bilinear_scanline_##cputype##_##name##_##op,    \
+                       src_type, uint8_t, dst_type, COVER,                    \
+                       FLAG_HAVE_NON_SOLID_MASK)                              \
+FAST_BILINEAR_MAINLOOP_COMMON (cputype##_##name##_none_##op,                  \
+                       scaled_bilinear_scanline_##cputype##_##name##_##op,    \
+                       src_type, uint8_t, dst_type, NONE,                     \
+                       FLAG_HAVE_NON_SOLID_MASK)                              \
+FAST_BILINEAR_MAINLOOP_COMMON (cputype##_##name##_pad_##op,                   \
+                       scaled_bilinear_scanline_##cputype##_##name##_##op,    \
+                       src_type, uint8_t, dst_type, PAD,                      \
+                       FLAG_HAVE_NON_SOLID_MASK)                              \
+FAST_BILINEAR_MAINLOOP_COMMON (cputype##_##name##_normal_##op,                \
+                       scaled_bilinear_scanline_##cputype##_##name##_##op,    \
+                       src_type, uint8_t, dst_type, NORMAL,                   \
+                       FLAG_HAVE_NON_SOLID_MASK)
+
+
 #endif
diff --git a/pixman/pixman/pixman-arm-neon-asm-bilinear.S b/pixman/pixman/pixman-arm-neon-asm-bilinear.S
new file mode 100644
index 000000000..e37b5c298
--- /dev/null
+++ b/pixman/pixman/pixman-arm-neon-asm-bilinear.S
@@ -0,0 +1,1368 @@
+/*
+ * Copyright © 2011 SCore Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Author:  Siarhei Siamashka (siarhei.siamashka@nokia.com)
+ * Author:  Taekyun Kim (tkq.kim@samsung.com)
+ */
+
+/*
+ * This file contains scaled bilinear scanline functions implemented
+ * using older siarhei's bilinear macro template.
+ *
+ * << General scanline function procedures >>
+ *  1. bilinear interpolate source pixels
+ *  2. load mask pixels
+ *  3. load destination pixels
+ *  4. duplicate mask to fill whole register
+ *  5. interleave source & destination pixels
+ *  6. apply mask to source pixels
+ *  7. combine source & destination pixels
+ *  8, Deinterleave final result
+ *  9. store destination pixels
+ *
+ * All registers with single number (i.e. src0, tmp0) are 64-bits registers.
+ * Registers with double numbers(src01, dst01) are 128-bits registers.
+ * All temp registers can be used freely outside the code block.
+ * Assume that symbol(register .req) OUT and MASK are defined at caller of these macro blocks.
+ *
+ * Remarks
+ *  There can be lots of pipeline stalls inside code block and between code blocks.
+ *  Further optimizations will be done by new macro templates using head/tail_head/tail scheme.
+ */
+
+/* Prevent the stack from becoming executable for no reason... */
+#if defined(__linux__) && defined (__ELF__)
+.section .note.GNU-stack,"",%progbits
+#endif
+
+.text
+.fpu neon
+.arch armv7a
+.object_arch armv4
+.eabi_attribute 10, 0
+.eabi_attribute 12, 0
+.arm
+.altmacro
+.p2align 2
+
+#include "pixman-private.h"
+#include "pixman-arm-neon-asm.h"
+
+/*
+ * Bilinear macros from pixman-arm-neon-asm.S
+ */
+
+/* Supplementary macro for setting function attributes */
+.macro pixman_asm_function fname
+    .func fname
+    .global fname
+#ifdef __ELF__
+    .hidden fname
+    .type fname, %function
+#endif
+fname:
+.endm
+
+/*
+ * Bilinear scaling support code which tries to provide pixel fetching, color
+ * format conversion, and interpolation as separate macros which can be used
+ * as the basic building blocks for constructing bilinear scanline functions.
+ */
+
+.macro bilinear_load_8888 reg1, reg2, tmp
+    mov       TMP1, X, asr #16
+    add       X, X, UX
+    add       TMP1, TOP, TMP1, asl #2
+    vld1.32   {reg1}, [TMP1], STRIDE
+    vld1.32   {reg2}, [TMP1]
+.endm
+
+.macro bilinear_load_0565 reg1, reg2, tmp
+    mov       TMP1, X, asr #16
+    add       X, X, UX
+    add       TMP1, TOP, TMP1, asl #1
+    vld1.32   {reg2[0]}, [TMP1], STRIDE
+    vld1.32   {reg2[1]}, [TMP1]
+    convert_four_0565_to_x888_packed reg2, reg1, reg2, tmp
+.endm
+
+.macro bilinear_load_and_vertical_interpolate_two_8888 \
+                    acc1, acc2, reg1, reg2, reg3, reg4, tmp1, tmp2
+
+    bilinear_load_8888 reg1, reg2, tmp1
+    vmull.u8  acc1, reg1, d28
+    vmlal.u8  acc1, reg2, d29
+    bilinear_load_8888 reg3, reg4, tmp2
+    vmull.u8  acc2, reg3, d28
+    vmlal.u8  acc2, reg4, d29
+.endm
+
+.macro bilinear_load_and_vertical_interpolate_four_8888 \
+                xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi \
+                yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi
+
+    bilinear_load_and_vertical_interpolate_two_8888 \
+                xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi
+    bilinear_load_and_vertical_interpolate_two_8888 \
+                yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi
+.endm
+
+.macro bilinear_load_and_vertical_interpolate_two_0565 \
+                acc1, acc2, reg1, reg2, reg3, reg4, acc2lo, acc2hi
+
+    mov       TMP1, X, asr #16
+    add       X, X, UX
+    add       TMP1, TOP, TMP1, asl #1
+    mov       TMP2, X, asr #16
+    add       X, X, UX
+    add       TMP2, TOP, TMP2, asl #1
+    vld1.32   {acc2lo[0]}, [TMP1], STRIDE
+    vld1.32   {acc2hi[0]}, [TMP2], STRIDE
+    vld1.32   {acc2lo[1]}, [TMP1]
+    vld1.32   {acc2hi[1]}, [TMP2]
+    convert_0565_to_x888 acc2, reg3, reg2, reg1
+    vzip.u8   reg1, reg3
+    vzip.u8   reg2, reg4
+    vzip.u8   reg3, reg4
+    vzip.u8   reg1, reg2
+    vmull.u8  acc1, reg1, d28
+    vmlal.u8  acc1, reg2, d29
+    vmull.u8  acc2, reg3, d28
+    vmlal.u8  acc2, reg4, d29
+.endm
+
+.macro bilinear_load_and_vertical_interpolate_four_0565 \
+                xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi \
+                yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi
+
+    mov       TMP1, X, asr #16
+    add       X, X, UX
+    add       TMP1, TOP, TMP1, asl #1
+    mov       TMP2, X, asr #16
+    add       X, X, UX
+    add       TMP2, TOP, TMP2, asl #1
+    vld1.32   {xacc2lo[0]}, [TMP1], STRIDE
+    vld1.32   {xacc2hi[0]}, [TMP2], STRIDE
+    vld1.32   {xacc2lo[1]}, [TMP1]
+    vld1.32   {xacc2hi[1]}, [TMP2]
+    convert_0565_to_x888 xacc2, xreg3, xreg2, xreg1
+    mov       TMP1, X, asr #16
+    add       X, X, UX
+    add       TMP1, TOP, TMP1, asl #1
+    mov       TMP2, X, asr #16
+    add       X, X, UX
+    add       TMP2, TOP, TMP2, asl #1
+    vld1.32   {yacc2lo[0]}, [TMP1], STRIDE
+    vzip.u8   xreg1, xreg3
+    vld1.32   {yacc2hi[0]}, [TMP2], STRIDE
+    vzip.u8   xreg2, xreg4
+    vld1.32   {yacc2lo[1]}, [TMP1]
+    vzip.u8   xreg3, xreg4
+    vld1.32   {yacc2hi[1]}, [TMP2]
+    vzip.u8   xreg1, xreg2
+    convert_0565_to_x888 yacc2, yreg3, yreg2, yreg1
+    vmull.u8  xacc1, xreg1, d28
+    vzip.u8   yreg1, yreg3
+    vmlal.u8  xacc1, xreg2, d29
+    vzip.u8   yreg2, yreg4
+    vmull.u8  xacc2, xreg3, d28
+    vzip.u8   yreg3, yreg4
+    vmlal.u8  xacc2, xreg4, d29
+    vzip.u8   yreg1, yreg2
+    vmull.u8  yacc1, yreg1, d28
+    vmlal.u8  yacc1, yreg2, d29
+    vmull.u8  yacc2, yreg3, d28
+    vmlal.u8  yacc2, yreg4, d29
+.endm
+
+.macro bilinear_store_8888 numpix, tmp1, tmp2
+.if numpix == 4
+    vst1.32   {d0, d1}, [OUT]!
+.elseif numpix == 2
+    vst1.32   {d0}, [OUT]!
+.elseif numpix == 1
+    vst1.32   {d0[0]}, [OUT, :32]!
+.else
+    .error bilinear_store_8888 numpix is unsupported
+.endif
+.endm
+
+.macro bilinear_store_0565 numpix, tmp1, tmp2
+    vuzp.u8 d0, d1
+    vuzp.u8 d2, d3
+    vuzp.u8 d1, d3
+    vuzp.u8 d0, d2
+    convert_8888_to_0565 d2, d1, d0, q1, tmp1, tmp2
+.if numpix == 4
+    vst1.16   {d2}, [OUT]!
+.elseif numpix == 2
+    vst1.32   {d2[0]}, [OUT]!
+.elseif numpix == 1
+    vst1.16   {d2[0]}, [OUT]!
+.else
+    .error bilinear_store_0565 numpix is unsupported
+.endif
+.endm
+
+
+/*
+ * Macros for loading mask pixels into register 'mask'.
+ * vdup must be done in somewhere else.
+ */
+.macro bilinear_load_mask_x numpix, mask
+.endm
+
+.macro bilinear_load_mask_8 numpix, mask
+.if numpix == 4
+    vld1.32     {mask[0]}, [MASK]!
+.elseif numpix == 2
+    vld1.16     {mask[0]}, [MASK]!
+.elseif numpix == 1
+    vld1.8      {mask[0]}, [MASK]!
+.else
+    .error bilinear_load_mask_8 numpix is unsupported
+.endif
+    pld         [MASK, #prefetch_offset]
+.endm
+
+.macro bilinear_load_mask mask_fmt, numpix, mask
+    bilinear_load_mask_&mask_fmt numpix, mask
+.endm
+
+
+/*
+ * Macros for loading destination pixels into register 'dst0' and 'dst1'.
+ * Interleave should be done somewhere else.
+ */
+.macro bilinear_load_dst_0565_src numpix, dst0, dst1, dst01
+.endm
+
+.macro bilinear_load_dst_8888_src numpix, dst0, dst1, dst01
+.endm
+
+.macro bilinear_load_dst_8888 numpix, dst0, dst1, dst01
+.if numpix == 4
+    vld1.32     {dst0, dst1}, [OUT]
+.elseif numpix == 2
+    vld1.32     {dst0}, [OUT]
+.elseif numpix == 1
+    vld1.32     {dst0[0]}, [OUT]
+.else
+    .error bilinear_load_dst_8888 numpix is unsupported
+.endif
+    pld         [OUT, #(prefetch_offset * 4)]
+.endm
+
+.macro bilinear_load_dst_8888_over numpix, dst0, dst1, dst01
+    bilinear_load_dst_8888 numpix, dst0, dst1, dst01
+.endm
+
+.macro bilinear_load_dst_8888_add numpix, dst0, dst1, dst01
+    bilinear_load_dst_8888 numpix, dst0, dst1, dst01
+.endm
+
+.macro bilinear_load_dst dst_fmt, op, numpix, dst0, dst1, dst01
+    bilinear_load_dst_&dst_fmt&_&op numpix, dst0, dst1, dst01
+.endm
+
+/*
+ * Macros for duplicating partially loaded mask to fill entire register.
+ * We will apply mask to interleaved source pixels, that is
+ *  (r0, r1, r2, r3, g0, g1, g2, g3) x (m0, m1, m2, m3, m0, m1, m2, m3)
+ *  (b0, b1, b2, b3, a0, a1, a2, a3) x (m0, m1, m2, m3, m0, m1, m2, m3)
+ * So, we need to duplicate loaded mask into whole register.
+ *
+ * For two pixel case
+ *  (r0, r1, x, x, g0, g1, x, x) x (m0, m1, m0, m1, m0, m1, m0, m1)
+ *  (b0, b1, x, x, a0, a1, x, x) x (m0, m1, m0, m1, m0, m1, m0, m1)
+ * We can do some optimizations for this including last pixel cases.
+ */
+.macro bilinear_duplicate_mask_x numpix, mask
+.endm
+
+.macro bilinear_duplicate_mask_8 numpix, mask
+.if numpix == 4
+    vdup.32     mask, mask[0]
+.elseif numpix == 2
+    vdup.16     mask, mask[0]
+.elseif numpix == 1
+    vdup.8      mask, mask[0]
+.else
+    .error bilinear_duplicate_mask_8 is unsupported
+.endif
+.endm
+
+.macro bilinear_duplicate_mask mask_fmt, numpix, mask
+    bilinear_duplicate_mask_&mask_fmt numpix, mask
+.endm
+
+/*
+ * Macros for interleaving src and dst pixels to rrrr gggg bbbb aaaa form.
+ * Interleave should be done when maks is enabled or operator is 'over'.
+ */
+.macro bilinear_interleave src0, src1, dst0, dst1
+    vuzp.8      src0, src1
+    vuzp.8      dst0, dst1
+    vuzp.8      src0, src1
+    vuzp.8      dst0, dst1
+.endm
+
+.macro bilinear_interleave_src_dst_x_src \
+                numpix, src0, src1, src01, dst0, dst1, dst01
+.endm
+
+.macro bilinear_interleave_src_dst_x_over \
+                numpix, src0, src1, src01, dst0, dst1, dst01
+
+    bilinear_interleave src0, src1, dst0, dst1
+.endm
+
+.macro bilinear_interleave_src_dst_x_add \
+                numpix, src0, src1, src01, dst0, dst1, dst01
+.endm
+
+.macro bilinear_interleave_src_dst_8_src \
+                numpix, src0, src1, src01, dst0, dst1, dst01
+
+    bilinear_interleave src0, src1, dst0, dst1
+.endm
+
+.macro bilinear_interleave_src_dst_8_over \
+                numpix, src0, src1, src01, dst0, dst1, dst01
+
+    bilinear_interleave src0, src1, dst0, dst1
+.endm
+
+.macro bilinear_interleave_src_dst_8_add \
+                numpix, src0, src1, src01, dst0, dst1, dst01
+
+    bilinear_interleave src0, src1, dst0, dst1
+.endm
+
+.macro bilinear_interleave_src_dst \
+                mask_fmt, op, numpix, src0, src1, src01, dst0, dst1, dst01
+
+    bilinear_interleave_src_dst_&mask_fmt&_&op \
+                numpix, src0, src1, src01, dst0, dst1, dst01
+.endm
+
+
+/*
+ * Macros for applying masks to src pixels. (see combine_mask_u() function)
+ * src, dst should be in interleaved form.
+ * mask register should be in form (m0, m1, m2, m3).
+ */
+.macro bilinear_apply_mask_to_src_x \
+                numpix, src0, src1, src01, mask, \
+                tmp01, tmp23, tmp45, tmp67
+.endm
+
+.macro bilinear_apply_mask_to_src_8 \
+                numpix, src0, src1, src01, mask, \
+                tmp01, tmp23, tmp45, tmp67
+
+    vmull.u8        tmp01, src0, mask
+    vmull.u8        tmp23, src1, mask
+    /* bubbles */
+    vrshr.u16       tmp45, tmp01, #8
+    vrshr.u16       tmp67, tmp23, #8
+    /* bubbles */
+    vraddhn.u16     src0, tmp45, tmp01
+    vraddhn.u16     src1, tmp67, tmp23
+.endm
+
+.macro bilinear_apply_mask_to_src \
+                mask_fmt, numpix, src0, src1, src01, mask, \
+                tmp01, tmp23, tmp45, tmp67
+
+    bilinear_apply_mask_to_src_&mask_fmt \
+                numpix, src0, src1, src01, mask, \
+                tmp01, tmp23, tmp45, tmp67
+.endm
+
+
+/*
+ * Macros for combining src and destination pixels.
+ * Interleave or not is depending on operator 'op'.
+ */
+.macro bilinear_combine_src \
+                numpix, src0, src1, src01, dst0, dst1, dst01, \
+                tmp01, tmp23, tmp45, tmp67, tmp8
+.endm
+
+.macro bilinear_combine_over \
+                numpix, src0, src1, src01, dst0, dst1, dst01, \
+                tmp01, tmp23, tmp45, tmp67, tmp8
+
+    vdup.32     tmp8, src1[1]
+    /* bubbles */
+    vmvn.8      tmp8, tmp8
+    /* bubbles */
+    vmull.u8    tmp01, dst0, tmp8
+    /* bubbles */
+    vmull.u8    tmp23, dst1, tmp8
+    /* bubbles */
+    vrshr.u16   tmp45, tmp01, #8
+    vrshr.u16   tmp67, tmp23, #8
+    /* bubbles */
+    vraddhn.u16 dst0, tmp45, tmp01
+    vraddhn.u16 dst1, tmp67, tmp23
+    /* bubbles */
+    vqadd.u8    src01, dst01, src01
+.endm
+
+.macro bilinear_combine_add \
+                numpix, src0, src1, src01, dst0, dst1, dst01, \
+                tmp01, tmp23, tmp45, tmp67, tmp8
+
+    vqadd.u8    src01, dst01, src01
+.endm
+
+.macro bilinear_combine \
+                op, numpix, src0, src1, src01, dst0, dst1, dst01, \
+                tmp01, tmp23, tmp45, tmp67, tmp8
+
+    bilinear_combine_&op \
+                numpix, src0, src1, src01, dst0, dst1, dst01, \
+                tmp01, tmp23, tmp45, tmp67, tmp8
+.endm
+
+/*
+ * Macros for final deinterleaving of destination pixels if needed.
+ */
+.macro bilinear_deinterleave numpix, dst0, dst1, dst01
+    vuzp.8      dst0, dst1
+    /* bubbles */
+    vuzp.8      dst0, dst1
+.endm
+
+.macro bilinear_deinterleave_dst_x_src numpix, dst0, dst1, dst01
+.endm
+
+.macro bilinear_deinterleave_dst_x_over numpix, dst0, dst1, dst01
+    bilinear_deinterleave numpix, dst0, dst1, dst01
+.endm
+
+.macro bilinear_deinterleave_dst_x_add numpix, dst0, dst1, dst01
+.endm
+
+.macro bilinear_deinterleave_dst_8_src numpix, dst0, dst1, dst01
+    bilinear_deinterleave numpix, dst0, dst1, dst01
+.endm
+
+.macro bilinear_deinterleave_dst_8_over numpix, dst0, dst1, dst01
+    bilinear_deinterleave numpix, dst0, dst1, dst01
+.endm
+
+.macro bilinear_deinterleave_dst_8_add numpix, dst0, dst1, dst01
+    bilinear_deinterleave numpix, dst0, dst1, dst01
+.endm
+
+.macro bilinear_deinterleave_dst mask_fmt, op, numpix, dst0, dst1, dst01
+    bilinear_deinterleave_dst_&mask_fmt&_&op numpix, dst0, dst1, dst01
+.endm
+
+
+.macro bilinear_interpolate_last_pixel src_fmt, mask_fmt, dst_fmt, op
+    bilinear_load_&src_fmt d0, d1, d2
+    bilinear_load_mask mask_fmt, 1, d4
+    bilinear_load_dst dst_fmt, op, 1, d18, d19, q9
+    vmull.u8  q1, d0, d28
+    vmlal.u8  q1, d1, d29
+    /* 5 cycles bubble */
+    vshll.u16 q0, d2, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q0, d2, d30
+    vmlal.u16 q0, d3, d30
+    /* 5 cycles bubble */
+    bilinear_duplicate_mask mask_fmt, 1, d4
+    vshrn.u32 d0, q0, #(2 * BILINEAR_INTERPOLATION_BITS)
+    /* 3 cycles bubble */
+    vmovn.u16 d0, q0
+    /* 1 cycle bubble */
+    bilinear_interleave_src_dst \
+                mask_fmt, op, 1, d0, d1, q0, d18, d19, q9
+    bilinear_apply_mask_to_src \
+                mask_fmt, 1, d0, d1, q0, d4, \
+                q3, q8, q10, q11
+    bilinear_combine \
+                op, 1, d0, d1, q0, d18, d19, q9, \
+                q3, q8, q10, q11, d5
+    bilinear_deinterleave_dst mask_fmt, op, 1, d0, d1, q0
+    bilinear_store_&dst_fmt 1, q2, q3
+.endm
+
+.macro bilinear_interpolate_two_pixels src_fmt, mask_fmt, dst_fmt, op
+    bilinear_load_and_vertical_interpolate_two_&src_fmt \
+                q1, q11, d0, d1, d20, d21, d22, d23
+    bilinear_load_mask mask_fmt, 2, d4
+    bilinear_load_dst dst_fmt, op, 2, d18, d19, q9
+    vshll.u16 q0, d2, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q0, d2, d30
+    vmlal.u16 q0, d3, d30
+    vshll.u16 q10, d22, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q10, d22, d31
+    vmlal.u16 q10, d23, d31
+    vshrn.u32 d0, q0, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32 d1, q10, #(2 * BILINEAR_INTERPOLATION_BITS)
+    bilinear_duplicate_mask mask_fmt, 2, d4
+    vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vadd.u16  q12, q12, q13
+    vmovn.u16 d0, q0
+    bilinear_interleave_src_dst \
+                mask_fmt, op, 2, d0, d1, q0, d18, d19, q9
+    bilinear_apply_mask_to_src \
+                mask_fmt, 2, d0, d1, q0, d4, \
+                q3, q8, q10, q11
+    bilinear_combine \
+                op, 2, d0, d1, q0, d18, d19, q9, \
+                q3, q8, q10, q11, d5
+    bilinear_deinterleave_dst mask_fmt, op, 2, d0, d1, q0
+    bilinear_store_&dst_fmt 2, q2, q3
+.endm
+
+.macro bilinear_interpolate_four_pixels src_fmt, mask_fmt, dst_fmt, op
+    bilinear_load_and_vertical_interpolate_four_&src_fmt \
+                q1, q11, d0, d1, d20, d21, d22, d23 \
+                q3, q9,  d4, d5, d16, d17, d18, d19
+    pld       [TMP1, PF_OFFS]
+    sub       TMP1, TMP1, STRIDE
+    vshll.u16 q0, d2, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q0, d2, d30
+    vmlal.u16 q0, d3, d30
+    vshll.u16 q10, d22, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q10, d22, d31
+    vmlal.u16 q10, d23, d31
+    vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vshll.u16 q2, d6, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q2, d6, d30
+    vmlal.u16 q2, d7, d30
+    vshll.u16 q8, d18, #BILINEAR_INTERPOLATION_BITS
+    bilinear_load_mask mask_fmt, 4, d22
+    bilinear_load_dst dst_fmt, op, 4, d2, d3, q1
+    pld       [TMP1, PF_OFFS]
+    vmlsl.u16 q8, d18, d31
+    vmlal.u16 q8, d19, d31
+    vadd.u16  q12, q12, q13
+    vshrn.u32 d0, q0, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32 d1, q10, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32 d4, q2, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32 d5, q8, #(2 * BILINEAR_INTERPOLATION_BITS)
+    bilinear_duplicate_mask mask_fmt, 4, d22
+    vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vmovn.u16 d0, q0
+    vmovn.u16 d1, q2
+    vadd.u16  q12, q12, q13
+    bilinear_interleave_src_dst \
+                mask_fmt, op, 4, d0, d1, q0, d2, d3, q1
+    bilinear_apply_mask_to_src \
+                mask_fmt, 4, d0, d1, q0, d22, \
+                q3, q8, q9, q10
+    bilinear_combine \
+                op, 4, d0, d1, q0, d2, d3, q1, \
+                q3, q8, q9, q10, d23
+    bilinear_deinterleave_dst mask_fmt, op, 4, d0, d1, q0
+    bilinear_store_&dst_fmt 4, q2, q3
+.endm
+
+.set BILINEAR_FLAG_USE_MASK,		1
+.set BILINEAR_FLAG_USE_ALL_NEON_REGS,	2
+
+/*
+ * Main template macro for generating NEON optimized bilinear scanline functions.
+ *
+ * Bilinear scanline generator macro take folling arguments:
+ *  fname			- name of the function to generate
+ *  src_fmt			- source color format (8888 or 0565)
+ *  dst_fmt			- destination color format (8888 or 0565)
+ *  src/dst_bpp_shift		- (1 << bpp_shift) is the size of src/dst pixel in bytes
+ *  process_last_pixel		- code block that interpolate one pixel and does not
+ *				  update horizontal weight
+ *  process_two_pixels		- code block that interpolate two pixels and update
+ *				  horizontal weight
+ *  process_four_pixels		- code block that interpolate four pixels and update
+ *				  horizontal weight
+ *  process_pixblock_head	- head part of middle loop
+ *  process_pixblock_tail	- tail part of middle loop
+ *  process_pixblock_tail_head	- tail_head of middle loop
+ *  pixblock_size		- number of pixels processed in a single middle loop
+ *  prefetch_distance		- prefetch in the source image by that many pixels ahead
+ */
+
+.macro generate_bilinear_scanline_func \
+	fname, \
+	src_fmt, dst_fmt, src_bpp_shift, dst_bpp_shift, \
+	bilinear_process_last_pixel, \
+	bilinear_process_two_pixels, \
+	bilinear_process_four_pixels, \
+	bilinear_process_pixblock_head, \
+	bilinear_process_pixblock_tail, \
+	bilinear_process_pixblock_tail_head, \
+	pixblock_size, \
+	prefetch_distance, \
+	flags
+
+pixman_asm_function fname
+.if pixblock_size == 8
+.elseif pixblock_size == 4
+.else
+    .error unsupported pixblock size
+.endif
+
+.if ((flags) & BILINEAR_FLAG_USE_MASK) == 0
+    OUT       .req    r0
+    TOP       .req    r1
+    BOTTOM    .req    r2
+    WT        .req    r3
+    WB        .req    r4
+    X         .req    r5
+    UX        .req    r6
+    WIDTH     .req    ip
+    TMP1      .req    r3
+    TMP2      .req    r4
+    PF_OFFS   .req    r7
+    TMP3      .req    r8
+    TMP4      .req    r9
+    STRIDE    .req    r2
+
+    mov		ip, sp
+    push	{r4, r5, r6, r7, r8, r9}
+    mov		PF_OFFS, #prefetch_distance
+    ldmia	ip, {WB, X, UX, WIDTH}
+.else
+    OUT       .req      r0
+    MASK      .req      r1
+    TOP       .req      r2
+    BOTTOM    .req      r3
+    WT        .req      r4
+    WB        .req      r5
+    X         .req      r6
+    UX        .req      r7
+    WIDTH     .req      ip
+    TMP1      .req      r4
+    TMP2      .req      r5
+    PF_OFFS   .req      r8
+    TMP3      .req      r9
+    TMP4      .req      r10
+    STRIDE    .req      r3
+
+    .set prefetch_offset, prefetch_distance
+
+    mov       ip, sp
+    push      {r4, r5, r6, r7, r8, r9, r10, ip}
+    mov       PF_OFFS, #prefetch_distance
+    ldmia     ip, {WT, WB, X, UX, WIDTH}
+.endif
+
+    mul       PF_OFFS, PF_OFFS, UX
+
+.if ((flags) & BILINEAR_FLAG_USE_ALL_NEON_REGS) != 0
+    vpush     {d8-d15}
+.endif
+
+    sub	      STRIDE, BOTTOM, TOP
+    .unreq    BOTTOM
+
+    cmp       WIDTH, #0
+    ble       3f
+
+    vdup.u16  q12, X
+    vdup.u16  q13, UX
+    vdup.u8   d28, WT
+    vdup.u8   d29, WB
+    vadd.u16  d25, d25, d26
+
+    /* ensure good destination alignment  */
+    cmp       WIDTH, #1
+    blt       0f
+    tst       OUT, #(1 << dst_bpp_shift)
+    beq       0f
+    vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vadd.u16  q12, q12, q13
+    bilinear_process_last_pixel
+    sub       WIDTH, WIDTH, #1
+0:
+    vadd.u16  q13, q13, q13
+    vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vadd.u16  q12, q12, q13
+
+    cmp       WIDTH, #2
+    blt       0f
+    tst       OUT, #(1 << (dst_bpp_shift + 1))
+    beq       0f
+    bilinear_process_two_pixels
+    sub       WIDTH, WIDTH, #2
+0:
+.if pixblock_size == 8
+    cmp       WIDTH, #4
+    blt       0f
+    tst       OUT, #(1 << (dst_bpp_shift + 2))
+    beq       0f
+    bilinear_process_four_pixels
+    sub       WIDTH, WIDTH, #4
+0:
+.endif
+    subs      WIDTH, WIDTH, #pixblock_size
+    blt       1f
+    mov       PF_OFFS, PF_OFFS, asr #(16 - src_bpp_shift)
+    bilinear_process_pixblock_head
+    subs      WIDTH, WIDTH, #pixblock_size
+    blt       5f
+0:
+    bilinear_process_pixblock_tail_head
+    subs      WIDTH, WIDTH, #pixblock_size
+    bge       0b
+5:
+    bilinear_process_pixblock_tail
+1:
+.if pixblock_size == 8
+    tst       WIDTH, #4
+    beq       2f
+    bilinear_process_four_pixels
+2:
+.endif
+    /* handle the remaining trailing pixels */
+    tst       WIDTH, #2
+    beq       2f
+    bilinear_process_two_pixels
+2:
+    tst       WIDTH, #1
+    beq       3f
+    bilinear_process_last_pixel
+3:
+.if ((flags) & BILINEAR_FLAG_USE_ALL_NEON_REGS) != 0
+    vpop      {d8-d15}
+.endif
+
+.if ((flags) & BILINEAR_FLAG_USE_MASK) == 0
+    pop       {r4, r5, r6, r7, r8, r9}
+.else
+    pop       {r4, r5, r6, r7, r8, r9, r10, ip}
+.endif
+    bx        lr
+
+    .unreq    OUT
+    .unreq    TOP
+    .unreq    WT
+    .unreq    WB
+    .unreq    X
+    .unreq    UX
+    .unreq    WIDTH
+    .unreq    TMP1
+    .unreq    TMP2
+    .unreq    PF_OFFS
+    .unreq    TMP3
+    .unreq    TMP4
+    .unreq    STRIDE
+.if ((flags) & BILINEAR_FLAG_USE_MASK) != 0
+    .unreq    MASK
+.endif
+
+.endfunc
+
+.endm
+
+/* src_8888_8_8888 */
+.macro bilinear_src_8888_8_8888_process_last_pixel
+    bilinear_interpolate_last_pixel 8888, 8, 8888, src
+.endm
+
+.macro bilinear_src_8888_8_8888_process_two_pixels
+    bilinear_interpolate_two_pixels 8888, 8, 8888, src
+.endm
+
+.macro bilinear_src_8888_8_8888_process_four_pixels
+    bilinear_interpolate_four_pixels 8888, 8, 8888, src
+.endm
+
+.macro bilinear_src_8888_8_8888_process_pixblock_head
+    bilinear_src_8888_8_8888_process_four_pixels
+.endm
+
+.macro bilinear_src_8888_8_8888_process_pixblock_tail
+.endm
+
+.macro bilinear_src_8888_8_8888_process_pixblock_tail_head
+    bilinear_src_8888_8_8888_process_pixblock_tail
+    bilinear_src_8888_8_8888_process_pixblock_head
+.endm
+
+/* src_8888_8_0565 */
+.macro bilinear_src_8888_8_0565_process_last_pixel
+    bilinear_interpolate_last_pixel 8888, 8, 0565, src
+.endm
+
+.macro bilinear_src_8888_8_0565_process_two_pixels
+    bilinear_interpolate_two_pixels 8888, 8, 0565, src
+.endm
+
+.macro bilinear_src_8888_8_0565_process_four_pixels
+    bilinear_interpolate_four_pixels 8888, 8, 0565, src
+.endm
+
+.macro bilinear_src_8888_8_0565_process_pixblock_head
+    bilinear_src_8888_8_0565_process_four_pixels
+.endm
+
+.macro bilinear_src_8888_8_0565_process_pixblock_tail
+.endm
+
+.macro bilinear_src_8888_8_0565_process_pixblock_tail_head
+    bilinear_src_8888_8_0565_process_pixblock_tail
+    bilinear_src_8888_8_0565_process_pixblock_head
+.endm
+
+/* src_0565_8_x888 */
+.macro bilinear_src_0565_8_x888_process_last_pixel
+    bilinear_interpolate_last_pixel 0565, 8, 8888, src
+.endm
+
+.macro bilinear_src_0565_8_x888_process_two_pixels
+    bilinear_interpolate_two_pixels 0565, 8, 8888, src
+.endm
+
+.macro bilinear_src_0565_8_x888_process_four_pixels
+    bilinear_interpolate_four_pixels 0565, 8, 8888, src
+.endm
+
+.macro bilinear_src_0565_8_x888_process_pixblock_head
+    bilinear_src_0565_8_x888_process_four_pixels
+.endm
+
+.macro bilinear_src_0565_8_x888_process_pixblock_tail
+.endm
+
+.macro bilinear_src_0565_8_x888_process_pixblock_tail_head
+    bilinear_src_0565_8_x888_process_pixblock_tail
+    bilinear_src_0565_8_x888_process_pixblock_head
+.endm
+
+/* src_0565_8_0565 */
+.macro bilinear_src_0565_8_0565_process_last_pixel
+    bilinear_interpolate_last_pixel 0565, 8, 0565, src
+.endm
+
+.macro bilinear_src_0565_8_0565_process_two_pixels
+    bilinear_interpolate_two_pixels 0565, 8, 0565, src
+.endm
+
+.macro bilinear_src_0565_8_0565_process_four_pixels
+    bilinear_interpolate_four_pixels 0565, 8, 0565, src
+.endm
+
+.macro bilinear_src_0565_8_0565_process_pixblock_head
+    bilinear_src_0565_8_0565_process_four_pixels
+.endm
+
+.macro bilinear_src_0565_8_0565_process_pixblock_tail
+.endm
+
+.macro bilinear_src_0565_8_0565_process_pixblock_tail_head
+    bilinear_src_0565_8_0565_process_pixblock_tail
+    bilinear_src_0565_8_0565_process_pixblock_head
+.endm
+
+/* over_8888_8888 */
+.macro bilinear_over_8888_8888_process_last_pixel
+    bilinear_interpolate_last_pixel 8888, x, 8888, over
+.endm
+
+.macro bilinear_over_8888_8888_process_two_pixels
+    bilinear_interpolate_two_pixels 8888, x, 8888, over
+.endm
+
+.macro bilinear_over_8888_8888_process_four_pixels
+    bilinear_interpolate_four_pixels 8888, x, 8888, over
+.endm
+
+.macro bilinear_over_8888_8888_process_pixblock_head
+    mov         TMP1, X, asr #16
+    add         X, X, UX
+    add         TMP1, TOP, TMP1, asl #2
+    mov         TMP2, X, asr #16
+    add         X, X, UX
+    add         TMP2, TOP, TMP2, asl #2
+
+    vld1.32     {d22}, [TMP1], STRIDE
+    vld1.32     {d23}, [TMP1]
+    mov         TMP3, X, asr #16
+    add         X, X, UX
+    add         TMP3, TOP, TMP3, asl #2
+    vmull.u8    q8, d22, d28
+    vmlal.u8    q8, d23, d29
+
+    vld1.32     {d22}, [TMP2], STRIDE
+    vld1.32     {d23}, [TMP2]
+    mov         TMP4, X, asr #16
+    add         X, X, UX
+    add         TMP4, TOP, TMP4, asl #2
+    vmull.u8    q9, d22, d28
+    vmlal.u8    q9, d23, d29
+
+    vld1.32     {d22}, [TMP3], STRIDE
+    vld1.32     {d23}, [TMP3]
+    vmull.u8    q10, d22, d28
+    vmlal.u8    q10, d23, d29
+
+    vshll.u16   q0, d16, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16   q0, d16, d30
+    vmlal.u16   q0, d17, d30
+
+    pld         [TMP4, PF_OFFS]
+    vld1.32     {d16}, [TMP4], STRIDE
+    vld1.32     {d17}, [TMP4]
+    pld         [TMP4, PF_OFFS]
+    vmull.u8    q11, d16, d28
+    vmlal.u8    q11, d17, d29
+
+    vshll.u16   q1, d18, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16   q1, d18, d31
+    vmlal.u16   q1, d19, d31
+    vshr.u16    q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vadd.u16    q12, q12, q13
+.endm
+
+.macro bilinear_over_8888_8888_process_pixblock_tail
+    vshll.u16   q2, d20, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16   q2, d20, d30
+    vmlal.u16   q2, d21, d30
+    vshll.u16   q3, d22, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16   q3, d22, d31
+    vmlal.u16   q3, d23, d31
+    vshrn.u32   d0, q0, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32   d1, q1, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vld1.32     {d2, d3}, [OUT, :128]
+    pld         [OUT, #(prefetch_offset * 4)]
+    vshrn.u32   d4, q2, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshr.u16    q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32   d5, q3, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vmovn.u16   d6, q0
+    vmovn.u16   d7, q2
+    vuzp.8      d6, d7
+    vuzp.8      d2, d3
+    vuzp.8      d6, d7
+    vuzp.8      d2, d3
+    vdup.32     d4, d7[1]
+    vmvn.8      d4, d4
+    vmull.u8    q11, d2, d4
+    vmull.u8    q2, d3, d4
+    vrshr.u16   q1, q11, #8
+    vrshr.u16   q10, q2, #8
+    vraddhn.u16 d2, q1, q11
+    vraddhn.u16 d3, q10, q2
+    vqadd.u8    q3, q1, q3
+    vuzp.8      d6, d7
+    vuzp.8      d6, d7
+    vadd.u16    q12, q12, q13
+    vst1.32     {d6, d7}, [OUT, :128]!
+.endm
+
+.macro bilinear_over_8888_8888_process_pixblock_tail_head
+                                            vshll.u16   q2, d20, #BILINEAR_INTERPOLATION_BITS
+    mov         TMP1, X, asr #16
+    add         X, X, UX
+    add         TMP1, TOP, TMP1, asl #2
+                                            vmlsl.u16   q2, d20, d30
+    mov         TMP2, X, asr #16
+    add         X, X, UX
+    add         TMP2, TOP, TMP2, asl #2
+                                            vmlal.u16   q2, d21, d30
+                                            vshll.u16   q3, d22, #BILINEAR_INTERPOLATION_BITS
+    vld1.32     {d20}, [TMP1], STRIDE
+                                            vmlsl.u16   q3, d22, d31
+                                            vmlal.u16   q3, d23, d31
+    vld1.32     {d21}, [TMP1]
+    vmull.u8    q8, d20, d28
+    vmlal.u8    q8, d21, d29
+                                            vshrn.u32   d0, q0, #(2 * BILINEAR_INTERPOLATION_BITS)
+                                            vshrn.u32   d1, q1, #(2 * BILINEAR_INTERPOLATION_BITS)
+                                            vld1.32     {d2, d3}, [OUT, :128]
+                                            pld         [OUT, PF_OFFS]
+                                            vshrn.u32   d4, q2, #(2 * BILINEAR_INTERPOLATION_BITS)
+                                            vshr.u16    q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vld1.32     {d22}, [TMP2], STRIDE
+                                            vshrn.u32   d5, q3, #(2 * BILINEAR_INTERPOLATION_BITS)
+                                            vmovn.u16   d6, q0
+    vld1.32     {d23}, [TMP2]
+    vmull.u8    q9, d22, d28
+    mov         TMP3, X, asr #16
+    add         X, X, UX
+    add         TMP3, TOP, TMP3, asl #2
+    mov         TMP4, X, asr #16
+    add         X, X, UX
+    add         TMP4, TOP, TMP4, asl #2
+    vmlal.u8    q9, d23, d29
+                                            vmovn.u16   d7, q2
+    vld1.32     {d22}, [TMP3], STRIDE
+                                            vuzp.8      d6, d7
+                                            vuzp.8      d2, d3
+                                            vuzp.8      d6, d7
+                                            vuzp.8      d2, d3
+                                            vdup.32     d4, d7[1]
+    vld1.32     {d23}, [TMP3]
+                                            vmvn.8      d4, d4
+    vmull.u8    q10, d22, d28
+    vmlal.u8    q10, d23, d29
+                                            vmull.u8    q11, d2, d4
+                                            vmull.u8    q2, d3, d4
+    vshll.u16   q0, d16, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16   q0, d16, d30
+                                            vrshr.u16   q1, q11, #8
+    vmlal.u16   q0, d17, d30
+                                            vrshr.u16   q8, q2, #8
+                                            vraddhn.u16 d2, q1, q11
+                                            vraddhn.u16 d3, q8, q2
+    pld         [TMP4, PF_OFFS]
+    vld1.32     {d16}, [TMP4], STRIDE
+                                            vqadd.u8    q3, q1, q3
+    vld1.32     {d17}, [TMP4]
+    pld         [TMP4, PF_OFFS]
+    vmull.u8    q11, d16, d28
+    vmlal.u8    q11, d17, d29
+                                            vuzp.8      d6, d7
+    vshll.u16   q1, d18, #BILINEAR_INTERPOLATION_BITS
+                                            vuzp.8      d6, d7
+    vmlsl.u16   q1, d18, d31
+                                            vadd.u16    q12, q12, q13
+    vmlal.u16   q1, d19, d31
+    vshr.u16    q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vadd.u16    q12, q12, q13
+                                            vst1.32     {d6, d7}, [OUT, :128]!
+.endm
+
+/* over_8888_8_8888 */
+.macro bilinear_over_8888_8_8888_process_last_pixel
+    bilinear_interpolate_last_pixel 8888, 8, 8888, over
+.endm
+
+.macro bilinear_over_8888_8_8888_process_two_pixels
+    bilinear_interpolate_two_pixels 8888, 8, 8888, over
+.endm
+
+.macro bilinear_over_8888_8_8888_process_four_pixels
+    bilinear_interpolate_four_pixels 8888, 8, 8888, over
+.endm
+
+.macro bilinear_over_8888_8_8888_process_pixblock_head
+    mov         TMP1, X, asr #16
+    add         X, X, UX
+    add         TMP1, TOP, TMP1, asl #2
+    vld1.32     {d0}, [TMP1], STRIDE
+    mov         TMP2, X, asr #16
+    add         X, X, UX
+    add         TMP2, TOP, TMP2, asl #2
+    vld1.32     {d1}, [TMP1]
+    mov         TMP3, X, asr #16
+    add         X, X, UX
+    add         TMP3, TOP, TMP3, asl #2
+    vld1.32     {d2}, [TMP2], STRIDE
+    mov         TMP4, X, asr #16
+    add         X, X, UX
+    add         TMP4, TOP, TMP4, asl #2
+    vld1.32     {d3}, [TMP2]
+    vmull.u8    q2, d0, d28
+    vmull.u8    q3, d2, d28
+    vmlal.u8    q2, d1, d29
+    vmlal.u8    q3, d3, d29
+    vshll.u16   q0, d4, #BILINEAR_INTERPOLATION_BITS
+    vshll.u16   q1, d6, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16   q0, d4, d30
+    vmlsl.u16   q1, d6, d31
+    vmlal.u16   q0, d5, d30
+    vmlal.u16   q1, d7, d31
+    vshrn.u32   d0, q0, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32   d1, q1, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vld1.32     {d2}, [TMP3], STRIDE
+    vld1.32     {d3}, [TMP3]
+    pld         [TMP4, PF_OFFS]
+    vld1.32     {d4}, [TMP4], STRIDE
+    vld1.32     {d5}, [TMP4]
+    pld         [TMP4, PF_OFFS]
+    vmull.u8    q3, d2, d28
+    vmlal.u8    q3, d3, d29
+    vmull.u8    q1, d4, d28
+    vmlal.u8    q1, d5, d29
+    vshr.u16    q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vld1.32     {d22[0]}, [MASK]!
+    pld         [MASK, #prefetch_offset]
+    vadd.u16    q12, q12, q13
+    vmovn.u16   d16, q0
+.endm
+
+.macro bilinear_over_8888_8_8888_process_pixblock_tail
+    vshll.u16   q9, d6, #BILINEAR_INTERPOLATION_BITS
+    vshll.u16   q10, d2, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16   q9, d6, d30
+    vmlsl.u16   q10, d2, d31
+    vmlal.u16   q9, d7, d30
+    vmlal.u16   q10, d3, d31
+    vshr.u16    q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vadd.u16    q12, q12, q13
+    vdup.32     d22, d22[0]
+    vshrn.u32   d18, q9, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32   d19, q10, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vmovn.u16   d17, q9
+    vld1.32     {d18, d19}, [OUT, :128]
+    pld         [OUT, PF_OFFS]
+    vuzp.8      d16, d17
+    vuzp.8      d18, d19
+    vuzp.8      d16, d17
+    vuzp.8      d18, d19
+    vmull.u8    q10, d16, d22
+    vmull.u8    q11, d17, d22
+    vrsra.u16   q10, q10, #8
+    vrsra.u16   q11, q11, #8
+    vrshrn.u16  d16, q10, #8
+    vrshrn.u16  d17, q11, #8
+    vdup.32     d22, d17[1]
+    vmvn.8      d22, d22
+    vmull.u8    q10, d18, d22
+    vmull.u8    q11, d19, d22
+    vrshr.u16   q9, q10, #8
+    vrshr.u16   q0, q11, #8
+    vraddhn.u16 d18, q9, q10
+    vraddhn.u16 d19, q0, q11
+    vqadd.u8    q9, q8, q9
+    vuzp.8      d18, d19
+    vuzp.8      d18, d19
+    vst1.32     {d18, d19}, [OUT, :128]!
+.endm
+
+.macro bilinear_over_8888_8_8888_process_pixblock_tail_head
+                                            vshll.u16   q9, d6, #BILINEAR_INTERPOLATION_BITS
+    mov         TMP1, X, asr #16
+    add         X, X, UX
+    add         TMP1, TOP, TMP1, asl #2
+                                            vshll.u16   q10, d2, #BILINEAR_INTERPOLATION_BITS
+    vld1.32     {d0}, [TMP1], STRIDE
+    mov         TMP2, X, asr #16
+    add         X, X, UX
+    add         TMP2, TOP, TMP2, asl #2
+                                            vmlsl.u16   q9, d6, d30
+                                            vmlsl.u16   q10, d2, d31
+    vld1.32     {d1}, [TMP1]
+    mov         TMP3, X, asr #16
+    add         X, X, UX
+    add         TMP3, TOP, TMP3, asl #2
+                                            vmlal.u16   q9, d7, d30
+                                            vmlal.u16   q10, d3, d31
+    vld1.32     {d2}, [TMP2], STRIDE
+    mov         TMP4, X, asr #16
+    add         X, X, UX
+    add         TMP4, TOP, TMP4, asl #2
+                                            vshr.u16    q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+                                            vadd.u16    q12, q12, q13
+    vld1.32     {d3}, [TMP2]
+                                            vdup.32     d22, d22[0]
+                                            vshrn.u32   d18, q9, #(2 * BILINEAR_INTERPOLATION_BITS)
+                                            vshrn.u32   d19, q10, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vmull.u8    q2, d0, d28
+    vmull.u8    q3, d2, d28
+                                            vmovn.u16   d17, q9
+                                            vld1.32     {d18, d19}, [OUT, :128]
+                                            pld         [OUT, #(prefetch_offset * 4)]
+    vmlal.u8    q2, d1, d29
+    vmlal.u8    q3, d3, d29
+                                            vuzp.8      d16, d17
+                                            vuzp.8      d18, d19
+    vshll.u16   q0, d4, #BILINEAR_INTERPOLATION_BITS
+    vshll.u16   q1, d6, #BILINEAR_INTERPOLATION_BITS
+                                            vuzp.8      d16, d17
+                                            vuzp.8      d18, d19
+    vmlsl.u16   q0, d4, d30
+    vmlsl.u16   q1, d6, d31
+                                            vmull.u8    q10, d16, d22
+                                            vmull.u8    q11, d17, d22
+    vmlal.u16   q0, d5, d30
+    vmlal.u16   q1, d7, d31
+                                            vrsra.u16   q10, q10, #8
+                                            vrsra.u16   q11, q11, #8
+    vshrn.u32   d0, q0, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32   d1, q1, #(2 * BILINEAR_INTERPOLATION_BITS)
+                                            vrshrn.u16  d16, q10, #8
+                                            vrshrn.u16  d17, q11, #8
+    vld1.32     {d2}, [TMP3], STRIDE
+                                            vdup.32     d22, d17[1]
+    vld1.32     {d3}, [TMP3]
+                                            vmvn.8      d22, d22
+    pld         [TMP4, PF_OFFS]
+    vld1.32     {d4}, [TMP4], STRIDE
+                                            vmull.u8    q10, d18, d22
+                                            vmull.u8    q11, d19, d22
+    vld1.32     {d5}, [TMP4]
+    pld         [TMP4, PF_OFFS]
+    vmull.u8    q3, d2, d28
+                                            vrshr.u16   q9, q10, #8
+                                            vrshr.u16   q15, q11, #8
+    vmlal.u8    q3, d3, d29
+    vmull.u8    q1, d4, d28
+                                            vraddhn.u16 d18, q9, q10
+                                            vraddhn.u16 d19, q15, q11
+    vmlal.u8    q1, d5, d29
+    vshr.u16    q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+                                            vqadd.u8    q9, q8, q9
+    vld1.32     {d22[0]}, [MASK]!
+                                            vuzp.8      d18, d19
+    vadd.u16    q12, q12, q13
+                                            vuzp.8      d18, d19
+    vmovn.u16   d16, q0
+                                            vst1.32     {d18, d19}, [OUT, :128]!
+.endm
+
+/* add_8888_8888 */
+.macro bilinear_add_8888_8888_process_last_pixel
+    bilinear_interpolate_last_pixel 8888, x, 8888, add
+.endm
+
+.macro bilinear_add_8888_8888_process_two_pixels
+    bilinear_interpolate_two_pixels 8888, x, 8888, add
+.endm
+
+.macro bilinear_add_8888_8888_process_four_pixels
+    bilinear_interpolate_four_pixels 8888, x, 8888, add
+.endm
+
+.macro bilinear_add_8888_8888_process_pixblock_head
+    bilinear_add_8888_8888_process_four_pixels
+.endm
+
+.macro bilinear_add_8888_8888_process_pixblock_tail
+.endm
+
+.macro bilinear_add_8888_8888_process_pixblock_tail_head
+    bilinear_add_8888_8888_process_pixblock_tail
+    bilinear_add_8888_8888_process_pixblock_head
+.endm
+
+/* add_8888_8_8888 */
+.macro bilinear_add_8888_8_8888_process_last_pixel
+    bilinear_interpolate_last_pixel 8888, 8, 8888, add
+.endm
+
+.macro bilinear_add_8888_8_8888_process_two_pixels
+    bilinear_interpolate_two_pixels 8888, 8, 8888, add
+.endm
+
+.macro bilinear_add_8888_8_8888_process_four_pixels
+    bilinear_interpolate_four_pixels 8888, 8, 8888, add
+.endm
+
+.macro bilinear_add_8888_8_8888_process_pixblock_head
+    bilinear_add_8888_8_8888_process_four_pixels
+.endm
+
+.macro bilinear_add_8888_8_8888_process_pixblock_tail
+.endm
+
+.macro bilinear_add_8888_8_8888_process_pixblock_tail_head
+    bilinear_add_8888_8_8888_process_pixblock_tail
+    bilinear_add_8888_8_8888_process_pixblock_head
+.endm
+
+
+/* Bilinear scanline functions */
+generate_bilinear_scanline_func \
+    pixman_scaled_bilinear_scanline_8888_8_8888_SRC_asm_neon, \
+    8888, 8888, 2, 2, \
+    bilinear_src_8888_8_8888_process_last_pixel, \
+    bilinear_src_8888_8_8888_process_two_pixels, \
+    bilinear_src_8888_8_8888_process_four_pixels, \
+    bilinear_src_8888_8_8888_process_pixblock_head, \
+    bilinear_src_8888_8_8888_process_pixblock_tail, \
+    bilinear_src_8888_8_8888_process_pixblock_tail_head, \
+    4, 28, BILINEAR_FLAG_USE_MASK
+
+generate_bilinear_scanline_func \
+    pixman_scaled_bilinear_scanline_8888_8_0565_SRC_asm_neon, \
+    8888, 0565, 2, 1, \
+    bilinear_src_8888_8_0565_process_last_pixel, \
+    bilinear_src_8888_8_0565_process_two_pixels, \
+    bilinear_src_8888_8_0565_process_four_pixels, \
+    bilinear_src_8888_8_0565_process_pixblock_head, \
+    bilinear_src_8888_8_0565_process_pixblock_tail, \
+    bilinear_src_8888_8_0565_process_pixblock_tail_head, \
+    4, 28, BILINEAR_FLAG_USE_MASK
+
+generate_bilinear_scanline_func \
+    pixman_scaled_bilinear_scanline_0565_8_x888_SRC_asm_neon, \
+    0565, 8888, 1, 2, \
+    bilinear_src_0565_8_x888_process_last_pixel, \
+    bilinear_src_0565_8_x888_process_two_pixels, \
+    bilinear_src_0565_8_x888_process_four_pixels, \
+    bilinear_src_0565_8_x888_process_pixblock_head, \
+    bilinear_src_0565_8_x888_process_pixblock_tail, \
+    bilinear_src_0565_8_x888_process_pixblock_tail_head, \
+    4, 28, BILINEAR_FLAG_USE_MASK
+
+generate_bilinear_scanline_func \
+    pixman_scaled_bilinear_scanline_0565_8_0565_SRC_asm_neon, \
+    0565, 0565, 1, 1, \
+    bilinear_src_0565_8_0565_process_last_pixel, \
+    bilinear_src_0565_8_0565_process_two_pixels, \
+    bilinear_src_0565_8_0565_process_four_pixels, \
+    bilinear_src_0565_8_0565_process_pixblock_head, \
+    bilinear_src_0565_8_0565_process_pixblock_tail, \
+    bilinear_src_0565_8_0565_process_pixblock_tail_head, \
+    4, 28, BILINEAR_FLAG_USE_MASK
+
+generate_bilinear_scanline_func \
+    pixman_scaled_bilinear_scanline_8888_8888_OVER_asm_neon, \
+    8888, 8888, 2, 2, \
+    bilinear_over_8888_8888_process_last_pixel, \
+    bilinear_over_8888_8888_process_two_pixels, \
+    bilinear_over_8888_8888_process_four_pixels, \
+    bilinear_over_8888_8888_process_pixblock_head, \
+    bilinear_over_8888_8888_process_pixblock_tail, \
+    bilinear_over_8888_8888_process_pixblock_tail_head, \
+    4, 28, 0
+
+generate_bilinear_scanline_func \
+    pixman_scaled_bilinear_scanline_8888_8_8888_OVER_asm_neon, \
+    8888, 8888, 2, 2, \
+    bilinear_over_8888_8_8888_process_last_pixel, \
+    bilinear_over_8888_8_8888_process_two_pixels, \
+    bilinear_over_8888_8_8888_process_four_pixels, \
+    bilinear_over_8888_8_8888_process_pixblock_head, \
+    bilinear_over_8888_8_8888_process_pixblock_tail, \
+    bilinear_over_8888_8_8888_process_pixblock_tail_head, \
+    4, 28, BILINEAR_FLAG_USE_MASK
+
+generate_bilinear_scanline_func \
+    pixman_scaled_bilinear_scanline_8888_8888_ADD_asm_neon, \
+    8888, 8888, 2, 2, \
+    bilinear_add_8888_8888_process_last_pixel, \
+    bilinear_add_8888_8888_process_two_pixels, \
+    bilinear_add_8888_8888_process_four_pixels, \
+    bilinear_add_8888_8888_process_pixblock_head, \
+    bilinear_add_8888_8888_process_pixblock_tail, \
+    bilinear_add_8888_8888_process_pixblock_tail_head, \
+    4, 28, 0
+
+generate_bilinear_scanline_func \
+    pixman_scaled_bilinear_scanline_8888_8_8888_ADD_asm_neon, \
+    8888, 8888, 2, 2, \
+    bilinear_add_8888_8_8888_process_last_pixel, \
+    bilinear_add_8888_8_8888_process_two_pixels, \
+    bilinear_add_8888_8_8888_process_four_pixels, \
+    bilinear_add_8888_8_8888_process_pixblock_head, \
+    bilinear_add_8888_8_8888_process_pixblock_tail, \
+    bilinear_add_8888_8_8888_process_pixblock_tail_head, \
+    4, 28, BILINEAR_FLAG_USE_MASK
diff --git a/pixman/pixman/pixman-arm-neon-asm.S b/pixman/pixman/pixman-arm-neon-asm.S
index 51bc347bc..187197dc3 100644
--- a/pixman/pixman/pixman-arm-neon-asm.S
+++ b/pixman/pixman/pixman-arm-neon-asm.S
@@ -47,7 +47,9 @@
     .eabi_attribute 12, 0 /* suppress Tag_Advanced_SIMD_arch */
     .arm
     .altmacro
+    .p2align 2
 
+#include "pixman-private.h"
 #include "pixman-arm-neon-asm.h"
 
 /* Global configuration options and preferences */
@@ -253,7 +255,7 @@
     vld1.16     {d4, d5}, [DST_R, :128]!
         vqadd.u8    q9, q0, q11
     vshrn.u16   d6, q2, #8
-    vld4.8      {d0, d1, d2, d3}, [SRC]!
+    fetch_src_pixblock
     vshrn.u16   d7, q2, #3
     vsli.u16    q2, q2, #5
         vshll.u8    q14, d16, #8
@@ -295,7 +297,7 @@
     pixman_composite_over_8888_0565_process_pixblock_tail
     vst1.16     {d28, d29}, [DST_W, :128]!
     vld1.16     {d4, d5}, [DST_R, :128]!
-    vld4.32     {d0, d1, d2, d3}, [SRC]!
+    fetch_src_pixblock
     pixman_composite_over_8888_0565_process_pixblock_head
     cache_preload 8, 8
 .endm
@@ -388,6 +390,7 @@ generate_composite_function \
     vld1.16     {d4, d5}, [DST_R, :128]!
     vst1.16     {d28, d29}, [DST_W, :128]!
     pixman_composite_over_n_0565_process_pixblock_head
+    cache_preload 8, 8
 .endm
 
 .macro pixman_composite_over_n_0565_init
@@ -432,7 +435,7 @@ generate_composite_function \
         vsri.u16    q14, q8, #5
                                     PF add PF_X, PF_X, #8
                                     PF tst PF_CTL, #0xF
-    vld4.8      {d0, d1, d2, d3}, [SRC]!
+    fetch_src_pixblock
                                     PF addne PF_X, PF_X, #8
                                     PF subne PF_CTL, PF_CTL, #1
         vsri.u16    q14, q9, #11
@@ -477,7 +480,7 @@ generate_composite_function \
 .macro pixman_composite_src_0565_8888_process_pixblock_tail_head
     pixman_composite_src_0565_8888_process_pixblock_tail
     vst4.8     {d28, d29, d30, d31}, [DST_W, :128]!
-    vld1.16    {d0, d1}, [SRC]!
+    fetch_src_pixblock
     pixman_composite_src_0565_8888_process_pixblock_head
     cache_preload 8, 8
 .endm
@@ -495,16 +498,16 @@ generate_composite_function \
 
 /******************************************************************************/
 
-.macro pixman_composite_add_8000_8000_process_pixblock_head
+.macro pixman_composite_add_8_8_process_pixblock_head
     vqadd.u8    q14, q0, q2
     vqadd.u8    q15, q1, q3
 .endm
 
-.macro pixman_composite_add_8000_8000_process_pixblock_tail
+.macro pixman_composite_add_8_8_process_pixblock_tail
 .endm
 
-.macro pixman_composite_add_8000_8000_process_pixblock_tail_head
-    vld1.8      {d0, d1, d2, d3}, [SRC]!
+.macro pixman_composite_add_8_8_process_pixblock_tail_head
+    fetch_src_pixblock
                                     PF add PF_X, PF_X, #32
                                     PF tst PF_CTL, #0xF
     vld1.8      {d4, d5, d6, d7}, [DST_R, :128]!
@@ -523,26 +526,26 @@ generate_composite_function \
 .endm
 
 generate_composite_function \
-    pixman_composite_add_8000_8000_asm_neon, 8, 0, 8, \
+    pixman_composite_add_8_8_asm_neon, 8, 0, 8, \
     FLAG_DST_READWRITE, \
     32, /* number of pixels, processed in a single block */ \
     10, /* prefetch distance */ \
     default_init, \
     default_cleanup, \
-    pixman_composite_add_8000_8000_process_pixblock_head, \
-    pixman_composite_add_8000_8000_process_pixblock_tail, \
-    pixman_composite_add_8000_8000_process_pixblock_tail_head
+    pixman_composite_add_8_8_process_pixblock_head, \
+    pixman_composite_add_8_8_process_pixblock_tail, \
+    pixman_composite_add_8_8_process_pixblock_tail_head
 
 /******************************************************************************/
 
 .macro pixman_composite_add_8888_8888_process_pixblock_tail_head
-    vld1.8      {d0, d1, d2, d3}, [SRC]!
+    fetch_src_pixblock
                                     PF add PF_X, PF_X, #8
                                     PF tst PF_CTL, #0xF
-    vld1.8      {d4, d5, d6, d7}, [DST_R, :128]!
+    vld1.32     {d4, d5, d6, d7}, [DST_R, :128]!
                                     PF addne PF_X, PF_X, #8
                                     PF subne PF_CTL, PF_CTL, #1
-        vst1.8      {d28, d29, d30, d31}, [DST_W, :128]!
+        vst1.32     {d28, d29, d30, d31}, [DST_W, :128]!
                                     PF cmp PF_X, ORIG_W
                                     PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]
                                     PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]
@@ -561,8 +564,8 @@ generate_composite_function \
     10, /* prefetch distance */ \
     default_init, \
     default_cleanup, \
-    pixman_composite_add_8000_8000_process_pixblock_head, \
-    pixman_composite_add_8000_8000_process_pixblock_tail, \
+    pixman_composite_add_8_8_process_pixblock_head, \
+    pixman_composite_add_8_8_process_pixblock_tail, \
     pixman_composite_add_8888_8888_process_pixblock_tail_head
 
 generate_composite_function_single_scanline \
@@ -571,13 +574,13 @@ generate_composite_function_single_scanline \
     8, /* number of pixels, processed in a single block */ \
     default_init, \
     default_cleanup, \
-    pixman_composite_add_8000_8000_process_pixblock_head, \
-    pixman_composite_add_8000_8000_process_pixblock_tail, \
+    pixman_composite_add_8_8_process_pixblock_head, \
+    pixman_composite_add_8_8_process_pixblock_tail, \
     pixman_composite_add_8888_8888_process_pixblock_tail_head
 
 /******************************************************************************/
 
-.macro pixman_composite_over_8888_8888_process_pixblock_head
+.macro pixman_composite_out_reverse_8888_8888_process_pixblock_head
     vmvn.8      d24, d3  /* get inverted alpha */
     /* do alpha blending */
     vmull.u8    q8, d24, d4
@@ -586,7 +589,7 @@ generate_composite_function_single_scanline \
     vmull.u8    q11, d24, d7
 .endm
 
-.macro pixman_composite_over_8888_8888_process_pixblock_tail
+.macro pixman_composite_out_reverse_8888_8888_process_pixblock_tail
     vrshr.u16   q14, q8, #8
     vrshr.u16   q15, q9, #8
     vrshr.u16   q12, q10, #8
@@ -595,6 +598,56 @@ generate_composite_function_single_scanline \
     vraddhn.u16 d29, q15, q9
     vraddhn.u16 d30, q12, q10
     vraddhn.u16 d31, q13, q11
+.endm
+
+.macro pixman_composite_out_reverse_8888_8888_process_pixblock_tail_head
+    vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!
+        vrshr.u16   q14, q8, #8
+                                    PF add PF_X, PF_X, #8
+                                    PF tst PF_CTL, #0xF
+        vrshr.u16   q15, q9, #8
+        vrshr.u16   q12, q10, #8
+        vrshr.u16   q13, q11, #8
+                                    PF addne PF_X, PF_X, #8
+                                    PF subne PF_CTL, PF_CTL, #1
+        vraddhn.u16 d28, q14, q8
+        vraddhn.u16 d29, q15, q9
+                                    PF cmp PF_X, ORIG_W
+        vraddhn.u16 d30, q12, q10
+        vraddhn.u16 d31, q13, q11
+    fetch_src_pixblock
+                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]
+    vmvn.8      d22, d3
+                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]
+        vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
+                                    PF subge PF_X, PF_X, ORIG_W
+    vmull.u8    q8, d22, d4
+                                    PF subges PF_CTL, PF_CTL, #0x10
+    vmull.u8    q9, d22, d5
+                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!
+    vmull.u8    q10, d22, d6
+                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!
+    vmull.u8    q11, d22, d7
+.endm
+
+generate_composite_function_single_scanline \
+    pixman_composite_scanline_out_reverse_asm_neon, 32, 0, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_out_reverse_8888_8888_process_pixblock_head, \
+    pixman_composite_out_reverse_8888_8888_process_pixblock_tail, \
+    pixman_composite_out_reverse_8888_8888_process_pixblock_tail_head
+
+/******************************************************************************/
+
+.macro pixman_composite_over_8888_8888_process_pixblock_head
+    pixman_composite_out_reverse_8888_8888_process_pixblock_head
+.endm
+
+.macro pixman_composite_over_8888_8888_process_pixblock_tail
+    pixman_composite_out_reverse_8888_8888_process_pixblock_tail
     vqadd.u8    q14, q0, q14
     vqadd.u8    q15, q1, q15
 .endm
@@ -616,7 +669,7 @@ generate_composite_function_single_scanline \
         vraddhn.u16 d31, q13, q11
         vqadd.u8    q14, q0, q14
         vqadd.u8    q15, q1, q15
-    vld4.8      {d0, d1, d2, d3}, [SRC]!
+    fetch_src_pixblock
                                     PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]
     vmvn.8      d22, d3
                                     PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]
@@ -654,12 +707,55 @@ generate_composite_function_single_scanline \
 
 /******************************************************************************/
 
-/* TODO: expand macros and do better instructions scheduling */
+.macro pixman_composite_over_n_8888_process_pixblock_head
+    /* deinterleaved source pixels in {d0, d1, d2, d3} */
+    /* inverted alpha in {d24} */
+    /* destination pixels in {d4, d5, d6, d7} */
+    vmull.u8    q8, d24, d4
+    vmull.u8    q9, d24, d5
+    vmull.u8    q10, d24, d6
+    vmull.u8    q11, d24, d7
+.endm
+
+.macro pixman_composite_over_n_8888_process_pixblock_tail
+    vrshr.u16   q14, q8, #8
+    vrshr.u16   q15, q9, #8
+    vrshr.u16   q2, q10, #8
+    vrshr.u16   q3, q11, #8
+    vraddhn.u16 d28, q14, q8
+    vraddhn.u16 d29, q15, q9
+    vraddhn.u16 d30, q2, q10
+    vraddhn.u16 d31, q3, q11
+    vqadd.u8    q14, q0, q14
+    vqadd.u8    q15, q1, q15
+.endm
+
 .macro pixman_composite_over_n_8888_process_pixblock_tail_head
-    pixman_composite_over_8888_8888_process_pixblock_tail
+        vrshr.u16   q14, q8, #8
+        vrshr.u16   q15, q9, #8
+        vrshr.u16   q2, q10, #8
+        vrshr.u16   q3, q11, #8
+        vraddhn.u16 d28, q14, q8
+        vraddhn.u16 d29, q15, q9
+        vraddhn.u16 d30, q2, q10
+        vraddhn.u16 d31, q3, q11
     vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!
-    vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
-    pixman_composite_over_8888_8888_process_pixblock_head
+        vqadd.u8    q14, q0, q14
+                                    PF add PF_X, PF_X, #8
+                                    PF tst PF_CTL, #0x0F
+                                    PF addne PF_X, PF_X, #8
+                                    PF subne PF_CTL, PF_CTL, #1
+        vqadd.u8    q15, q1, q15
+                                    PF cmp PF_X, ORIG_W
+    vmull.u8    q8, d24, d4
+                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]
+    vmull.u8    q9, d24, d5
+                                    PF subge PF_X, PF_X, ORIG_W
+    vmull.u8    q10, d24, d6
+                                    PF subges PF_CTL, PF_CTL, #0x10
+    vmull.u8    q11, d24, d7
+                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!
+        vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
 .endm
 
 .macro pixman_composite_over_n_8888_init
@@ -669,6 +765,7 @@ generate_composite_function_single_scanline \
     vdup.8      d1, d3[1]
     vdup.8      d2, d3[2]
     vdup.8      d3, d3[3]
+    vmvn.8      d24, d3  /* get inverted alpha */
 .endm
 
 generate_composite_function \
@@ -739,61 +836,112 @@ generate_composite_function \
 
 /******************************************************************************/
 
-.macro pixman_composite_over_n_8_0565_process_pixblock_head
-    /* in */
-    vmull.u8    q0, d24, d8
-    vmull.u8    q1, d24, d9
-    vmull.u8    q6, d24, d10
-    vmull.u8    q7, d24, d11
-    vrshr.u16   q10, q0, #8
-    vrshr.u16   q11, q1, #8
-    vrshr.u16   q12, q6, #8
-    vrshr.u16   q13, q7, #8
-    vraddhn.u16 d0, q0, q10
-    vraddhn.u16 d1, q1, q11
-    vraddhn.u16 d2, q6, q12
-    vraddhn.u16 d3, q7, q13
-
-    vshrn.u16   d6, q2, #8
-    vshrn.u16   d7, q2, #3
-    vsli.u16    q2, q2, #5
-    vsri.u8     d6, d6, #5
-    vmvn.8      d3, d3
-    vsri.u8     d7, d7, #6
-    vshrn.u16   d30, q2, #2
-    /* now do alpha blending */
-    vmull.u8    q10, d3, d6
-    vmull.u8    q11, d3, d7
-    vmull.u8    q12, d3, d30
-    vrshr.u16   q13, q10, #8
-    vrshr.u16   q3, q11, #8
-    vrshr.u16   q15, q12, #8
-    vraddhn.u16 d20, q10, q13
-    vraddhn.u16 d23, q11, q3
-    vraddhn.u16 d22, q12, q15
-.endm
+.macro pixman_composite_over_8888_8_0565_process_pixblock_head
+    vmull.u8    q0,  d24, d8    /* IN for SRC pixels (part1) */
+    vmull.u8    q1,  d24, d9
+    vmull.u8    q6,  d24, d10
+    vmull.u8    q7,  d24, d11
+        vshrn.u16   d6,  q2, #8 /* convert DST_R data to 32-bpp (part1) */
+        vshrn.u16   d7,  q2, #3
+        vsli.u16    q2,  q2, #5
+    vrshr.u16   q8,  q0,  #8    /* IN for SRC pixels (part2) */
+    vrshr.u16   q9,  q1,  #8
+    vrshr.u16   q10, q6,  #8
+    vrshr.u16   q11, q7,  #8
+    vraddhn.u16 d0,  q0,  q8
+    vraddhn.u16 d1,  q1,  q9
+    vraddhn.u16 d2,  q6,  q10
+    vraddhn.u16 d3,  q7,  q11
+        vsri.u8     d6,  d6, #5 /* convert DST_R data to 32-bpp (part2) */
+        vsri.u8     d7,  d7, #6
+    vmvn.8      d3,  d3
+        vshrn.u16   d30, q2, #2
+    vmull.u8    q8,  d3, d6     /* now do alpha blending */
+    vmull.u8    q9,  d3, d7
+    vmull.u8    q10, d3, d30
+.endm
+
+.macro pixman_composite_over_8888_8_0565_process_pixblock_tail
+    /* 3 cycle bubble (after vmull.u8) */
+    vrshr.u16   q13, q8,  #8
+    vrshr.u16   q11, q9,  #8
+    vrshr.u16   q15, q10, #8
+    vraddhn.u16 d16, q8,  q13
+    vraddhn.u16 d27, q9,  q11
+    vraddhn.u16 d26, q10, q15
+    vqadd.u8    d16, d2,  d16
+    /* 1 cycle bubble */
+    vqadd.u8    q9,  q0,  q13
+    vshll.u8    q14, d16, #8    /* convert to 16bpp */
+    vshll.u8    q8,  d19, #8
+    vshll.u8    q9,  d18, #8
+    vsri.u16    q14, q8,  #5
+    /* 1 cycle bubble */
+    vsri.u16    q14, q9,  #11
+.endm
+
+.macro pixman_composite_over_8888_8_0565_process_pixblock_tail_head
+    vld1.16     {d4, d5}, [DST_R, :128]!
+    vshrn.u16   d6,  q2,  #8
+    fetch_mask_pixblock
+    vshrn.u16   d7,  q2,  #3
+    fetch_src_pixblock
+    vmull.u8    q6,  d24, d10
+        vrshr.u16   q13, q8,  #8
+        vrshr.u16   q11, q9,  #8
+        vrshr.u16   q15, q10, #8
+        vraddhn.u16 d16, q8,  q13
+        vraddhn.u16 d27, q9,  q11
+        vraddhn.u16 d26, q10, q15
+        vqadd.u8    d16, d2,  d16
+    vmull.u8    q1,  d24, d9
+        vqadd.u8    q9,  q0,  q13
+        vshll.u8    q14, d16, #8
+    vmull.u8    q0,  d24, d8
+        vshll.u8    q8,  d19, #8
+        vshll.u8    q9,  d18, #8
+        vsri.u16    q14, q8,  #5
+    vmull.u8    q7,  d24, d11
+        vsri.u16    q14, q9,  #11
 
-.macro pixman_composite_over_n_8_0565_process_pixblock_tail
-    vqadd.u8    d16, d2, d20
-    vqadd.u8    q9, q0, q11
-    /* convert to r5g6b5 */
-    vshll.u8    q14, d16, #8
-    vshll.u8    q8, d19, #8
-    vshll.u8    q9, d18, #8
-    vsri.u16    q14, q8, #5
-    vsri.u16    q14, q9, #11
-.endm
+    cache_preload 8, 8
 
-/* TODO: expand macros and do better instructions scheduling */
-.macro pixman_composite_over_n_8_0565_process_pixblock_tail_head
-    pixman_composite_over_n_8_0565_process_pixblock_tail
+    vsli.u16    q2,  q2,  #5
+    vrshr.u16   q8,  q0,  #8
+    vrshr.u16   q9,  q1,  #8
+    vrshr.u16   q10, q6,  #8
+    vrshr.u16   q11, q7,  #8
+    vraddhn.u16 d0,  q0,  q8
+    vraddhn.u16 d1,  q1,  q9
+    vraddhn.u16 d2,  q6,  q10
+    vraddhn.u16 d3,  q7,  q11
+    vsri.u8     d6,  d6,  #5
+    vsri.u8     d7,  d7,  #6
+    vmvn.8      d3,  d3
+    vshrn.u16   d30, q2,  #2
     vst1.16     {d28, d29}, [DST_W, :128]!
-    vld1.16     {d4, d5}, [DST_R, :128]!
-    vld1.8      {d24}, [MASK]!
-    cache_preload 8, 8
-    pixman_composite_over_n_8_0565_process_pixblock_head
+    vmull.u8    q8,  d3,  d6
+    vmull.u8    q9,  d3,  d7
+    vmull.u8    q10, d3,  d30
 .endm
 
+generate_composite_function \
+    pixman_composite_over_8888_8_0565_asm_neon, 32, 8, 16, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    default_init_need_all_regs, \
+    default_cleanup_need_all_regs, \
+    pixman_composite_over_8888_8_0565_process_pixblock_head, \
+    pixman_composite_over_8888_8_0565_process_pixblock_tail, \
+    pixman_composite_over_8888_8_0565_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    4,  /* dst_r_basereg */ \
+    8,  /* src_basereg   */ \
+    24  /* mask_basereg  */
+
+/******************************************************************************/
+
 /*
  * This function needs a special initialization of solid mask.
  * Solid source pixel data is fetched from stack at ARGS_STACK_OFFSET
@@ -825,9 +973,37 @@ generate_composite_function \
     5, /* prefetch distance */ \
     pixman_composite_over_n_8_0565_init, \
     pixman_composite_over_n_8_0565_cleanup, \
-    pixman_composite_over_n_8_0565_process_pixblock_head, \
-    pixman_composite_over_n_8_0565_process_pixblock_tail, \
-    pixman_composite_over_n_8_0565_process_pixblock_tail_head
+    pixman_composite_over_8888_8_0565_process_pixblock_head, \
+    pixman_composite_over_8888_8_0565_process_pixblock_tail, \
+    pixman_composite_over_8888_8_0565_process_pixblock_tail_head
+
+/******************************************************************************/
+
+.macro pixman_composite_over_8888_n_0565_init
+    add         DUMMY, sp, #(ARGS_STACK_OFFSET + 8)
+    vpush       {d8-d15}
+    vld1.32     {d24[0]}, [DUMMY]
+    vdup.8      d24, d24[3]
+.endm
+
+.macro pixman_composite_over_8888_n_0565_cleanup
+    vpop        {d8-d15}
+.endm
+
+generate_composite_function \
+    pixman_composite_over_8888_n_0565_asm_neon, 32, 0, 16, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_over_8888_n_0565_init, \
+    pixman_composite_over_8888_n_0565_cleanup, \
+    pixman_composite_over_8888_8_0565_process_pixblock_head, \
+    pixman_composite_over_8888_8_0565_process_pixblock_tail, \
+    pixman_composite_over_8888_8_0565_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    4,  /* dst_r_basereg */ \
+    8,  /* src_basereg   */ \
+    24  /* mask_basereg  */
 
 /******************************************************************************/
 
@@ -839,7 +1015,7 @@ generate_composite_function \
 
 .macro pixman_composite_src_0565_0565_process_pixblock_tail_head
     vst1.16 {d0, d1, d2, d3}, [DST_W, :128]!
-    vld1.16 {d0, d1, d2, d3}, [SRC]!
+    fetch_src_pixblock
     cache_preload 16, 16
 .endm
 
@@ -985,7 +1161,7 @@ generate_composite_function \
 
 .macro pixman_composite_src_8888_8888_process_pixblock_tail_head
     vst1.32 {d0, d1, d2, d3}, [DST_W, :128]!
-    vld1.32 {d0, d1, d2, d3}, [SRC]!
+    fetch_src_pixblock
     cache_preload 8, 8
 .endm
 
@@ -1016,7 +1192,7 @@ generate_composite_function \
 
 .macro pixman_composite_src_x888_8888_process_pixblock_tail_head
     vst1.32 {d0, d1, d2, d3}, [DST_W, :128]!
-    vld1.32 {d0, d1, d2, d3}, [SRC]!
+    fetch_src_pixblock
     vorr     q0, q0, q2
     vorr     q1, q1, q2
     cache_preload 8, 8
@@ -1044,6 +1220,145 @@ generate_composite_function \
 
 /******************************************************************************/
 
+.macro pixman_composite_src_n_8_8888_process_pixblock_head
+    /* expecting solid source in {d0, d1, d2, d3} */
+    /* mask is in d24 (d25, d26, d27 are unused) */
+
+    /* in */
+    vmull.u8    q8, d24, d0
+    vmull.u8    q9, d24, d1
+    vmull.u8    q10, d24, d2
+    vmull.u8    q11, d24, d3
+    vrsra.u16   q8, q8, #8
+    vrsra.u16   q9, q9, #8
+    vrsra.u16   q10, q10, #8
+    vrsra.u16   q11, q11, #8
+.endm
+
+.macro pixman_composite_src_n_8_8888_process_pixblock_tail
+    vrshrn.u16  d28, q8, #8
+    vrshrn.u16  d29, q9, #8
+    vrshrn.u16  d30, q10, #8
+    vrshrn.u16  d31, q11, #8
+.endm
+
+.macro pixman_composite_src_n_8_8888_process_pixblock_tail_head
+    fetch_mask_pixblock
+                                    PF add PF_X, PF_X, #8
+        vrshrn.u16  d28, q8, #8
+                                    PF tst PF_CTL, #0x0F
+        vrshrn.u16  d29, q9, #8
+                                    PF addne PF_X, PF_X, #8
+        vrshrn.u16  d30, q10, #8
+                                    PF subne PF_CTL, PF_CTL, #1
+        vrshrn.u16  d31, q11, #8
+                                    PF cmp PF_X, ORIG_W
+    vmull.u8    q8, d24, d0
+                                    PF pld, [PF_MASK, PF_X, lsl #mask_bpp_shift]
+    vmull.u8    q9, d24, d1
+                                    PF subge PF_X, PF_X, ORIG_W
+    vmull.u8    q10, d24, d2
+                                    PF subges PF_CTL, PF_CTL, #0x10
+    vmull.u8    q11, d24, d3
+                                    PF ldrgeb DUMMY, [PF_MASK, MASK_STRIDE, lsl #mask_bpp_shift]!
+        vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
+    vrsra.u16   q8, q8, #8
+    vrsra.u16   q9, q9, #8
+    vrsra.u16   q10, q10, #8
+    vrsra.u16   q11, q11, #8
+.endm
+
+.macro pixman_composite_src_n_8_8888_init
+    add         DUMMY, sp, #ARGS_STACK_OFFSET
+    vld1.32     {d3[0]}, [DUMMY]
+    vdup.8      d0, d3[0]
+    vdup.8      d1, d3[1]
+    vdup.8      d2, d3[2]
+    vdup.8      d3, d3[3]
+.endm
+
+.macro pixman_composite_src_n_8_8888_cleanup
+.endm
+
+generate_composite_function \
+    pixman_composite_src_n_8_8888_asm_neon, 0, 8, 32, \
+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_src_n_8_8888_init, \
+    pixman_composite_src_n_8_8888_cleanup, \
+    pixman_composite_src_n_8_8888_process_pixblock_head, \
+    pixman_composite_src_n_8_8888_process_pixblock_tail, \
+    pixman_composite_src_n_8_8888_process_pixblock_tail_head, \
+
+/******************************************************************************/
+
+.macro pixman_composite_src_n_8_8_process_pixblock_head
+    vmull.u8    q0, d24, d16
+    vmull.u8    q1, d25, d16
+    vmull.u8    q2, d26, d16
+    vmull.u8    q3, d27, d16
+    vrsra.u16   q0, q0,  #8
+    vrsra.u16   q1, q1,  #8
+    vrsra.u16   q2, q2,  #8
+    vrsra.u16   q3, q3,  #8
+.endm
+
+.macro pixman_composite_src_n_8_8_process_pixblock_tail
+    vrshrn.u16  d28, q0, #8
+    vrshrn.u16  d29, q1, #8
+    vrshrn.u16  d30, q2, #8
+    vrshrn.u16  d31, q3, #8
+.endm
+
+.macro pixman_composite_src_n_8_8_process_pixblock_tail_head
+    fetch_mask_pixblock
+                                    PF add PF_X, PF_X, #8
+        vrshrn.u16  d28, q0, #8
+                                    PF tst PF_CTL, #0x0F
+        vrshrn.u16  d29, q1, #8
+                                    PF addne PF_X, PF_X, #8
+        vrshrn.u16  d30, q2, #8
+                                    PF subne PF_CTL, PF_CTL, #1
+        vrshrn.u16  d31, q3, #8
+                                    PF cmp PF_X, ORIG_W
+    vmull.u8    q0,  d24, d16
+                                    PF pld, [PF_MASK, PF_X, lsl #mask_bpp_shift]
+    vmull.u8    q1,  d25, d16
+                                    PF subge PF_X, PF_X, ORIG_W
+    vmull.u8    q2,  d26, d16
+                                    PF subges PF_CTL, PF_CTL, #0x10
+    vmull.u8    q3,  d27, d16
+                                    PF ldrgeb DUMMY, [PF_MASK, MASK_STRIDE, lsl #mask_bpp_shift]!
+        vst1.8      {d28, d29, d30, d31}, [DST_W, :128]!
+    vrsra.u16   q0, q0,  #8
+    vrsra.u16   q1, q1,  #8
+    vrsra.u16   q2, q2,  #8
+    vrsra.u16   q3, q3,  #8
+.endm
+
+.macro pixman_composite_src_n_8_8_init
+    add         DUMMY, sp, #ARGS_STACK_OFFSET
+    vld1.32     {d16[0]}, [DUMMY]
+    vdup.8      d16, d16[3]
+.endm
+
+.macro pixman_composite_src_n_8_8_cleanup
+.endm
+
+generate_composite_function \
+    pixman_composite_src_n_8_8_asm_neon, 0, 8, 8, \
+    FLAG_DST_WRITEONLY, \
+    32, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_src_n_8_8_init, \
+    pixman_composite_src_n_8_8_cleanup, \
+    pixman_composite_src_n_8_8_process_pixblock_head, \
+    pixman_composite_src_n_8_8_process_pixblock_tail, \
+    pixman_composite_src_n_8_8_process_pixblock_tail_head
+
+/******************************************************************************/
+
 .macro pixman_composite_over_n_8_8888_process_pixblock_head
     /* expecting deinterleaved source data in {d8, d9, d10, d11} */
     /* d8 - blue, d9 - green, d10 - red, d11 - alpha */
@@ -1051,49 +1366,83 @@ generate_composite_function \
     /* mask is in d24 (d25, d26, d27 are unused) */
 
     /* in */
-    vmull.u8    q0, d24, d8
-    vmull.u8    q1, d24, d9
-    vmull.u8    q6, d24, d10
-    vmull.u8    q7, d24, d11
-    vrshr.u16   q10, q0, #8
-    vrshr.u16   q11, q1, #8
-    vrshr.u16   q12, q6, #8
-    vrshr.u16   q13, q7, #8
-    vraddhn.u16 d0, q0, q10
-    vraddhn.u16 d1, q1, q11
-    vraddhn.u16 d2, q6, q12
-    vraddhn.u16 d3, q7, q13
-    vmvn.8      d24, d3  /* get inverted alpha */
+    vmull.u8    q6, d24, d8
+    vmull.u8    q7, d24, d9
+    vmull.u8    q8, d24, d10
+    vmull.u8    q9, d24, d11
+    vrshr.u16   q10, q6, #8
+    vrshr.u16   q11, q7, #8
+    vrshr.u16   q12, q8, #8
+    vrshr.u16   q13, q9, #8
+    vraddhn.u16 d0, q6, q10
+    vraddhn.u16 d1, q7, q11
+    vraddhn.u16 d2, q8, q12
+    vraddhn.u16 d3, q9, q13
+    vmvn.8      d25, d3  /* get inverted alpha */
     /* source:      d0 - blue, d1 - green, d2 - red, d3 - alpha */
     /* destination: d4 - blue, d5 - green, d6 - red, d7 - alpha */
     /* now do alpha blending */
-    vmull.u8    q8, d24, d4
-    vmull.u8    q9, d24, d5
-    vmull.u8    q10, d24, d6
-    vmull.u8    q11, d24, d7
+    vmull.u8    q8, d25, d4
+    vmull.u8    q9, d25, d5
+    vmull.u8    q10, d25, d6
+    vmull.u8    q11, d25, d7
 .endm
 
 .macro pixman_composite_over_n_8_8888_process_pixblock_tail
     vrshr.u16   q14, q8, #8
     vrshr.u16   q15, q9, #8
-    vrshr.u16   q12, q10, #8
-    vrshr.u16   q13, q11, #8
+    vrshr.u16   q6, q10, #8
+    vrshr.u16   q7, q11, #8
     vraddhn.u16 d28, q14, q8
     vraddhn.u16 d29, q15, q9
-    vraddhn.u16 d30, q12, q10
-    vraddhn.u16 d31, q13, q11
+    vraddhn.u16 d30, q6, q10
+    vraddhn.u16 d31, q7, q11
     vqadd.u8    q14, q0, q14
     vqadd.u8    q15, q1, q15
 .endm
 
-/* TODO: expand macros and do better instructions scheduling */
 .macro pixman_composite_over_n_8_8888_process_pixblock_tail_head
-    pixman_composite_over_n_8_8888_process_pixblock_tail
-    vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
+        vrshr.u16   q14, q8, #8
     vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!
-    vld1.8      {d24}, [MASK]!
-    cache_preload 8, 8
-    pixman_composite_over_n_8_8888_process_pixblock_head
+        vrshr.u16   q15, q9, #8
+    fetch_mask_pixblock
+        vrshr.u16   q6, q10, #8
+                                    PF add PF_X, PF_X, #8
+        vrshr.u16   q7, q11, #8
+                                    PF tst PF_CTL, #0x0F
+        vraddhn.u16 d28, q14, q8
+                                    PF addne PF_X, PF_X, #8
+        vraddhn.u16 d29, q15, q9
+                                    PF subne PF_CTL, PF_CTL, #1
+        vraddhn.u16 d30, q6, q10
+                                    PF cmp PF_X, ORIG_W
+        vraddhn.u16 d31, q7, q11
+                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]
+    vmull.u8    q6, d24, d8
+                                    PF pld, [PF_MASK, PF_X, lsl #mask_bpp_shift]
+    vmull.u8    q7, d24, d9
+                                    PF subge PF_X, PF_X, ORIG_W
+    vmull.u8    q8, d24, d10
+                                    PF subges PF_CTL, PF_CTL, #0x10
+    vmull.u8    q9, d24, d11
+                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!
+        vqadd.u8    q14, q0, q14
+                                    PF ldrgeb DUMMY, [PF_MASK, MASK_STRIDE, lsl #mask_bpp_shift]!
+        vqadd.u8    q15, q1, q15
+    vrshr.u16   q10, q6, #8
+    vrshr.u16   q11, q7, #8
+    vrshr.u16   q12, q8, #8
+    vrshr.u16   q13, q9, #8
+    vraddhn.u16 d0, q6, q10
+    vraddhn.u16 d1, q7, q11
+    vraddhn.u16 d2, q8, q12
+    vraddhn.u16 d3, q9, q13
+        vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
+    vmvn.8      d25, d3
+    vmull.u8    q8, d25, d4
+    vmull.u8    q9, d25, d5
+    vmull.u8    q10, d25, d6
+    vmull.u8    q11, d25, d7
 .endm
 
 .macro pixman_composite_over_n_8_8888_init
@@ -1123,6 +1472,74 @@ generate_composite_function \
 
 /******************************************************************************/
 
+.macro pixman_composite_over_n_8_8_process_pixblock_head
+    vmull.u8    q0,  d24, d8
+    vmull.u8    q1,  d25, d8
+    vmull.u8    q6,  d26, d8
+    vmull.u8    q7,  d27, d8
+    vrshr.u16   q10, q0,  #8
+    vrshr.u16   q11, q1,  #8
+    vrshr.u16   q12, q6,  #8
+    vrshr.u16   q13, q7,  #8
+    vraddhn.u16 d0,  q0,  q10
+    vraddhn.u16 d1,  q1,  q11
+    vraddhn.u16 d2,  q6,  q12
+    vraddhn.u16 d3,  q7,  q13
+    vmvn.8      q12, q0
+    vmvn.8      q13, q1
+    vmull.u8    q8,  d24, d4
+    vmull.u8    q9,  d25, d5
+    vmull.u8    q10, d26, d6
+    vmull.u8    q11, d27, d7
+.endm
+
+.macro pixman_composite_over_n_8_8_process_pixblock_tail
+    vrshr.u16   q14, q8,  #8
+    vrshr.u16   q15, q9,  #8
+    vrshr.u16   q12, q10, #8
+    vrshr.u16   q13, q11, #8
+    vraddhn.u16 d28, q14, q8
+    vraddhn.u16 d29, q15, q9
+    vraddhn.u16 d30, q12, q10
+    vraddhn.u16 d31, q13, q11
+    vqadd.u8    q14, q0,  q14
+    vqadd.u8    q15, q1,  q15
+.endm
+
+/* TODO: expand macros and do better instructions scheduling */
+.macro pixman_composite_over_n_8_8_process_pixblock_tail_head
+    vld1.8      {d4, d5, d6, d7}, [DST_R, :128]!
+    pixman_composite_over_n_8_8_process_pixblock_tail
+    fetch_mask_pixblock
+    cache_preload 32, 32
+    vst1.8      {d28, d29, d30, d31}, [DST_W, :128]!
+    pixman_composite_over_n_8_8_process_pixblock_head
+.endm
+
+.macro pixman_composite_over_n_8_8_init
+    add         DUMMY, sp, #ARGS_STACK_OFFSET
+    vpush       {d8-d15}
+    vld1.32     {d8[0]}, [DUMMY]
+    vdup.8      d8, d8[3]
+.endm
+
+.macro pixman_composite_over_n_8_8_cleanup
+    vpop        {d8-d15}
+.endm
+
+generate_composite_function \
+    pixman_composite_over_n_8_8_asm_neon, 0, 8, 8, \
+    FLAG_DST_READWRITE, \
+    32, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_over_n_8_8_init, \
+    pixman_composite_over_n_8_8_cleanup, \
+    pixman_composite_over_n_8_8_process_pixblock_head, \
+    pixman_composite_over_n_8_8_process_pixblock_tail, \
+    pixman_composite_over_n_8_8_process_pixblock_tail_head
+
+/******************************************************************************/
+
 .macro pixman_composite_over_n_8888_8888_ca_process_pixblock_head
     /*
      * 'combine_mask_ca' replacement
@@ -1159,11 +1576,10 @@ generate_composite_function \
      *
      * output: updated dest in {d28, d29, d30, d31}
      */
-    vmvn.8      d24, d24
-    vmvn.8      d25, d25
+    vmvn.8      q12, q12
+    vmvn.8      d26, d26
     vmull.u8    q8,  d24, d4
     vmull.u8    q9,  d25, d5
-    vmvn.8      d26, d26
     vmvn.8      d27, d3
     vmull.u8    q10, d26, d6
     vmull.u8    q11, d27, d7
@@ -1193,7 +1609,7 @@ generate_composite_function \
         vraddhn.u16 d29, q15, q9
         vraddhn.u16 d30, q6, q10
         vraddhn.u16 d31, q7, q11
-    vld4.8      {d24, d25, d26, d27}, [MASK]!
+    fetch_mask_pixblock
         vqadd.u8    q14, q0, q14
         vqadd.u8    q15, q1, q15
     cache_preload 8, 8
@@ -1228,6 +1644,227 @@ generate_composite_function \
 
 /******************************************************************************/
 
+.macro pixman_composite_over_n_8888_0565_ca_process_pixblock_head
+    /*
+     * 'combine_mask_ca' replacement
+     *
+     * input:  solid src (n) in {d8,  d9,  d10, d11}  [B, G, R, A]
+     *         mask in          {d24, d25, d26}       [B, G, R]
+     * output: updated src in   {d0,  d1,  d2 }       [B, G, R]
+     *         updated mask in  {d24, d25, d26}       [B, G, R]
+     */
+    vmull.u8    q0,  d24, d8
+    vmull.u8    q1,  d25, d9
+    vmull.u8    q6,  d26, d10
+    vmull.u8    q9,  d11, d25
+    vmull.u8    q12, d11, d24
+    vmull.u8    q13, d11, d26
+    vrshr.u16   q8,  q0,  #8
+    vrshr.u16   q10, q1,  #8
+    vrshr.u16   q11, q6,  #8
+    vraddhn.u16 d0,  q0,  q8
+    vraddhn.u16 d1,  q1,  q10
+    vraddhn.u16 d2,  q6,  q11
+    vrshr.u16   q11, q12, #8
+    vrshr.u16   q8,  q9,  #8
+    vrshr.u16   q6,  q13, #8
+    vraddhn.u16 d24, q12, q11
+    vraddhn.u16 d25, q9,  q8
+    /*
+     * convert 8 r5g6b5 pixel data from {d4, d5} to planar 8-bit format
+     * and put data into d16 - blue, d17 - green, d18 - red
+     */
+       vshrn.u16   d17, q2,  #3
+       vshrn.u16   d18, q2,  #8
+    vraddhn.u16 d26, q13, q6
+       vsli.u16    q2,  q2,  #5
+       vsri.u8     d18, d18, #5
+       vsri.u8     d17, d17, #6
+    /*
+     * 'combine_over_ca' replacement
+     *
+     * output: updated dest in d16 - blue, d17 - green, d18 - red
+     */
+    vmvn.8      q12, q12
+       vshrn.u16   d16, q2,  #2
+    vmvn.8      d26, d26
+    vmull.u8    q6,  d16, d24
+    vmull.u8    q7,  d17, d25
+    vmull.u8    q11, d18, d26
+.endm
+
+.macro pixman_composite_over_n_8888_0565_ca_process_pixblock_tail
+    /* ... continue 'combine_over_ca' replacement */
+    vrshr.u16   q10, q6,  #8
+    vrshr.u16   q14, q7,  #8
+    vrshr.u16   q15, q11, #8
+    vraddhn.u16 d16, q10, q6
+    vraddhn.u16 d17, q14, q7
+    vraddhn.u16 d18, q15, q11
+    vqadd.u8    q8,  q0,  q8
+    vqadd.u8    d18, d2,  d18
+    /*
+     * convert the results in d16, d17, d18 to r5g6b5 and store
+     * them into {d28, d29}
+     */
+    vshll.u8    q14, d18, #8
+    vshll.u8    q10, d17, #8
+    vshll.u8    q15, d16, #8
+    vsri.u16    q14, q10, #5
+    vsri.u16    q14, q15, #11
+.endm
+
+.macro pixman_composite_over_n_8888_0565_ca_process_pixblock_tail_head
+    fetch_mask_pixblock
+        vrshr.u16   q10, q6, #8
+        vrshr.u16   q14, q7, #8
+    vld1.16     {d4, d5}, [DST_R, :128]!
+        vrshr.u16   q15, q11, #8
+        vraddhn.u16 d16, q10, q6
+        vraddhn.u16 d17, q14, q7
+        vraddhn.u16 d22, q15, q11
+            /* process_pixblock_head */
+            /*
+             * 'combine_mask_ca' replacement
+             *
+             * input:  solid src (n) in {d8,  d9,  d10, d11}  [B, G, R, A]
+             *         mask in          {d24, d25, d26}       [B, G, R]
+             * output: updated src in   {d0,  d1,  d2 }       [B, G, R]
+             *         updated mask in  {d24, d25, d26}       [B, G, R]
+             */
+            vmull.u8    q6,  d26, d10
+        vqadd.u8    q8,  q0, q8
+            vmull.u8    q0,  d24, d8
+        vqadd.u8    d22, d2, d22
+            vmull.u8    q1,  d25, d9
+        /*
+         * convert the result in d16, d17, d22 to r5g6b5 and store
+         * it into {d28, d29}
+         */
+        vshll.u8    q14, d22, #8
+        vshll.u8    q10, d17, #8
+        vshll.u8    q15, d16, #8
+            vmull.u8    q9,  d11, d25
+        vsri.u16    q14, q10, #5
+            vmull.u8    q12, d11, d24
+            vmull.u8    q13, d11, d26
+        vsri.u16    q14, q15, #11
+    cache_preload 8, 8
+            vrshr.u16   q8,  q0,  #8
+            vrshr.u16   q10, q1,  #8
+            vrshr.u16   q11, q6,  #8
+            vraddhn.u16 d0,  q0,  q8
+            vraddhn.u16 d1,  q1,  q10
+            vraddhn.u16 d2,  q6,  q11
+            vrshr.u16   q11, q12, #8
+            vrshr.u16   q8,  q9,  #8
+            vrshr.u16   q6,  q13, #8
+            vraddhn.u16 d24, q12, q11
+            vraddhn.u16 d25, q9,  q8
+                /*
+                 * convert 8 r5g6b5 pixel data from {d4, d5} to planar
+	         * 8-bit format and put data into d16 - blue, d17 - green,
+	         * d18 - red
+                 */
+                vshrn.u16   d17, q2,  #3
+                vshrn.u16   d18, q2,  #8
+            vraddhn.u16 d26, q13, q6
+                vsli.u16    q2,  q2,  #5
+                vsri.u8     d17, d17, #6
+                vsri.u8     d18, d18, #5
+            /*
+             * 'combine_over_ca' replacement
+             *
+             * output: updated dest in d16 - blue, d17 - green, d18 - red
+             */
+            vmvn.8      q12, q12
+                vshrn.u16   d16, q2,  #2
+            vmvn.8      d26, d26
+            vmull.u8    q7,  d17, d25
+            vmull.u8    q6,  d16, d24
+            vmull.u8    q11, d18, d26
+    vst1.16     {d28, d29}, [DST_W, :128]!
+.endm
+
+.macro pixman_composite_over_n_8888_0565_ca_init
+    add         DUMMY, sp, #ARGS_STACK_OFFSET
+    vpush       {d8-d15}
+    vld1.32     {d11[0]}, [DUMMY]
+    vdup.8      d8, d11[0]
+    vdup.8      d9, d11[1]
+    vdup.8      d10, d11[2]
+    vdup.8      d11, d11[3]
+.endm
+
+.macro pixman_composite_over_n_8888_0565_ca_cleanup
+    vpop        {d8-d15}
+.endm
+
+generate_composite_function \
+    pixman_composite_over_n_8888_0565_ca_asm_neon, 0, 32, 16, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_over_n_8888_0565_ca_init, \
+    pixman_composite_over_n_8888_0565_ca_cleanup, \
+    pixman_composite_over_n_8888_0565_ca_process_pixblock_head, \
+    pixman_composite_over_n_8888_0565_ca_process_pixblock_tail, \
+    pixman_composite_over_n_8888_0565_ca_process_pixblock_tail_head
+
+/******************************************************************************/
+
+.macro pixman_composite_in_n_8_process_pixblock_head
+    /* expecting source data in {d0, d1, d2, d3} */
+    /* and destination data in {d4, d5, d6, d7} */
+    vmull.u8    q8,  d4,  d3
+    vmull.u8    q9,  d5,  d3
+    vmull.u8    q10, d6,  d3
+    vmull.u8    q11, d7,  d3
+.endm
+
+.macro pixman_composite_in_n_8_process_pixblock_tail
+    vrshr.u16   q14, q8,  #8
+    vrshr.u16   q15, q9,  #8
+    vrshr.u16   q12, q10, #8
+    vrshr.u16   q13, q11, #8
+    vraddhn.u16 d28, q8,  q14
+    vraddhn.u16 d29, q9,  q15
+    vraddhn.u16 d30, q10, q12
+    vraddhn.u16 d31, q11, q13
+.endm
+
+.macro pixman_composite_in_n_8_process_pixblock_tail_head
+    pixman_composite_in_n_8_process_pixblock_tail
+    vld1.8      {d4, d5, d6, d7}, [DST_R, :128]!
+    cache_preload 32, 32
+    pixman_composite_in_n_8_process_pixblock_head
+    vst1.8      {d28, d29, d30, d31}, [DST_W, :128]!
+.endm
+
+.macro pixman_composite_in_n_8_init
+    add         DUMMY, sp, #ARGS_STACK_OFFSET
+    vld1.32     {d3[0]}, [DUMMY]
+    vdup.8      d3, d3[3]
+.endm
+
+.macro pixman_composite_in_n_8_cleanup
+.endm
+
+generate_composite_function \
+    pixman_composite_in_n_8_asm_neon, 0, 0, 8, \
+    FLAG_DST_READWRITE, \
+    32, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_in_n_8_init, \
+    pixman_composite_in_n_8_cleanup, \
+    pixman_composite_in_n_8_process_pixblock_head, \
+    pixman_composite_in_n_8_process_pixblock_tail, \
+    pixman_composite_in_n_8_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    4,  /* dst_r_basereg */ \
+    0,  /* src_basereg   */ \
+    24  /* mask_basereg  */
+
 .macro pixman_composite_add_n_8_8_process_pixblock_head
     /* expecting source data in {d8, d9, d10, d11} */
     /* d8 - blue, d9 - green, d10 - red, d11 - alpha */
@@ -1257,7 +1894,7 @@ generate_composite_function \
     pixman_composite_add_n_8_8_process_pixblock_tail
     vst1.8      {d28, d29, d30, d31}, [DST_W, :128]!
     vld1.8      {d4, d5, d6, d7}, [DST_R, :128]!
-    vld1.8      {d24, d25, d26, d27}, [MASK]!
+    fetch_mask_pixblock
     cache_preload 32, 32
     pixman_composite_add_n_8_8_process_pixblock_head
 .endm
@@ -1314,8 +1951,8 @@ generate_composite_function \
     pixman_composite_add_8_8_8_process_pixblock_tail
     vst1.8      {d28, d29, d30, d31}, [DST_W, :128]!
     vld1.8      {d4, d5, d6, d7}, [DST_R, :128]!
-    vld1.8      {d24, d25, d26, d27}, [MASK]!
-    vld1.8      {d0, d1, d2, d3}, [SRC]!
+    fetch_mask_pixblock
+    fetch_src_pixblock
     cache_preload 32, 32
     pixman_composite_add_8_8_8_process_pixblock_head
 .endm
@@ -1343,34 +1980,50 @@ generate_composite_function \
     /* expecting source data in {d0, d1, d2, d3} */
     /* destination data in {d4, d5, d6, d7} */
     /* mask in {d24, d25, d26, d27} */
-    vmull.u8    q8, d27, d0
-    vmull.u8    q9, d27, d1
+    vmull.u8    q8,  d27, d0
+    vmull.u8    q9,  d27, d1
     vmull.u8    q10, d27, d2
     vmull.u8    q11, d27, d3
-    vrshr.u16   q0, q8, #8
-    vrshr.u16   q1, q9, #8
-    vrshr.u16   q12, q10, #8
-    vrshr.u16   q13, q11, #8
-    vraddhn.u16 d0, q0, q8
-    vraddhn.u16 d1, q1, q9
-    vraddhn.u16 d2, q12, q10
-    vraddhn.u16 d3, q13, q11
-    vqadd.u8    q14, q0, q2
-    vqadd.u8    q15, q1, q3
+    /* 1 cycle bubble */
+    vrsra.u16   q8,  q8,  #8
+    vrsra.u16   q9,  q9,  #8
+    vrsra.u16   q10, q10, #8
+    vrsra.u16   q11, q11, #8
 .endm
 
 .macro pixman_composite_add_8888_8888_8888_process_pixblock_tail
+    /* 2 cycle bubble */
+    vrshrn.u16  d28, q8,  #8
+    vrshrn.u16  d29, q9,  #8
+    vrshrn.u16  d30, q10, #8
+    vrshrn.u16  d31, q11, #8
+    vqadd.u8    q14, q2,  q14
+    /* 1 cycle bubble */
+    vqadd.u8    q15, q3,  q15
 .endm
 
-/* TODO: expand macros and do better instructions scheduling */
 .macro pixman_composite_add_8888_8888_8888_process_pixblock_tail_head
-    pixman_composite_add_8888_8888_8888_process_pixblock_tail
-    vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
+    fetch_src_pixblock
+        vrshrn.u16  d28, q8,  #8
+    fetch_mask_pixblock
+        vrshrn.u16  d29, q9,  #8
+    vmull.u8    q8,  d27, d0
+        vrshrn.u16  d30, q10, #8
+    vmull.u8    q9,  d27, d1
+        vrshrn.u16  d31, q11, #8
+    vmull.u8    q10, d27, d2
+        vqadd.u8    q14, q2,  q14
+    vmull.u8    q11, d27, d3
+        vqadd.u8    q15, q3,  q15
+    vrsra.u16   q8,  q8,  #8
     vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!
-    vld4.8      {d24, d25, d26, d27}, [MASK]!
-    vld4.8      {d0, d1, d2, d3}, [SRC]!
+    vrsra.u16   q9,  q9,  #8
+        vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
+    vrsra.u16   q10, q10, #8
+
     cache_preload 8, 8
-    pixman_composite_add_8888_8888_8888_process_pixblock_head
+
+    vrsra.u16   q11, q11, #8
 .endm
 
 generate_composite_function \
@@ -1396,7 +2049,79 @@ generate_composite_function_single_scanline \
 
 /******************************************************************************/
 
-.macro pixman_composite_over_8888_n_8888_process_pixblock_head
+generate_composite_function \
+    pixman_composite_add_8888_8_8888_asm_neon, 32, 8, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_add_8888_8888_8888_process_pixblock_head, \
+    pixman_composite_add_8888_8888_8888_process_pixblock_tail, \
+    pixman_composite_add_8888_8888_8888_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    4,  /* dst_r_basereg */ \
+    0,  /* src_basereg   */ \
+    27  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_add_n_8_8888_init
+    add         DUMMY, sp, #ARGS_STACK_OFFSET
+    vld1.32     {d3[0]}, [DUMMY]
+    vdup.8      d0, d3[0]
+    vdup.8      d1, d3[1]
+    vdup.8      d2, d3[2]
+    vdup.8      d3, d3[3]
+.endm
+
+.macro pixman_composite_add_n_8_8888_cleanup
+.endm
+
+generate_composite_function \
+    pixman_composite_add_n_8_8888_asm_neon, 0, 8, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_add_n_8_8888_init, \
+    pixman_composite_add_n_8_8888_cleanup, \
+    pixman_composite_add_8888_8888_8888_process_pixblock_head, \
+    pixman_composite_add_8888_8888_8888_process_pixblock_tail, \
+    pixman_composite_add_8888_8888_8888_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    4,  /* dst_r_basereg */ \
+    0,  /* src_basereg   */ \
+    27  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_add_8888_n_8888_init
+    add         DUMMY, sp, #(ARGS_STACK_OFFSET + 8)
+    vld1.32     {d27[0]}, [DUMMY]
+    vdup.8      d27, d27[3]
+.endm
+
+.macro pixman_composite_add_8888_n_8888_cleanup
+.endm
+
+generate_composite_function \
+    pixman_composite_add_8888_n_8888_asm_neon, 32, 0, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_add_8888_n_8888_init, \
+    pixman_composite_add_8888_n_8888_cleanup, \
+    pixman_composite_add_8888_8888_8888_process_pixblock_head, \
+    pixman_composite_add_8888_8888_8888_process_pixblock_tail, \
+    pixman_composite_add_8888_8888_8888_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    4,  /* dst_r_basereg */ \
+    0,  /* src_basereg   */ \
+    27  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_out_reverse_8888_n_8888_process_pixblock_head
     /* expecting source data in {d0, d1, d2, d3} */
     /* destination data in {d4, d5, d6, d7} */
     /* solid mask is in d15 */
@@ -1422,7 +2147,7 @@ generate_composite_function_single_scanline \
     vmull.u8    q11, d24, d7
 .endm
 
-.macro pixman_composite_over_8888_n_8888_process_pixblock_tail
+.macro pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail
     vrshr.u16   q14, q8, #8
     vrshr.u16   q15, q9, #8
     vrshr.u16   q12, q10, #8
@@ -1431,6 +2156,41 @@ generate_composite_function_single_scanline \
     vraddhn.u16 d29, q15, q9
     vraddhn.u16 d30, q12, q10
     vraddhn.u16 d31, q13, q11
+.endm
+
+/* TODO: expand macros and do better instructions scheduling */
+.macro pixman_composite_out_reverse_8888_8888_8888_process_pixblock_tail_head
+    vld4.8     {d4, d5, d6, d7}, [DST_R, :128]!
+    pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail
+    fetch_src_pixblock
+    cache_preload 8, 8
+    fetch_mask_pixblock
+    pixman_composite_out_reverse_8888_n_8888_process_pixblock_head
+    vst4.8     {d28, d29, d30, d31}, [DST_W, :128]!
+.endm
+
+generate_composite_function_single_scanline \
+    pixman_composite_scanline_out_reverse_mask_asm_neon, 32, 32, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    default_init_need_all_regs, \
+    default_cleanup_need_all_regs, \
+    pixman_composite_out_reverse_8888_n_8888_process_pixblock_head, \
+    pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail, \
+    pixman_composite_out_reverse_8888_8888_8888_process_pixblock_tail_head \
+    28, /* dst_w_basereg */ \
+    4,  /* dst_r_basereg */ \
+    0,  /* src_basereg   */ \
+    12  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_over_8888_n_8888_process_pixblock_head
+    pixman_composite_out_reverse_8888_n_8888_process_pixblock_head
+.endm
+
+.macro pixman_composite_over_8888_n_8888_process_pixblock_tail
+    pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail
     vqadd.u8    q14, q0, q14
     vqadd.u8    q15, q1, q15
 .endm
@@ -1439,7 +2199,7 @@ generate_composite_function_single_scanline \
 .macro pixman_composite_over_8888_n_8888_process_pixblock_tail_head
     vld4.8     {d4, d5, d6, d7}, [DST_R, :128]!
     pixman_composite_over_8888_n_8888_process_pixblock_tail
-    vld4.8     {d0, d1, d2, d3}, [SRC]!
+    fetch_src_pixblock
     cache_preload 8, 8
     pixman_composite_over_8888_n_8888_process_pixblock_head
     vst4.8     {d28, d29, d30, d31}, [DST_W, :128]!
@@ -1473,28 +2233,20 @@ generate_composite_function \
 .macro pixman_composite_over_8888_8888_8888_process_pixblock_tail_head
     vld4.8     {d4, d5, d6, d7}, [DST_R, :128]!
     pixman_composite_over_8888_n_8888_process_pixblock_tail
-    vld4.8     {d0, d1, d2, d3}, [SRC]!
+    fetch_src_pixblock
     cache_preload 8, 8
-    vld4.8     {d12, d13, d14, d15}, [MASK]!
+    fetch_mask_pixblock
     pixman_composite_over_8888_n_8888_process_pixblock_head
     vst4.8     {d28, d29, d30, d31}, [DST_W, :128]!
 .endm
 
-.macro pixman_composite_over_8888_8888_8888_init
-    vpush       {d8-d15}
-.endm
-
-.macro pixman_composite_over_8888_8888_8888_cleanup
-    vpop        {d8-d15}
-.endm
-
 generate_composite_function \
     pixman_composite_over_8888_8888_8888_asm_neon, 32, 32, 32, \
     FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
     8, /* number of pixels, processed in a single block */ \
     5, /* prefetch distance */ \
-    pixman_composite_over_8888_8888_8888_init, \
-    pixman_composite_over_8888_8888_8888_cleanup, \
+    default_init_need_all_regs, \
+    default_cleanup_need_all_regs, \
     pixman_composite_over_8888_n_8888_process_pixblock_head, \
     pixman_composite_over_8888_n_8888_process_pixblock_tail, \
     pixman_composite_over_8888_8888_8888_process_pixblock_tail_head \
@@ -1507,8 +2259,8 @@ generate_composite_function_single_scanline \
     pixman_composite_scanline_over_mask_asm_neon, 32, 32, 32, \
     FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
     8, /* number of pixels, processed in a single block */ \
-    pixman_composite_over_8888_8888_8888_init, \
-    pixman_composite_over_8888_8888_8888_cleanup, \
+    default_init_need_all_regs, \
+    default_cleanup_need_all_regs, \
     pixman_composite_over_8888_n_8888_process_pixblock_head, \
     pixman_composite_over_8888_n_8888_process_pixblock_tail, \
     pixman_composite_over_8888_8888_8888_process_pixblock_tail_head \
@@ -1523,28 +2275,20 @@ generate_composite_function_single_scanline \
 .macro pixman_composite_over_8888_8_8888_process_pixblock_tail_head
     vld4.8     {d4, d5, d6, d7}, [DST_R, :128]!
     pixman_composite_over_8888_n_8888_process_pixblock_tail
-    vld4.8     {d0, d1, d2, d3}, [SRC]!
+    fetch_src_pixblock
     cache_preload 8, 8
-    vld1.8     {d15}, [MASK]!
+    fetch_mask_pixblock
     pixman_composite_over_8888_n_8888_process_pixblock_head
     vst4.8     {d28, d29, d30, d31}, [DST_W, :128]!
 .endm
 
-.macro pixman_composite_over_8888_8_8888_init
-    vpush       {d8-d15}
-.endm
-
-.macro pixman_composite_over_8888_8_8888_cleanup
-    vpop        {d8-d15}
-.endm
-
 generate_composite_function \
     pixman_composite_over_8888_8_8888_asm_neon, 32, 8, 32, \
     FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
     8, /* number of pixels, processed in a single block */ \
     5, /* prefetch distance */ \
-    pixman_composite_over_8888_8_8888_init, \
-    pixman_composite_over_8888_8_8888_cleanup, \
+    default_init_need_all_regs, \
+    default_cleanup_need_all_regs, \
     pixman_composite_over_8888_n_8888_process_pixblock_head, \
     pixman_composite_over_8888_n_8888_process_pixblock_tail, \
     pixman_composite_over_8888_8_8888_process_pixblock_tail_head \
@@ -1563,7 +2307,7 @@ generate_composite_function \
 
 .macro pixman_composite_src_0888_0888_process_pixblock_tail_head
     vst3.8 {d0, d1, d2}, [DST_W]!
-    vld3.8 {d0, d1, d2}, [SRC]!
+    fetch_src_pixblock
     cache_preload 8, 8
 .endm
 
@@ -1593,7 +2337,7 @@ generate_composite_function \
 
 .macro pixman_composite_src_0888_8888_rev_process_pixblock_tail_head
     vst4.8 {d0, d1, d2, d3}, [DST_W]!
-    vld3.8 {d0, d1, d2}, [SRC]!
+    fetch_src_pixblock
     vswp   d0, d2
     cache_preload 8, 8
 .endm
@@ -1632,7 +2376,7 @@ generate_composite_function \
 
 .macro pixman_composite_src_0888_0565_rev_process_pixblock_tail_head
         vshll.u8    q14, d0, #8
-    vld3.8 {d0, d1, d2}, [SRC]!
+    fetch_src_pixblock
         vsri.u16    q14, q8, #5
         vsri.u16    q14, q9, #11
     vshll.u8    q8, d1, #8
@@ -1678,7 +2422,7 @@ generate_composite_function \
         vswp        d3, d31
         vrshr.u16   q12, q9, #8
         vrshr.u16   q13, q10, #8
-    vld4.8 {d0, d1, d2, d3}, [SRC]!
+    fetch_src_pixblock
         vraddhn.u16 d30, q11, q8
                                     PF add PF_X, PF_X, #8
                                     PF tst PF_CTL, #0xF
@@ -1711,3 +2455,1183 @@ generate_composite_function \
     0, /* dst_r_basereg */ \
     0, /* src_basereg   */ \
     0  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_src_rpixbuf_8888_process_pixblock_head
+    vmull.u8    q8, d3, d0
+    vmull.u8    q9, d3, d1
+    vmull.u8    q10, d3, d2
+.endm
+
+.macro pixman_composite_src_rpixbuf_8888_process_pixblock_tail
+    vrshr.u16   q11, q8, #8
+    vswp        d3, d31
+    vrshr.u16   q12, q9, #8
+    vrshr.u16   q13, q10, #8
+    vraddhn.u16 d28, q11, q8
+    vraddhn.u16 d29, q12, q9
+    vraddhn.u16 d30, q13, q10
+.endm
+
+.macro pixman_composite_src_rpixbuf_8888_process_pixblock_tail_head
+        vrshr.u16   q11, q8, #8
+        vswp        d3, d31
+        vrshr.u16   q12, q9, #8
+        vrshr.u16   q13, q10, #8
+    fetch_src_pixblock
+        vraddhn.u16 d28, q11, q8
+                                    PF add PF_X, PF_X, #8
+                                    PF tst PF_CTL, #0xF
+                                    PF addne PF_X, PF_X, #8
+                                    PF subne PF_CTL, PF_CTL, #1
+        vraddhn.u16 d29, q12, q9
+        vraddhn.u16 d30, q13, q10
+    vmull.u8    q8, d3, d0
+    vmull.u8    q9, d3, d1
+    vmull.u8    q10, d3, d2
+        vst4.8 {d28, d29, d30, d31}, [DST_W, :128]!
+                                    PF cmp PF_X, ORIG_W
+                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]
+                                    PF subge PF_X, PF_X, ORIG_W
+                                    PF subges PF_CTL, PF_CTL, #0x10
+                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!
+.endm
+
+generate_composite_function \
+    pixman_composite_src_rpixbuf_8888_asm_neon, 32, 0, 32, \
+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    10, /* prefetch distance */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_src_rpixbuf_8888_process_pixblock_head, \
+    pixman_composite_src_rpixbuf_8888_process_pixblock_tail, \
+    pixman_composite_src_rpixbuf_8888_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    0, /* dst_r_basereg */ \
+    0, /* src_basereg   */ \
+    0  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_over_0565_8_0565_process_pixblock_head
+    /* mask is in d15 */
+    convert_0565_to_x888 q4, d2, d1, d0
+    convert_0565_to_x888 q5, d6, d5, d4
+    /* source pixel data is in      {d0, d1, d2, XX} */
+    /* destination pixel data is in {d4, d5, d6, XX} */
+    vmvn.8      d7,  d15
+    vmull.u8    q6,  d15, d2
+    vmull.u8    q5,  d15, d1
+    vmull.u8    q4,  d15, d0
+    vmull.u8    q8,  d7,  d4
+    vmull.u8    q9,  d7,  d5
+    vmull.u8    q13, d7,  d6
+    vrshr.u16   q12, q6,  #8
+    vrshr.u16   q11, q5,  #8
+    vrshr.u16   q10, q4,  #8
+    vraddhn.u16 d2,  q6,  q12
+    vraddhn.u16 d1,  q5,  q11
+    vraddhn.u16 d0,  q4,  q10
+.endm
+
+.macro pixman_composite_over_0565_8_0565_process_pixblock_tail
+    vrshr.u16   q14, q8,  #8
+    vrshr.u16   q15, q9,  #8
+    vrshr.u16   q12, q13, #8
+    vraddhn.u16 d28, q14, q8
+    vraddhn.u16 d29, q15, q9
+    vraddhn.u16 d30, q12, q13
+    vqadd.u8    q0,  q0,  q14
+    vqadd.u8    q1,  q1,  q15
+    /* 32bpp result is in {d0, d1, d2, XX} */
+    convert_8888_to_0565 d2, d1, d0, q14, q15, q3
+.endm
+
+/* TODO: expand macros and do better instructions scheduling */
+.macro pixman_composite_over_0565_8_0565_process_pixblock_tail_head
+    fetch_mask_pixblock
+    pixman_composite_over_0565_8_0565_process_pixblock_tail
+    fetch_src_pixblock
+    vld1.16    {d10, d11}, [DST_R, :128]!
+    cache_preload 8, 8
+    pixman_composite_over_0565_8_0565_process_pixblock_head
+    vst1.16    {d28, d29}, [DST_W, :128]!
+.endm
+
+generate_composite_function \
+    pixman_composite_over_0565_8_0565_asm_neon, 16, 8, 16, \
+    FLAG_DST_READWRITE, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    default_init_need_all_regs, \
+    default_cleanup_need_all_regs, \
+    pixman_composite_over_0565_8_0565_process_pixblock_head, \
+    pixman_composite_over_0565_8_0565_process_pixblock_tail, \
+    pixman_composite_over_0565_8_0565_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    10,  /* dst_r_basereg */ \
+    8,  /* src_basereg   */ \
+    15  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_over_0565_n_0565_init
+    add         DUMMY, sp, #(ARGS_STACK_OFFSET + 8)
+    vpush       {d8-d15}
+    vld1.32     {d15[0]}, [DUMMY]
+    vdup.8      d15, d15[3]
+.endm
+
+.macro pixman_composite_over_0565_n_0565_cleanup
+    vpop        {d8-d15}
+.endm
+
+generate_composite_function \
+    pixman_composite_over_0565_n_0565_asm_neon, 16, 0, 16, \
+    FLAG_DST_READWRITE, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_over_0565_n_0565_init, \
+    pixman_composite_over_0565_n_0565_cleanup, \
+    pixman_composite_over_0565_8_0565_process_pixblock_head, \
+    pixman_composite_over_0565_8_0565_process_pixblock_tail, \
+    pixman_composite_over_0565_8_0565_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    10, /* dst_r_basereg */ \
+    8,  /* src_basereg   */ \
+    15  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_add_0565_8_0565_process_pixblock_head
+    /* mask is in d15 */
+    convert_0565_to_x888 q4, d2, d1, d0
+    convert_0565_to_x888 q5, d6, d5, d4
+    /* source pixel data is in      {d0, d1, d2, XX} */
+    /* destination pixel data is in {d4, d5, d6, XX} */
+    vmull.u8    q6,  d15, d2
+    vmull.u8    q5,  d15, d1
+    vmull.u8    q4,  d15, d0
+    vrshr.u16   q12, q6,  #8
+    vrshr.u16   q11, q5,  #8
+    vrshr.u16   q10, q4,  #8
+    vraddhn.u16 d2,  q6,  q12
+    vraddhn.u16 d1,  q5,  q11
+    vraddhn.u16 d0,  q4,  q10
+.endm
+
+.macro pixman_composite_add_0565_8_0565_process_pixblock_tail
+    vqadd.u8    q0,  q0,  q2
+    vqadd.u8    q1,  q1,  q3
+    /* 32bpp result is in {d0, d1, d2, XX} */
+    convert_8888_to_0565 d2, d1, d0, q14, q15, q3
+.endm
+
+/* TODO: expand macros and do better instructions scheduling */
+.macro pixman_composite_add_0565_8_0565_process_pixblock_tail_head
+    fetch_mask_pixblock
+    pixman_composite_add_0565_8_0565_process_pixblock_tail
+    fetch_src_pixblock
+    vld1.16    {d10, d11}, [DST_R, :128]!
+    cache_preload 8, 8
+    pixman_composite_add_0565_8_0565_process_pixblock_head
+    vst1.16    {d28, d29}, [DST_W, :128]!
+.endm
+
+generate_composite_function \
+    pixman_composite_add_0565_8_0565_asm_neon, 16, 8, 16, \
+    FLAG_DST_READWRITE, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    default_init_need_all_regs, \
+    default_cleanup_need_all_regs, \
+    pixman_composite_add_0565_8_0565_process_pixblock_head, \
+    pixman_composite_add_0565_8_0565_process_pixblock_tail, \
+    pixman_composite_add_0565_8_0565_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    10, /* dst_r_basereg */ \
+    8,  /* src_basereg   */ \
+    15  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_out_reverse_8_0565_process_pixblock_head
+    /* mask is in d15 */
+    convert_0565_to_x888 q5, d6, d5, d4
+    /* destination pixel data is in {d4, d5, d6, xx} */
+    vmvn.8      d24, d15 /* get inverted alpha */
+    /* now do alpha blending */
+    vmull.u8    q8, d24, d4
+    vmull.u8    q9, d24, d5
+    vmull.u8    q10, d24, d6
+.endm
+
+.macro pixman_composite_out_reverse_8_0565_process_pixblock_tail
+    vrshr.u16   q14, q8, #8
+    vrshr.u16   q15, q9, #8
+    vrshr.u16   q12, q10, #8
+    vraddhn.u16 d0, q14, q8
+    vraddhn.u16 d1, q15, q9
+    vraddhn.u16 d2, q12, q10
+    /* 32bpp result is in {d0, d1, d2, XX} */
+    convert_8888_to_0565 d2, d1, d0, q14, q15, q3
+.endm
+
+/* TODO: expand macros and do better instructions scheduling */
+.macro pixman_composite_out_reverse_8_0565_process_pixblock_tail_head
+    fetch_src_pixblock
+    pixman_composite_out_reverse_8_0565_process_pixblock_tail
+    vld1.16    {d10, d11}, [DST_R, :128]!
+    cache_preload 8, 8
+    pixman_composite_out_reverse_8_0565_process_pixblock_head
+    vst1.16    {d28, d29}, [DST_W, :128]!
+.endm
+
+generate_composite_function \
+    pixman_composite_out_reverse_8_0565_asm_neon, 8, 0, 16, \
+    FLAG_DST_READWRITE, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    default_init_need_all_regs, \
+    default_cleanup_need_all_regs, \
+    pixman_composite_out_reverse_8_0565_process_pixblock_head, \
+    pixman_composite_out_reverse_8_0565_process_pixblock_tail, \
+    pixman_composite_out_reverse_8_0565_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    10, /* dst_r_basereg */ \
+    15, /* src_basereg   */ \
+    0   /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_out_reverse_8_8888_process_pixblock_head
+    /* src is in d0 */
+    /* destination pixel data is in {d4, d5, d6, d7} */
+    vmvn.8      d1, d0 /* get inverted alpha */
+    /* now do alpha blending */
+    vmull.u8    q8, d1, d4
+    vmull.u8    q9, d1, d5
+    vmull.u8    q10, d1, d6
+    vmull.u8    q11, d1, d7
+.endm
+
+.macro pixman_composite_out_reverse_8_8888_process_pixblock_tail
+    vrshr.u16   q14, q8, #8
+    vrshr.u16   q15, q9, #8
+    vrshr.u16   q12, q10, #8
+    vrshr.u16   q13, q11, #8
+    vraddhn.u16 d28, q14, q8
+    vraddhn.u16 d29, q15, q9
+    vraddhn.u16 d30, q12, q10
+    vraddhn.u16 d31, q13, q11
+    /* 32bpp result is in {d28, d29, d30, d31} */
+.endm
+
+/* TODO: expand macros and do better instructions scheduling */
+.macro pixman_composite_out_reverse_8_8888_process_pixblock_tail_head
+    fetch_src_pixblock
+    pixman_composite_out_reverse_8_8888_process_pixblock_tail
+    vld4.8    {d4, d5, d6, d7}, [DST_R, :128]!
+    cache_preload 8, 8
+    pixman_composite_out_reverse_8_8888_process_pixblock_head
+    vst4.8    {d28, d29, d30, d31}, [DST_W, :128]!
+.endm
+
+generate_composite_function \
+    pixman_composite_out_reverse_8_8888_asm_neon, 8, 0, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_out_reverse_8_8888_process_pixblock_head, \
+    pixman_composite_out_reverse_8_8888_process_pixblock_tail, \
+    pixman_composite_out_reverse_8_8888_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    4, /* dst_r_basereg */ \
+    0, /* src_basereg   */ \
+    0   /* mask_basereg  */
+
+/******************************************************************************/
+
+generate_composite_function_nearest_scanline \
+    pixman_scaled_nearest_scanline_8888_8888_OVER_asm_neon, 32, 0, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_over_8888_8888_process_pixblock_head, \
+    pixman_composite_over_8888_8888_process_pixblock_tail, \
+    pixman_composite_over_8888_8888_process_pixblock_tail_head
+
+generate_composite_function_nearest_scanline \
+    pixman_scaled_nearest_scanline_8888_0565_OVER_asm_neon, 32, 0, 16, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_over_8888_0565_process_pixblock_head, \
+    pixman_composite_over_8888_0565_process_pixblock_tail, \
+    pixman_composite_over_8888_0565_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    4,  /* dst_r_basereg */ \
+    0,  /* src_basereg   */ \
+    24  /* mask_basereg  */
+
+generate_composite_function_nearest_scanline \
+    pixman_scaled_nearest_scanline_8888_0565_SRC_asm_neon, 32, 0, 16, \
+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_src_8888_0565_process_pixblock_head, \
+    pixman_composite_src_8888_0565_process_pixblock_tail, \
+    pixman_composite_src_8888_0565_process_pixblock_tail_head
+
+generate_composite_function_nearest_scanline \
+    pixman_scaled_nearest_scanline_0565_8888_SRC_asm_neon, 16, 0, 32, \
+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_src_0565_8888_process_pixblock_head, \
+    pixman_composite_src_0565_8888_process_pixblock_tail, \
+    pixman_composite_src_0565_8888_process_pixblock_tail_head
+
+generate_composite_function_nearest_scanline \
+    pixman_scaled_nearest_scanline_8888_8_0565_OVER_asm_neon, 32, 8, 16, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    default_init_need_all_regs, \
+    default_cleanup_need_all_regs, \
+    pixman_composite_over_8888_8_0565_process_pixblock_head, \
+    pixman_composite_over_8888_8_0565_process_pixblock_tail, \
+    pixman_composite_over_8888_8_0565_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    4,  /* dst_r_basereg */ \
+    8,  /* src_basereg   */ \
+    24  /* mask_basereg  */
+
+generate_composite_function_nearest_scanline \
+    pixman_scaled_nearest_scanline_0565_8_0565_OVER_asm_neon, 16, 8, 16, \
+    FLAG_DST_READWRITE, \
+    8, /* number of pixels, processed in a single block */ \
+    default_init_need_all_regs, \
+    default_cleanup_need_all_regs, \
+    pixman_composite_over_0565_8_0565_process_pixblock_head, \
+    pixman_composite_over_0565_8_0565_process_pixblock_tail, \
+    pixman_composite_over_0565_8_0565_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    10,  /* dst_r_basereg */ \
+    8,  /* src_basereg   */ \
+    15  /* mask_basereg  */
+
+/******************************************************************************/
+
+/* Supplementary macro for setting function attributes */
+.macro pixman_asm_function fname
+    .func fname
+    .global fname
+#ifdef __ELF__
+    .hidden fname
+    .type fname, %function
+#endif
+fname:
+.endm
+
+/*
+ * Bilinear scaling support code which tries to provide pixel fetching, color
+ * format conversion, and interpolation as separate macros which can be used
+ * as the basic building blocks for constructing bilinear scanline functions.
+ */
+
+.macro bilinear_load_8888 reg1, reg2, tmp
+    mov       TMP1, X, asr #16
+    add       X, X, UX
+    add       TMP1, TOP, TMP1, asl #2
+    vld1.32   {reg1}, [TMP1], STRIDE
+    vld1.32   {reg2}, [TMP1]
+.endm
+
+.macro bilinear_load_0565 reg1, reg2, tmp
+    mov       TMP1, X, asr #16
+    add       X, X, UX
+    add       TMP1, TOP, TMP1, asl #1
+    vld1.32   {reg2[0]}, [TMP1], STRIDE
+    vld1.32   {reg2[1]}, [TMP1]
+    convert_four_0565_to_x888_packed reg2, reg1, reg2, tmp
+.endm
+
+.macro bilinear_load_and_vertical_interpolate_two_8888 \
+                    acc1, acc2, reg1, reg2, reg3, reg4, tmp1, tmp2
+
+    bilinear_load_8888 reg1, reg2, tmp1
+    vmull.u8  acc1, reg1, d28
+    vmlal.u8  acc1, reg2, d29
+    bilinear_load_8888 reg3, reg4, tmp2
+    vmull.u8  acc2, reg3, d28
+    vmlal.u8  acc2, reg4, d29
+.endm
+
+.macro bilinear_load_and_vertical_interpolate_four_8888 \
+                xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi \
+                yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi
+
+    bilinear_load_and_vertical_interpolate_two_8888 \
+                xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi
+    bilinear_load_and_vertical_interpolate_two_8888 \
+                yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi
+.endm
+
+.macro bilinear_load_and_vertical_interpolate_two_0565 \
+                acc1, acc2, reg1, reg2, reg3, reg4, acc2lo, acc2hi
+
+    mov       TMP1, X, asr #16
+    add       X, X, UX
+    add       TMP1, TOP, TMP1, asl #1
+    mov       TMP2, X, asr #16
+    add       X, X, UX
+    add       TMP2, TOP, TMP2, asl #1
+    vld1.32   {acc2lo[0]}, [TMP1], STRIDE
+    vld1.32   {acc2hi[0]}, [TMP2], STRIDE
+    vld1.32   {acc2lo[1]}, [TMP1]
+    vld1.32   {acc2hi[1]}, [TMP2]
+    convert_0565_to_x888 acc2, reg3, reg2, reg1
+    vzip.u8   reg1, reg3
+    vzip.u8   reg2, reg4
+    vzip.u8   reg3, reg4
+    vzip.u8   reg1, reg2
+    vmull.u8  acc1, reg1, d28
+    vmlal.u8  acc1, reg2, d29
+    vmull.u8  acc2, reg3, d28
+    vmlal.u8  acc2, reg4, d29
+.endm
+
+.macro bilinear_load_and_vertical_interpolate_four_0565 \
+                xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi \
+                yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi
+
+    mov       TMP1, X, asr #16
+    add       X, X, UX
+    add       TMP1, TOP, TMP1, asl #1
+    mov       TMP2, X, asr #16
+    add       X, X, UX
+    add       TMP2, TOP, TMP2, asl #1
+    vld1.32   {xacc2lo[0]}, [TMP1], STRIDE
+    vld1.32   {xacc2hi[0]}, [TMP2], STRIDE
+    vld1.32   {xacc2lo[1]}, [TMP1]
+    vld1.32   {xacc2hi[1]}, [TMP2]
+    convert_0565_to_x888 xacc2, xreg3, xreg2, xreg1
+    mov       TMP1, X, asr #16
+    add       X, X, UX
+    add       TMP1, TOP, TMP1, asl #1
+    mov       TMP2, X, asr #16
+    add       X, X, UX
+    add       TMP2, TOP, TMP2, asl #1
+    vld1.32   {yacc2lo[0]}, [TMP1], STRIDE
+    vzip.u8   xreg1, xreg3
+    vld1.32   {yacc2hi[0]}, [TMP2], STRIDE
+    vzip.u8   xreg2, xreg4
+    vld1.32   {yacc2lo[1]}, [TMP1]
+    vzip.u8   xreg3, xreg4
+    vld1.32   {yacc2hi[1]}, [TMP2]
+    vzip.u8   xreg1, xreg2
+    convert_0565_to_x888 yacc2, yreg3, yreg2, yreg1
+    vmull.u8  xacc1, xreg1, d28
+    vzip.u8   yreg1, yreg3
+    vmlal.u8  xacc1, xreg2, d29
+    vzip.u8   yreg2, yreg4
+    vmull.u8  xacc2, xreg3, d28
+    vzip.u8   yreg3, yreg4
+    vmlal.u8  xacc2, xreg4, d29
+    vzip.u8   yreg1, yreg2
+    vmull.u8  yacc1, yreg1, d28
+    vmlal.u8  yacc1, yreg2, d29
+    vmull.u8  yacc2, yreg3, d28
+    vmlal.u8  yacc2, yreg4, d29
+.endm
+
+.macro bilinear_store_8888 numpix, tmp1, tmp2
+.if numpix == 4
+    vst1.32   {d0, d1}, [OUT, :128]!
+.elseif numpix == 2
+    vst1.32   {d0}, [OUT, :64]!
+.elseif numpix == 1
+    vst1.32   {d0[0]}, [OUT, :32]!
+.else
+    .error bilinear_store_8888 numpix is unsupported
+.endif
+.endm
+
+.macro bilinear_store_0565 numpix, tmp1, tmp2
+    vuzp.u8 d0, d1
+    vuzp.u8 d2, d3
+    vuzp.u8 d1, d3
+    vuzp.u8 d0, d2
+    convert_8888_to_0565 d2, d1, d0, q1, tmp1, tmp2
+.if numpix == 4
+    vst1.16   {d2}, [OUT, :64]!
+.elseif numpix == 2
+    vst1.32   {d2[0]}, [OUT, :32]!
+.elseif numpix == 1
+    vst1.16   {d2[0]}, [OUT, :16]!
+.else
+    .error bilinear_store_0565 numpix is unsupported
+.endif
+.endm
+
+.macro bilinear_interpolate_last_pixel src_fmt, dst_fmt
+    bilinear_load_&src_fmt d0, d1, d2
+    vmull.u8  q1, d0, d28
+    vmlal.u8  q1, d1, d29
+    /* 5 cycles bubble */
+    vshll.u16 q0, d2, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q0, d2, d30
+    vmlal.u16 q0, d3, d30
+    /* 5 cycles bubble */
+    vshrn.u32 d0, q0, #(2 * BILINEAR_INTERPOLATION_BITS)
+    /* 3 cycles bubble */
+    vmovn.u16 d0, q0
+    /* 1 cycle bubble */
+    bilinear_store_&dst_fmt 1, q2, q3
+.endm
+
+.macro bilinear_interpolate_two_pixels src_fmt, dst_fmt
+    bilinear_load_and_vertical_interpolate_two_&src_fmt \
+                q1, q11, d0, d1, d20, d21, d22, d23
+    vshll.u16 q0, d2, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q0, d2, d30
+    vmlal.u16 q0, d3, d30
+    vshll.u16 q10, d22, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q10, d22, d31
+    vmlal.u16 q10, d23, d31
+    vshrn.u32 d0, q0, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32 d1, q10, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vadd.u16  q12, q12, q13
+    vmovn.u16 d0, q0
+    bilinear_store_&dst_fmt 2, q2, q3
+.endm
+
+.macro bilinear_interpolate_four_pixels src_fmt, dst_fmt
+    bilinear_load_and_vertical_interpolate_four_&src_fmt \
+                q1, q11, d0, d1, d20, d21, d22, d23 \
+                q3, q9,  d4, d5, d16, d17, d18, d19
+    pld       [TMP1, PF_OFFS]
+    sub       TMP1, TMP1, STRIDE
+    vshll.u16 q0, d2, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q0, d2, d30
+    vmlal.u16 q0, d3, d30
+    vshll.u16 q10, d22, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q10, d22, d31
+    vmlal.u16 q10, d23, d31
+    vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vshll.u16 q2, d6, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q2, d6, d30
+    vmlal.u16 q2, d7, d30
+    vshll.u16 q8, d18, #BILINEAR_INTERPOLATION_BITS
+    pld       [TMP2, PF_OFFS]
+    vmlsl.u16 q8, d18, d31
+    vmlal.u16 q8, d19, d31
+    vadd.u16  q12, q12, q13
+    vshrn.u32 d0, q0, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32 d1, q10, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32 d4, q2, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32 d5, q8, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vmovn.u16 d0, q0
+    vmovn.u16 d1, q2
+    vadd.u16  q12, q12, q13
+    bilinear_store_&dst_fmt 4, q2, q3
+.endm
+
+.macro bilinear_interpolate_four_pixels_head src_fmt, dst_fmt
+.ifdef have_bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt
+    bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt&_head
+.else
+    bilinear_interpolate_four_pixels src_fmt, dst_fmt
+.endif
+.endm
+
+.macro bilinear_interpolate_four_pixels_tail src_fmt, dst_fmt
+.ifdef have_bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt
+    bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt&_tail
+.endif
+.endm
+
+.macro bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt
+.ifdef have_bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt
+    bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt&_tail_head
+.else
+    bilinear_interpolate_four_pixels src_fmt, dst_fmt
+.endif
+.endm
+
+.macro bilinear_interpolate_eight_pixels_head src_fmt, dst_fmt
+.ifdef have_bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt
+    bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt&_head
+.else
+    bilinear_interpolate_four_pixels_head src_fmt, dst_fmt
+    bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt
+.endif
+.endm
+
+.macro bilinear_interpolate_eight_pixels_tail src_fmt, dst_fmt
+.ifdef have_bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt
+    bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt&_tail
+.else
+    bilinear_interpolate_four_pixels_tail src_fmt, dst_fmt
+.endif
+.endm
+
+.macro bilinear_interpolate_eight_pixels_tail_head src_fmt, dst_fmt
+.ifdef have_bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt
+    bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt&_tail_head
+.else
+    bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt
+    bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt
+.endif
+.endm
+
+.set BILINEAR_FLAG_UNROLL_4,          0
+.set BILINEAR_FLAG_UNROLL_8,          1
+.set BILINEAR_FLAG_USE_ALL_NEON_REGS, 2
+
+/*
+ * Main template macro for generating NEON optimized bilinear scanline
+ * functions.
+ *
+ * Bilinear scanline scaler macro template uses the following arguments:
+ *  fname             - name of the function to generate
+ *  src_fmt           - source color format (8888 or 0565)
+ *  dst_fmt           - destination color format (8888 or 0565)
+ *  bpp_shift         - (1 << bpp_shift) is the size of source pixel in bytes
+ *  prefetch_distance - prefetch in the source image by that many
+ *                      pixels ahead
+ */
+
+.macro generate_bilinear_scanline_func fname, src_fmt, dst_fmt, \
+                                       src_bpp_shift, dst_bpp_shift, \
+                                       prefetch_distance, flags
+
+pixman_asm_function fname
+    OUT       .req      r0
+    TOP       .req      r1
+    BOTTOM    .req      r2
+    WT        .req      r3
+    WB        .req      r4
+    X         .req      r5
+    UX        .req      r6
+    WIDTH     .req      ip
+    TMP1      .req      r3
+    TMP2      .req      r4
+    PF_OFFS   .req      r7
+    TMP3      .req      r8
+    TMP4      .req      r9
+    STRIDE    .req      r2
+
+    mov       ip, sp
+    push      {r4, r5, r6, r7, r8, r9}
+    mov       PF_OFFS, #prefetch_distance
+    ldmia     ip, {WB, X, UX, WIDTH}
+    mul       PF_OFFS, PF_OFFS, UX
+
+.if ((flags) & BILINEAR_FLAG_USE_ALL_NEON_REGS) != 0
+    vpush     {d8-d15}
+.endif
+
+    sub       STRIDE, BOTTOM, TOP
+    .unreq    BOTTOM
+
+    cmp       WIDTH, #0
+    ble       3f
+
+    vdup.u16  q12, X
+    vdup.u16  q13, UX
+    vdup.u8   d28, WT
+    vdup.u8   d29, WB
+    vadd.u16  d25, d25, d26
+
+    /* ensure good destination alignment  */
+    cmp       WIDTH, #1
+    blt       0f
+    tst       OUT, #(1 << dst_bpp_shift)
+    beq       0f
+    vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vadd.u16  q12, q12, q13
+    bilinear_interpolate_last_pixel src_fmt, dst_fmt
+    sub       WIDTH, WIDTH, #1
+0:
+    vadd.u16  q13, q13, q13
+    vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vadd.u16  q12, q12, q13
+
+    cmp       WIDTH, #2
+    blt       0f
+    tst       OUT, #(1 << (dst_bpp_shift + 1))
+    beq       0f
+    bilinear_interpolate_two_pixels src_fmt, dst_fmt
+    sub       WIDTH, WIDTH, #2
+0:
+.if ((flags) & BILINEAR_FLAG_UNROLL_8) != 0
+/*********** 8 pixels per iteration *****************/
+    cmp       WIDTH, #4
+    blt       0f
+    tst       OUT, #(1 << (dst_bpp_shift + 2))
+    beq       0f
+    bilinear_interpolate_four_pixels src_fmt, dst_fmt
+    sub       WIDTH, WIDTH, #4
+0:
+    subs      WIDTH, WIDTH, #8
+    blt       1f
+    mov       PF_OFFS, PF_OFFS, asr #(16 - src_bpp_shift)
+    bilinear_interpolate_eight_pixels_head src_fmt, dst_fmt
+    subs      WIDTH, WIDTH, #8
+    blt       5f
+0:
+    bilinear_interpolate_eight_pixels_tail_head src_fmt, dst_fmt
+    subs      WIDTH, WIDTH, #8
+    bge       0b
+5:
+    bilinear_interpolate_eight_pixels_tail src_fmt, dst_fmt
+1:
+    tst       WIDTH, #4
+    beq       2f
+    bilinear_interpolate_four_pixels src_fmt, dst_fmt
+2:
+.else
+/*********** 4 pixels per iteration *****************/
+    subs      WIDTH, WIDTH, #4
+    blt       1f
+    mov       PF_OFFS, PF_OFFS, asr #(16 - src_bpp_shift)
+    bilinear_interpolate_four_pixels_head src_fmt, dst_fmt
+    subs      WIDTH, WIDTH, #4
+    blt       5f
+0:
+    bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt
+    subs      WIDTH, WIDTH, #4
+    bge       0b
+5:
+    bilinear_interpolate_four_pixels_tail src_fmt, dst_fmt
+1:
+/****************************************************/
+.endif
+    /* handle the remaining trailing pixels */
+    tst       WIDTH, #2
+    beq       2f
+    bilinear_interpolate_two_pixels src_fmt, dst_fmt
+2:
+    tst       WIDTH, #1
+    beq       3f
+    bilinear_interpolate_last_pixel src_fmt, dst_fmt
+3:
+.if ((flags) & BILINEAR_FLAG_USE_ALL_NEON_REGS) != 0
+    vpop      {d8-d15}
+.endif
+    pop       {r4, r5, r6, r7, r8, r9}
+    bx        lr
+
+    .unreq    OUT
+    .unreq    TOP
+    .unreq    WT
+    .unreq    WB
+    .unreq    X
+    .unreq    UX
+    .unreq    WIDTH
+    .unreq    TMP1
+    .unreq    TMP2
+    .unreq    PF_OFFS
+    .unreq    TMP3
+    .unreq    TMP4
+    .unreq    STRIDE
+.endfunc
+
+.endm
+
+/*****************************************************************************/
+
+.set have_bilinear_interpolate_four_pixels_8888_8888, 1
+
+.macro bilinear_interpolate_four_pixels_8888_8888_head
+    mov       TMP1, X, asr #16
+    add       X, X, UX
+    add       TMP1, TOP, TMP1, asl #2
+    mov       TMP2, X, asr #16
+    add       X, X, UX
+    add       TMP2, TOP, TMP2, asl #2
+
+    vld1.32   {d22}, [TMP1], STRIDE
+    vld1.32   {d23}, [TMP1]
+    mov       TMP3, X, asr #16
+    add       X, X, UX
+    add       TMP3, TOP, TMP3, asl #2
+    vmull.u8  q8, d22, d28
+    vmlal.u8  q8, d23, d29
+
+    vld1.32   {d22}, [TMP2], STRIDE
+    vld1.32   {d23}, [TMP2]
+    mov       TMP4, X, asr #16
+    add       X, X, UX
+    add       TMP4, TOP, TMP4, asl #2
+    vmull.u8  q9, d22, d28
+    vmlal.u8  q9, d23, d29
+
+    vld1.32   {d22}, [TMP3], STRIDE
+    vld1.32   {d23}, [TMP3]
+    vmull.u8  q10, d22, d28
+    vmlal.u8  q10, d23, d29
+
+    vshll.u16 q0, d16, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q0, d16, d30
+    vmlal.u16 q0, d17, d30
+
+    pld       [TMP4, PF_OFFS]
+    vld1.32   {d16}, [TMP4], STRIDE
+    vld1.32   {d17}, [TMP4]
+    pld       [TMP4, PF_OFFS]
+    vmull.u8  q11, d16, d28
+    vmlal.u8  q11, d17, d29
+
+    vshll.u16 q1, d18, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q1, d18, d31
+.endm
+
+.macro bilinear_interpolate_four_pixels_8888_8888_tail
+    vmlal.u16 q1, d19, d31
+    vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vshll.u16 q2, d20, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q2, d20, d30
+    vmlal.u16 q2, d21, d30
+    vshll.u16 q3, d22, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q3, d22, d31
+    vmlal.u16 q3, d23, d31
+    vadd.u16  q12, q12, q13
+    vshrn.u32 d0, q0, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32 d1, q1, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32 d4, q2, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32 d5, q3, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vmovn.u16 d6, q0
+    vmovn.u16 d7, q2
+    vadd.u16  q12, q12, q13
+    vst1.32   {d6, d7}, [OUT, :128]!
+.endm
+
+.macro bilinear_interpolate_four_pixels_8888_8888_tail_head
+    mov       TMP1, X, asr #16
+    add       X, X, UX
+    add       TMP1, TOP, TMP1, asl #2
+    mov       TMP2, X, asr #16
+    add       X, X, UX
+    add       TMP2, TOP, TMP2, asl #2
+        vmlal.u16 q1, d19, d31
+        vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+        vshll.u16 q2, d20, #BILINEAR_INTERPOLATION_BITS
+        vmlsl.u16 q2, d20, d30
+        vmlal.u16 q2, d21, d30
+        vshll.u16 q3, d22, #BILINEAR_INTERPOLATION_BITS
+    vld1.32   {d20}, [TMP1], STRIDE
+        vmlsl.u16 q3, d22, d31
+        vmlal.u16 q3, d23, d31
+    vld1.32   {d21}, [TMP1]
+    vmull.u8  q8, d20, d28
+    vmlal.u8  q8, d21, d29
+        vshrn.u32 d0, q0, #(2 * BILINEAR_INTERPOLATION_BITS)
+        vshrn.u32 d1, q1, #(2 * BILINEAR_INTERPOLATION_BITS)
+        vshrn.u32 d4, q2, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vld1.32   {d22}, [TMP2], STRIDE
+        vshrn.u32 d5, q3, #(2 * BILINEAR_INTERPOLATION_BITS)
+        vadd.u16  q12, q12, q13
+    vld1.32   {d23}, [TMP2]
+    vmull.u8  q9, d22, d28
+    mov       TMP3, X, asr #16
+    add       X, X, UX
+    add       TMP3, TOP, TMP3, asl #2
+    mov       TMP4, X, asr #16
+    add       X, X, UX
+    add       TMP4, TOP, TMP4, asl #2
+    vmlal.u8  q9, d23, d29
+    vld1.32   {d22}, [TMP3], STRIDE
+        vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vld1.32   {d23}, [TMP3]
+    vmull.u8  q10, d22, d28
+    vmlal.u8  q10, d23, d29
+        vmovn.u16 d6, q0
+    vshll.u16 q0, d16, #BILINEAR_INTERPOLATION_BITS
+        vmovn.u16 d7, q2
+    vmlsl.u16 q0, d16, d30
+    vmlal.u16 q0, d17, d30
+    pld       [TMP4, PF_OFFS]
+    vld1.32   {d16}, [TMP4], STRIDE
+        vadd.u16  q12, q12, q13
+    vld1.32   {d17}, [TMP4]
+    pld       [TMP4, PF_OFFS]
+    vmull.u8  q11, d16, d28
+    vmlal.u8  q11, d17, d29
+        vst1.32   {d6, d7}, [OUT, :128]!
+    vshll.u16 q1, d18, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q1, d18, d31
+.endm
+
+/*****************************************************************************/
+
+.set have_bilinear_interpolate_eight_pixels_8888_0565, 1
+
+.macro bilinear_interpolate_eight_pixels_8888_0565_head
+    mov       TMP1, X, asr #16
+    add       X, X, UX
+    add       TMP1, TOP, TMP1, asl #2
+    mov       TMP2, X, asr #16
+    add       X, X, UX
+    add       TMP2, TOP, TMP2, asl #2
+    vld1.32   {d20}, [TMP1], STRIDE
+    vld1.32   {d21}, [TMP1]
+    vmull.u8  q8, d20, d28
+    vmlal.u8  q8, d21, d29
+    vld1.32   {d22}, [TMP2], STRIDE
+    vld1.32   {d23}, [TMP2]
+    vmull.u8  q9, d22, d28
+    mov       TMP3, X, asr #16
+    add       X, X, UX
+    add       TMP3, TOP, TMP3, asl #2
+    mov       TMP4, X, asr #16
+    add       X, X, UX
+    add       TMP4, TOP, TMP4, asl #2
+    vmlal.u8  q9, d23, d29
+    vld1.32   {d22}, [TMP3], STRIDE
+    vld1.32   {d23}, [TMP3]
+    vmull.u8  q10, d22, d28
+    vmlal.u8  q10, d23, d29
+    vshll.u16 q0, d16, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q0, d16, d30
+    vmlal.u16 q0, d17, d30
+    pld       [TMP4, PF_OFFS]
+    vld1.32   {d16}, [TMP4], STRIDE
+    vld1.32   {d17}, [TMP4]
+    pld       [TMP4, PF_OFFS]
+    vmull.u8  q11, d16, d28
+    vmlal.u8  q11, d17, d29
+    vshll.u16 q1, d18, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q1, d18, d31
+
+    mov       TMP1, X, asr #16
+    add       X, X, UX
+    add       TMP1, TOP, TMP1, asl #2
+    mov       TMP2, X, asr #16
+    add       X, X, UX
+    add       TMP2, TOP, TMP2, asl #2
+        vmlal.u16 q1, d19, d31
+        vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+        vshll.u16 q2, d20, #BILINEAR_INTERPOLATION_BITS
+        vmlsl.u16 q2, d20, d30
+        vmlal.u16 q2, d21, d30
+        vshll.u16 q3, d22, #BILINEAR_INTERPOLATION_BITS
+    vld1.32   {d20}, [TMP1], STRIDE
+        vmlsl.u16 q3, d22, d31
+        vmlal.u16 q3, d23, d31
+    vld1.32   {d21}, [TMP1]
+    vmull.u8  q8, d20, d28
+    vmlal.u8  q8, d21, d29
+        vshrn.u32 d0, q0, #(2 * BILINEAR_INTERPOLATION_BITS)
+        vshrn.u32 d1, q1, #(2 * BILINEAR_INTERPOLATION_BITS)
+        vshrn.u32 d4, q2, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vld1.32   {d22}, [TMP2], STRIDE
+        vshrn.u32 d5, q3, #(2 * BILINEAR_INTERPOLATION_BITS)
+        vadd.u16  q12, q12, q13
+    vld1.32   {d23}, [TMP2]
+    vmull.u8  q9, d22, d28
+    mov       TMP3, X, asr #16
+    add       X, X, UX
+    add       TMP3, TOP, TMP3, asl #2
+    mov       TMP4, X, asr #16
+    add       X, X, UX
+    add       TMP4, TOP, TMP4, asl #2
+    vmlal.u8  q9, d23, d29
+    vld1.32   {d22}, [TMP3], STRIDE
+        vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vld1.32   {d23}, [TMP3]
+    vmull.u8  q10, d22, d28
+    vmlal.u8  q10, d23, d29
+        vmovn.u16 d8, q0
+    vshll.u16 q0, d16, #BILINEAR_INTERPOLATION_BITS
+        vmovn.u16 d9, q2
+    vmlsl.u16 q0, d16, d30
+    vmlal.u16 q0, d17, d30
+    pld       [TMP4, PF_OFFS]
+    vld1.32   {d16}, [TMP4], STRIDE
+        vadd.u16  q12, q12, q13
+    vld1.32   {d17}, [TMP4]
+    pld       [TMP4, PF_OFFS]
+    vmull.u8  q11, d16, d28
+    vmlal.u8  q11, d17, d29
+    vshll.u16 q1, d18, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q1, d18, d31
+.endm
+
+.macro bilinear_interpolate_eight_pixels_8888_0565_tail
+    vmlal.u16 q1, d19, d31
+    vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vshll.u16 q2, d20, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q2, d20, d30
+    vmlal.u16 q2, d21, d30
+    vshll.u16 q3, d22, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q3, d22, d31
+    vmlal.u16 q3, d23, d31
+    vadd.u16  q12, q12, q13
+    vshrn.u32 d0, q0, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32 d1, q1, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32 d4, q2, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vshrn.u32 d5, q3, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vmovn.u16 d10, q0
+    vmovn.u16 d11, q2
+    vadd.u16  q12, q12, q13
+
+    vuzp.u8   d8, d9
+    vuzp.u8   d10, d11
+    vuzp.u8   d9, d11
+    vuzp.u8   d8, d10
+    vshll.u8  q6, d9, #8
+    vshll.u8  q5, d10, #8
+    vshll.u8  q7, d8, #8
+    vsri.u16  q5, q6, #5
+    vsri.u16  q5, q7, #11
+    vst1.32   {d10, d11}, [OUT, :128]!
+.endm
+
+.macro bilinear_interpolate_eight_pixels_8888_0565_tail_head
+    mov       TMP1, X, asr #16
+    add       X, X, UX
+    add       TMP1, TOP, TMP1, asl #2
+    mov       TMP2, X, asr #16
+    add       X, X, UX
+    add       TMP2, TOP, TMP2, asl #2
+        vmlal.u16 q1, d19, d31
+        vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+            vuzp.u8 d8, d9
+        vshll.u16 q2, d20, #BILINEAR_INTERPOLATION_BITS
+        vmlsl.u16 q2, d20, d30
+        vmlal.u16 q2, d21, d30
+        vshll.u16 q3, d22, #BILINEAR_INTERPOLATION_BITS
+    vld1.32   {d20}, [TMP1], STRIDE
+        vmlsl.u16 q3, d22, d31
+        vmlal.u16 q3, d23, d31
+    vld1.32   {d21}, [TMP1]
+    vmull.u8  q8, d20, d28
+    vmlal.u8  q8, d21, d29
+        vshrn.u32 d0, q0, #(2 * BILINEAR_INTERPOLATION_BITS)
+        vshrn.u32 d1, q1, #(2 * BILINEAR_INTERPOLATION_BITS)
+        vshrn.u32 d4, q2, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vld1.32   {d22}, [TMP2], STRIDE
+        vshrn.u32 d5, q3, #(2 * BILINEAR_INTERPOLATION_BITS)
+        vadd.u16  q12, q12, q13
+    vld1.32   {d23}, [TMP2]
+    vmull.u8  q9, d22, d28
+    mov       TMP3, X, asr #16
+    add       X, X, UX
+    add       TMP3, TOP, TMP3, asl #2
+    mov       TMP4, X, asr #16
+    add       X, X, UX
+    add       TMP4, TOP, TMP4, asl #2
+    vmlal.u8  q9, d23, d29
+    vld1.32   {d22}, [TMP3], STRIDE
+        vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vld1.32   {d23}, [TMP3]
+    vmull.u8  q10, d22, d28
+    vmlal.u8  q10, d23, d29
+        vmovn.u16 d10, q0
+    vshll.u16 q0, d16, #BILINEAR_INTERPOLATION_BITS
+        vmovn.u16 d11, q2
+    vmlsl.u16 q0, d16, d30
+    vmlal.u16 q0, d17, d30
+    pld       [TMP4, PF_OFFS]
+    vld1.32   {d16}, [TMP4], STRIDE
+        vadd.u16  q12, q12, q13
+    vld1.32   {d17}, [TMP4]
+    pld       [TMP4, PF_OFFS]
+    vmull.u8  q11, d16, d28
+    vmlal.u8  q11, d17, d29
+            vuzp.u8 d10, d11
+    vshll.u16 q1, d18, #BILINEAR_INTERPOLATION_BITS
+    vmlsl.u16 q1, d18, d31
+
+    mov       TMP1, X, asr #16
+    add       X, X, UX
+    add       TMP1, TOP, TMP1, asl #2
+    mov       TMP2, X, asr #16
+    add       X, X, UX
+    add       TMP2, TOP, TMP2, asl #2
+        vmlal.u16 q1, d19, d31
+            vuzp.u8 d9, d11
+        vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+        vshll.u16 q2, d20, #BILINEAR_INTERPOLATION_BITS
+            vuzp.u8 d8, d10
+        vmlsl.u16 q2, d20, d30
+        vmlal.u16 q2, d21, d30
+        vshll.u16 q3, d22, #BILINEAR_INTERPOLATION_BITS
+    vld1.32   {d20}, [TMP1], STRIDE
+        vmlsl.u16 q3, d22, d31
+        vmlal.u16 q3, d23, d31
+    vld1.32   {d21}, [TMP1]
+    vmull.u8  q8, d20, d28
+    vmlal.u8  q8, d21, d29
+            vshll.u8  q6, d9, #8
+            vshll.u8  q5, d10, #8
+            vshll.u8  q7, d8, #8
+        vshrn.u32 d0, q0, #(2 * BILINEAR_INTERPOLATION_BITS)
+            vsri.u16  q5, q6, #5
+        vshrn.u32 d1, q1, #(2 * BILINEAR_INTERPOLATION_BITS)
+            vsri.u16  q5, q7, #11
+        vshrn.u32 d4, q2, #(2 * BILINEAR_INTERPOLATION_BITS)
+    vld1.32   {d22}, [TMP2], STRIDE
+        vshrn.u32 d5, q3, #(2 * BILINEAR_INTERPOLATION_BITS)
+        vadd.u16  q12, q12, q13
+    vld1.32   {d23}, [TMP2]
+    vmull.u8  q9, d22, d28
+    mov       TMP3, X, asr #16
+    add       X, X, UX
+    add       TMP3, TOP, TMP3, asl #2
+    mov       TMP4, X, asr #16
+    add       X, X, UX
+    add       TMP4, TOP, TMP4, asl #2
+    vmlal.u8  q9, d23, d29
+    vld1.32   {d22}, [TMP3], STRIDE
+        vshr.u16  q15, q12, #(16 - BILINEAR_INTERPOLATION_BITS)
+    vld1.32   {d23}, [TMP3]
+    vmull.u8  q10, d22, d28
+    vmlal.u8  q10, d23, d29
+        vmovn.u16 d8, q0
+    vshll.u16 q0, d16, #BILINEAR_INTERPOLATION_BITS
+        vmovn.u16 d9, q2
+    vmlsl.u16 q0, d16, d30
+    vmlal.u16 q0, d17, d30
+    pld       [TMP4, PF_OFFS]
+    vld1.32   {d16}, [TMP4], STRIDE
+        vadd.u16  q12, q12, q13
+    vld1.32   {d17}, [TMP4]
+    pld       [TMP4, PF_OFFS]
+    vmull.u8  q11, d16, d28
+    vmlal.u8  q11, d17, d29
+    vshll.u16 q1, d18, #BILINEAR_INTERPOLATION_BITS
+            vst1.32   {d10, d11}, [OUT, :128]!
+    vmlsl.u16 q1, d18, d31
+.endm
+/*****************************************************************************/
+
+generate_bilinear_scanline_func \
+    pixman_scaled_bilinear_scanline_8888_8888_SRC_asm_neon, 8888, 8888, \
+    2, 2, 28, BILINEAR_FLAG_UNROLL_4
+
+generate_bilinear_scanline_func \
+    pixman_scaled_bilinear_scanline_8888_0565_SRC_asm_neon, 8888, 0565, \
+    2, 1, 28, BILINEAR_FLAG_UNROLL_8 | BILINEAR_FLAG_USE_ALL_NEON_REGS
+
+generate_bilinear_scanline_func \
+    pixman_scaled_bilinear_scanline_0565_x888_SRC_asm_neon, 0565, 8888, \
+    1, 2, 28, BILINEAR_FLAG_UNROLL_4
+
+generate_bilinear_scanline_func \
+    pixman_scaled_bilinear_scanline_0565_0565_SRC_asm_neon, 0565, 0565, \
+    1, 1, 28, BILINEAR_FLAG_UNROLL_4
diff --git a/pixman/pixman/pixman-arm-neon-asm.h b/pixman/pixman/pixman-arm-neon-asm.h
index 56c3faebf..d0d92d74c 100644
--- a/pixman/pixman/pixman-arm-neon-asm.h
+++ b/pixman/pixman/pixman-arm-neon-asm.h
@@ -205,6 +205,137 @@
 .endif
 .endm
 
+/*
+ * Pixel fetcher for nearest scaling (needs TMP1, TMP2, VX, UNIT_X register
+ * aliases to be defined)
+ */
+.macro pixld1_s elem_size, reg1, mem_operand
+.if elem_size == 16
+    mov     TMP1, VX, asr #16
+    adds    VX, VX, UNIT_X
+5:  subpls  VX, VX, SRC_WIDTH_FIXED
+    bpl     5b
+    add     TMP1, mem_operand, TMP1, asl #1
+    mov     TMP2, VX, asr #16
+    adds    VX, VX, UNIT_X
+5:  subpls  VX, VX, SRC_WIDTH_FIXED
+    bpl     5b
+    add     TMP2, mem_operand, TMP2, asl #1
+    vld1.16 {d&reg1&[0]}, [TMP1, :16]
+    mov     TMP1, VX, asr #16
+    adds    VX, VX, UNIT_X
+5:  subpls  VX, VX, SRC_WIDTH_FIXED
+    bpl     5b
+    add     TMP1, mem_operand, TMP1, asl #1
+    vld1.16 {d&reg1&[1]}, [TMP2, :16]
+    mov     TMP2, VX, asr #16
+    adds    VX, VX, UNIT_X
+5:  subpls  VX, VX, SRC_WIDTH_FIXED
+    bpl     5b
+    add     TMP2, mem_operand, TMP2, asl #1
+    vld1.16 {d&reg1&[2]}, [TMP1, :16]
+    vld1.16 {d&reg1&[3]}, [TMP2, :16]
+.elseif elem_size == 32
+    mov     TMP1, VX, asr #16
+    adds    VX, VX, UNIT_X
+5:  subpls  VX, VX, SRC_WIDTH_FIXED
+    bpl     5b
+    add     TMP1, mem_operand, TMP1, asl #2
+    mov     TMP2, VX, asr #16
+    adds    VX, VX, UNIT_X
+5:  subpls  VX, VX, SRC_WIDTH_FIXED
+    bpl     5b
+    add     TMP2, mem_operand, TMP2, asl #2
+    vld1.32 {d&reg1&[0]}, [TMP1, :32]
+    vld1.32 {d&reg1&[1]}, [TMP2, :32]
+.else
+    .error "unsupported"
+.endif
+.endm
+
+.macro pixld2_s elem_size, reg1, reg2, mem_operand
+.if 0 /* elem_size == 32 */
+    mov     TMP1, VX, asr #16
+    add     VX, VX, UNIT_X, asl #1
+    add     TMP1, mem_operand, TMP1, asl #2
+    mov     TMP2, VX, asr #16
+    sub     VX, VX, UNIT_X
+    add     TMP2, mem_operand, TMP2, asl #2
+    vld1.32 {d&reg1&[0]}, [TMP1, :32]
+    mov     TMP1, VX, asr #16
+    add     VX, VX, UNIT_X, asl #1
+    add     TMP1, mem_operand, TMP1, asl #2
+    vld1.32 {d&reg2&[0]}, [TMP2, :32]
+    mov     TMP2, VX, asr #16
+    add     VX, VX, UNIT_X
+    add     TMP2, mem_operand, TMP2, asl #2
+    vld1.32 {d&reg1&[1]}, [TMP1, :32]
+    vld1.32 {d&reg2&[1]}, [TMP2, :32]
+.else
+    pixld1_s elem_size, reg1, mem_operand
+    pixld1_s elem_size, reg2, mem_operand
+.endif
+.endm
+
+.macro pixld0_s elem_size, reg1, idx, mem_operand
+.if elem_size == 16
+    mov     TMP1, VX, asr #16
+    adds    VX, VX, UNIT_X
+5:  subpls  VX, VX, SRC_WIDTH_FIXED
+    bpl     5b
+    add     TMP1, mem_operand, TMP1, asl #1
+    vld1.16 {d&reg1&[idx]}, [TMP1, :16]
+.elseif elem_size == 32
+    mov     TMP1, VX, asr #16
+    adds    VX, VX, UNIT_X
+5:  subpls  VX, VX, SRC_WIDTH_FIXED
+    bpl     5b
+    add     TMP1, mem_operand, TMP1, asl #2
+    vld1.32 {d&reg1&[idx]}, [TMP1, :32]
+.endif
+.endm
+
+.macro pixld_s_internal numbytes, elem_size, basereg, mem_operand
+.if numbytes == 32
+    pixld2_s elem_size, %(basereg+4), %(basereg+5), mem_operand
+    pixld2_s elem_size, %(basereg+6), %(basereg+7), mem_operand
+    pixdeinterleave elem_size, %(basereg+4)
+.elseif numbytes == 16
+    pixld2_s elem_size, %(basereg+2), %(basereg+3), mem_operand
+.elseif numbytes == 8
+    pixld1_s elem_size, %(basereg+1), mem_operand
+.elseif numbytes == 4
+    .if elem_size == 32
+        pixld0_s elem_size, %(basereg+0), 1, mem_operand
+    .elseif elem_size == 16
+        pixld0_s elem_size, %(basereg+0), 2, mem_operand
+        pixld0_s elem_size, %(basereg+0), 3, mem_operand
+    .else
+        pixld0_s elem_size, %(basereg+0), 4, mem_operand
+        pixld0_s elem_size, %(basereg+0), 5, mem_operand
+        pixld0_s elem_size, %(basereg+0), 6, mem_operand
+        pixld0_s elem_size, %(basereg+0), 7, mem_operand
+    .endif
+.elseif numbytes == 2
+    .if elem_size == 16
+        pixld0_s elem_size, %(basereg+0), 1, mem_operand
+    .else
+        pixld0_s elem_size, %(basereg+0), 2, mem_operand
+        pixld0_s elem_size, %(basereg+0), 3, mem_operand
+    .endif
+.elseif numbytes == 1
+    pixld0_s elem_size, %(basereg+0), 1, mem_operand
+.else
+    .error "unsupported size: numbytes"
+.endif
+.endm
+
+.macro pixld_s numpix, bpp, basereg, mem_operand
+.if bpp > 0
+    pixld_s_internal %(numpix * bpp / 8), %(bpp), basereg, mem_operand
+.endif
+.endm
+
 .macro vuzp8 reg1, reg2
     vuzp.8 d&reg1, d&reg2
 .endm
@@ -254,7 +385,7 @@
  * execute simultaneously with NEON and be completely shadowed by it. Thus
  * we get no performance overhead at all (*). This looks like a very nice
  * feature of Cortex-A8, if used wisely. We don't have a hardware prefetcher,
- * but still can implement some rather advanced prefetch logic in sofware
+ * but still can implement some rather advanced prefetch logic in software
  * for almost zero cost!
  *
  * (*) The overhead of the prefetcher is visible when running some trivial
@@ -316,6 +447,11 @@
 .endif
 .endm
 
+.macro fetch_mask_pixblock
+    pixld       pixblock_size, mask_bpp, \
+                (mask_basereg - pixblock_size * mask_bpp / 64), MASK
+.endm
+
 /*
  * Macro which is used to process leading pixels until destination
  * pointer is properly aligned (at 16 bytes boundary). When destination
@@ -335,7 +471,7 @@ local skip1
     tst         DST_R, #lowbit
     beq         1f
 .endif
-    pixld       (lowbit * 8 / dst_w_bpp), src_bpp, src_basereg, SRC
+    pixld_src   (lowbit * 8 / dst_w_bpp), src_bpp, src_basereg, SRC
     pixld       (lowbit * 8 / dst_w_bpp), mask_bpp, mask_basereg, MASK
 .if dst_r_bpp > 0
     pixld_a     (lowbit * 8 / dst_r_bpp), dst_r_bpp, dst_r_basereg, DST_R
@@ -397,7 +533,7 @@ local skip1
 .if pixblock_size > chunk_size
     tst         W, #chunk_size
     beq         1f
-    pixld       chunk_size, src_bpp, src_basereg, SRC
+    pixld_src   chunk_size, src_bpp, src_basereg, SRC
     pixld       chunk_size, mask_bpp, mask_basereg, MASK
 .if dst_aligned_flag != 0
     pixld_a     chunk_size, dst_r_bpp, dst_r_basereg, DST_R
@@ -531,6 +667,13 @@ fname:
     .set src_basereg, src_basereg_
     .set mask_basereg, mask_basereg_
 
+    .macro pixld_src x:vararg
+        pixld x
+    .endm
+    .macro fetch_src_pixblock
+        pixld_src   pixblock_size, src_bpp, \
+                    (src_basereg - pixblock_size * src_bpp / 64), SRC
+    .endm
 /*
  * Assign symbolic names to registers
  */
@@ -696,8 +839,7 @@ fname:
     /* Implement "head (tail_head) ... (tail_head) tail" loop pattern */
     pixld_a     pixblock_size, dst_r_bpp, \
                 (dst_r_basereg - pixblock_size * dst_r_bpp / 64), DST_R
-    pixld       pixblock_size, src_bpp, \
-                (src_basereg - pixblock_size * src_bpp / 64), SRC
+    fetch_src_pixblock
     pixld       pixblock_size, mask_bpp, \
                 (mask_basereg - pixblock_size * mask_bpp / 64), MASK
     PF add      PF_X, PF_X, #pixblock_size
@@ -739,8 +881,7 @@ fname:
     beq         1f
     pixld       pixblock_size, dst_r_bpp, \
                 (dst_r_basereg - pixblock_size * dst_r_bpp / 64), DST_R
-    pixld       pixblock_size, src_bpp, \
-                (src_basereg - pixblock_size * src_bpp / 64), SRC
+    fetch_src_pixblock
     pixld       pixblock_size, mask_bpp, \
                 (mask_basereg - pixblock_size * mask_bpp / 64), MASK
     process_pixblock_head
@@ -761,6 +902,9 @@ fname:
     cleanup
     pop         {r4-r12, pc}  /* exit */
 
+    .purgem     fetch_src_pixblock
+    .purgem     pixld_src
+
     .unreq      SRC
     .unreq      MASK
     .unreq      DST_R
@@ -784,7 +928,8 @@ fname:
  * A simplified variant of function generation template for a single
  * scanline processing (for implementing pixman combine functions)
  */
-.macro generate_composite_function_single_scanline fname, \
+.macro generate_composite_function_scanline        use_nearest_scaling, \
+                                                   fname, \
                                                    src_bpp_, \
                                                    mask_bpp_, \
                                                    dst_w_bpp_, \
@@ -821,15 +966,47 @@ fname:
     .set dst_r_basereg, dst_r_basereg_
     .set src_basereg, src_basereg_
     .set mask_basereg, mask_basereg_
-/*
- * Assign symbolic names to registers
- */
+
+.if use_nearest_scaling != 0
+    /*
+     * Assign symbolic names to registers for nearest scaling
+     */
+    W           .req        r0
+    DST_W       .req        r1
+    SRC         .req        r2
+    VX          .req        r3
+    UNIT_X      .req        ip
+    MASK        .req        lr
+    TMP1        .req        r4
+    TMP2        .req        r5
+    DST_R       .req        r6
+    SRC_WIDTH_FIXED .req        r7
+
+    .macro pixld_src x:vararg
+        pixld_s x
+    .endm
+
+    ldr         UNIT_X, [sp]
+    push        {r4-r8, lr}
+    ldr         SRC_WIDTH_FIXED, [sp, #(24 + 4)]
+    .if mask_bpp != 0
+    ldr         MASK, [sp, #(24 + 8)]
+    .endif
+.else
+    /*
+     * Assign symbolic names to registers
+     */
     W           .req        r0      /* width (is updated during processing) */
     DST_W       .req        r1      /* destination buffer pointer for writes */
     SRC         .req        r2      /* source buffer pointer */
     DST_R       .req        ip      /* destination buffer pointer for reads */
     MASK        .req        r3      /* mask pointer */
 
+    .macro pixld_src x:vararg
+        pixld x
+    .endm
+.endif
+
 .if (((flags) & FLAG_DST_READWRITE) != 0)
     .set dst_r_bpp, dst_w_bpp
 .else
@@ -841,6 +1018,11 @@ fname:
     .set DEINTERLEAVE_32BPP_ENABLED, 0
 .endif
 
+    .macro fetch_src_pixblock
+        pixld_src   pixblock_size, src_bpp, \
+                    (src_basereg - pixblock_size * src_bpp / 64), SRC
+    .endm
+
     init
     mov         DST_R, DST_W
 
@@ -857,8 +1039,7 @@ fname:
     /* Implement "head (tail_head) ... (tail_head) tail" loop pattern */
     pixld_a     pixblock_size, dst_r_bpp, \
                 (dst_r_basereg - pixblock_size * dst_r_bpp / 64), DST_R
-    pixld       pixblock_size, src_bpp, \
-                (src_basereg - pixblock_size * src_bpp / 64), SRC
+    fetch_src_pixblock
     pixld       pixblock_size, mask_bpp, \
                 (mask_basereg - pixblock_size * mask_bpp / 64), MASK
     process_pixblock_head
@@ -880,7 +1061,11 @@ fname:
                             process_pixblock_tail_head
 
     cleanup
-    bx         lr  /* exit */
+.if use_nearest_scaling != 0
+    pop         {r4-r8, pc}  /* exit */
+.else
+    bx          lr  /* exit */
+.endif
 8:
     /* Process the remaining trailing pixels in the scanline (dst unaligned) */
     process_trailing_pixels 0, 0, \
@@ -889,6 +1074,22 @@ fname:
                             process_pixblock_tail_head
 
     cleanup
+
+.if use_nearest_scaling != 0
+    pop         {r4-r8, pc}  /* exit */
+
+    .unreq      DST_R
+    .unreq      SRC
+    .unreq      W
+    .unreq      VX
+    .unreq      UNIT_X
+    .unreq      TMP1
+    .unreq      TMP2
+    .unreq      DST_W
+    .unreq      MASK
+    .unreq      SRC_WIDTH_FIXED
+
+.else
     bx          lr  /* exit */
 
     .unreq      SRC
@@ -896,11 +1097,100 @@ fname:
     .unreq      DST_R
     .unreq      DST_W
     .unreq      W
+.endif
+
+    .purgem     fetch_src_pixblock
+    .purgem     pixld_src
+
     .endfunc
 .endm
 
+.macro generate_composite_function_single_scanline x:vararg
+    generate_composite_function_scanline 0, x
+.endm
+
+.macro generate_composite_function_nearest_scanline x:vararg
+    generate_composite_function_scanline 1, x
+.endm
+
+/* Default prologue/epilogue, nothing special needs to be done */
+
 .macro default_init
 .endm
 
 .macro default_cleanup
 .endm
+
+/*
+ * Prologue/epilogue variant which additionally saves/restores d8-d15
+ * registers (they need to be saved/restored by callee according to ABI).
+ * This is required if the code needs to use all the NEON registers.
+ */
+
+.macro default_init_need_all_regs
+    vpush       {d8-d15}
+.endm
+
+.macro default_cleanup_need_all_regs
+    vpop        {d8-d15}
+.endm
+
+/******************************************************************************/
+
+/*
+ * Conversion of 8 r5g6b6 pixels packed in 128-bit register (in)
+ * into a planar a8r8g8b8 format (with a, r, g, b color components
+ * stored into 64-bit registers out_a, out_r, out_g, out_b respectively).
+ *
+ * Warning: the conversion is destructive and the original
+ *          value (in) is lost.
+ */
+.macro convert_0565_to_8888 in, out_a, out_r, out_g, out_b
+    vshrn.u16   out_r, in,    #8
+    vshrn.u16   out_g, in,    #3
+    vsli.u16    in,    in,    #5
+    vmov.u8     out_a, #255
+    vsri.u8     out_r, out_r, #5
+    vsri.u8     out_g, out_g, #6
+    vshrn.u16   out_b, in,    #2
+.endm
+
+.macro convert_0565_to_x888 in, out_r, out_g, out_b
+    vshrn.u16   out_r, in,    #8
+    vshrn.u16   out_g, in,    #3
+    vsli.u16    in,    in,    #5
+    vsri.u8     out_r, out_r, #5
+    vsri.u8     out_g, out_g, #6
+    vshrn.u16   out_b, in,    #2
+.endm
+
+/*
+ * Conversion from planar a8r8g8b8 format (with a, r, g, b color components
+ * in 64-bit registers in_a, in_r, in_g, in_b respectively) into 8 r5g6b6
+ * pixels packed in 128-bit register (out). Requires two temporary 128-bit
+ * registers (tmp1, tmp2)
+ */
+.macro convert_8888_to_0565 in_r, in_g, in_b, out, tmp1, tmp2
+    vshll.u8    tmp1, in_g, #8
+    vshll.u8    out, in_r, #8
+    vshll.u8    tmp2, in_b, #8
+    vsri.u16    out, tmp1, #5
+    vsri.u16    out, tmp2, #11
+.endm
+
+/*
+ * Conversion of four r5g6b5 pixels (in) to four x8r8g8b8 pixels
+ * returned in (out0, out1) registers pair. Requires one temporary
+ * 64-bit register (tmp). 'out1' and 'in' may overlap, the original
+ * value from 'in' is lost
+ */
+.macro convert_four_0565_to_x888_packed in, out0, out1, tmp
+    vshl.u16    out0, in,   #5  /* G top 6 bits */
+    vshl.u16    tmp,  in,   #11 /* B top 5 bits */
+    vsri.u16    in,   in,   #5  /* R is ready in top bits */
+    vsri.u16    out0, out0, #6  /* G is ready in top bits */
+    vsri.u16    tmp,  tmp,  #5  /* B is ready in top bits */
+    vshr.u16    out1, in,   #8  /* R is in place */
+    vsri.u16    out0, tmp,  #8  /* G & B is in place */
+    vzip.u16    out0, out1      /* everything is in place */
+.endm
diff --git a/pixman/pixman/pixman-arm-neon.c b/pixman/pixman/pixman-arm-neon.c
index 6808b3658..60e9c78d2 100644
--- a/pixman/pixman/pixman-arm-neon.c
+++ b/pixman/pixman/pixman-arm-neon.c
@@ -52,7 +52,9 @@ PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0888_0565_rev,
                                    uint8_t, 3, uint16_t, 1)
 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_pixbuf_8888,
                                    uint32_t, 1, uint32_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, add_8000_8000,
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_rpixbuf_8888,
+                                   uint32_t, 1, uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, add_8_8,
                                    uint8_t, 1, uint8_t, 1)
 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, add_8888_8888,
                                    uint32_t, 1, uint32_t, 1)
@@ -60,34 +62,104 @@ PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, over_8888_0565,
                                    uint32_t, 1, uint16_t, 1)
 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, over_8888_8888,
                                    uint32_t, 1, uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, out_reverse_8_0565,
+                                   uint8_t, 1, uint16_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, out_reverse_8_8888,
+                                   uint8_t, 1, uint32_t, 1)
 
-PIXMAN_ARM_BIND_FAST_PATH_N_DST (neon, over_n_0565,
+PIXMAN_ARM_BIND_FAST_PATH_N_DST (SKIP_ZERO_SRC, neon, over_n_0565,
                                  uint16_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_N_DST (neon, over_n_8888,
+PIXMAN_ARM_BIND_FAST_PATH_N_DST (SKIP_ZERO_SRC, neon, over_n_8888,
                                  uint32_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_N_DST (neon, over_reverse_n_8888,
+PIXMAN_ARM_BIND_FAST_PATH_N_DST (SKIP_ZERO_SRC, neon, over_reverse_n_8888,
                                  uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_N_DST (0, neon, in_n_8,
+                                 uint8_t, 1)
 
-PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (neon, over_n_8_0565,
+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, neon, over_n_8_0565,
                                       uint8_t, 1, uint16_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (neon, over_n_8_8888,
+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, neon, over_n_8_8888,
                                       uint8_t, 1, uint32_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (neon, over_n_8888_8888_ca,
+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, neon, over_n_8888_8888_ca,
                                       uint32_t, 1, uint32_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (neon, add_n_8_8,
+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, neon, over_n_8888_0565_ca,
+				      uint32_t, 1, uint16_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, neon, over_n_8_8,
+                                      uint8_t, 1, uint8_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, neon, add_n_8_8,
+                                      uint8_t, 1, uint8_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, neon, add_n_8_8888,
+                                      uint8_t, 1, uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (0, neon, src_n_8_8888,
+                                      uint8_t, 1, uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (0, neon, src_n_8_8,
                                       uint8_t, 1, uint8_t, 1)
 
-PIXMAN_ARM_BIND_FAST_PATH_SRC_N_DST (neon, over_8888_n_8888,
+PIXMAN_ARM_BIND_FAST_PATH_SRC_N_DST (SKIP_ZERO_MASK, neon, over_8888_n_8888,
+                                     uint32_t, 1, uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_N_DST (SKIP_ZERO_MASK, neon, over_8888_n_0565,
+                                     uint32_t, 1, uint16_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_N_DST (SKIP_ZERO_MASK, neon, over_0565_n_0565,
+                                     uint16_t, 1, uint16_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_N_DST (SKIP_ZERO_MASK, neon, add_8888_n_8888,
                                      uint32_t, 1, uint32_t, 1)
 
 PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, add_8_8_8,
                                         uint8_t, 1, uint8_t, 1, uint8_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, add_0565_8_0565,
+                                        uint16_t, 1, uint8_t, 1, uint16_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, add_8888_8_8888,
+                                        uint32_t, 1, uint8_t, 1, uint32_t, 1)
 PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, add_8888_8888_8888,
                                         uint32_t, 1, uint32_t, 1, uint32_t, 1)
 PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, over_8888_8_8888,
                                         uint32_t, 1, uint8_t, 1, uint32_t, 1)
 PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, over_8888_8888_8888,
                                         uint32_t, 1, uint32_t, 1, uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, over_8888_8_0565,
+                                        uint32_t, 1, uint8_t, 1, uint16_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, over_0565_8_0565,
+                                        uint16_t, 1, uint8_t, 1, uint16_t, 1)
+
+PIXMAN_ARM_BIND_SCALED_NEAREST_SRC_DST (neon, 8888_8888, OVER,
+                                        uint32_t, uint32_t)
+PIXMAN_ARM_BIND_SCALED_NEAREST_SRC_DST (neon, 8888_0565, OVER,
+                                        uint32_t, uint16_t)
+PIXMAN_ARM_BIND_SCALED_NEAREST_SRC_DST (neon, 8888_0565, SRC,
+                                        uint32_t, uint16_t)
+PIXMAN_ARM_BIND_SCALED_NEAREST_SRC_DST (neon, 0565_8888, SRC,
+                                        uint16_t, uint32_t)
+
+PIXMAN_ARM_BIND_SCALED_NEAREST_SRC_A8_DST (SKIP_ZERO_SRC, neon, 8888_8_0565,
+                                           OVER, uint32_t, uint16_t)
+PIXMAN_ARM_BIND_SCALED_NEAREST_SRC_A8_DST (SKIP_ZERO_SRC, neon, 0565_8_0565,
+                                           OVER, uint16_t, uint16_t)
+
+PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_DST (0, neon, 8888_8888, SRC,
+                                         uint32_t, uint32_t)
+PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_DST (0, neon, 8888_0565, SRC,
+                                         uint32_t, uint16_t)
+PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_DST (0, neon, 0565_x888, SRC,
+                                         uint16_t, uint32_t)
+PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_DST (0, neon, 0565_0565, SRC,
+                                         uint16_t, uint16_t)
+PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_DST (SKIP_ZERO_SRC, neon, 8888_8888, OVER,
+                                         uint32_t, uint32_t)
+PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_DST (SKIP_ZERO_SRC, neon, 8888_8888, ADD,
+                                         uint32_t, uint32_t)
+
+PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_A8_DST (0, neon, 8888_8_8888, SRC,
+                                            uint32_t, uint32_t)
+PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_A8_DST (0, neon, 8888_8_0565, SRC,
+                                            uint32_t, uint16_t)
+PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_A8_DST (0, neon, 0565_8_x888, SRC,
+                                            uint16_t, uint32_t)
+PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_A8_DST (0, neon, 0565_8_0565, SRC,
+                                            uint16_t, uint16_t)
+PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_A8_DST (SKIP_ZERO_SRC, neon, 8888_8_8888, OVER,
+                                            uint32_t, uint32_t)
+PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_A8_DST (SKIP_ZERO_SRC, neon, 8888_8_8888, ADD,
+                                            uint32_t, uint32_t)
 
 void
 pixman_composite_src_n_8_asm_neon (int32_t   w,
@@ -111,14 +183,15 @@ pixman_composite_src_n_8888_asm_neon (int32_t   w,
                                       uint32_t  src);
 
 static pixman_bool_t
-pixman_fill_neon (uint32_t *bits,
-                  int       stride,
-                  int       bpp,
-                  int       x,
-                  int       y,
-                  int       width,
-                  int       height,
-                  uint32_t  _xor)
+arm_neon_fill (pixman_implementation_t *imp,
+               uint32_t *               bits,
+               int                      stride,
+               int                      bpp,
+               int                      x,
+               int                      y,
+               int                      width,
+               int                      height,
+	       uint32_t                 _xor)
 {
     /* stride is always multiple of 32bit units in pixman */
     uint32_t byte_stride = stride * sizeof(uint32_t);
@@ -155,18 +228,19 @@ pixman_fill_neon (uint32_t *bits,
 }
 
 static pixman_bool_t
-pixman_blt_neon (uint32_t *src_bits,
-                 uint32_t *dst_bits,
-                 int       src_stride,
-                 int       dst_stride,
-                 int       src_bpp,
-                 int       dst_bpp,
-                 int       src_x,
-                 int       src_y,
-                 int       dst_x,
-                 int       dst_y,
-                 int       width,
-                 int       height)
+arm_neon_blt (pixman_implementation_t *imp,
+              uint32_t *               src_bits,
+              uint32_t *               dst_bits,
+              int                      src_stride,
+              int                      dst_stride,
+              int                      src_bpp,
+              int                      dst_bpp,
+              int                      src_x,
+              int                      src_y,
+              int                      dest_x,
+              int                      dest_y,
+              int                      width,
+              int                      height)
 {
     if (src_bpp != dst_bpp)
 	return FALSE;
@@ -177,7 +251,7 @@ pixman_blt_neon (uint32_t *src_bits,
 	pixman_composite_src_0565_0565_asm_neon (
 		width, height,
 		(uint16_t *)(((char *) dst_bits) +
-		dst_y * dst_stride * 4 + dst_x * 2), dst_stride * 2,
+		dest_y * dst_stride * 4 + dest_x * 2), dst_stride * 2,
 		(uint16_t *)(((char *) src_bits) +
 		src_y * src_stride * 4 + src_x * 2), src_stride * 2);
 	return TRUE;
@@ -185,7 +259,7 @@ pixman_blt_neon (uint32_t *src_bits,
 	pixman_composite_src_8888_8888_asm_neon (
 		width, height,
 		(uint32_t *)(((char *) dst_bits) +
-		dst_y * dst_stride * 4 + dst_x * 4), dst_stride,
+		dest_y * dst_stride * 4 + dest_x * 4), dst_stride,
 		(uint32_t *)(((char *) src_bits) +
 		src_y * src_stride * 4 + src_x * 4), src_stride);
 	return TRUE;
@@ -218,6 +292,16 @@ static const pixman_fast_path_t arm_neon_fast_paths[] =
     PIXMAN_STD_FAST_PATH (SRC,  b8g8r8,   null,     x8r8g8b8, neon_composite_src_0888_8888_rev),
     PIXMAN_STD_FAST_PATH (SRC,  b8g8r8,   null,     r5g6b5,   neon_composite_src_0888_0565_rev),
     PIXMAN_STD_FAST_PATH (SRC,  pixbuf,   pixbuf,   a8r8g8b8, neon_composite_src_pixbuf_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  pixbuf,   pixbuf,   a8b8g8r8, neon_composite_src_rpixbuf_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  rpixbuf,  rpixbuf,  a8r8g8b8, neon_composite_src_rpixbuf_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  rpixbuf,  rpixbuf,  a8b8g8r8, neon_composite_src_pixbuf_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  solid,    a8,       a8r8g8b8, neon_composite_src_n_8_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  solid,    a8,       x8r8g8b8, neon_composite_src_n_8_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  solid,    a8,       a8b8g8r8, neon_composite_src_n_8_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  solid,    a8,       x8b8g8r8, neon_composite_src_n_8_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  solid,    a8,       a8,       neon_composite_src_n_8_8),
+
+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       a8,       neon_composite_over_n_8_8),
     PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       r5g6b5,   neon_composite_over_n_8_0565),
     PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       b5g6r5,   neon_composite_over_n_8_0565),
     PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       a8r8g8b8, neon_composite_over_n_8_8888),
@@ -231,12 +315,22 @@ static const pixman_fast_path_t arm_neon_fast_paths[] =
     PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, x8r8g8b8, neon_composite_over_n_8888_8888_ca),
     PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, a8b8g8r8, neon_composite_over_n_8888_8888_ca),
     PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, x8b8g8r8, neon_composite_over_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, r5g6b5,   neon_composite_over_n_8888_0565_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, b5g6r5,   neon_composite_over_n_8888_0565_ca),
     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid,    a8r8g8b8, neon_composite_over_8888_n_8888),
     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid,    x8r8g8b8, neon_composite_over_8888_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid,    r5g6b5,   neon_composite_over_8888_n_0565),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, solid,    b5g6r5,   neon_composite_over_8888_n_0565),
+    PIXMAN_STD_FAST_PATH (OVER, r5g6b5,   solid,    r5g6b5,   neon_composite_over_0565_n_0565),
+    PIXMAN_STD_FAST_PATH (OVER, b5g6r5,   solid,    b5g6r5,   neon_composite_over_0565_n_0565),
     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8,       a8r8g8b8, neon_composite_over_8888_8_8888),
     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8,       x8r8g8b8, neon_composite_over_8888_8_8888),
     PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8,       a8b8g8r8, neon_composite_over_8888_8_8888),
     PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8,       x8b8g8r8, neon_composite_over_8888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8,       r5g6b5,   neon_composite_over_8888_8_0565),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8,       b5g6r5,   neon_composite_over_8888_8_0565),
+    PIXMAN_STD_FAST_PATH (OVER, r5g6b5,   a8,       r5g6b5,   neon_composite_over_0565_8_0565),
+    PIXMAN_STD_FAST_PATH (OVER, b5g6r5,   a8,       b5g6r5,   neon_composite_over_0565_8_0565),
     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, a8r8g8b8, neon_composite_over_8888_8888_8888),
     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null,     r5g6b5,   neon_composite_over_8888_0565),
     PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null,     b5g6r5,   neon_composite_over_8888_0565),
@@ -247,63 +341,88 @@ static const pixman_fast_path_t arm_neon_fast_paths[] =
     PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, null,     a8r8g8b8, neon_composite_src_x888_8888),
     PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, null,     a8b8g8r8, neon_composite_src_x888_8888),
     PIXMAN_STD_FAST_PATH (ADD,  solid,    a8,       a8,       neon_composite_add_n_8_8),
+    PIXMAN_STD_FAST_PATH (ADD,  solid,    a8,       a8r8g8b8, neon_composite_add_n_8_8888),
+    PIXMAN_STD_FAST_PATH (ADD,  solid,    a8,       a8b8g8r8, neon_composite_add_n_8_8888),
     PIXMAN_STD_FAST_PATH (ADD,  a8,       a8,       a8,       neon_composite_add_8_8_8),
+    PIXMAN_STD_FAST_PATH (ADD,  r5g6b5,   a8,       r5g6b5,   neon_composite_add_0565_8_0565),
+    PIXMAN_STD_FAST_PATH (ADD,  b5g6r5,   a8,       b5g6r5,   neon_composite_add_0565_8_0565),
+    PIXMAN_STD_FAST_PATH (ADD,  a8r8g8b8, a8,       a8r8g8b8, neon_composite_add_8888_8_8888),
+    PIXMAN_STD_FAST_PATH (ADD,  a8b8g8r8, a8,       a8b8g8r8, neon_composite_add_8888_8_8888),
     PIXMAN_STD_FAST_PATH (ADD,  a8r8g8b8, a8r8g8b8, a8r8g8b8, neon_composite_add_8888_8888_8888),
-    PIXMAN_STD_FAST_PATH (ADD,  a8,       null,     a8,       neon_composite_add_8000_8000),
+    PIXMAN_STD_FAST_PATH (ADD,  a8r8g8b8, solid,    a8r8g8b8, neon_composite_add_8888_n_8888),
+    PIXMAN_STD_FAST_PATH (ADD,  a8b8g8r8, solid,    a8b8g8r8, neon_composite_add_8888_n_8888),
+    PIXMAN_STD_FAST_PATH (ADD,  a8,       null,     a8,       neon_composite_add_8_8),
     PIXMAN_STD_FAST_PATH (ADD,  a8r8g8b8, null,     a8r8g8b8, neon_composite_add_8888_8888),
     PIXMAN_STD_FAST_PATH (ADD,  a8b8g8r8, null,     a8b8g8r8, neon_composite_add_8888_8888),
+    PIXMAN_STD_FAST_PATH (IN,   solid,    null,     a8,       neon_composite_in_n_8),
     PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8r8g8b8, neon_composite_over_reverse_n_8888),
     PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8b8g8r8, neon_composite_over_reverse_n_8888),
+    PIXMAN_STD_FAST_PATH (OUT_REVERSE,  a8,    null, r5g6b5,   neon_composite_out_reverse_8_0565),
+    PIXMAN_STD_FAST_PATH (OUT_REVERSE,  a8,    null, b5g6r5,   neon_composite_out_reverse_8_0565),
+    PIXMAN_STD_FAST_PATH (OUT_REVERSE,  a8,    null, a8r8g8b8, neon_composite_out_reverse_8_8888),
+    PIXMAN_STD_FAST_PATH (OUT_REVERSE,  a8,    null, a8b8g8r8, neon_composite_out_reverse_8_8888),
 
-    { PIXMAN_OP_NONE },
-};
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, neon_8888_8888),
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (OVER, a8b8g8r8, a8b8g8r8, neon_8888_8888),
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, neon_8888_8888),
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (OVER, a8b8g8r8, x8b8g8r8, neon_8888_8888),
 
-static pixman_bool_t
-arm_neon_blt (pixman_implementation_t *imp,
-              uint32_t *               src_bits,
-              uint32_t *               dst_bits,
-              int                      src_stride,
-              int                      dst_stride,
-              int                      src_bpp,
-              int                      dst_bpp,
-              int                      src_x,
-              int                      src_y,
-              int                      dst_x,
-              int                      dst_y,
-              int                      width,
-              int                      height)
-{
-    if (!pixman_blt_neon (
-            src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
-            src_x, src_y, dst_x, dst_y, width, height))
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (OVER, a8r8g8b8, r5g6b5, neon_8888_0565),
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (OVER, a8b8g8r8, b5g6r5, neon_8888_0565),
 
-    {
-	return _pixman_implementation_blt (
-	    imp->delegate,
-	    src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
-	    src_x, src_y, dst_x, dst_y, width, height);
-    }
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, a8r8g8b8, r5g6b5, neon_8888_0565),
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, x8r8g8b8, r5g6b5, neon_8888_0565),
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, a8b8g8r8, b5g6r5, neon_8888_0565),
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, x8b8g8r8, b5g6r5, neon_8888_0565),
 
-    return TRUE;
-}
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, b5g6r5, x8b8g8r8, neon_0565_8888),
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, r5g6b5, x8r8g8b8, neon_0565_8888),
+    /* Note: NONE repeat is not supported yet */
+    SIMPLE_NEAREST_FAST_PATH_COVER (SRC, r5g6b5, a8r8g8b8, neon_0565_8888),
+    SIMPLE_NEAREST_FAST_PATH_COVER (SRC, b5g6r5, a8b8g8r8, neon_0565_8888),
+    SIMPLE_NEAREST_FAST_PATH_PAD (SRC, r5g6b5, a8r8g8b8, neon_0565_8888),
+    SIMPLE_NEAREST_FAST_PATH_PAD (SRC, b5g6r5, a8b8g8r8, neon_0565_8888),
 
-static pixman_bool_t
-arm_neon_fill (pixman_implementation_t *imp,
-               uint32_t *               bits,
-               int                      stride,
-               int                      bpp,
-               int                      x,
-               int                      y,
-               int                      width,
-               int                      height,
-               uint32_t xor)
-{
-    if (pixman_fill_neon (bits, stride, bpp, x, y, width, height, xor))
-	return TRUE;
+    PIXMAN_ARM_SIMPLE_NEAREST_A8_MASK_FAST_PATH (OVER, a8r8g8b8, r5g6b5, neon_8888_8_0565),
+    PIXMAN_ARM_SIMPLE_NEAREST_A8_MASK_FAST_PATH (OVER, a8b8g8r8, b5g6r5, neon_8888_8_0565),
 
-    return _pixman_implementation_fill (
-	imp->delegate, bits, stride, bpp, x, y, width, height, xor);
-}
+    PIXMAN_ARM_SIMPLE_NEAREST_A8_MASK_FAST_PATH (OVER, r5g6b5, r5g6b5, neon_0565_8_0565),
+    PIXMAN_ARM_SIMPLE_NEAREST_A8_MASK_FAST_PATH (OVER, b5g6r5, b5g6r5, neon_0565_8_0565),
+
+    SIMPLE_BILINEAR_FAST_PATH (SRC, a8r8g8b8, a8r8g8b8, neon_8888_8888),
+    SIMPLE_BILINEAR_FAST_PATH (SRC, a8r8g8b8, x8r8g8b8, neon_8888_8888),
+    SIMPLE_BILINEAR_FAST_PATH (SRC, x8r8g8b8, x8r8g8b8, neon_8888_8888),
+
+    SIMPLE_BILINEAR_FAST_PATH (SRC, a8r8g8b8, r5g6b5, neon_8888_0565),
+    SIMPLE_BILINEAR_FAST_PATH (SRC, x8r8g8b8, r5g6b5, neon_8888_0565),
+
+    SIMPLE_BILINEAR_FAST_PATH (SRC, r5g6b5, x8r8g8b8, neon_0565_x888),
+    SIMPLE_BILINEAR_FAST_PATH (SRC, r5g6b5, r5g6b5, neon_0565_0565),
+
+    SIMPLE_BILINEAR_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, neon_8888_8888),
+    SIMPLE_BILINEAR_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, neon_8888_8888),
+
+    SIMPLE_BILINEAR_FAST_PATH (ADD, a8r8g8b8, a8r8g8b8, neon_8888_8888),
+    SIMPLE_BILINEAR_FAST_PATH (ADD, a8r8g8b8, x8r8g8b8, neon_8888_8888),
+
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, a8r8g8b8, a8r8g8b8, neon_8888_8_8888),
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, a8r8g8b8, x8r8g8b8, neon_8888_8_8888),
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, x8r8g8b8, x8r8g8b8, neon_8888_8_8888),
+
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, a8r8g8b8, r5g6b5, neon_8888_8_0565),
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, x8r8g8b8, r5g6b5, neon_8888_8_0565),
+
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, r5g6b5, x8r8g8b8, neon_0565_8_x888),
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, r5g6b5, r5g6b5, neon_0565_8_0565),
+
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, neon_8888_8_8888),
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, neon_8888_8_8888),
+
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (ADD, a8r8g8b8, a8r8g8b8, neon_8888_8_8888),
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (ADD, a8r8g8b8, x8r8g8b8, neon_8888_8_8888),
+
+    { PIXMAN_OP_NONE },
+};
 
 #define BIND_COMBINE_U(name)                                             \
 void                                                                     \
@@ -334,16 +453,17 @@ neon_combine_##name##_u (pixman_implementation_t *imp,                   \
 
 BIND_COMBINE_U (over)
 BIND_COMBINE_U (add)
+BIND_COMBINE_U (out_reverse)
 
 pixman_implementation_t *
-_pixman_implementation_create_arm_neon (void)
+_pixman_implementation_create_arm_neon (pixman_implementation_t *fallback)
 {
-    pixman_implementation_t *general = _pixman_implementation_create_fast_path ();
     pixman_implementation_t *imp =
-	_pixman_implementation_create (general, arm_neon_fast_paths);
+	_pixman_implementation_create (fallback, arm_neon_fast_paths);
 
     imp->combine_32[PIXMAN_OP_OVER] = neon_combine_over_u;
     imp->combine_32[PIXMAN_OP_ADD] = neon_combine_add_u;
+    imp->combine_32[PIXMAN_OP_OUT_REVERSE] = neon_combine_out_reverse_u;
 
     imp->blt = arm_neon_blt;
     imp->fill = arm_neon_fill;
diff --git a/pixman/pixman/pixman-arm-simd-asm-scaled.S b/pixman/pixman/pixman-arm-simd-asm-scaled.S
new file mode 100644
index 000000000..711099548
--- /dev/null
+++ b/pixman/pixman/pixman-arm-simd-asm-scaled.S
@@ -0,0 +1,165 @@
+/*
+ * Copyright © 2008 Mozilla Corporation
+ * Copyright © 2010 Nokia Corporation
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Mozilla Corporation not be used in
+ * advertising or publicity pertaining to distribution of the software without
+ * specific, written prior permission.  Mozilla Corporation makes no
+ * representations about the suitability of this software for any purpose.  It
+ * is provided "as is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ *
+ * Author:  Jeff Muizelaar (jeff@infidigm.net)
+ *
+ */
+
+/* Prevent the stack from becoming executable */
+#if defined(__linux__) && defined(__ELF__)
+.section .note.GNU-stack,"",%progbits
+#endif
+
+	.text
+	.arch armv6
+	.object_arch armv4
+	.arm
+	.altmacro
+	.p2align 2
+
+/* Supplementary macro for setting function attributes */
+.macro pixman_asm_function fname
+	.func fname
+	.global fname
+#ifdef __ELF__
+	.hidden fname
+	.type fname, %function
+#endif
+fname:
+.endm
+
+/*
+ * Note: This code is only using armv5te instructions (not even armv6),
+ *       but is scheduled for ARM Cortex-A8 pipeline. So it might need to
+ *       be split into a few variants, tuned for each microarchitecture.
+ *
+ * TODO: In order to get good performance on ARM9/ARM11 cores (which don't
+ * have efficient write combining), it needs to be changed to use 16-byte
+ * aligned writes using STM instruction.
+ *
+ * Nearest scanline scaler macro template uses the following arguments:
+ *  fname                     - name of the function to generate
+ *  bpp_shift                 - (1 << bpp_shift) is the size of pixel in bytes
+ *  t                         - type suffix for LDR/STR instructions
+ *  prefetch_distance         - prefetch in the source image by that many
+ *                              pixels ahead
+ *  prefetch_braking_distance - stop prefetching when that many pixels are
+ *                              remaining before the end of scanline
+ */
+
+.macro generate_nearest_scanline_func fname, bpp_shift, t,      \
+                                      prefetch_distance,        \
+                                      prefetch_braking_distance
+
+pixman_asm_function fname
+	W		.req	r0
+	DST		.req	r1
+	SRC		.req	r2
+	VX		.req	r3
+	UNIT_X		.req	ip
+	TMP1		.req	r4
+	TMP2		.req	r5
+	VXMASK		.req	r6
+	PF_OFFS		.req	r7
+	SRC_WIDTH_FIXED	.req	r8
+
+	ldr	UNIT_X, [sp]
+	push	{r4, r5, r6, r7, r8, r10}
+	mvn	VXMASK, #((1 << bpp_shift) - 1)
+	ldr	SRC_WIDTH_FIXED, [sp, #28]
+
+	/* define helper macro */
+	.macro	scale_2_pixels
+		ldr&t	TMP1, [SRC, TMP1]
+		and	TMP2, VXMASK, VX, asr #(16 - bpp_shift)
+		adds	VX, VX, UNIT_X
+		str&t	TMP1, [DST], #(1 << bpp_shift)
+9:		subpls	VX, VX, SRC_WIDTH_FIXED
+		bpl	9b
+
+		ldr&t	TMP2, [SRC, TMP2]
+		and	TMP1, VXMASK, VX, asr #(16 - bpp_shift)
+		adds	VX, VX, UNIT_X
+		str&t	TMP2, [DST], #(1 << bpp_shift)
+9:		subpls	VX, VX, SRC_WIDTH_FIXED
+		bpl	9b
+	.endm
+
+	/* now do the scaling */
+	and	TMP1, VXMASK, VX, asr #(16 - bpp_shift)
+	adds	VX, VX, UNIT_X
+9:	subpls	VX, VX, SRC_WIDTH_FIXED
+	bpl	9b
+	subs	W, W, #(8 + prefetch_braking_distance)
+	blt	2f
+	/* calculate prefetch offset */
+	mov	PF_OFFS, #prefetch_distance
+	mla	PF_OFFS, UNIT_X, PF_OFFS, VX
+1:	/* main loop, process 8 pixels per iteration with prefetch */
+	pld	[SRC, PF_OFFS, asr #(16 - bpp_shift)]
+	add	PF_OFFS, UNIT_X, lsl #3
+	scale_2_pixels
+	scale_2_pixels
+	scale_2_pixels
+	scale_2_pixels
+	subs	W, W, #8
+	bge	1b
+2:
+	subs	W, W, #(4 - 8 - prefetch_braking_distance)
+	blt	2f
+1:	/* process the remaining pixels */
+	scale_2_pixels
+	scale_2_pixels
+	subs	W, W, #4
+	bge	1b
+2:
+	tst	W, #2
+	beq	2f
+	scale_2_pixels
+2:
+	tst	W, #1
+	ldrne&t	TMP1, [SRC, TMP1]
+	strne&t	TMP1, [DST]
+	/* cleanup helper macro */
+	.purgem	scale_2_pixels
+	.unreq	DST
+	.unreq	SRC
+	.unreq	W
+	.unreq	VX
+	.unreq	UNIT_X
+	.unreq	TMP1
+	.unreq	TMP2
+	.unreq	VXMASK
+	.unreq	PF_OFFS
+	.unreq  SRC_WIDTH_FIXED
+	/* return */
+	pop	{r4, r5, r6, r7, r8, r10}
+	bx	lr
+.endfunc
+.endm
+
+generate_nearest_scanline_func \
+    pixman_scaled_nearest_scanline_0565_0565_SRC_asm_armv6, 1, h, 80, 32
+
+generate_nearest_scanline_func \
+    pixman_scaled_nearest_scanline_8888_8888_SRC_asm_armv6, 2,  , 48, 32
diff --git a/pixman/pixman/pixman-arm-simd-asm.S b/pixman/pixman/pixman-arm-simd-asm.S
index 1a1a0d641..c20968879 100644
--- a/pixman/pixman/pixman-arm-simd-asm.S
+++ b/pixman/pixman/pixman-arm-simd-asm.S
@@ -1,13 +1,14 @@
 /*
- * Copyright © 2008 Mozilla Corporation
+ * Copyright © 2012 Raspberry Pi Foundation
+ * Copyright © 2012 RISC OS Open Ltd
  *
  * Permission to use, copy, modify, distribute, and sell this software and its
  * documentation for any purpose is hereby granted without fee, provided that
  * the above copyright notice appear in all copies and that both that
  * copyright notice and this permission notice appear in supporting
- * documentation, and that the name of Mozilla Corporation not be used in
+ * documentation, and that the name of the copyright holders not be used in
  * advertising or publicity pertaining to distribution of the software without
- * specific, written prior permission.  Mozilla Corporation makes no
+ * specific, written prior permission.  The copyright holders make no
  * representations about the suitability of this software for any purpose.  It
  * is provided "as is" without express or implied warranty.
  *
@@ -20,7 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
  * SOFTWARE.
  *
- * Author:  Jeff Muizelaar (jeff@infidigm.net)
+ * Author:  Ben Avison (bavison@riscosopen.org)
  *
  */
 
@@ -34,297 +35,579 @@
 	.object_arch armv4
 	.arm
 	.altmacro
+	.p2align 2
 
-/* Supplementary macro for setting function attributes */
-.macro pixman_asm_function fname
-	.func fname
-	.global fname
-#ifdef __ELF__
-	.hidden fname
-	.type fname, %function
-#endif
-fname:
+#include "pixman-arm-simd-asm.h"
+
+/* A head macro should do all processing which results in an output of up to
+ * 16 bytes, as far as the final load instruction. The corresponding tail macro
+ * should complete the processing of the up-to-16 bytes. The calling macro will
+ * sometimes choose to insert a preload or a decrement of X between them.
+ *   cond           ARM condition code for code block
+ *   numbytes       Number of output bytes that should be generated this time
+ *   firstreg       First WK register in which to place output
+ *   unaligned_src  Whether to use non-wordaligned loads of source image
+ *   unaligned_mask Whether to use non-wordaligned loads of mask image
+ *   preload        If outputting 16 bytes causes 64 bytes to be read, whether an extra preload should be output
+ */
+
+.macro blit_init
+        line_saved_regs STRIDE_D, STRIDE_S
 .endm
 
-/*
- * The code below was generated by gcc 4.3.4 from the commented out
- * functions in 'pixman-arm-simd.c' file with the following optimization
- * options: "-O3 -mcpu=arm1136jf-s -fomit-frame-pointer"
- *
- * TODO: replace gcc generated code with hand tuned versions because
- * the code quality is not very good, introduce symbolic register
- * aliases for better readability and maintainability.
+.macro blit_process_head   cond, numbytes, firstreg, unaligned_src, unaligned_mask, preload
+        pixld   cond, numbytes, firstreg, SRC, unaligned_src
+.endm
+
+.macro blit_inner_loop  process_head, process_tail, unaligned_src, unaligned_mask, dst_alignment
+    WK4     .req    STRIDE_D
+    WK5     .req    STRIDE_S
+    WK6     .req    MASK
+    WK7     .req    STRIDE_M
+110:    pixld   , 16, 0, SRC, unaligned_src
+        pixld   , 16, 4, SRC, unaligned_src
+        pld     [SRC, SCRATCH]
+        pixst   , 16, 0, DST
+        pixst   , 16, 4, DST
+        subs    X, X, #32*8/src_bpp
+        bhs     110b
+    .unreq  WK4
+    .unreq  WK5
+    .unreq  WK6
+    .unreq  WK7
+.endm
+
+generate_composite_function \
+    pixman_composite_src_8888_8888_asm_armv6, 32, 0, 32, \
+    FLAG_DST_WRITEONLY | FLAG_COND_EXEC | FLAG_SPILL_LINE_VARS_WIDE | FLAG_PROCESS_PRESERVES_SCRATCH, \
+    4, /* prefetch distance */ \
+    blit_init, \
+    nop_macro, /* newline */ \
+    nop_macro, /* cleanup */ \
+    blit_process_head, \
+    nop_macro, /* process tail */ \
+    blit_inner_loop
+
+generate_composite_function \
+    pixman_composite_src_0565_0565_asm_armv6, 16, 0, 16, \
+    FLAG_DST_WRITEONLY | FLAG_COND_EXEC | FLAG_SPILL_LINE_VARS_WIDE | FLAG_PROCESS_PRESERVES_SCRATCH, \
+    4, /* prefetch distance */ \
+    blit_init, \
+    nop_macro, /* newline */ \
+    nop_macro, /* cleanup */ \
+    blit_process_head, \
+    nop_macro, /* process tail */ \
+    blit_inner_loop
+
+generate_composite_function \
+    pixman_composite_src_8_8_asm_armv6, 8, 0, 8, \
+    FLAG_DST_WRITEONLY | FLAG_COND_EXEC | FLAG_SPILL_LINE_VARS_WIDE | FLAG_PROCESS_PRESERVES_SCRATCH, \
+    3, /* prefetch distance */ \
+    blit_init, \
+    nop_macro, /* newline */ \
+    nop_macro, /* cleanup */ \
+    blit_process_head, \
+    nop_macro, /* process tail */ \
+    blit_inner_loop
+
+/******************************************************************************/
+
+.macro src_n_8888_init
+        ldr     SRC, [sp, #ARGS_STACK_OFFSET]
+        mov     STRIDE_S, SRC
+        mov     MASK, SRC
+        mov     STRIDE_M, SRC
+.endm
+
+.macro src_n_0565_init
+        ldrh    SRC, [sp, #ARGS_STACK_OFFSET]
+        orr     SRC, SRC, lsl #16
+        mov     STRIDE_S, SRC
+        mov     MASK, SRC
+        mov     STRIDE_M, SRC
+.endm
+
+.macro src_n_8_init
+        ldrb    SRC, [sp, #ARGS_STACK_OFFSET]
+        orr     SRC, SRC, lsl #8
+        orr     SRC, SRC, lsl #16
+        mov     STRIDE_S, SRC
+        mov     MASK, SRC
+        mov     STRIDE_M, SRC
+.endm
+
+.macro fill_process_tail  cond, numbytes, firstreg
+    WK4     .req    SRC
+    WK5     .req    STRIDE_S
+    WK6     .req    MASK
+    WK7     .req    STRIDE_M
+        pixst   cond, numbytes, 4, DST
+    .unreq  WK4
+    .unreq  WK5
+    .unreq  WK6
+    .unreq  WK7
+.endm
+
+generate_composite_function \
+    pixman_composite_src_n_8888_asm_armv6, 0, 0, 32, \
+    FLAG_DST_WRITEONLY | FLAG_COND_EXEC | FLAG_PROCESS_PRESERVES_PSR | FLAG_PROCESS_DOES_STORE | FLAG_PROCESS_PRESERVES_SCRATCH \
+    0, /* prefetch distance doesn't apply */ \
+    src_n_8888_init \
+    nop_macro, /* newline */ \
+    nop_macro /* cleanup */ \
+    nop_macro /* process head */ \
+    fill_process_tail
+
+generate_composite_function \
+    pixman_composite_src_n_0565_asm_armv6, 0, 0, 16, \
+    FLAG_DST_WRITEONLY | FLAG_COND_EXEC | FLAG_PROCESS_PRESERVES_PSR | FLAG_PROCESS_DOES_STORE | FLAG_PROCESS_PRESERVES_SCRATCH \
+    0, /* prefetch distance doesn't apply */ \
+    src_n_0565_init \
+    nop_macro, /* newline */ \
+    nop_macro /* cleanup */ \
+    nop_macro /* process head */ \
+    fill_process_tail
+
+generate_composite_function \
+    pixman_composite_src_n_8_asm_armv6, 0, 0, 8, \
+    FLAG_DST_WRITEONLY | FLAG_COND_EXEC | FLAG_PROCESS_PRESERVES_PSR | FLAG_PROCESS_DOES_STORE | FLAG_PROCESS_PRESERVES_SCRATCH \
+    0, /* prefetch distance doesn't apply */ \
+    src_n_8_init \
+    nop_macro, /* newline */ \
+    nop_macro /* cleanup */ \
+    nop_macro /* process head */ \
+    fill_process_tail
+
+/******************************************************************************/
+
+.macro src_x888_8888_pixel, cond, reg
+        orr&cond WK&reg, WK&reg, #0xFF000000
+.endm
+
+.macro pixman_composite_src_x888_8888_process_head   cond, numbytes, firstreg, unaligned_src, unaligned_mask, preload
+        pixld   cond, numbytes, firstreg, SRC, unaligned_src
+.endm
+
+.macro pixman_composite_src_x888_8888_process_tail   cond, numbytes, firstreg
+        src_x888_8888_pixel cond, %(firstreg+0)
+ .if numbytes >= 8
+        src_x888_8888_pixel cond, %(firstreg+1)
+  .if numbytes == 16
+        src_x888_8888_pixel cond, %(firstreg+2)
+        src_x888_8888_pixel cond, %(firstreg+3)
+  .endif
+ .endif
+.endm
+
+generate_composite_function \
+    pixman_composite_src_x888_8888_asm_armv6, 32, 0, 32, \
+    FLAG_DST_WRITEONLY | FLAG_COND_EXEC | FLAG_PROCESS_PRESERVES_SCRATCH, \
+    3, /* prefetch distance */ \
+    nop_macro, /* init */ \
+    nop_macro, /* newline */ \
+    nop_macro, /* cleanup */ \
+    pixman_composite_src_x888_8888_process_head, \
+    pixman_composite_src_x888_8888_process_tail
+
+/******************************************************************************/
+
+.macro src_0565_8888_init
+        /* Hold loop invariants in MASK and STRIDE_M */
+        ldr     MASK, =0x07E007E0
+        mov     STRIDE_M, #0xFF000000
+        /* Set GE[3:0] to 1010 so SEL instructions do what we want */
+        ldr     SCRATCH, =0x80008000
+        uadd8   SCRATCH, SCRATCH, SCRATCH
+.endm
+
+.macro src_0565_8888_2pixels, reg1, reg2
+        and     SCRATCH, WK&reg1, MASK             @ 00000GGGGGG0000000000gggggg00000
+        bic     WK&reg2, WK&reg1, MASK             @ RRRRR000000BBBBBrrrrr000000bbbbb
+        orr     SCRATCH, SCRATCH, SCRATCH, lsr #6  @ 00000GGGGGGGGGGGG0000ggggggggggg
+        mov     WK&reg1, WK&reg2, lsl #16          @ rrrrr000000bbbbb0000000000000000
+        mov     SCRATCH, SCRATCH, ror #19          @ GGGG0000ggggggggggg00000GGGGGGGG
+        bic     WK&reg2, WK&reg2, WK&reg1, lsr #16 @ RRRRR000000BBBBB0000000000000000
+        orr     WK&reg1, WK&reg1, WK&reg1, lsr #5  @ rrrrrrrrrr0bbbbbbbbbb00000000000
+        orr     WK&reg2, WK&reg2, WK&reg2, lsr #5  @ RRRRRRRRRR0BBBBBBBBBB00000000000
+        pkhtb   WK&reg1, WK&reg1, WK&reg1, asr #5  @ rrrrrrrr--------bbbbbbbb--------
+        sel     WK&reg1, WK&reg1, SCRATCH          @ rrrrrrrrggggggggbbbbbbbb--------
+        mov     SCRATCH, SCRATCH, ror #16          @ ggg00000GGGGGGGGGGGG0000gggggggg
+        pkhtb   WK&reg2, WK&reg2, WK&reg2, asr #5  @ RRRRRRRR--------BBBBBBBB--------
+        sel     WK&reg2, WK&reg2, SCRATCH          @ RRRRRRRRGGGGGGGGBBBBBBBB--------
+        orr     WK&reg1, STRIDE_M, WK&reg1, lsr #8 @ 11111111rrrrrrrrggggggggbbbbbbbb
+        orr     WK&reg2, STRIDE_M, WK&reg2, lsr #8 @ 11111111RRRRRRRRGGGGGGGGBBBBBBBB
+.endm
+
+/* This version doesn't need STRIDE_M, but is one instruction longer.
+   It would however be preferable for an XRGB target, since we could knock off the last 2 instructions, but is that a common case?
+        and     SCRATCH, WK&reg1, MASK             @ 00000GGGGGG0000000000gggggg00000
+        bic     WK&reg1, WK&reg1, MASK             @ RRRRR000000BBBBBrrrrr000000bbbbb
+        orr     SCRATCH, SCRATCH, SCRATCH, lsr #6  @ 00000GGGGGGGGGGGG0000ggggggggggg
+        mov     WK&reg2, WK&reg1, lsr #16          @ 0000000000000000RRRRR000000BBBBB
+        mov     SCRATCH, SCRATCH, ror #27          @ GGGGGGGGGGGG0000ggggggggggg00000
+        bic     WK&reg1, WK&reg1, WK&reg2, lsl #16 @ 0000000000000000rrrrr000000bbbbb
+        mov     WK&reg2, WK&reg2, lsl #3           @ 0000000000000RRRRR000000BBBBB000
+        mov     WK&reg1, WK&reg1, lsl #3           @ 0000000000000rrrrr000000bbbbb000
+        orr     WK&reg2, WK&reg2, WK&reg2, lsr #5  @ 0000000000000RRRRRRRRRR0BBBBBBBB
+        orr     WK&reg1, WK&reg1, WK&reg1, lsr #5  @ 0000000000000rrrrrrrrrr0bbbbbbbb
+        pkhbt   WK&reg2, WK&reg2, WK&reg2, lsl #5  @ --------RRRRRRRR--------BBBBBBBB
+        pkhbt   WK&reg1, WK&reg1, WK&reg1, lsl #5  @ --------rrrrrrrr--------bbbbbbbb
+        sel     WK&reg2, SCRATCH, WK&reg2          @ --------RRRRRRRRGGGGGGGGBBBBBBBB
+        sel     WK&reg1, SCRATCH, WK&reg1          @ --------rrrrrrrrggggggggbbbbbbbb
+        orr     WK&reg2, WK&reg2, #0xFF000000      @ 11111111RRRRRRRRGGGGGGGGBBBBBBBB
+        orr     WK&reg1, WK&reg1, #0xFF000000      @ 11111111rrrrrrrrggggggggbbbbbbbb
+*/
+
+.macro src_0565_8888_1pixel, reg
+        bic     SCRATCH, WK&reg, MASK              @ 0000000000000000rrrrr000000bbbbb
+        and     WK&reg, WK&reg, MASK               @ 000000000000000000000gggggg00000
+        mov     SCRATCH, SCRATCH, lsl #3           @ 0000000000000rrrrr000000bbbbb000
+        mov     WK&reg, WK&reg, lsl #5             @ 0000000000000000gggggg0000000000
+        orr     SCRATCH, SCRATCH, SCRATCH, lsr #5  @ 0000000000000rrrrrrrrrr0bbbbbbbb
+        orr     WK&reg, WK&reg, WK&reg, lsr #6     @ 000000000000000gggggggggggg00000
+        pkhbt   SCRATCH, SCRATCH, SCRATCH, lsl #5  @ --------rrrrrrrr--------bbbbbbbb
+        sel     WK&reg, WK&reg, SCRATCH            @ --------rrrrrrrrggggggggbbbbbbbb
+        orr     WK&reg, WK&reg, #0xFF000000        @ 11111111rrrrrrrrggggggggbbbbbbbb
+.endm
+
+.macro src_0565_8888_process_head   cond, numbytes, firstreg, unaligned_src, unaligned_mask, preload
+ .if numbytes == 16
+        pixldst ld,, 8, firstreg, %(firstreg+2),,, SRC, unaligned_src
+ .elseif numbytes == 8
+        pixld   , 4, firstreg, SRC, unaligned_src
+ .elseif numbytes == 4
+        pixld   , 2, firstreg, SRC, unaligned_src
+ .endif
+.endm
+
+.macro src_0565_8888_process_tail   cond, numbytes, firstreg
+ .if numbytes == 16
+        src_0565_8888_2pixels firstreg, %(firstreg+1)
+        src_0565_8888_2pixels %(firstreg+2), %(firstreg+3)
+ .elseif numbytes == 8
+        src_0565_8888_2pixels firstreg, %(firstreg+1)
+ .else
+        src_0565_8888_1pixel firstreg
+ .endif
+.endm
+
+generate_composite_function \
+    pixman_composite_src_0565_8888_asm_armv6, 16, 0, 32, \
+    FLAG_DST_WRITEONLY | FLAG_BRANCH_OVER, \
+    3, /* prefetch distance */ \
+    src_0565_8888_init, \
+    nop_macro, /* newline */ \
+    nop_macro, /* cleanup */ \
+    src_0565_8888_process_head, \
+    src_0565_8888_process_tail
+
+/******************************************************************************/
+
+.macro add_8_8_8pixels  cond, dst1, dst2
+        uqadd8&cond  WK&dst1, WK&dst1, MASK
+        uqadd8&cond  WK&dst2, WK&dst2, STRIDE_M
+.endm
+
+.macro add_8_8_4pixels  cond, dst
+        uqadd8&cond  WK&dst, WK&dst, MASK
+.endm
+
+.macro add_8_8_process_head  cond, numbytes, firstreg, unaligned_src, unaligned_mask, preload
+    WK4     .req    MASK
+    WK5     .req    STRIDE_M
+ .if numbytes == 16
+        pixld   cond, 8, 4, SRC, unaligned_src
+        pixld   cond, 16, firstreg, DST, 0
+        add_8_8_8pixels cond, firstreg, %(firstreg+1)
+        pixld   cond, 8, 4, SRC, unaligned_src
+ .else
+        pixld   cond, numbytes, 4, SRC, unaligned_src
+        pixld   cond, numbytes, firstreg, DST, 0
+ .endif
+    .unreq  WK4
+    .unreq  WK5
+.endm
+
+.macro add_8_8_process_tail  cond, numbytes, firstreg
+ .if numbytes == 16
+        add_8_8_8pixels cond, %(firstreg+2), %(firstreg+3)
+ .elseif numbytes == 8
+        add_8_8_8pixels cond, firstreg, %(firstreg+1)
+ .else
+        add_8_8_4pixels cond, firstreg
+ .endif
+.endm
+
+generate_composite_function \
+    pixman_composite_add_8_8_asm_armv6, 8, 0, 8, \
+    FLAG_DST_READWRITE | FLAG_BRANCH_OVER | FLAG_PROCESS_PRESERVES_SCRATCH, \
+    2, /* prefetch distance */ \
+    nop_macro, /* init */ \
+    nop_macro, /* newline */ \
+    nop_macro, /* cleanup */ \
+    add_8_8_process_head, \
+    add_8_8_process_tail
+
+/******************************************************************************/
+
+.macro over_8888_8888_init
+        /* Hold loop invariant in MASK */
+        ldr     MASK, =0x00800080
+        /* Set GE[3:0] to 0101 so SEL instructions do what we want */
+        uadd8   SCRATCH, MASK, MASK
+        line_saved_regs STRIDE_D, STRIDE_S, ORIG_W
+.endm
+
+.macro over_8888_8888_process_head  cond, numbytes, firstreg, unaligned_src, unaligned_mask, preload
+    WK4     .req    STRIDE_D
+    WK5     .req    STRIDE_S
+    WK6     .req    STRIDE_M
+    WK7     .req    ORIG_W
+        pixld   , numbytes, %(4+firstreg), SRC, unaligned_src
+        pixld   , numbytes, firstreg, DST, 0
+    .unreq  WK4
+    .unreq  WK5
+    .unreq  WK6
+    .unreq  WK7
+.endm
+
+.macro over_8888_8888_check_transparent  numbytes, reg0, reg1, reg2, reg3
+        /* Since these colours a premultiplied by alpha, only 0 indicates transparent (any other colour with 0 in the alpha byte is luminous) */
+        teq     WK&reg0, #0
+ .if numbytes > 4
+        teqeq   WK&reg1, #0
+  .if numbytes > 8
+        teqeq   WK&reg2, #0
+        teqeq   WK&reg3, #0
+  .endif
+ .endif
+.endm
+
+.macro over_8888_8888_prepare  next
+        mov     WK&next, WK&next, lsr #24
+.endm
+
+.macro over_8888_8888_1pixel src, dst, offset, next
+        /* src = destination component multiplier */
+        rsb     WK&src, WK&src, #255
+        /* Split even/odd bytes of dst into SCRATCH/dst */
+        uxtb16  SCRATCH, WK&dst
+        uxtb16  WK&dst, WK&dst, ror #8
+        /* Multiply through, adding 0.5 to the upper byte of result for rounding */
+        mla     SCRATCH, SCRATCH, WK&src, MASK
+        mla     WK&dst, WK&dst, WK&src, MASK
+        /* Where we would have had a stall between the result of the first MLA and the shifter input,
+         * reload the complete source pixel */
+        ldr     WK&src, [SRC, #offset]
+        /* Multiply by 257/256 to approximate 256/255 */
+        uxtab16 SCRATCH, SCRATCH, SCRATCH, ror #8
+        /* In this stall, start processing the next pixel */
+ .if offset < -4
+        mov     WK&next, WK&next, lsr #24
+ .endif
+        uxtab16 WK&dst, WK&dst, WK&dst, ror #8
+        /* Recombine even/odd bytes of multiplied destination */
+        mov     SCRATCH, SCRATCH, ror #8
+        sel     WK&dst, SCRATCH, WK&dst
+        /* Saturated add of source to multiplied destination */
+        uqadd8  WK&dst, WK&dst, WK&src
+.endm
+
+.macro over_8888_8888_process_tail  cond, numbytes, firstreg
+    WK4     .req    STRIDE_D
+    WK5     .req    STRIDE_S
+    WK6     .req    STRIDE_M
+    WK7     .req    ORIG_W
+        over_8888_8888_check_transparent numbytes, %(4+firstreg), %(5+firstreg), %(6+firstreg), %(7+firstreg)
+        beq     10f
+        over_8888_8888_prepare  %(4+firstreg)
+ .set PROCESS_REG, firstreg
+ .set PROCESS_OFF, -numbytes
+ .rept numbytes / 4
+        over_8888_8888_1pixel %(4+PROCESS_REG), %(0+PROCESS_REG), PROCESS_OFF, %(5+PROCESS_REG)
+  .set PROCESS_REG, PROCESS_REG+1
+  .set PROCESS_OFF, PROCESS_OFF+4
+ .endr
+        pixst   , numbytes, firstreg, DST
+10:
+    .unreq  WK4
+    .unreq  WK5
+    .unreq  WK6
+    .unreq  WK7
+.endm
+
+generate_composite_function \
+    pixman_composite_over_8888_8888_asm_armv6, 32, 0, 32 \
+    FLAG_DST_READWRITE | FLAG_BRANCH_OVER | FLAG_PROCESS_CORRUPTS_PSR | FLAG_PROCESS_DOES_STORE | FLAG_SPILL_LINE_VARS \
+    2, /* prefetch distance */ \
+    over_8888_8888_init, \
+    nop_macro, /* newline */ \
+    nop_macro, /* cleanup */ \
+    over_8888_8888_process_head, \
+    over_8888_8888_process_tail
+
+/******************************************************************************/
+
+/* Multiply each byte of a word by a byte.
+ * Useful when there aren't any obvious ways to fill the stalls with other instructions.
+ * word  Register containing 4 bytes
+ * byte  Register containing byte multiplier (bits 8-31 must be 0)
+ * tmp   Scratch register
+ * half  Register containing the constant 0x00800080
+ * GE[3:0] bits must contain 0101
  */
+.macro mul_8888_8  word, byte, tmp, half
+        /* Split even/odd bytes of word apart */
+        uxtb16  tmp, word
+        uxtb16  word, word, ror #8
+        /* Multiply bytes together with rounding, then by 257/256 */
+        mla     tmp, tmp, byte, half
+        mla     word, word, byte, half /* 1 stall follows */
+        uxtab16 tmp, tmp, tmp, ror #8  /* 1 stall follows */
+        uxtab16 word, word, word, ror #8
+        /* Recombine bytes */
+        mov     tmp, tmp, ror #8
+        sel     word, tmp, word
+.endm
+
+/******************************************************************************/
+
+.macro over_8888_n_8888_init
+        /* Mask is constant */
+        ldr     MASK, [sp, #ARGS_STACK_OFFSET+8]
+        /* Hold loop invariant in STRIDE_M */
+        ldr     STRIDE_M, =0x00800080
+        /* We only want the alpha bits of the constant mask */
+        mov     MASK, MASK, lsr #24
+        /* Set GE[3:0] to 0101 so SEL instructions do what we want */
+        uadd8   SCRATCH, STRIDE_M, STRIDE_M
+        line_saved_regs Y, STRIDE_D, STRIDE_S, ORIG_W
+.endm
+
+.macro over_8888_n_8888_process_head  cond, numbytes, firstreg, unaligned_src, unaligned_mask, preload
+    WK4     .req    Y
+    WK5     .req    STRIDE_D
+    WK6     .req    STRIDE_S
+    WK7     .req    ORIG_W
+        pixld   , numbytes, %(4+(firstreg%2)), SRC, unaligned_src
+        pixld   , numbytes, firstreg, DST, 0
+    .unreq  WK4
+    .unreq  WK5
+    .unreq  WK6
+    .unreq  WK7
+.endm
+
+.macro over_8888_n_8888_1pixel src, dst
+        mul_8888_8  WK&src, MASK, SCRATCH, STRIDE_M
+        sub     WK7, WK6, WK&src, lsr #24
+        mul_8888_8  WK&dst, WK7, SCRATCH, STRIDE_M
+        uqadd8  WK&dst, WK&dst, WK&src
+.endm
+
+.macro over_8888_n_8888_process_tail  cond, numbytes, firstreg
+    WK4     .req    Y
+    WK5     .req    STRIDE_D
+    WK6     .req    STRIDE_S
+    WK7     .req    ORIG_W
+        over_8888_8888_check_transparent numbytes, %(4+(firstreg%2)), %(5+(firstreg%2)), %(6+firstreg), %(7+firstreg)
+        beq     10f
+        mov     WK6, #255
+ .set PROCESS_REG, firstreg
+ .rept numbytes / 4
+  .if numbytes == 16 && PROCESS_REG == 2
+        /* We're using WK6 and WK7 as temporaries, so half way through
+         * 4 pixels, reload the second two source pixels but this time
+         * into WK4 and WK5 */
+        ldmdb   SRC, {WK4, WK5}
+  .endif
+        over_8888_n_8888_1pixel  %(4+(PROCESS_REG%2)), %(PROCESS_REG)
+  .set PROCESS_REG, PROCESS_REG+1
+ .endr
+        pixst   , numbytes, firstreg, DST
+10:
+    .unreq  WK4
+    .unreq  WK5
+    .unreq  WK6
+    .unreq  WK7
+.endm
+
+generate_composite_function \
+    pixman_composite_over_8888_n_8888_asm_armv6, 32, 0, 32 \
+    FLAG_DST_READWRITE | FLAG_BRANCH_OVER | FLAG_PROCESS_CORRUPTS_PSR | FLAG_PROCESS_DOES_STORE | FLAG_SPILL_LINE_VARS \
+    2, /* prefetch distance */ \
+    over_8888_n_8888_init, \
+    nop_macro, /* newline */ \
+    nop_macro, /* cleanup */ \
+    over_8888_n_8888_process_head, \
+    over_8888_n_8888_process_tail
+
+/******************************************************************************/
+
+.macro over_n_8_8888_init
+        /* Source is constant, but splitting it into even/odd bytes is a loop invariant */
+        ldr     SRC, [sp, #ARGS_STACK_OFFSET]
+        /* Not enough registers to hold this constant, but we still use it here to set GE[3:0] */
+        ldr     SCRATCH, =0x00800080
+        uxtb16  STRIDE_S, SRC
+        uxtb16  SRC, SRC, ror #8
+        /* Set GE[3:0] to 0101 so SEL instructions do what we want */
+        uadd8   SCRATCH, SCRATCH, SCRATCH
+        line_saved_regs Y, STRIDE_D, STRIDE_M, ORIG_W
+.endm
+
+.macro over_n_8_8888_newline
+        ldr     STRIDE_D, =0x00800080
+        b       1f
+ .ltorg
+1:
+.endm
+
+.macro over_n_8_8888_process_head  cond, numbytes, firstreg, unaligned_src, unaligned_mask, preload
+    WK4     .req    STRIDE_M
+        pixld   , numbytes/4, 4, MASK, unaligned_mask
+        pixld   , numbytes, firstreg, DST, 0
+    .unreq  WK4
+.endm
+
+.macro over_n_8_8888_1pixel src, dst
+        uxtb    Y, WK4, ror #src*8
+        /* Trailing part of multiplication of source */
+        mla     SCRATCH, STRIDE_S, Y, STRIDE_D
+        mla     Y, SRC, Y, STRIDE_D
+        mov     ORIG_W, #255
+        uxtab16 SCRATCH, SCRATCH, SCRATCH, ror #8
+        uxtab16 Y, Y, Y, ror #8
+        mov     SCRATCH, SCRATCH, ror #8
+        sub     ORIG_W, ORIG_W, Y, lsr #24
+        sel     Y, SCRATCH, Y
+        /* Then multiply the destination */
+        mul_8888_8  WK&dst, ORIG_W, SCRATCH, STRIDE_D
+        uqadd8  WK&dst, WK&dst, Y
+.endm
+
+.macro over_n_8_8888_process_tail  cond, numbytes, firstreg
+    WK4     .req    STRIDE_M
+        teq     WK4, #0
+        beq     10f
+ .set PROCESS_REG, firstreg
+ .rept numbytes / 4
+        over_n_8_8888_1pixel  %(PROCESS_REG-firstreg), %(PROCESS_REG)
+  .set PROCESS_REG, PROCESS_REG+1
+ .endr
+        pixst   , numbytes, firstreg, DST
+10:
+    .unreq  WK4
+.endm
+
+generate_composite_function \
+    pixman_composite_over_n_8_8888_asm_armv6, 0, 8, 32 \
+    FLAG_DST_READWRITE | FLAG_BRANCH_OVER | FLAG_PROCESS_CORRUPTS_PSR | FLAG_PROCESS_DOES_STORE | FLAG_SPILL_LINE_VARS \
+    2, /* prefetch distance */ \
+    over_n_8_8888_init, \
+    over_n_8_8888_newline, \
+    nop_macro, /* cleanup */ \
+    over_n_8_8888_process_head, \
+    over_n_8_8888_process_tail
+
+/******************************************************************************/
 
-pixman_asm_function pixman_composite_add_8000_8000_asm_armv6
-	push	{r4, r5, r6, r7, r8, r9, r10, r11}
-	mov	r10, r1
-	sub	sp, sp, #4
-	subs	r10, r10, #1
-	mov	r11, r0
-	mov	r8, r2
-	str	r3, [sp]
-	ldr	r7, [sp, #36]
-	bcc	0f
-6:	cmp	r11, #0
-	beq	1f
-	orr	r3, r8, r7
-	tst	r3, #3
-	beq	2f
-	mov	r1, r8
-	mov	r0, r7
-	mov	r12, r11
-	b	3f
-5:	tst	r3, #3
-	beq	4f
-3:	ldrb	r2, [r0], #1
-	subs	r12, r12, #1
-	ldrb	r3, [r1]
-	uqadd8	r3, r2, r3
-	strb	r3, [r1], #1
-	orr	r3, r1, r0
-	bne	5b
-1:	ldr	r3, [sp]
-	add	r8, r8, r3
-	ldr	r3, [sp, #40]
-	add	r7, r7, r3
-10:	subs	r10, r10, #1
-	bcs	6b
-0:	add	sp, sp, #4
-	pop	{r4, r5, r6, r7, r8, r9, r10, r11}
-	bx	lr
-2:	mov	r12, r11
-	mov	r1, r8
-	mov	r0, r7
-4:	cmp	r12, #3
-	subgt	r6, r12, #4
-	movgt	r9, r12
-	lsrgt	r5, r6, #2
-	addgt	r3, r5, #1
-	movgt	r12, #0
-	lslgt	r4, r3, #2
-	ble	7f
-8:	ldr	r3, [r0, r12]
-	ldr	r2, [r1, r12]
-	uqadd8	r3, r3, r2
-	str	r3, [r1, r12]
-	add	r12, r12, #4
-	cmp	r12, r4
-	bne	8b
-	sub	r3, r9, #4
-	bic	r3, r3, #3
-	add	r3, r3, #4
-	subs	r12, r6, r5, lsl #2
-	add	r1, r1, r3
-	add	r0, r0, r3
-	beq	1b
-7:	mov	r4, #0
-9:	ldrb	r3, [r1, r4]
-	ldrb	r2, [r0, r4]
-	uqadd8	r3, r2, r3
-	strb	r3, [r1, r4]
-	add	r4, r4, #1
-	cmp	r4, r12
-	bne	9b
-	ldr	r3, [sp]
-	add	r8, r8, r3
-	ldr	r3, [sp, #40]
-	add	r7, r7, r3
-	b	10b
-.endfunc
-
-pixman_asm_function pixman_composite_over_8888_8888_asm_armv6
-	push	{r4, r5, r6, r7, r8, r9, r10, r11}
-	sub	sp, sp, #20
-	cmp	r1, #0
-	mov	r12, r2
-	str	r1, [sp, #12]
-	str	r0, [sp, #16]
-	ldr	r2, [sp, #52]
-	beq	0f
-	lsl	r3, r3, #2
-	str	r3, [sp]
-	ldr	r3, [sp, #56]
-	mov	r10, #0
-	lsl	r3, r3, #2
-	str	r3, [sp, #8]
-	mov	r11, r3
-	b	1f
-6:	ldr	r11, [sp, #8]
-1:	ldr	r9, [sp]
-	mov	r0, r12
-	add	r12, r12, r9
-	mov	r1, r2
-	str	r12, [sp, #4]
-	add	r2, r2, r11
-	ldr	r12, [sp, #16]
-	ldr	r3, =0x00800080
-	ldr	r9, =0xff00ff00
-	mov	r11, #255
-	cmp	r12, #0
-	beq	4f
-5:	ldr	r5, [r1], #4
-	ldr	r4, [r0]
-	sub	r8, r11, r5, lsr #24
-	uxtb16	r6, r4
-	uxtb16	r7, r4, ror #8
-	mla	r6, r6, r8, r3
-	mla	r7, r7, r8, r3
-	uxtab16	r6, r6, r6, ror #8
-	uxtab16	r7, r7, r7, ror #8
-	and	r7, r7, r9
-	uxtab16	r6, r7, r6, ror #8
-	uqadd8	r5, r6, r5
-	str	r5, [r0], #4
-	subs	r12, r12, #1
-	bne	5b
-4:	ldr	r3, [sp, #12]
-	add	r10, r10, #1
-	cmp	r10, r3
-	ldr	r12, [sp, #4]
-	bne	6b
-0:	add	sp, sp, #20
-	pop	{r4, r5, r6, r7, r8, r9, r10, r11}
-	bx	lr
-.endfunc
-
-pixman_asm_function pixman_composite_over_8888_n_8888_asm_armv6
-	push	{r4, r5, r6, r7, r8, r9, r10, r11}
-	sub	sp, sp, #28
-	cmp	r1, #0
-	str	r1, [sp, #12]
-	ldrb	r1, [sp, #71]
-	mov	r12, r2
-	str	r0, [sp, #16]
-	ldr	r2, [sp, #60]
-	str	r1, [sp, #24]
-	beq	0f
-	lsl	r3, r3, #2
-	str	r3, [sp, #20]
-	ldr	r3, [sp, #64]
-	mov	r10, #0
-	lsl	r3, r3, #2
-	str	r3, [sp, #8]
-	mov	r11, r3
-	b	1f
-5:	ldr	r11, [sp, #8]
-1:	ldr	r4, [sp, #20]
-	mov	r0, r12
-	mov	r1, r2
-	add	r12, r12, r4
-	add	r2, r2, r11
-	str	r12, [sp]
-	str	r2, [sp, #4]
-	ldr	r12, [sp, #16]
-	ldr	r2, =0x00800080
-	ldr	r3, [sp, #24]
-	mov	r11, #255
-	cmp	r12, #0
-	beq	3f
-4:	ldr	r5, [r1], #4
-	ldr	r4, [r0]
-	uxtb16	r6, r5
-	uxtb16	r7, r5, ror #8
-	mla	r6, r6, r3, r2
-	mla	r7, r7, r3, r2
-	uxtab16	r6, r6, r6, ror #8
-	uxtab16	r7, r7, r7, ror #8
-	uxtb16	r6, r6, ror #8
-	uxtb16	r7, r7, ror #8
-	orr	r5, r6, r7, lsl #8
-	uxtb16	r6, r4
-	uxtb16	r7, r4, ror #8
-	sub	r8, r11, r5, lsr #24
-	mla	r6, r6, r8, r2
-	mla	r7, r7, r8, r2
-	uxtab16	r6, r6, r6, ror #8
-	uxtab16	r7, r7, r7, ror #8
-	uxtb16	r6, r6, ror #8
-	uxtb16	r7, r7, ror #8
-	orr	r6, r6, r7, lsl #8
-	uqadd8	r5, r6, r5
-	str	r5, [r0], #4
-	subs	r12, r12, #1
-	bne	4b
-3:	ldr	r1, [sp, #12]
-	add	r10, r10, #1
-	cmp	r10, r1
-	ldr	r12, [sp]
-	ldr	r2, [sp, #4]
-	bne	5b
-0:	add	sp, sp, #28
-	pop	{r4, r5, r6, r7, r8, r9, r10, r11}
-	bx	lr
-.endfunc
-
-pixman_asm_function pixman_composite_over_n_8_8888_asm_armv6
-	push	{r4, r5, r6, r7, r8, r9, r10, r11}
-	sub	sp, sp, #28
-	cmp	r1, #0
-	ldr	r9, [sp, #60]
-	str	r1, [sp, #12]
-	bic	r1, r9, #-16777216
-	str	r1, [sp, #20]
-	mov	r12, r2
-	lsr	r1, r9, #8
-	ldr	r2, [sp, #20]
-	bic	r1, r1, #-16777216
-	bic	r2, r2, #65280
-	bic	r1, r1, #65280
-	str	r2, [sp, #20]
-	str	r0, [sp, #16]
-	str	r1, [sp, #4]
-	ldr	r2, [sp, #68]
-	beq	0f
-	lsl	r3, r3, #2
-	str	r3, [sp, #24]
-	mov	r0, #0
-	b	1f
-5:	ldr	r3, [sp, #24]
-1:	ldr	r4, [sp, #72]
-	mov	r10, r12
-	mov	r1, r2
-	add	r12, r12, r3
-	add	r2, r2, r4
-	str	r12, [sp, #8]
-	str	r2, [sp]
-	ldr	r12, [sp, #16]
-	ldr	r11, =0x00800080
-	ldr	r2, [sp, #4]
-	ldr	r3, [sp, #20]
-	cmp	r12, #0
-	beq	3f
-4:	ldrb	r5, [r1], #1
-	ldr	r4, [r10]
-	mla	r6, r3, r5, r11
-	mla	r7, r2, r5, r11
-	uxtab16	r6, r6, r6, ror #8
-	uxtab16	r7, r7, r7, ror #8
-	uxtb16	r6, r6, ror #8
-	uxtb16	r7, r7, ror #8
-	orr	r5, r6, r7, lsl #8
-	uxtb16	r6, r4
-	uxtb16	r7, r4, ror #8
-	mvn	r8, r5
-	lsr	r8, r8, #24
-	mla	r6, r6, r8, r11
-	mla	r7, r7, r8, r11
-	uxtab16	r6, r6, r6, ror #8
-	uxtab16	r7, r7, r7, ror #8
-	uxtb16	r6, r6, ror #8
-	uxtb16	r7, r7, ror #8
-	orr	r6, r6, r7, lsl #8
-	uqadd8	r5, r6, r5
-	str	r5, [r10], #4
-	subs	r12, r12, #1
-	bne	4b
-3:	ldr	r4, [sp, #12]
-	add	r0, r0, #1
-	cmp	r0, r4
-	ldr	r12, [sp, #8]
-	ldr	r2, [sp]
-	bne	5b
-0:	add	sp, sp, #28
-	pop	{r4, r5, r6, r7, r8, r9, r10, r11}
-	bx	lr
-.endfunc
diff --git a/pixman/pixman/pixman-arm-simd-asm.h b/pixman/pixman/pixman-arm-simd-asm.h
new file mode 100644
index 000000000..65436062b
--- /dev/null
+++ b/pixman/pixman/pixman-arm-simd-asm.h
@@ -0,0 +1,908 @@
+/*
+ * Copyright © 2012 Raspberry Pi Foundation
+ * Copyright © 2012 RISC OS Open Ltd
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of the copyright holders not be used in
+ * advertising or publicity pertaining to distribution of the software without
+ * specific, written prior permission.  The copyright holders make no
+ * representations about the suitability of this software for any purpose.  It
+ * is provided "as is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ *
+ * Author:  Ben Avison (bavison@riscosopen.org)
+ *
+ */
+
+/*
+ * Because the alignment of pixel data to cachelines, and even the number of
+ * cachelines per row can vary from row to row, and because of the need to
+ * preload each scanline once and only once, this prefetch strategy treats
+ * each row of pixels independently. When a pixel row is long enough, there
+ * are three distinct phases of prefetch:
+ * * an inner loop section, where each time a cacheline of data is
+ *    processed, another cacheline is preloaded (the exact distance ahead is
+ *    determined empirically using profiling results from lowlevel-blt-bench)
+ * * a leading section, where enough cachelines are preloaded to ensure no
+ *    cachelines escape being preloaded when the inner loop starts
+ * * a trailing section, where a limited number (0 or more) of cachelines
+ *    are preloaded to deal with data (if any) that hangs off the end of the
+ *    last iteration of the inner loop, plus any trailing bytes that were not
+ *    enough to make up one whole iteration of the inner loop
+ * 
+ * There are (in general) three distinct code paths, selected between
+ * depending upon how long the pixel row is. If it is long enough that there
+ * is at least one iteration of the inner loop (as described above) then
+ * this is described as the "wide" case. If it is shorter than that, but
+ * there are still enough bytes output that there is at least one 16-byte-
+ * long, 16-byte-aligned write to the destination (the optimum type of
+ * write), then this is the "medium" case. If it is not even this long, then
+ * this is the "narrow" case, and there is no attempt to align writes to
+ * 16-byte boundaries. In the "medium" and "narrow" cases, all the
+ * cachelines containing data from the pixel row are prefetched up-front.
+ */
+
+/*
+ * Determine whether we put the arguments on the stack for debugging.
+ */
+#undef DEBUG_PARAMS
+
+/*
+ * Bit flags for 'generate_composite_function' macro which are used
+ * to tune generated functions behavior.
+ */
+.set FLAG_DST_WRITEONLY,         0
+.set FLAG_DST_READWRITE,         1
+.set FLAG_COND_EXEC,             0
+.set FLAG_BRANCH_OVER,           2
+.set FLAG_PROCESS_PRESERVES_PSR, 0
+.set FLAG_PROCESS_CORRUPTS_PSR,  4
+.set FLAG_PROCESS_DOESNT_STORE,  0
+.set FLAG_PROCESS_DOES_STORE,    8 /* usually because it needs to conditionally skip it */
+.set FLAG_NO_SPILL_LINE_VARS,        0
+.set FLAG_SPILL_LINE_VARS_WIDE,      16
+.set FLAG_SPILL_LINE_VARS_NON_WIDE,  32
+.set FLAG_SPILL_LINE_VARS,           48
+.set FLAG_PROCESS_CORRUPTS_SCRATCH,  0
+.set FLAG_PROCESS_PRESERVES_SCRATCH, 64
+
+/*
+ * Offset into stack where mask and source pointer/stride can be accessed.
+ */
+#ifdef DEBUG_PARAMS
+.set ARGS_STACK_OFFSET,        (9*4+9*4)
+#else
+.set ARGS_STACK_OFFSET,        (9*4)
+#endif
+
+/*
+ * Constants for selecting preferable prefetch type.
+ */
+.set PREFETCH_TYPE_NONE,       0
+.set PREFETCH_TYPE_STANDARD,   1
+
+/*
+ * Definitions of macros for load/store of pixel data.
+ */
+
+.macro pixldst op, cond=al, numbytes, reg0, reg1, reg2, reg3, base, unaligned=0
+ .if numbytes == 16
+  .if unaligned == 1
+        op&r&cond    WK&reg0, [base], #4
+        op&r&cond    WK&reg1, [base], #4
+        op&r&cond    WK&reg2, [base], #4
+        op&r&cond    WK&reg3, [base], #4
+  .else
+        op&m&cond&ia base!, {WK&reg0,WK&reg1,WK&reg2,WK&reg3}
+  .endif
+ .elseif numbytes == 8
+  .if unaligned == 1
+        op&r&cond    WK&reg0, [base], #4
+        op&r&cond    WK&reg1, [base], #4
+  .else
+        op&m&cond&ia base!, {WK&reg0,WK&reg1}
+  .endif
+ .elseif numbytes == 4
+        op&r&cond    WK&reg0, [base], #4
+ .elseif numbytes == 2
+        op&r&cond&h  WK&reg0, [base], #2
+ .elseif numbytes == 1
+        op&r&cond&b  WK&reg0, [base], #1
+ .else
+  .error "unsupported size: numbytes"
+ .endif
+.endm
+
+.macro pixst_baseupdated cond, numbytes, reg0, reg1, reg2, reg3, base
+ .if numbytes == 16
+        stm&cond&db base, {WK&reg0,WK&reg1,WK&reg2,WK&reg3}
+ .elseif numbytes == 8
+        stm&cond&db base, {WK&reg0,WK&reg1}
+ .elseif numbytes == 4
+        str&cond    WK&reg0, [base, #-4]
+ .elseif numbytes == 2
+        str&cond&h  WK&reg0, [base, #-2]
+ .elseif numbytes == 1
+        str&cond&b  WK&reg0, [base, #-1]
+ .else
+  .error "unsupported size: numbytes"
+ .endif
+.endm
+
+.macro pixld cond, numbytes, firstreg, base, unaligned
+        pixldst ld, cond, numbytes, %(firstreg+0), %(firstreg+1), %(firstreg+2), %(firstreg+3), base, unaligned
+.endm
+
+.macro pixst cond, numbytes, firstreg, base
+ .if (flags) & FLAG_DST_READWRITE
+        pixst_baseupdated cond, numbytes, %(firstreg+0), %(firstreg+1), %(firstreg+2), %(firstreg+3), base
+ .else
+        pixldst st, cond, numbytes, %(firstreg+0), %(firstreg+1), %(firstreg+2), %(firstreg+3), base
+ .endif
+.endm
+
+.macro PF a, x:vararg
+ .if (PREFETCH_TYPE_CURRENT == PREFETCH_TYPE_STANDARD)
+        a x
+ .endif
+.endm
+
+
+.macro preload_leading_step1  bpp, ptr, base
+/* If the destination is already 16-byte aligned, then we need to preload
+ * between 0 and prefetch_distance (inclusive) cache lines ahead so there
+ * are no gaps when the inner loop starts.
+ */
+ .if bpp > 0
+        PF  bic,    ptr, base, #31
+  .set OFFSET, 0
+  .rept prefetch_distance+1
+        PF  pld,    [ptr, #OFFSET]
+   .set OFFSET, OFFSET+32
+  .endr
+ .endif
+.endm
+
+.macro preload_leading_step2  bpp, bpp_shift, ptr, base
+/* However, if the destination is not 16-byte aligned, we may need to
+ * preload more cache lines than that. The question we need to ask is:
+ * are the bytes corresponding to the leading pixels more than the amount
+ * by which the source pointer will be rounded down for preloading, and if
+ * so, by how many cache lines? Effectively, we want to calculate
+ *     leading_bytes = ((-dst)&15)*src_bpp/dst_bpp
+ *     inner_loop_offset = (src+leading_bytes)&31
+ *     extra_needed = leading_bytes - inner_loop_offset
+ * and test if extra_needed is <= 0, <= 32, or > 32 (where > 32 is only
+ * possible when there are 4 src bytes for every 1 dst byte).
+ */
+ .if bpp > 0
+  .ifc base,DST
+        /* The test can be simplified further when preloading the destination */
+        PF  tst,    base, #16
+        PF  beq,    61f
+  .else
+   .if bpp/dst_w_bpp == 4
+        PF  add,    SCRATCH, base, WK0, lsl #bpp_shift-dst_bpp_shift
+        PF  and,    SCRATCH, SCRATCH, #31
+        PF  rsb,    SCRATCH, SCRATCH, WK0, lsl #bpp_shift-dst_bpp_shift
+        PF  sub,    SCRATCH, SCRATCH, #1    /* so now ranges are -16..-1 / 0..31 / 32..63 */
+        PF  movs,   SCRATCH, SCRATCH, #32-6 /* so this sets         NC   /  nc   /   Nc   */
+        PF  bcs,    61f
+        PF  bpl,    60f
+        PF  pld,    [ptr, #32*(prefetch_distance+2)]
+   .else
+        PF  mov,    SCRATCH, base, lsl #32-5
+        PF  add,    SCRATCH, SCRATCH, WK0, lsl #32-5+bpp_shift-dst_bpp_shift
+        PF  rsbs,   SCRATCH, SCRATCH, WK0, lsl #32-5+bpp_shift-dst_bpp_shift
+        PF  bls,    61f
+   .endif
+  .endif
+60:     PF  pld,    [ptr, #32*(prefetch_distance+1)]
+61:
+ .endif
+.endm
+
+#define IS_END_OF_GROUP(INDEX,SIZE) ((SIZE) < 2 || ((INDEX) & ~((INDEX)+1)) & ((SIZE)/2))
+.macro preload_middle   bpp, base, scratch_holds_offset
+ .if bpp > 0
+        /* prefetch distance = 256/bpp, stm distance = 128/dst_w_bpp */
+  .if IS_END_OF_GROUP(SUBBLOCK,256/128*dst_w_bpp/bpp)
+   .if scratch_holds_offset
+        PF  pld,    [base, SCRATCH]
+   .else
+        PF  bic,    SCRATCH, base, #31
+        PF  pld,    [SCRATCH, #32*prefetch_distance]
+   .endif
+  .endif
+ .endif
+.endm
+
+.macro preload_trailing  bpp, bpp_shift, base
+ .if bpp > 0
+  .if bpp*pix_per_block > 256
+        /* Calculations are more complex if more than one fetch per block */
+        PF  and,    WK1, base, #31
+        PF  add,    WK1, WK1, WK0, lsl #bpp_shift
+        PF  add,    WK1, WK1, #32*(bpp*pix_per_block/256-1)*(prefetch_distance+1)
+        PF  bic,    SCRATCH, base, #31
+80:     PF  pld,    [SCRATCH, #32*(prefetch_distance+1)]
+        PF  add,    SCRATCH, SCRATCH, #32
+        PF  subs,   WK1, WK1, #32
+        PF  bhi,    80b
+  .else
+        /* If exactly one fetch per block, then we need either 0, 1 or 2 extra preloads */
+        PF  mov,    SCRATCH, base, lsl #32-5
+        PF  adds,   SCRATCH, SCRATCH, X, lsl #32-5+bpp_shift
+        PF  adceqs, SCRATCH, SCRATCH, #0
+        /* The instruction above has two effects: ensures Z is only
+         * set if C was clear (so Z indicates that both shifted quantities
+         * were 0), and clears C if Z was set (so C indicates that the sum
+         * of the shifted quantities was greater and not equal to 32) */
+        PF  beq,    82f
+        PF  bic,    SCRATCH, base, #31
+        PF  bcc,    81f
+        PF  pld,    [SCRATCH, #32*(prefetch_distance+2)]
+81:     PF  pld,    [SCRATCH, #32*(prefetch_distance+1)]
+82:
+  .endif
+ .endif
+.endm
+
+
+.macro preload_line    narrow_case, bpp, bpp_shift, base
+/* "narrow_case" - just means that the macro was invoked from the "narrow"
+ *    code path rather than the "medium" one - because in the narrow case,
+ *    the row of pixels is known to output no more than 30 bytes, then
+ *    (assuming the source pixels are no wider than the the destination
+ *    pixels) they cannot possibly straddle more than 2 32-byte cachelines,
+ *    meaning there's no need for a loop.
+ * "bpp" - number of bits per pixel in the channel (source, mask or
+ *    destination) that's being preloaded, or 0 if this channel is not used
+ *    for reading
+ * "bpp_shift" - log2 of ("bpp"/8) (except if "bpp"=0 of course)
+ * "base" - base address register of channel to preload (SRC, MASK or DST)
+ */
+ .if bpp > 0
+  .if narrow_case && (bpp <= dst_w_bpp)
+        /* In these cases, each line for each channel is in either 1 or 2 cache lines */
+        PF  bic,    WK0, base, #31
+        PF  pld,    [WK0]
+        PF  add,    WK1, base, X, LSL #bpp_shift
+        PF  sub,    WK1, WK1, #1
+        PF  bic,    WK1, WK1, #31
+        PF  cmp,    WK1, WK0
+        PF  beq,    90f
+        PF  pld,    [WK1]
+90:
+  .else
+        PF  bic,    WK0, base, #31
+        PF  pld,    [WK0]
+        PF  add,    WK1, base, X, lsl #bpp_shift
+        PF  sub,    WK1, WK1, #1
+        PF  bic,    WK1, WK1, #31
+        PF  cmp,    WK1, WK0
+        PF  beq,    92f
+91:     PF  add,    WK0, WK0, #32
+        PF  cmp,    WK0, WK1
+        PF  pld,    [WK0]
+        PF  bne,    91b
+92:
+  .endif
+ .endif
+.endm
+
+
+.macro conditional_process1_helper  cond, process_head, process_tail, numbytes, firstreg, unaligned_src, unaligned_mask, decrementx
+        process_head  cond, numbytes, firstreg, unaligned_src, unaligned_mask, 0
+ .if decrementx
+        sub&cond X, X, #8*numbytes/dst_w_bpp
+ .endif
+        process_tail  cond, numbytes, firstreg
+ .if !((flags) & FLAG_PROCESS_DOES_STORE)
+        pixst   cond, numbytes, firstreg, DST
+ .endif
+.endm
+
+.macro conditional_process1  cond, process_head, process_tail, numbytes, firstreg, unaligned_src, unaligned_mask, decrementx
+ .if (flags) & FLAG_BRANCH_OVER
+  .ifc cond,mi
+        bpl     100f
+  .endif
+  .ifc cond,cs
+        bcc     100f
+  .endif
+  .ifc cond,ne
+        beq     100f
+  .endif
+        conditional_process1_helper  , process_head, process_tail, numbytes, firstreg, unaligned_src, unaligned_mask, decrementx
+100:
+ .else
+        conditional_process1_helper  cond, process_head, process_tail, numbytes, firstreg, unaligned_src, unaligned_mask, decrementx
+ .endif
+.endm
+
+.macro conditional_process2  test, cond1, cond2, process_head, process_tail, numbytes1, numbytes2, firstreg1, firstreg2, unaligned_src, unaligned_mask, decrementx
+ .if (flags) & (FLAG_DST_READWRITE | FLAG_BRANCH_OVER | FLAG_PROCESS_CORRUPTS_PSR | FLAG_PROCESS_DOES_STORE)
+        /* Can't interleave reads and writes */
+        test
+        conditional_process1  cond1, process_head, process_tail, numbytes1, firstreg1, unaligned_src, unaligned_mask, decrementx
+  .if (flags) & FLAG_PROCESS_CORRUPTS_PSR
+        test
+  .endif
+        conditional_process1  cond2, process_head, process_tail, numbytes2, firstreg2, unaligned_src, unaligned_mask, decrementx
+ .else
+        /* Can interleave reads and writes for better scheduling */
+        test
+        process_head  cond1, numbytes1, firstreg1, unaligned_src, unaligned_mask, 0
+        process_head  cond2, numbytes2, firstreg2, unaligned_src, unaligned_mask, 0
+  .if decrementx
+        sub&cond1 X, X, #8*numbytes1/dst_w_bpp
+        sub&cond2 X, X, #8*numbytes2/dst_w_bpp
+  .endif
+        process_tail  cond1, numbytes1, firstreg1
+        process_tail  cond2, numbytes2, firstreg2
+        pixst   cond1, numbytes1, firstreg1, DST
+        pixst   cond2, numbytes2, firstreg2, DST
+ .endif
+.endm
+
+
+.macro test_bits_1_0_ptr
+        movs    SCRATCH, WK0, lsl #32-1  /* C,N = bits 1,0 of DST */
+.endm
+
+.macro test_bits_3_2_ptr
+        movs    SCRATCH, WK0, lsl #32-3  /* C,N = bits 3, 2 of DST */
+.endm
+
+.macro leading_15bytes  process_head, process_tail
+        /* On entry, WK0 bits 0-3 = number of bytes until destination is 16-byte aligned */
+        /* Use unaligned loads in all cases for simplicity */
+ .if dst_w_bpp == 8
+        conditional_process2  test_bits_1_0_ptr, mi, cs, process_head, process_tail, 1, 2, 1, 2, 1, 1, 1
+ .elseif dst_w_bpp == 16
+        test_bits_1_0_ptr
+        conditional_process1  cs, process_head, process_tail, 2, 2, 1, 1, 1
+ .endif
+        conditional_process2  test_bits_3_2_ptr, mi, cs, process_head, process_tail, 4, 8, 1, 2, 1, 1, 1
+.endm
+
+.macro test_bits_3_2_pix
+        movs    SCRATCH, X, lsl #dst_bpp_shift+32-3
+.endm
+
+.macro test_bits_1_0_pix
+ .if dst_w_bpp == 8
+        movs    SCRATCH, X, lsl #dst_bpp_shift+32-1
+ .else
+        movs    SCRATCH, X, lsr #1
+ .endif
+.endm
+
+.macro trailing_15bytes  process_head, process_tail, unaligned_src, unaligned_mask
+        conditional_process2  test_bits_3_2_pix, cs, mi, process_head, process_tail, 8, 4, 0, 2, unaligned_src, unaligned_mask, 0
+ .if dst_w_bpp == 16
+        test_bits_1_0_pix
+        conditional_process1  cs, process_head, process_tail, 2, 0, unaligned_src, unaligned_mask, 0
+ .elseif dst_w_bpp == 8
+        conditional_process2  test_bits_1_0_pix, cs, mi, process_head, process_tail, 2, 1, 0, 1, unaligned_src, unaligned_mask, 0
+ .endif
+.endm
+
+
+.macro wide_case_inner_loop  process_head, process_tail, unaligned_src, unaligned_mask, dst_alignment
+110:
+ .set SUBBLOCK, 0 /* this is a count of STMs; there can be up to 8 STMs per block */
+ .rept pix_per_block*dst_w_bpp/128
+        process_head  , 16, 0, unaligned_src, unaligned_mask, 1
+  .if (src_bpp > 0) && (mask_bpp == 0) && ((flags) & FLAG_PROCESS_PRESERVES_SCRATCH)
+        preload_middle  src_bpp, SRC, 1
+  .elseif (src_bpp == 0) && (mask_bpp > 0) && ((flags) & FLAG_PROCESS_PRESERVES_SCRATCH)
+        preload_middle  mask_bpp, MASK, 1
+  .else
+        preload_middle  src_bpp, SRC, 0
+        preload_middle  mask_bpp, MASK, 0
+  .endif
+  .if (dst_r_bpp > 0) && ((SUBBLOCK % 2) == 0)
+        /* Because we know that writes are 16-byte aligned, it's relatively easy to ensure that
+         * destination prefetches are 32-byte aligned. It's also the easiest channel to offset
+         * preloads for, to achieve staggered prefetches for multiple channels, because there are
+         * always two STMs per prefetch, so there is always an opposite STM on which to put the
+         * preload. Note, no need to BIC the base register here */
+        PF  pld,    [DST, #32*prefetch_distance - dst_alignment]
+  .endif
+        process_tail  , 16, 0
+  .if !((flags) & FLAG_PROCESS_DOES_STORE)
+        pixst   , 16, 0, DST
+  .endif
+  .set SUBBLOCK, SUBBLOCK+1
+ .endr
+        subs    X, X, #pix_per_block
+        bhs     110b
+.endm
+
+.macro wide_case_inner_loop_and_trailing_pixels  process_head, process_tail, process_inner_loop, exit_label, unaligned_src, unaligned_mask
+        /* Destination now 16-byte aligned; we have at least one block before we have to stop preloading */
+ .if dst_r_bpp > 0
+        tst     DST, #16
+        bne     111f
+        process_inner_loop  process_head, process_tail, unaligned_src, unaligned_mask, 16
+        b       112f
+111:
+ .endif
+        process_inner_loop  process_head, process_tail, unaligned_src, unaligned_mask, 0
+112:
+        /* Just before the final (prefetch_distance+1) 32-byte blocks, deal with final preloads */
+ .if (src_bpp*pix_per_block > 256) || (mask_bpp*pix_per_block > 256) || (dst_r_bpp*pix_per_block > 256)
+        PF  and,    WK0, X, #pix_per_block-1
+ .endif
+        preload_trailing  src_bpp, src_bpp_shift, SRC
+        preload_trailing  mask_bpp, mask_bpp_shift, MASK
+        preload_trailing  dst_r_bpp, dst_bpp_shift, DST
+        add     X, X, #(prefetch_distance+2)*pix_per_block - 128/dst_w_bpp
+        /* The remainder of the line is handled identically to the medium case */
+        medium_case_inner_loop_and_trailing_pixels  process_head, process_tail,, exit_label, unaligned_src, unaligned_mask
+.endm
+
+.macro medium_case_inner_loop_and_trailing_pixels  process_head, process_tail, unused, exit_label, unaligned_src, unaligned_mask
+120:
+        process_head  , 16, 0, unaligned_src, unaligned_mask, 0
+        process_tail  , 16, 0
+ .if !((flags) & FLAG_PROCESS_DOES_STORE)
+        pixst   , 16, 0, DST
+ .endif
+        subs    X, X, #128/dst_w_bpp
+        bhs     120b
+        /* Trailing pixels */
+        tst     X, #128/dst_w_bpp - 1
+        beq     exit_label
+        trailing_15bytes  process_head, process_tail, unaligned_src, unaligned_mask
+.endm
+
+.macro narrow_case_inner_loop_and_trailing_pixels  process_head, process_tail, unused, exit_label, unaligned_src, unaligned_mask
+        tst     X, #16*8/dst_w_bpp
+        conditional_process1  ne, process_head, process_tail, 16, 0, unaligned_src, unaligned_mask, 0
+        /* Trailing pixels */
+        /* In narrow case, it's relatively unlikely to be aligned, so let's do without a branch here */
+        trailing_15bytes  process_head, process_tail, unaligned_src, unaligned_mask
+.endm
+
+.macro switch_on_alignment  action, process_head, process_tail, process_inner_loop, exit_label
+ /* Note that if we're reading the destination, it's already guaranteed to be aligned at this point */
+ .if mask_bpp == 8 || mask_bpp == 16
+        tst     MASK, #3
+        bne     141f
+ .endif
+  .if src_bpp == 8 || src_bpp == 16
+        tst     SRC, #3
+        bne     140f
+  .endif
+        action  process_head, process_tail, process_inner_loop, exit_label, 0, 0
+  .if src_bpp == 8 || src_bpp == 16
+        b       exit_label
+140:
+        action  process_head, process_tail, process_inner_loop, exit_label, 1, 0
+  .endif
+ .if mask_bpp == 8 || mask_bpp == 16
+        b       exit_label
+141:
+  .if src_bpp == 8 || src_bpp == 16
+        tst     SRC, #3
+        bne     142f
+  .endif
+        action  process_head, process_tail, process_inner_loop, exit_label, 0, 1
+  .if src_bpp == 8 || src_bpp == 16
+        b       exit_label
+142:
+        action  process_head, process_tail, process_inner_loop, exit_label, 1, 1
+  .endif
+ .endif
+.endm
+
+
+.macro end_of_line      restore_x, vars_spilled, loop_label, last_one
+ .if vars_spilled
+        /* Sadly, GAS doesn't seem have an equivalent of the DCI directive? */
+        /* This is ldmia sp,{} */
+        .word   0xE89D0000 | LINE_SAVED_REGS
+ .endif
+        subs    Y, Y, #1
+ .if vars_spilled
+  .if (LINE_SAVED_REGS) & (1<<1)
+        str     Y, [sp]
+  .endif
+ .endif
+        add     DST, DST, STRIDE_D
+ .if src_bpp > 0
+        add     SRC, SRC, STRIDE_S
+ .endif
+ .if mask_bpp > 0
+        add     MASK, MASK, STRIDE_M
+ .endif
+ .if restore_x
+        mov     X, ORIG_W
+ .endif
+        bhs     loop_label
+ .ifc "last_one",""
+  .if vars_spilled
+        b       197f
+  .else
+        b       198f
+  .endif
+ .else
+  .if (!vars_spilled) && ((flags) & FLAG_SPILL_LINE_VARS)
+        b       198f
+  .endif
+ .endif
+.endm
+
+
+.macro generate_composite_function fname, \
+                                   src_bpp_, \
+                                   mask_bpp_, \
+                                   dst_w_bpp_, \
+                                   flags_, \
+                                   prefetch_distance_, \
+                                   init, \
+                                   newline, \
+                                   cleanup, \
+                                   process_head, \
+                                   process_tail, \
+                                   process_inner_loop
+
+ .func fname
+ .global fname
+ /* For ELF format also set function visibility to hidden */
+#ifdef __ELF__
+ .hidden fname
+ .type fname, %function
+#endif
+
+/*
+ * Make some macro arguments globally visible and accessible
+ * from other macros
+ */
+ .set src_bpp, src_bpp_
+ .set mask_bpp, mask_bpp_
+ .set dst_w_bpp, dst_w_bpp_
+ .set flags, flags_
+ .set prefetch_distance, prefetch_distance_
+
+/*
+ * Select prefetch type for this function.
+ */
+ .if prefetch_distance == 0
+  .set PREFETCH_TYPE_CURRENT, PREFETCH_TYPE_NONE
+ .else
+  .set PREFETCH_TYPE_CURRENT, PREFETCH_TYPE_STANDARD
+ .endif
+
+ .if src_bpp == 32
+  .set src_bpp_shift, 2
+ .elseif src_bpp == 24
+  .set src_bpp_shift, 0
+ .elseif src_bpp == 16
+  .set src_bpp_shift, 1
+ .elseif src_bpp == 8
+  .set src_bpp_shift, 0
+ .elseif src_bpp == 0
+  .set src_bpp_shift, -1
+ .else
+  .error "requested src bpp (src_bpp) is not supported"
+ .endif
+
+ .if mask_bpp == 32
+  .set mask_bpp_shift, 2
+ .elseif mask_bpp == 24
+  .set mask_bpp_shift, 0
+ .elseif mask_bpp == 8
+  .set mask_bpp_shift, 0
+ .elseif mask_bpp == 0
+  .set mask_bpp_shift, -1
+ .else
+  .error "requested mask bpp (mask_bpp) is not supported"
+ .endif
+
+ .if dst_w_bpp == 32
+  .set dst_bpp_shift, 2
+ .elseif dst_w_bpp == 24
+  .set dst_bpp_shift, 0
+ .elseif dst_w_bpp == 16
+  .set dst_bpp_shift, 1
+ .elseif dst_w_bpp == 8
+  .set dst_bpp_shift, 0
+ .else
+  .error "requested dst bpp (dst_w_bpp) is not supported"
+ .endif
+
+ .if (((flags) & FLAG_DST_READWRITE) != 0)
+  .set dst_r_bpp, dst_w_bpp
+ .else
+  .set dst_r_bpp, 0
+ .endif
+
+ .set pix_per_block, 16*8/dst_w_bpp
+ .if src_bpp != 0
+  .if 32*8/src_bpp > pix_per_block
+   .set pix_per_block, 32*8/src_bpp
+  .endif
+ .endif
+ .if mask_bpp != 0
+  .if 32*8/mask_bpp > pix_per_block
+   .set pix_per_block, 32*8/mask_bpp
+  .endif
+ .endif
+ .if dst_r_bpp != 0
+  .if 32*8/dst_r_bpp > pix_per_block
+   .set pix_per_block, 32*8/dst_r_bpp
+  .endif
+ .endif
+
+/* The standard entry conditions set up by pixman-arm-common.h are:
+ * r0 = width (pixels)
+ * r1 = height (rows)
+ * r2 = pointer to top-left pixel of destination
+ * r3 = destination stride (pixels)
+ * [sp] = source pixel value, or pointer to top-left pixel of source
+ * [sp,#4] = 0 or source stride (pixels)
+ * The following arguments are unused for non-mask operations
+ * [sp,#8] = mask pixel value, or pointer to top-left pixel of mask
+ * [sp,#12] = 0 or mask stride (pixels)
+ */
+
+/*
+ * Assign symbolic names to registers
+ */
+    X           .req    r0  /* pixels to go on this line */
+    Y           .req    r1  /* lines to go */
+    DST         .req    r2  /* destination pixel pointer */
+    STRIDE_D    .req    r3  /* destination stride (bytes, minus width) */
+    SRC         .req    r4  /* source pixel pointer */
+    STRIDE_S    .req    r5  /* source stride (bytes, minus width) */
+    MASK        .req    r6  /* mask pixel pointer (if applicable) */
+    STRIDE_M    .req    r7  /* mask stride (bytes, minus width) */
+    WK0         .req    r8  /* pixel data registers */
+    WK1         .req    r9
+    WK2         .req    r10
+    WK3         .req    r11
+    SCRATCH     .req    r12
+    ORIG_W      .req    r14 /* width (pixels) */
+
+fname:
+        push    {r4-r11, lr}        /* save all registers */
+
+        subs    Y, Y, #1
+        blo     199f
+
+#ifdef DEBUG_PARAMS
+        sub     sp, sp, #9*4
+#endif
+
+ .if src_bpp > 0
+        ldr     SRC, [sp, #ARGS_STACK_OFFSET]
+        ldr     STRIDE_S, [sp, #ARGS_STACK_OFFSET+4]
+ .endif
+ .if mask_bpp > 0
+        ldr     MASK, [sp, #ARGS_STACK_OFFSET+8]
+        ldr     STRIDE_M, [sp, #ARGS_STACK_OFFSET+12]
+ .endif
+        
+#ifdef DEBUG_PARAMS
+        add     Y, Y, #1
+        stmia   sp, {r0-r7,pc}
+        sub     Y, Y, #1
+#endif
+
+        init
+        
+        lsl     STRIDE_D, #dst_bpp_shift /* stride in bytes */
+        sub     STRIDE_D, STRIDE_D, X, lsl #dst_bpp_shift
+ .if src_bpp > 0
+        lsl     STRIDE_S, #src_bpp_shift
+        sub     STRIDE_S, STRIDE_S, X, lsl #src_bpp_shift
+ .endif
+ .if mask_bpp > 0
+        lsl     STRIDE_M, #mask_bpp_shift
+        sub     STRIDE_M, STRIDE_M, X, lsl #mask_bpp_shift
+ .endif
+ 
+        /* Are we not even wide enough to have one 16-byte aligned 16-byte block write? */
+        cmp     X, #2*16*8/dst_w_bpp - 1
+        blo     170f
+ .if src_bpp || mask_bpp || dst_r_bpp /* Wide and medium cases are the same for fill */
+        /* To preload ahead on the current line, we need at least (prefetch_distance+2) 32-byte blocks on all prefetch channels */
+        cmp     X, #(prefetch_distance+3)*pix_per_block - 1
+        blo     160f
+
+        /* Wide case */
+        /* Adjust X so that the decrement instruction can also test for
+         * inner loop termination. We want it to stop when there are
+         * (prefetch_distance+1) complete blocks to go. */
+        sub     X, X, #(prefetch_distance+2)*pix_per_block
+        mov     ORIG_W, X
+  .if (flags) & FLAG_SPILL_LINE_VARS_WIDE
+        /* This is stmdb sp!,{} */
+        .word   0xE92D0000 | LINE_SAVED_REGS
+  .endif
+151:    /* New line */
+        newline
+        preload_leading_step1  src_bpp, WK1, SRC
+        preload_leading_step1  mask_bpp, WK2, MASK
+        preload_leading_step1  dst_r_bpp, WK3, DST
+        
+        tst     DST, #15
+        beq     154f
+        rsb     WK0, DST, #0 /* bits 0-3 = number of leading bytes until destination aligned */
+  .if (src_bpp != 0 && src_bpp != 2*dst_w_bpp) || (mask_bpp != 0 && mask_bpp != 2*dst_w_bpp)
+        PF  and,    WK0, WK0, #15
+  .endif
+
+        preload_leading_step2  src_bpp, src_bpp_shift, WK1, SRC
+        preload_leading_step2  mask_bpp, mask_bpp_shift, WK2, MASK
+        preload_leading_step2  dst_r_bpp, dst_bpp_shift, WK3, DST
+
+        leading_15bytes  process_head, process_tail
+        
+154:    /* Destination now 16-byte aligned; we have at least one prefetch on each channel as well as at least one 16-byte output block */
+ .if (src_bpp > 0) && (mask_bpp == 0) && ((flags) & FLAG_PROCESS_PRESERVES_SCRATCH)
+        and     SCRATCH, SRC, #31
+        rsb     SCRATCH, SCRATCH, #32*prefetch_distance
+ .elseif (src_bpp == 0) && (mask_bpp > 0) && ((flags) & FLAG_PROCESS_PRESERVES_SCRATCH)
+        and     SCRATCH, MASK, #31
+        rsb     SCRATCH, SCRATCH, #32*prefetch_distance
+ .endif
+ .ifc "process_inner_loop",""
+        switch_on_alignment  wide_case_inner_loop_and_trailing_pixels, process_head, process_tail, wide_case_inner_loop, 157f
+ .else
+        switch_on_alignment  wide_case_inner_loop_and_trailing_pixels, process_head, process_tail, process_inner_loop, 157f
+ .endif
+
+157:    /* Check for another line */
+        end_of_line 1, %((flags) & FLAG_SPILL_LINE_VARS_WIDE), 151b
+ .endif
+
+ .ltorg
+
+160:    /* Medium case */
+        mov     ORIG_W, X
+ .if (flags) & FLAG_SPILL_LINE_VARS_NON_WIDE
+        /* This is stmdb sp!,{} */
+        .word   0xE92D0000 | LINE_SAVED_REGS
+ .endif
+161:    /* New line */
+        newline
+        preload_line 0, src_bpp, src_bpp_shift, SRC  /* in: X, corrupts: WK0-WK1 */
+        preload_line 0, mask_bpp, mask_bpp_shift, MASK
+        preload_line 0, dst_r_bpp, dst_bpp_shift, DST
+        
+        sub     X, X, #128/dst_w_bpp     /* simplifies inner loop termination */
+        tst     DST, #15
+        beq     164f
+        rsb     WK0, DST, #0 /* bits 0-3 = number of leading bytes until destination aligned */
+        
+        leading_15bytes  process_head, process_tail
+        
+164:    /* Destination now 16-byte aligned; we have at least one 16-byte output block */
+        switch_on_alignment  medium_case_inner_loop_and_trailing_pixels, process_head, process_tail,, 167f
+        
+167:    /* Check for another line */
+        end_of_line 1, %((flags) & FLAG_SPILL_LINE_VARS_NON_WIDE), 161b
+
+ .ltorg
+
+170:    /* Narrow case, less than 31 bytes, so no guarantee of at least one 16-byte block */
+ .if dst_w_bpp < 32
+        mov     ORIG_W, X
+ .endif
+ .if (flags) & FLAG_SPILL_LINE_VARS_NON_WIDE
+        /* This is stmdb sp!,{} */
+        .word   0xE92D0000 | LINE_SAVED_REGS
+ .endif
+171:    /* New line */
+        newline
+        preload_line 1, src_bpp, src_bpp_shift, SRC  /* in: X, corrupts: WK0-WK1 */
+        preload_line 1, mask_bpp, mask_bpp_shift, MASK
+        preload_line 1, dst_r_bpp, dst_bpp_shift, DST
+        
+ .if dst_w_bpp == 8
+        tst     DST, #3
+        beq     174f
+172:    subs    X, X, #1
+        blo     177f
+        process_head  , 1, 0, 1, 1, 0
+        process_tail  , 1, 0
+  .if !((flags) & FLAG_PROCESS_DOES_STORE)
+        pixst   , 1, 0, DST
+  .endif
+        tst     DST, #3
+        bne     172b
+ .elseif dst_w_bpp == 16
+        tst     DST, #2
+        beq     174f
+        subs    X, X, #1
+        blo     177f
+        process_head  , 2, 0, 1, 1, 0
+        process_tail  , 2, 0
+  .if !((flags) & FLAG_PROCESS_DOES_STORE)
+        pixst   , 2, 0, DST
+  .endif
+ .endif
+
+174:    /* Destination now 4-byte aligned; we have 0 or more output bytes to go */
+        switch_on_alignment  narrow_case_inner_loop_and_trailing_pixels, process_head, process_tail,, 177f
+
+177:    /* Check for another line */
+        end_of_line %(dst_w_bpp < 32), %((flags) & FLAG_SPILL_LINE_VARS_NON_WIDE), 171b, last_one
+
+197:
+ .if (flags) & FLAG_SPILL_LINE_VARS
+        add     sp, sp, #LINE_SAVED_REG_COUNT*4
+ .endif
+198:
+        cleanup
+
+#ifdef DEBUG_PARAMS
+        add     sp, sp, #9*4 /* junk the debug copy of arguments */
+#endif
+199:
+        pop     {r4-r11, pc}  /* exit */
+
+ .ltorg
+
+    .unreq  X
+    .unreq  Y
+    .unreq  DST
+    .unreq  STRIDE_D
+    .unreq  SRC
+    .unreq  STRIDE_S
+    .unreq  MASK
+    .unreq  STRIDE_M
+    .unreq  WK0
+    .unreq  WK1
+    .unreq  WK2
+    .unreq  WK3
+    .unreq  SCRATCH
+    .unreq  ORIG_W
+    .endfunc
+.endm
+
+.macro line_saved_regs  x:vararg
+ .set LINE_SAVED_REGS, 0
+ .set LINE_SAVED_REG_COUNT, 0
+ .irp SAVED_REG,x
+  .ifc "SAVED_REG","Y"
+   .set LINE_SAVED_REGS, LINE_SAVED_REGS | (1<<1)
+   .set LINE_SAVED_REG_COUNT, LINE_SAVED_REG_COUNT + 1
+  .endif
+  .ifc "SAVED_REG","STRIDE_D"
+   .set LINE_SAVED_REGS, LINE_SAVED_REGS | (1<<3)
+   .set LINE_SAVED_REG_COUNT, LINE_SAVED_REG_COUNT + 1
+  .endif
+  .ifc "SAVED_REG","STRIDE_S"
+   .set LINE_SAVED_REGS, LINE_SAVED_REGS | (1<<5)
+   .set LINE_SAVED_REG_COUNT, LINE_SAVED_REG_COUNT + 1
+  .endif
+  .ifc "SAVED_REG","STRIDE_M"
+   .set LINE_SAVED_REGS, LINE_SAVED_REGS | (1<<7)
+   .set LINE_SAVED_REG_COUNT, LINE_SAVED_REG_COUNT + 1
+  .endif
+  .ifc "SAVED_REG","ORIG_W"
+   .set LINE_SAVED_REGS, LINE_SAVED_REGS | (1<<14)
+   .set LINE_SAVED_REG_COUNT, LINE_SAVED_REG_COUNT + 1
+  .endif
+ .endr
+.endm
+
+.macro nop_macro x:vararg
+.endm
diff --git a/pixman/pixman/pixman-arm-simd.c b/pixman/pixman/pixman-arm-simd.c
index 389c9e01a..af062e19d 100644
--- a/pixman/pixman/pixman-arm-simd.c
+++ b/pixman/pixman/pixman-arm-simd.c
@@ -29,365 +29,193 @@
 
 #include "pixman-private.h"
 #include "pixman-arm-common.h"
+#include "pixman-inlines.h"
 
-#if 0 /* This code was moved to 'pixman-arm-simd-asm.S' */
-
-void
-pixman_composite_add_8000_8000_asm_armv6 (int32_t  width,
-                                          int32_t  height,
-                                          uint8_t *dst_line,
-                                          int32_t  dst_stride,
-                                          uint8_t *src_line,
-                                          int32_t  src_stride)
-{
-    uint8_t *dst, *src;
-    int32_t w;
-    uint8_t s, d;
-
-    while (height--)
-    {
-	dst = dst_line;
-	dst_line += dst_stride;
-	src = src_line;
-	src_line += src_stride;
-	w = width;
-
-	/* ensure both src and dst are properly aligned before doing 32 bit reads
-	 * we'll stay in this loop if src and dst have differing alignments
-	 */
-	while (w && (((unsigned long)dst & 3) || ((unsigned long)src & 3)))
-	{
-	    s = *src;
-	    d = *dst;
-	    asm ("uqadd8 %0, %1, %2" : "+r" (d) : "r" (s));
-	    *dst = d;
-
-	    dst++;
-	    src++;
-	    w--;
-	}
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (armv6, src_8888_8888,
+		                   uint32_t, 1, uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (armv6, src_x888_8888,
+                                   uint32_t, 1, uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (armv6, src_0565_0565,
+                                   uint16_t, 1, uint16_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (armv6, src_8_8,
+                                   uint8_t, 1, uint8_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (armv6, src_0565_8888,
+                                   uint16_t, 1, uint32_t, 1)
 
-	while (w >= 4)
-	{
-	    asm ("uqadd8 %0, %1, %2"
-		 : "=r" (*(uint32_t*)dst)
-		 : "r" (*(uint32_t*)src), "r" (*(uint32_t*)dst));
-	    dst += 4;
-	    src += 4;
-	    w -= 4;
-	}
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (armv6, add_8_8,
+                                   uint8_t, 1, uint8_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (armv6, over_8888_8888,
+                                   uint32_t, 1, uint32_t, 1)
 
-	while (w)
-	{
-	    s = *src;
-	    d = *dst;
-	    asm ("uqadd8 %0, %1, %2" : "+r" (d) : "r" (s));
-	    *dst = d;
+PIXMAN_ARM_BIND_FAST_PATH_SRC_N_DST (SKIP_ZERO_MASK, armv6, over_8888_n_8888,
+                                     uint32_t, 1, uint32_t, 1)
 
-	    dst++;
-	    src++;
-	    w--;
-	}
-    }
+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, armv6, over_n_8_8888,
+                                      uint8_t, 1, uint32_t, 1)
 
-}
+PIXMAN_ARM_BIND_SCALED_NEAREST_SRC_DST (armv6, 0565_0565, SRC,
+                                        uint16_t, uint16_t)
+PIXMAN_ARM_BIND_SCALED_NEAREST_SRC_DST (armv6, 8888_8888, SRC,
+                                        uint32_t, uint32_t)
 
 void
-pixman_composite_over_8888_8888_asm_armv6 (int32_t   width,
-                                           int32_t   height,
-                                           uint32_t *dst_line,
-                                           int32_t   dst_stride,
-                                           uint32_t *src_line,
-                                           int32_t   src_stride)
-{
-    uint32_t    *dst;
-    uint32_t    *src;
-    int32_t w;
-    uint32_t component_half = 0x800080;
-    uint32_t upper_component_mask = 0xff00ff00;
-    uint32_t alpha_mask = 0xff;
-
-    while (height--)
-    {
-	dst = dst_line;
-	dst_line += dst_stride;
-	src = src_line;
-	src_line += src_stride;
-	w = width;
-
-/* #define inner_branch */
-	asm volatile (
-	    "cmp %[w], #0\n\t"
-	    "beq 2f\n\t"
-	    "1:\n\t"
-	    /* load src */
-	    "ldr r5, [%[src]], #4\n\t"
-#ifdef inner_branch
-	    /* We can avoid doing the multiplication in two cases: 0x0 or 0xff.
-	     * The 0x0 case also allows us to avoid doing an unecessary data
-	     * write which is more valuable so we only check for that
-	     */
-	    "cmp r5, #0\n\t"
-	    "beq 3f\n\t"
-
-	    /* = 255 - alpha */
-	    "sub r8, %[alpha_mask], r5, lsr #24\n\t"
-
-	    "ldr r4, [%[dest]] \n\t"
+pixman_composite_src_n_8888_asm_armv6 (int32_t   w,
+                                       int32_t   h,
+                                       uint32_t *dst,
+                                       int32_t   dst_stride,
+                                       uint32_t  src);
 
-#else
-	    "ldr r4, [%[dest]] \n\t"
-
-	    /* = 255 - alpha */
-	    "sub r8, %[alpha_mask], r5, lsr #24\n\t"
-#endif
-	    "uxtb16 r6, r4\n\t"
-	    "uxtb16 r7, r4, ror #8\n\t"
-
-	    /* multiply by 257 and divide by 65536 */
-	    "mla r6, r6, r8, %[component_half]\n\t"
-	    "mla r7, r7, r8, %[component_half]\n\t"
-
-	    "uxtab16 r6, r6, r6, ror #8\n\t"
-	    "uxtab16 r7, r7, r7, ror #8\n\t"
-
-	    /* recombine the 0xff00ff00 bytes of r6 and r7 */
-	    "and r7, r7, %[upper_component_mask]\n\t"
-	    "uxtab16 r6, r7, r6, ror #8\n\t"
-
-	    "uqadd8 r5, r6, r5\n\t"
-
-#ifdef inner_branch
-	    "3:\n\t"
-
-#endif
-	    "str r5, [%[dest]], #4\n\t"
-	    /* increment counter and jmp to top */
-	    "subs	%[w], %[w], #1\n\t"
-	    "bne	1b\n\t"
-	    "2:\n\t"
-	    : [w] "+r" (w), [dest] "+r" (dst), [src] "+r" (src)
-	    : [component_half] "r" (component_half), [upper_component_mask] "r" (upper_component_mask),
-	      [alpha_mask] "r" (alpha_mask)
-	    : "r4", "r5", "r6", "r7", "r8", "cc", "memory"
-	    );
-    }
-}
+void
+pixman_composite_src_n_0565_asm_armv6 (int32_t   w,
+                                       int32_t   h,
+                                       uint16_t *dst,
+                                       int32_t   dst_stride,
+                                       uint16_t  src);
 
 void
-pixman_composite_over_8888_n_8888_asm_armv6 (int32_t   width,
-                                             int32_t   height,
-                                             uint32_t *dst_line,
-                                             int32_t   dst_stride,
-                                             uint32_t *src_line,
-                                             int32_t   src_stride,
-                                             uint32_t  mask)
+pixman_composite_src_n_8_asm_armv6 (int32_t   w,
+                                    int32_t   h,
+                                    uint8_t  *dst,
+                                    int32_t   dst_stride,
+                                    uint8_t  src);
+
+static pixman_bool_t
+arm_simd_fill (pixman_implementation_t *imp,
+               uint32_t *               bits,
+               int                      stride, /* in 32-bit words */
+               int                      bpp,
+               int                      x,
+               int                      y,
+               int                      width,
+               int                      height,
+               uint32_t                 _xor)
 {
-    uint32_t *dst;
-    uint32_t *src;
-    int32_t w;
-    uint32_t component_half = 0x800080;
-    uint32_t alpha_mask = 0xff;
-
-    mask = (mask) >> 24;
+    /* stride is always multiple of 32bit units in pixman */
+    uint32_t byte_stride = stride * sizeof(uint32_t);
 
-    while (height--)
+    switch (bpp)
     {
-	dst = dst_line;
-	dst_line += dst_stride;
-	src = src_line;
-	src_line += src_stride;
-	w = width;
-
-/* #define inner_branch */
-	asm volatile (
-	    "cmp %[w], #0\n\t"
-	    "beq 2f\n\t"
-	    "1:\n\t"
-	    /* load src */
-	    "ldr r5, [%[src]], #4\n\t"
-#ifdef inner_branch
-	    /* We can avoid doing the multiplication in two cases: 0x0 or 0xff.
-	     * The 0x0 case also allows us to avoid doing an unecessary data
-	     * write which is more valuable so we only check for that
-	     */
-	    "cmp r5, #0\n\t"
-	    "beq 3f\n\t"
-
-#endif
-	    "ldr r4, [%[dest]] \n\t"
-
-	    "uxtb16 r6, r5\n\t"
-	    "uxtb16 r7, r5, ror #8\n\t"
-
-	    /* multiply by alpha (r8) then by 257 and divide by 65536 */
-	    "mla r6, r6, %[mask_alpha], %[component_half]\n\t"
-	    "mla r7, r7, %[mask_alpha], %[component_half]\n\t"
-
-	    "uxtab16 r6, r6, r6, ror #8\n\t"
-	    "uxtab16 r7, r7, r7, ror #8\n\t"
-
-	    "uxtb16 r6, r6, ror #8\n\t"
-	    "uxtb16 r7, r7, ror #8\n\t"
-
-	    /* recombine */
-	    "orr r5, r6, r7, lsl #8\n\t"
-
-	    "uxtb16 r6, r4\n\t"
-	    "uxtb16 r7, r4, ror #8\n\t"
-
-	    /* 255 - alpha */
-	    "sub r8, %[alpha_mask], r5, lsr #24\n\t"
-
-	    /* multiply by alpha (r8) then by 257 and divide by 65536 */
-	    "mla r6, r6, r8, %[component_half]\n\t"
-	    "mla r7, r7, r8, %[component_half]\n\t"
-
-	    "uxtab16 r6, r6, r6, ror #8\n\t"
-	    "uxtab16 r7, r7, r7, ror #8\n\t"
-
-	    "uxtb16 r6, r6, ror #8\n\t"
-	    "uxtb16 r7, r7, ror #8\n\t"
-
-	    /* recombine */
-	    "orr r6, r6, r7, lsl #8\n\t"
-
-	    "uqadd8 r5, r6, r5\n\t"
-
-#ifdef inner_branch
-	    "3:\n\t"
-
-#endif
-	    "str r5, [%[dest]], #4\n\t"
-	    /* increment counter and jmp to top */
-	    "subs	%[w], %[w], #1\n\t"
-	    "bne	1b\n\t"
-	    "2:\n\t"
-	    : [w] "+r" (w), [dest] "+r" (dst), [src] "+r" (src)
-	    : [component_half] "r" (component_half), [mask_alpha] "r" (mask),
-	      [alpha_mask] "r" (alpha_mask)
-	    : "r4", "r5", "r6", "r7", "r8", "r9", "cc", "memory"
-	    );
+    case 8:
+	pixman_composite_src_n_8_asm_armv6 (
+		width,
+		height,
+		(uint8_t *)(((char *) bits) + y * byte_stride + x),
+		byte_stride,
+		_xor & 0xff);
+	return TRUE;
+    case 16:
+	pixman_composite_src_n_0565_asm_armv6 (
+		width,
+		height,
+		(uint16_t *)(((char *) bits) + y * byte_stride + x * 2),
+		byte_stride / 2,
+		_xor & 0xffff);
+	return TRUE;
+    case 32:
+	pixman_composite_src_n_8888_asm_armv6 (
+		width,
+		height,
+		(uint32_t *)(((char *) bits) + y * byte_stride + x * 4),
+		byte_stride / 4,
+		_xor);
+	return TRUE;
+    default:
+	return FALSE;
     }
 }
 
-void
-pixman_composite_over_n_8_8888_asm_armv6 (int32_t   width,
-                                          int32_t   height,
-                                          uint32_t *dst_line,
-                                          int32_t   dst_stride,
-                                          uint32_t  src,
-                                          int32_t   unused,
-                                          uint8_t  *mask_line,
-                                          int32_t   mask_stride)
+static pixman_bool_t
+arm_simd_blt (pixman_implementation_t *imp,
+              uint32_t *               src_bits,
+              uint32_t *               dst_bits,
+              int                      src_stride, /* in 32-bit words */
+              int                      dst_stride, /* in 32-bit words */
+              int                      src_bpp,
+              int                      dst_bpp,
+              int                      src_x,
+              int                      src_y,
+              int                      dest_x,
+              int                      dest_y,
+              int                      width,
+              int                      height)
 {
-    uint32_t  srca;
-    uint32_t *dst;
-    uint8_t  *mask;
-    int32_t w;
-
-    srca = src >> 24;
+    if (src_bpp != dst_bpp)
+	return FALSE;
 
-    uint32_t component_mask = 0xff00ff;
-    uint32_t component_half = 0x800080;
-
-    uint32_t src_hi = (src >> 8) & component_mask;
-    uint32_t src_lo = src & component_mask;
-
-    while (height--)
+    switch (src_bpp)
     {
-	dst = dst_line;
-	dst_line += dst_stride;
-	mask = mask_line;
-	mask_line += mask_stride;
-	w = width;
-
-/* #define inner_branch */
-	asm volatile (
-	    "cmp %[w], #0\n\t"
-	    "beq 2f\n\t"
-	    "1:\n\t"
-	    /* load mask */
-	    "ldrb r5, [%[mask]], #1\n\t"
-#ifdef inner_branch
-	    /* We can avoid doing the multiplication in two cases: 0x0 or 0xff.
-	     * The 0x0 case also allows us to avoid doing an unecessary data
-	     * write which is more valuable so we only check for that
-	     */
-	    "cmp r5, #0\n\t"
-	    "beq 3f\n\t"
-
-#endif
-	    "ldr r4, [%[dest]] \n\t"
-
-	    /* multiply by alpha (r8) then by 257 and divide by 65536 */
-	    "mla r6, %[src_lo], r5, %[component_half]\n\t"
-	    "mla r7, %[src_hi], r5, %[component_half]\n\t"
-
-	    "uxtab16 r6, r6, r6, ror #8\n\t"
-	    "uxtab16 r7, r7, r7, ror #8\n\t"
-
-	    "uxtb16 r6, r6, ror #8\n\t"
-	    "uxtb16 r7, r7, ror #8\n\t"
-
-	    /* recombine */
-	    "orr r5, r6, r7, lsl #8\n\t"
-
-	    "uxtb16 r6, r4\n\t"
-	    "uxtb16 r7, r4, ror #8\n\t"
-
-	    /* we could simplify this to use 'sub' if we were
-	     * willing to give up a register for alpha_mask
-	     */
-	    "mvn r8, r5\n\t"
-	    "mov r8, r8, lsr #24\n\t"
-
-	    /* multiply by alpha (r8) then by 257 and divide by 65536 */
-	    "mla r6, r6, r8, %[component_half]\n\t"
-	    "mla r7, r7, r8, %[component_half]\n\t"
-
-	    "uxtab16 r6, r6, r6, ror #8\n\t"
-	    "uxtab16 r7, r7, r7, ror #8\n\t"
-
-	    "uxtb16 r6, r6, ror #8\n\t"
-	    "uxtb16 r7, r7, ror #8\n\t"
-
-	    /* recombine */
-	    "orr r6, r6, r7, lsl #8\n\t"
-
-	    "uqadd8 r5, r6, r5\n\t"
-
-#ifdef inner_branch
-	    "3:\n\t"
-
-#endif
-	    "str r5, [%[dest]], #4\n\t"
-	    /* increment counter and jmp to top */
-	    "subs	%[w], %[w], #1\n\t"
-	    "bne	1b\n\t"
-	    "2:\n\t"
-	    : [w] "+r" (w), [dest] "+r" (dst), [src] "+r" (src), [mask] "+r" (mask)
-	    : [component_half] "r" (component_half),
-	      [src_hi] "r" (src_hi), [src_lo] "r" (src_lo)
-	    : "r4", "r5", "r6", "r7", "r8", "cc", "memory");
+    case 8:
+        pixman_composite_src_8_8_asm_armv6 (
+                width, height,
+                (uint8_t *)(((char *) dst_bits) +
+                dest_y * dst_stride * 4 + dest_x * 1), dst_stride * 4,
+                (uint8_t *)(((char *) src_bits) +
+                src_y * src_stride * 4 + src_x * 1), src_stride * 4);
+        return TRUE;
+    case 16:
+	pixman_composite_src_0565_0565_asm_armv6 (
+		width, height,
+		(uint16_t *)(((char *) dst_bits) +
+		dest_y * dst_stride * 4 + dest_x * 2), dst_stride * 2,
+		(uint16_t *)(((char *) src_bits) +
+		src_y * src_stride * 4 + src_x * 2), src_stride * 2);
+	return TRUE;
+    case 32:
+	pixman_composite_src_8888_8888_asm_armv6 (
+		width, height,
+		(uint32_t *)(((char *) dst_bits) +
+		dest_y * dst_stride * 4 + dest_x * 4), dst_stride,
+		(uint32_t *)(((char *) src_bits) +
+		src_y * src_stride * 4 + src_x * 4), src_stride);
+	return TRUE;
+    default:
+	return FALSE;
     }
 }
 
-#endif
-
-PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (armv6, add_8000_8000,
-                                   uint8_t, 1, uint8_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (armv6, over_8888_8888,
-                                   uint32_t, 1, uint32_t, 1)
-
-PIXMAN_ARM_BIND_FAST_PATH_SRC_N_DST (armv6, over_8888_n_8888,
-                                     uint32_t, 1, uint32_t, 1)
-
-PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (armv6, over_n_8_8888,
-                                      uint8_t, 1, uint32_t, 1)
-
 static const pixman_fast_path_t arm_simd_fast_paths[] =
 {
+    PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, a8r8g8b8, armv6_composite_src_8888_8888),
+    PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, a8b8g8r8, armv6_composite_src_8888_8888),
+    PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, x8r8g8b8, armv6_composite_src_8888_8888),
+    PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, x8b8g8r8, armv6_composite_src_8888_8888),
+    PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, x8r8g8b8, armv6_composite_src_8888_8888),
+    PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, x8b8g8r8, armv6_composite_src_8888_8888),
+
+    PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, a8b8g8r8, armv6_composite_src_x888_8888),
+    PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, a8r8g8b8, armv6_composite_src_x888_8888),
+
+    PIXMAN_STD_FAST_PATH (SRC, r5g6b5, null, r5g6b5, armv6_composite_src_0565_0565),
+    PIXMAN_STD_FAST_PATH (SRC, b5g6r5, null, b5g6r5, armv6_composite_src_0565_0565),
+    PIXMAN_STD_FAST_PATH (SRC, a1r5g5b5, null, a1r5g5b5, armv6_composite_src_0565_0565),
+    PIXMAN_STD_FAST_PATH (SRC, a1b5g5r5, null, a1b5g5r5, armv6_composite_src_0565_0565),
+    PIXMAN_STD_FAST_PATH (SRC, a1r5g5b5, null, x1r5g5b5, armv6_composite_src_0565_0565),
+    PIXMAN_STD_FAST_PATH (SRC, a1b5g5r5, null, x1b5g5r5, armv6_composite_src_0565_0565),
+    PIXMAN_STD_FAST_PATH (SRC, x1r5g5b5, null, x1r5g5b5, armv6_composite_src_0565_0565),
+    PIXMAN_STD_FAST_PATH (SRC, x1b5g5r5, null, x1b5g5r5, armv6_composite_src_0565_0565),
+    PIXMAN_STD_FAST_PATH (SRC, a4r4g4b4, null, a4r4g4b4, armv6_composite_src_0565_0565),
+    PIXMAN_STD_FAST_PATH (SRC, a4b4g4r4, null, a4b4g4r4, armv6_composite_src_0565_0565),
+    PIXMAN_STD_FAST_PATH (SRC, a4r4g4b4, null, x4r4g4b4, armv6_composite_src_0565_0565),
+    PIXMAN_STD_FAST_PATH (SRC, a4b4g4r4, null, x4b4g4r4, armv6_composite_src_0565_0565),
+    PIXMAN_STD_FAST_PATH (SRC, x4r4g4b4, null, x4r4g4b4, armv6_composite_src_0565_0565),
+    PIXMAN_STD_FAST_PATH (SRC, x4b4g4r4, null, x4b4g4r4, armv6_composite_src_0565_0565),
+
+    PIXMAN_STD_FAST_PATH (SRC, a8, null, a8, armv6_composite_src_8_8),
+    PIXMAN_STD_FAST_PATH (SRC, r3g3b2, null, r3g3b2, armv6_composite_src_8_8),
+    PIXMAN_STD_FAST_PATH (SRC, b2g3r3, null, b2g3r3, armv6_composite_src_8_8),
+    PIXMAN_STD_FAST_PATH (SRC, a2r2g2b2, null, a2r2g2b2, armv6_composite_src_8_8),
+    PIXMAN_STD_FAST_PATH (SRC, a2b2g2r2, null, a2b2g2r2, armv6_composite_src_8_8),
+    PIXMAN_STD_FAST_PATH (SRC, c8, null, c8, armv6_composite_src_8_8),
+    PIXMAN_STD_FAST_PATH (SRC, g8, null, g8, armv6_composite_src_8_8),
+    PIXMAN_STD_FAST_PATH (SRC, x4a4, null, x4a4, armv6_composite_src_8_8),
+    PIXMAN_STD_FAST_PATH (SRC, x4c4, null, x4c4, armv6_composite_src_8_8),
+    PIXMAN_STD_FAST_PATH (SRC, x4g4, null, x4g4, armv6_composite_src_8_8),
+
+    PIXMAN_STD_FAST_PATH (SRC, r5g6b5, null, a8r8g8b8, armv6_composite_src_0565_8888),
+    PIXMAN_STD_FAST_PATH (SRC, r5g6b5, null, x8r8g8b8, armv6_composite_src_0565_8888),
+    PIXMAN_STD_FAST_PATH (SRC, b5g6r5, null, a8b8g8r8, armv6_composite_src_0565_8888),
+    PIXMAN_STD_FAST_PATH (SRC, b5g6r5, null, x8b8g8r8, armv6_composite_src_0565_8888),
+
     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, a8r8g8b8, armv6_composite_over_8888_8888),
     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, x8r8g8b8, armv6_composite_over_8888_8888),
     PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, a8b8g8r8, armv6_composite_over_8888_8888),
@@ -397,21 +225,33 @@ static const pixman_fast_path_t arm_simd_fast_paths[] =
     PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, solid, a8b8g8r8, armv6_composite_over_8888_n_8888),
     PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, solid, x8b8g8r8, armv6_composite_over_8888_n_8888),
 
-    PIXMAN_STD_FAST_PATH (ADD, a8, null, a8, armv6_composite_add_8000_8000),
+    PIXMAN_STD_FAST_PATH (ADD, a8, null, a8, armv6_composite_add_8_8),
 
     PIXMAN_STD_FAST_PATH (OVER, solid, a8, a8r8g8b8, armv6_composite_over_n_8_8888),
     PIXMAN_STD_FAST_PATH (OVER, solid, a8, x8r8g8b8, armv6_composite_over_n_8_8888),
     PIXMAN_STD_FAST_PATH (OVER, solid, a8, a8b8g8r8, armv6_composite_over_n_8_8888),
     PIXMAN_STD_FAST_PATH (OVER, solid, a8, x8b8g8r8, armv6_composite_over_n_8_8888),
 
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, r5g6b5, r5g6b5, armv6_0565_0565),
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, b5g6r5, b5g6r5, armv6_0565_0565),
+
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, a8r8g8b8, a8r8g8b8, armv6_8888_8888),
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, a8r8g8b8, x8r8g8b8, armv6_8888_8888),
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, x8r8g8b8, x8r8g8b8, armv6_8888_8888),
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, a8b8g8r8, a8b8g8r8, armv6_8888_8888),
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, a8b8g8r8, x8b8g8r8, armv6_8888_8888),
+    PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, x8b8g8r8, x8b8g8r8, armv6_8888_8888),
+
     { PIXMAN_OP_NONE },
 };
 
 pixman_implementation_t *
-_pixman_implementation_create_arm_simd (void)
+_pixman_implementation_create_arm_simd (pixman_implementation_t *fallback)
 {
-    pixman_implementation_t *general = _pixman_implementation_create_fast_path ();
-    pixman_implementation_t *imp = _pixman_implementation_create (general, arm_simd_fast_paths);
+    pixman_implementation_t *imp = _pixman_implementation_create (fallback, arm_simd_fast_paths);
+
+    imp->blt = arm_simd_blt;
+    imp->fill = arm_simd_fill;
 
     return imp;
 }
diff --git a/pixman/pixman/pixman-arm.c b/pixman/pixman/pixman-arm.c
new file mode 100644
index 000000000..23374e41c
--- /dev/null
+++ b/pixman/pixman/pixman-arm.c
@@ -0,0 +1,225 @@
+/*
+ * Copyright © 2000 SuSE, Inc.
+ * Copyright © 2007 Red Hat, Inc.
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of SuSE not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  SuSE makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as is"
+ * without express or implied warranty.
+ *
+ * SuSE DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL SuSE
+ * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "pixman-private.h"
+
+typedef enum
+{
+    ARM_V7		= (1 << 0),
+    ARM_V6		= (1 << 1),
+    ARM_VFP		= (1 << 2),
+    ARM_NEON		= (1 << 3),
+    ARM_IWMMXT		= (1 << 4)
+} arm_cpu_features_t;
+
+#if defined(USE_ARM_SIMD) || defined(USE_ARM_NEON) || defined(USE_ARM_IWMMXT)
+
+#if defined(_MSC_VER)
+
+/* Needed for EXCEPTION_ILLEGAL_INSTRUCTION */
+#include <windows.h>
+
+extern int pixman_msvc_try_arm_neon_op ();
+extern int pixman_msvc_try_arm_simd_op ();
+
+static arm_cpu_features_t
+detect_cpu_features (void)
+{
+    arm_cpu_features_t features = 0;
+
+    __try
+    {
+	pixman_msvc_try_arm_simd_op ();
+	features |= ARM_V6;
+    }
+    __except (GetExceptionCode () == EXCEPTION_ILLEGAL_INSTRUCTION)
+    {
+    }
+
+    __try
+    {
+	pixman_msvc_try_arm_neon_op ();
+	features |= ARM_NEON;
+    }
+    __except (GetExceptionCode () == EXCEPTION_ILLEGAL_INSTRUCTION)
+    {
+    }
+
+    return features;
+}
+
+#elif defined(__APPLE__) && defined(TARGET_OS_IPHONE) /* iOS */
+
+#include "TargetConditionals.h"
+
+static arm_cpu_features_t
+detect_cpu_features (void)
+{
+    arm_cpu_features_t features = 0;
+
+    features |= ARM_V6;
+
+    /* Detection of ARM NEON on iOS is fairly simple because iOS binaries
+     * contain separate executable images for each processor architecture.
+     * So all we have to do is detect the armv7 architecture build. The
+     * operating system automatically runs the armv7 binary for armv7 devices
+     * and the armv6 binary for armv6 devices.
+     */
+#if defined(__ARM_NEON__)
+    features |= ARM_NEON;
+#endif
+
+    return features;
+}
+
+#elif defined(__ANDROID__) || defined(ANDROID) /* Android */
+
+#include <cpu-features.h>
+
+static arm_cpu_features_t
+detect_cpu_features (void)
+{
+    arm_cpu_features_t features = 0;
+    AndroidCpuFamily cpu_family;
+    uint64_t cpu_features;
+
+    cpu_family = android_getCpuFamily();
+    cpu_features = android_getCpuFeatures();
+
+    if (cpu_family == ANDROID_CPU_FAMILY_ARM)
+    {
+	if (cpu_features & ANDROID_CPU_ARM_FEATURE_ARMv7)
+	    features |= ARM_V7;
+
+	if (cpu_features & ANDROID_CPU_ARM_FEATURE_VFPv3)
+	    features |= ARM_VFP;
+
+	if (cpu_features & ANDROID_CPU_ARM_FEATURE_NEON)
+	    features |= ARM_NEON;
+    }
+
+    return features;
+}
+
+#elif defined (__linux__) /* linux ELF */
+
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+#include <string.h>
+#include <elf.h>
+
+static arm_cpu_features_t
+detect_cpu_features (void)
+{
+    arm_cpu_features_t features = 0;
+    Elf32_auxv_t aux;
+    int fd;
+
+    fd = open ("/proc/self/auxv", O_RDONLY);
+    if (fd >= 0)
+    {
+	while (read (fd, &aux, sizeof(Elf32_auxv_t)) == sizeof(Elf32_auxv_t))
+	{
+	    if (aux.a_type == AT_HWCAP)
+	    {
+		uint32_t hwcap = aux.a_un.a_val;
+
+		/* hardcode these values to avoid depending on specific
+		 * versions of the hwcap header, e.g. HWCAP_NEON
+		 */
+		if ((hwcap & 64) != 0)
+		    features |= ARM_VFP;
+		if ((hwcap & 512) != 0)
+		    features |= ARM_IWMMXT;
+		/* this flag is only present on kernel 2.6.29 */
+		if ((hwcap & 4096) != 0)
+		    features |= ARM_NEON;
+	    }
+	    else if (aux.a_type == AT_PLATFORM)
+	    {
+		const char *plat = (const char*) aux.a_un.a_val;
+
+		if (strncmp (plat, "v7l", 3) == 0)
+		    features |= (ARM_V7 | ARM_V6);
+		else if (strncmp (plat, "v6l", 3) == 0)
+		    features |= ARM_V6;
+	    }
+	}
+	close (fd);
+    }
+
+    return features;
+}
+
+#else /* Unknown */
+
+static arm_cpu_features_t
+detect_cpu_features (void)
+{
+    return 0;
+}
+
+#endif /* Linux elf */
+
+static pixman_bool_t
+have_feature (arm_cpu_features_t feature)
+{
+    static pixman_bool_t initialized;
+    static arm_cpu_features_t features;
+
+    if (!initialized)
+    {
+	features = detect_cpu_features();
+	initialized = TRUE;
+    }
+
+    return (features & feature) == feature;
+}
+
+#endif /* USE_ARM_SIMD || USE_ARM_NEON || USE_ARM_IWMMXT */
+
+pixman_implementation_t *
+_pixman_arm_get_implementations (pixman_implementation_t *imp)
+{
+#ifdef USE_ARM_SIMD
+    if (!_pixman_disabled ("arm-simd") && have_feature (ARM_V6))
+	imp = _pixman_implementation_create_arm_simd (imp);
+#endif
+
+#ifdef USE_ARM_IWMMXT
+    if (!_pixman_disabled ("arm-iwmmxt") && have_feature (ARM_IWMMXT))
+	imp = _pixman_implementation_create_mmx (imp);
+#endif
+
+#ifdef USE_ARM_NEON
+    if (!_pixman_disabled ("arm-neon") && have_feature (ARM_NEON))
+	imp = _pixman_implementation_create_arm_neon (imp);
+#endif
+
+    return imp;
+}
diff --git a/pixman/pixman/pixman-bits-image.c b/pixman/pixman/pixman-bits-image.c
index 0225ae5aa..dcdcc6994 100644
--- a/pixman/pixman/pixman-bits-image.c
+++ b/pixman/pixman/pixman-bits-image.c
@@ -34,100 +34,27 @@
 #include <string.h>
 #include "pixman-private.h"
 #include "pixman-combine32.h"
+#include "pixman-inlines.h"
 
-/* Store functions */
-
-static void
-bits_image_store_scanline_32 (bits_image_t *  image,
-                              int             x,
-                              int             y,
-                              int             width,
-                              const uint32_t *buffer)
-{
-    image->store_scanline_raw_32 (image, x, y, width, buffer);
-
-    if (image->common.alpha_map)
-    {
-	x -= image->common.alpha_origin_x;
-	y -= image->common.alpha_origin_y;
-
-	bits_image_store_scanline_32 (image->common.alpha_map, x, y, width, buffer);
-    }
-}
-
-static void
-bits_image_store_scanline_64 (bits_image_t *  image,
-                              int             x,
-                              int             y,
-                              int             width,
-                              const uint32_t *buffer)
+static uint32_t *
+_pixman_image_get_scanline_generic_float (pixman_iter_t * iter,
+					  const uint32_t *mask)
 {
-    image->store_scanline_raw_64 (image, x, y, width, buffer);
+    pixman_iter_get_scanline_t fetch_32 = iter->data;
+    uint32_t *buffer = iter->buffer;
 
-    if (image->common.alpha_map)
-    {
-	x -= image->common.alpha_origin_x;
-	y -= image->common.alpha_origin_y;
-
-	bits_image_store_scanline_64 (image->common.alpha_map, x, y, width, buffer);
-    }
-}
+    fetch_32 (iter, NULL);
 
-void
-_pixman_image_store_scanline_32 (bits_image_t *  image,
-                                 int             x,
-                                 int             y,
-                                 int             width,
-                                 const uint32_t *buffer)
-{
-    image->store_scanline_32 (image, x, y, width, buffer);
-}
+    pixman_expand_to_float ((argb_t *)buffer, buffer, PIXMAN_a8r8g8b8, iter->width);
 
-void
-_pixman_image_store_scanline_64 (bits_image_t *  image,
-                                 int             x,
-                                 int             y,
-                                 int             width,
-                                 const uint32_t *buffer)
-{
-    image->store_scanline_64 (image, x, y, width, buffer);
+    return iter->buffer;
 }
 
 /* Fetch functions */
 
-static uint32_t
-bits_image_fetch_pixel_alpha (bits_image_t *image, int x, int y)
-{
-    uint32_t pixel;
-    uint32_t pixel_a;
-
-    pixel = image->fetch_pixel_raw_32 (image, x, y);
-
-    assert (image->common.alpha_map);
-
-    x -= image->common.alpha_origin_x;
-    y -= image->common.alpha_origin_y;
-
-    if (x < 0 || x >= image->common.alpha_map->width ||
-	y < 0 || y >= image->common.alpha_map->height)
-    {
-	pixel_a = 0;
-    }
-    else
-    {
-	pixel_a = image->common.alpha_map->fetch_pixel_raw_32 (
-	    image->common.alpha_map, x, y);
-	pixel_a = ALPHA_8 (pixel_a);
-    }
-
-    pixel &= 0x00ffffff;
-    pixel |= (pixel_a << 24);
-
-    return pixel;
-}
-
 static force_inline uint32_t
-get_pixel (bits_image_t *image, int x, int y, pixman_bool_t check_bounds)
+fetch_pixel_no_alpha (bits_image_t *image,
+		      int x, int y, pixman_bool_t check_bounds)
 {
     if (check_bounds &&
 	(x < 0 || x >= image->width || y < 0 || y >= image->height))
@@ -138,46 +65,22 @@ get_pixel (bits_image_t *image, int x, int y, pixman_bool_t check_bounds)
     return image->fetch_pixel_32 (image, x, y);
 }
 
-static force_inline void
-repeat (pixman_repeat_t repeat, int size, int *coord)
-{
-    switch (repeat)
-    {
-    case PIXMAN_REPEAT_NORMAL:
-	*coord = MOD (*coord, size);
-	break;
-
-    case PIXMAN_REPEAT_PAD:
-	*coord = CLIP (*coord, 0, size - 1);
-	break;
-
-    case PIXMAN_REPEAT_REFLECT:
-	*coord = MOD (*coord, size * 2);
-
-	if (*coord >= size)
-	    *coord = size * 2 - *coord - 1;
-	break;
-
-    case PIXMAN_REPEAT_NONE:
-	break;
-
-    default:
-        break;
-    }
-}
+typedef uint32_t (* get_pixel_t) (bits_image_t *image,
+				  int x, int y, pixman_bool_t check_bounds);
 
 static force_inline uint32_t
 bits_image_fetch_pixel_nearest (bits_image_t   *image,
 				pixman_fixed_t  x,
-				pixman_fixed_t  y)
+				pixman_fixed_t  y,
+				get_pixel_t	get_pixel)
 {
     int x0 = pixman_fixed_to_int (x - pixman_fixed_e);
     int y0 = pixman_fixed_to_int (y - pixman_fixed_e);
 
     if (image->common.repeat != PIXMAN_REPEAT_NONE)
     {
-	repeat (image->common.repeat, image->width, &x0);
-	repeat (image->common.repeat, image->height, &y0);
+	repeat (image->common.repeat, &x0, image->width);
+	repeat (image->common.repeat, &y0, image->height);
 
 	return get_pixel (image, x0, y0, FALSE);
     }
@@ -187,101 +90,11 @@ bits_image_fetch_pixel_nearest (bits_image_t   *image,
     }
 }
 
-#if SIZEOF_LONG > 4
-
-static force_inline uint32_t
-bilinear_interpolation (uint32_t tl, uint32_t tr,
-			uint32_t bl, uint32_t br,
-			int distx, int disty)
-{
-    uint64_t distxy, distxiy, distixy, distixiy;
-    uint64_t tl64, tr64, bl64, br64;
-    uint64_t f, r;
-
-    distxy = distx * disty;
-    distxiy = distx * (256 - disty);
-    distixy = (256 - distx) * disty;
-    distixiy = (256 - distx) * (256 - disty);
-
-    /* Alpha and Blue */
-    tl64 = tl & 0xff0000ff;
-    tr64 = tr & 0xff0000ff;
-    bl64 = bl & 0xff0000ff;
-    br64 = br & 0xff0000ff;
-
-    f = tl64 * distixiy + tr64 * distxiy + bl64 * distixy + br64 * distxy;
-    r = f & 0x0000ff0000ff0000ull;
-
-    /* Red and Green */
-    tl64 = tl;
-    tl64 = ((tl64 << 16) & 0x000000ff00000000ull) | (tl64 & 0x0000ff00ull);
-
-    tr64 = tr;
-    tr64 = ((tr64 << 16) & 0x000000ff00000000ull) | (tr64 & 0x0000ff00ull);
-
-    bl64 = bl;
-    bl64 = ((bl64 << 16) & 0x000000ff00000000ull) | (bl64 & 0x0000ff00ull);
-
-    br64 = br;
-    br64 = ((br64 << 16) & 0x000000ff00000000ull) | (br64 & 0x0000ff00ull);
-
-    f = tl64 * distixiy + tr64 * distxiy + bl64 * distixy + br64 * distxy;
-    r |= ((f >> 16) & 0x000000ff00000000ull) | (f & 0xff000000ull);
-
-    return (uint32_t)(r >> 16);
-}
-
-#else
-
-static force_inline uint32_t
-bilinear_interpolation (uint32_t tl, uint32_t tr,
-			uint32_t bl, uint32_t br,
-			int distx, int disty)
-{
-    int distxy, distxiy, distixy, distixiy;
-    uint32_t f, r;
-
-    distxy = distx * disty;
-    distxiy = (distx << 8) - distxy;	/* distx * (256 - disty) */
-    distixy = (disty << 8) - distxy;	/* disty * (256 - distx) */
-    distixiy =
-	256 * 256 - (disty << 8) -
-	(distx << 8) + distxy;		/* (256 - distx) * (256 - disty) */
-
-    /* Blue */
-    r = (tl & 0x000000ff) * distixiy + (tr & 0x000000ff) * distxiy
-      + (bl & 0x000000ff) * distixy  + (br & 0x000000ff) * distxy;
-
-    /* Green */
-    f = (tl & 0x0000ff00) * distixiy + (tr & 0x0000ff00) * distxiy
-      + (bl & 0x0000ff00) * distixy  + (br & 0x0000ff00) * distxy;
-    r |= f & 0xff000000;
-
-    tl >>= 16;
-    tr >>= 16;
-    bl >>= 16;
-    br >>= 16;
-    r >>= 16;
-
-    /* Red */
-    f = (tl & 0x000000ff) * distixiy + (tr & 0x000000ff) * distxiy
-      + (bl & 0x000000ff) * distixy  + (br & 0x000000ff) * distxy;
-    r |= f & 0x00ff0000;
-
-    /* Alpha */
-    f = (tl & 0x0000ff00) * distixiy + (tr & 0x0000ff00) * distxiy
-      + (bl & 0x0000ff00) * distixy  + (br & 0x0000ff00) * distxy;
-    r |= f & 0xff000000;
-
-    return r;
-}
-
-#endif
-
 static force_inline uint32_t
 bits_image_fetch_pixel_bilinear (bits_image_t   *image,
 				 pixman_fixed_t  x,
-				 pixman_fixed_t  y)
+				 pixman_fixed_t  y,
+				 get_pixel_t	 get_pixel)
 {
     pixman_repeat_t repeat_mode = image->common.repeat;
     int width = image->width;
@@ -293,8 +106,8 @@ bits_image_fetch_pixel_bilinear (bits_image_t   *image,
     x1 = x - pixman_fixed_1 / 2;
     y1 = y - pixman_fixed_1 / 2;
 
-    distx = (x1 >> 8) & 0xff;
-    disty = (y1 >> 8) & 0xff;
+    distx = pixman_fixed_to_bilinear_weight (x1);
+    disty = pixman_fixed_to_bilinear_weight (y1);
 
     x1 = pixman_fixed_to_int (x1);
     y1 = pixman_fixed_to_int (y1);
@@ -303,10 +116,10 @@ bits_image_fetch_pixel_bilinear (bits_image_t   *image,
 
     if (repeat_mode != PIXMAN_REPEAT_NONE)
     {
-	repeat (repeat_mode, width, &x1);
-	repeat (repeat_mode, height, &y1);
-	repeat (repeat_mode, width, &x2);
-	repeat (repeat_mode, height, &y2);
+	repeat (repeat_mode, &x1, width);
+	repeat (repeat_mode, &y1, height);
+	repeat (repeat_mode, &x2, width);
+	repeat (repeat_mode, &y2, height);
 
 	tl = get_pixel (image, x1, y1, FALSE);
 	bl = get_pixel (image, x1, y2, FALSE);
@@ -324,232 +137,22 @@ bits_image_fetch_pixel_bilinear (bits_image_t   *image,
     return bilinear_interpolation (tl, tr, bl, br, distx, disty);
 }
 
-static void
-bits_image_fetch_bilinear_no_repeat_8888 (pixman_image_t * ima,
-					  int              offset,
-					  int              line,
-					  int              width,
-					  uint32_t *       buffer,
-					  const uint32_t * mask,
-					  uint32_t         mask_bits)
-{
-    bits_image_t *bits = &ima->bits;
-    pixman_fixed_t x_top, x_bottom, x;
-    pixman_fixed_t ux_top, ux_bottom, ux;
-    pixman_vector_t v;
-    uint32_t top_mask, bottom_mask;
-    uint32_t *top_row;
-    uint32_t *bottom_row;
-    uint32_t *end;
-    uint32_t zero[2] = { 0, 0 };
-    int y, y1, y2;
-    int disty;
-    int mask_inc;
-    int w;
-
-    /* reference point is the center of the pixel */
-    v.vector[0] = pixman_int_to_fixed (offset) + pixman_fixed_1 / 2;
-    v.vector[1] = pixman_int_to_fixed (line) + pixman_fixed_1 / 2;
-    v.vector[2] = pixman_fixed_1;
-
-    if (!pixman_transform_point_3d (bits->common.transform, &v))
-	return;
-
-    ux = ux_top = ux_bottom = bits->common.transform->matrix[0][0];
-    x = x_top = x_bottom = v.vector[0] - pixman_fixed_1/2;
-
-    y = v.vector[1] - pixman_fixed_1/2;
-    disty = (y >> 8) & 0xff;
-
-    /* Load the pointers to the first and second lines from the source
-     * image that bilinear code must read.
-     *
-     * The main trick in this code is about the check if any line are
-     * outside of the image;
-     *
-     * When I realize that a line (any one) is outside, I change
-     * the pointer to a dummy area with zeros. Once I change this, I
-     * must be sure the pointer will not change, so I set the
-     * variables to each pointer increments inside the loop.
-     */
-    y1 = pixman_fixed_to_int (y);
-    y2 = y1 + 1;
-
-    if (y1 < 0 || y1 >= bits->height)
-    {
-	top_row = zero;
-	x_top = 0;
-	ux_top = 0;
-    }
-    else
-    {
-	top_row = bits->bits + y1 * bits->rowstride;
-	x_top = x;
-	ux_top = ux;
-    }
-
-    if (y2 < 0 || y2 >= bits->height)
-    {
-	bottom_row = zero;
-	x_bottom = 0;
-	ux_bottom = 0;
-    }
-    else
-    {
-	bottom_row = bits->bits + y2 * bits->rowstride;
-	x_bottom = x;
-	ux_bottom = ux;
-    }
-
-    /* Instead of checking whether the operation uses the mast in
-     * each loop iteration, verify this only once and prepare the
-     * variables to make the code smaller inside the loop.
-     */
-    if (!mask)
-    {
-        mask_inc = 0;
-        mask_bits = 1;
-        mask = &mask_bits;
-    }
-    else
-    {
-        /* If have a mask, prepare the variables to check it */
-        mask_inc = 1;
-    }
-
-    /* If both are zero, then the whole thing is zero */
-    if (top_row == zero && bottom_row == zero)
-    {
-	memset (buffer, 0, width * sizeof (uint32_t));
-	return;
-    }
-    else if (bits->format == PIXMAN_x8r8g8b8)
-    {
-	if (top_row == zero)
-	{
-	    top_mask = 0;
-	    bottom_mask = 0xff000000;
-	}
-	else if (bottom_row == zero)
-	{
-	    top_mask = 0xff000000;
-	    bottom_mask = 0;
-	}
-	else
-	{
-	    top_mask = 0xff000000;
-	    bottom_mask = 0xff000000;
-	}
-    }
-    else
-    {
-	top_mask = 0;
-	bottom_mask = 0;
-    }
-
-    end = buffer + width;
-
-    /* Zero fill to the left of the image */
-    while (buffer < end && x < pixman_fixed_minus_1)
-    {
-	*buffer++ = 0;
-	x += ux;
-	x_top += ux_top;
-	x_bottom += ux_bottom;
-	mask += mask_inc;
-    }
-
-    /* Left edge
-     */
-    while (buffer < end && x < 0)
-    {
-	uint32_t tr, br;
-	int32_t distx;
-
-	tr = top_row[pixman_fixed_to_int (x_top) + 1] | top_mask;
-	br = bottom_row[pixman_fixed_to_int (x_bottom) + 1] | bottom_mask;
-
-	distx = (x >> 8) & 0xff;
-
-	*buffer++ = bilinear_interpolation (0, tr, 0, br, distx, disty);
-
-	x += ux;
-	x_top += ux_top;
-	x_bottom += ux_bottom;
-	mask += mask_inc;
-    }
-
-    /* Main part */
-    w = pixman_int_to_fixed (bits->width - 1);
-
-    while (buffer < end  &&  x < w)
-    {
-	if (*mask)
-	{
-	    uint32_t tl, tr, bl, br;
-	    int32_t distx;
-
-	    tl = top_row [pixman_fixed_to_int (x_top)] | top_mask;
-	    tr = top_row [pixman_fixed_to_int (x_top) + 1] | top_mask;
-	    bl = bottom_row [pixman_fixed_to_int (x_bottom)] | bottom_mask;
-	    br = bottom_row [pixman_fixed_to_int (x_bottom) + 1] | bottom_mask;
-
-	    distx = (x >> 8) & 0xff;
-
-	    *buffer = bilinear_interpolation (tl, tr, bl, br, distx, disty);
-	}
-
-	buffer++;
-	x += ux;
-	x_top += ux_top;
-	x_bottom += ux_bottom;
-	mask += mask_inc;
-    }
-
-    /* Right Edge */
-    w = pixman_int_to_fixed (bits->width);
-    while (buffer < end  &&  x < w)
-    {
-	if (*mask)
-	{
-	    uint32_t tl, bl;
-	    int32_t distx;
-
-	    tl = top_row [pixman_fixed_to_int (x_top)] | top_mask;
-	    bl = bottom_row [pixman_fixed_to_int (x_bottom)] | bottom_mask;
-
-	    distx = (x >> 8) & 0xff;
-
-	    *buffer = bilinear_interpolation (tl, 0, bl, 0, distx, disty);
-	}
-
-	buffer++;
-	x += ux;
-	x_top += ux_top;
-	x_bottom += ux_bottom;
-	mask += mask_inc;
-    }
-
-    /* Zero fill to the left of the image */
-    while (buffer < end)
-	*buffer++ = 0;
-}
-
 static force_inline uint32_t
 bits_image_fetch_pixel_convolution (bits_image_t   *image,
 				    pixman_fixed_t  x,
-				    pixman_fixed_t  y)
+				    pixman_fixed_t  y,
+				    get_pixel_t     get_pixel)
 {
     pixman_fixed_t *params = image->common.filter_params;
     int x_off = (params[0] - pixman_fixed_1) >> 1;
     int y_off = (params[1] - pixman_fixed_1) >> 1;
     int32_t cwidth = pixman_fixed_to_int (params[0]);
     int32_t cheight = pixman_fixed_to_int (params[1]);
-    int32_t srtot, sgtot, sbtot, satot;
     int32_t i, j, x1, x2, y1, y2;
     pixman_repeat_t repeat_mode = image->common.repeat;
     int width = image->width;
     int height = image->height;
+    int srtot, sgtot, sbtot, satot;
 
     params += 2;
 
@@ -575,8 +178,8 @@ bits_image_fetch_pixel_convolution (bits_image_t   *image,
 
 		if (repeat_mode != PIXMAN_REPEAT_NONE)
 		{
-		    repeat (repeat_mode, width, &rx);
-		    repeat (repeat_mode, height, &ry);
+		    repeat (repeat_mode, &rx, width);
+		    repeat (repeat_mode, &ry, height);
 
 		    pixel = get_pixel (image, rx, ry, FALSE);
 		}
@@ -585,20 +188,118 @@ bits_image_fetch_pixel_convolution (bits_image_t   *image,
 		    pixel = get_pixel (image, rx, ry, TRUE);
 		}
 
-		srtot += RED_8 (pixel) * f;
-		sgtot += GREEN_8 (pixel) * f;
-		sbtot += BLUE_8 (pixel) * f;
-		satot += ALPHA_8 (pixel) * f;
+		srtot += (int)RED_8 (pixel) * f;
+		sgtot += (int)GREEN_8 (pixel) * f;
+		sbtot += (int)BLUE_8 (pixel) * f;
+		satot += (int)ALPHA_8 (pixel) * f;
 	    }
 
 	    params++;
 	}
     }
 
-    satot >>= 16;
-    srtot >>= 16;
-    sgtot >>= 16;
-    sbtot >>= 16;
+    satot = (satot + 0x8000) >> 16;
+    srtot = (srtot + 0x8000) >> 16;
+    sgtot = (sgtot + 0x8000) >> 16;
+    sbtot = (sbtot + 0x8000) >> 16;
+
+    satot = CLIP (satot, 0, 0xff);
+    srtot = CLIP (srtot, 0, 0xff);
+    sgtot = CLIP (sgtot, 0, 0xff);
+    sbtot = CLIP (sbtot, 0, 0xff);
+
+    return ((satot << 24) | (srtot << 16) | (sgtot <<  8) | (sbtot));
+}
+
+static uint32_t
+bits_image_fetch_pixel_separable_convolution (bits_image_t *image,
+                                              pixman_fixed_t x,
+                                              pixman_fixed_t y,
+                                              get_pixel_t    get_pixel)
+{
+    pixman_fixed_t *params = image->common.filter_params;
+    pixman_repeat_t repeat_mode = image->common.repeat;
+    int width = image->width;
+    int height = image->height;
+    int cwidth = pixman_fixed_to_int (params[0]);
+    int cheight = pixman_fixed_to_int (params[1]);
+    int x_phase_bits = pixman_fixed_to_int (params[2]);
+    int y_phase_bits = pixman_fixed_to_int (params[3]);
+    int x_phase_shift = 16 - x_phase_bits;
+    int y_phase_shift = 16 - y_phase_bits;
+    int x_off = ((cwidth << 16) - pixman_fixed_1) >> 1;
+    int y_off = ((cheight << 16) - pixman_fixed_1) >> 1;
+    pixman_fixed_t *y_params;
+    int srtot, sgtot, sbtot, satot;
+    int32_t x1, x2, y1, y2;
+    int32_t px, py;
+    int i, j;
+
+    /* Round x and y to the middle of the closest phase before continuing. This
+     * ensures that the convolution matrix is aligned right, since it was
+     * positioned relative to a particular phase (and not relative to whatever
+     * exact fraction we happen to get here).
+     */
+    x = ((x >> x_phase_shift) << x_phase_shift) + ((1 << x_phase_shift) >> 1);
+    y = ((y >> y_phase_shift) << y_phase_shift) + ((1 << y_phase_shift) >> 1);
+
+    px = (x & 0xffff) >> x_phase_shift;
+    py = (y & 0xffff) >> y_phase_shift;
+
+    y_params = params + 4 + (1 << x_phase_bits) * cwidth + py * cheight;
+
+    x1 = pixman_fixed_to_int (x - pixman_fixed_e - x_off);
+    y1 = pixman_fixed_to_int (y - pixman_fixed_e - y_off);
+    x2 = x1 + cwidth;
+    y2 = y1 + cheight;
+
+    srtot = sgtot = sbtot = satot = 0;
+
+    for (i = y1; i < y2; ++i)
+    {
+        pixman_fixed_48_16_t fy = *y_params++;
+        pixman_fixed_t *x_params = params + 4 + px * cwidth;
+
+        if (fy)
+        {
+            for (j = x1; j < x2; ++j)
+            {
+                pixman_fixed_t fx = *x_params++;
+		int rx = j;
+		int ry = i;
+
+                if (fx)
+                {
+                    pixman_fixed_t f;
+                    uint32_t pixel;
+
+                    if (repeat_mode != PIXMAN_REPEAT_NONE)
+                    {
+                        repeat (repeat_mode, &rx, width);
+                        repeat (repeat_mode, &ry, height);
+
+                        pixel = get_pixel (image, rx, ry, FALSE);
+                    }
+                    else
+                    {
+                        pixel = get_pixel (image, rx, ry, TRUE);
+		    }
+
+                    f = (fy * fx + 0x8000) >> 16;
+
+                    srtot += (int)RED_8 (pixel) * f;
+                    sgtot += (int)GREEN_8 (pixel) * f;
+                    sbtot += (int)BLUE_8 (pixel) * f;
+                    satot += (int)ALPHA_8 (pixel) * f;
+                }
+            }
+	}
+    }
+
+    satot = (satot + 0x8000) >> 16;
+    srtot = (srtot + 0x8000) >> 16;
+    sgtot = (sgtot + 0x8000) >> 16;
+    sbtot = (sbtot + 0x8000) >> 16;
 
     satot = CLIP (satot, 0, 0xff);
     srtot = CLIP (srtot, 0, 0xff);
@@ -611,25 +312,30 @@ bits_image_fetch_pixel_convolution (bits_image_t   *image,
 static force_inline uint32_t
 bits_image_fetch_pixel_filtered (bits_image_t *image,
 				 pixman_fixed_t x,
-				 pixman_fixed_t y)
+				 pixman_fixed_t y,
+				 get_pixel_t    get_pixel)
 {
     switch (image->common.filter)
     {
     case PIXMAN_FILTER_NEAREST:
     case PIXMAN_FILTER_FAST:
-	return bits_image_fetch_pixel_nearest (image, x, y);
+	return bits_image_fetch_pixel_nearest (image, x, y, get_pixel);
 	break;
 
     case PIXMAN_FILTER_BILINEAR:
     case PIXMAN_FILTER_GOOD:
     case PIXMAN_FILTER_BEST:
-	return bits_image_fetch_pixel_bilinear (image, x, y);
+	return bits_image_fetch_pixel_bilinear (image, x, y, get_pixel);
 	break;
 
     case PIXMAN_FILTER_CONVOLUTION:
-	return bits_image_fetch_pixel_convolution (image, x, y);
+	return bits_image_fetch_pixel_convolution (image, x, y, get_pixel);
 	break;
 
+    case PIXMAN_FILTER_SEPARABLE_CONVOLUTION:
+        return bits_image_fetch_pixel_separable_convolution (image, x, y, get_pixel);
+        break;
+
     default:
         break;
     }
@@ -637,17 +343,18 @@ bits_image_fetch_pixel_filtered (bits_image_t *image,
     return 0;
 }
 
-static void
-bits_image_fetch_transformed (pixman_image_t * image,
-                              int              offset,
-                              int              line,
-                              int              width,
-                              uint32_t *       buffer,
-                              const uint32_t * mask,
-                              uint32_t         mask_bits)
+static uint32_t *
+bits_image_fetch_affine_no_alpha (pixman_iter_t *  iter,
+				  const uint32_t * mask)
 {
-    pixman_fixed_t x, y, w;
-    pixman_fixed_t ux, uy, uw;
+    pixman_image_t *image  = iter->image;
+    int             offset = iter->x;
+    int             line   = iter->y++;
+    int             width  = iter->width;
+    uint32_t *      buffer = iter->buffer;
+
+    pixman_fixed_t x, y;
+    pixman_fixed_t ux, uy;
     pixman_vector_t v;
     int i;
 
@@ -656,77 +363,159 @@ bits_image_fetch_transformed (pixman_image_t * image,
     v.vector[1] = pixman_int_to_fixed (line) + pixman_fixed_1 / 2;
     v.vector[2] = pixman_fixed_1;
 
-    /* when using convolution filters or PIXMAN_REPEAT_PAD one
-     * might get here without a transform */
     if (image->common.transform)
     {
 	if (!pixman_transform_point_3d (image->common.transform, &v))
-	    return;
+	    return iter->buffer;
 
 	ux = image->common.transform->matrix[0][0];
 	uy = image->common.transform->matrix[1][0];
-	uw = image->common.transform->matrix[2][0];
     }
     else
     {
 	ux = pixman_fixed_1;
 	uy = 0;
-	uw = 0;
     }
 
     x = v.vector[0];
     y = v.vector[1];
-    w = v.vector[2];
 
-    if (w == pixman_fixed_1 && uw == 0) /* Affine */
+    for (i = 0; i < width; ++i)
     {
-	for (i = 0; i < width; ++i)
+	if (!mask || mask[i])
 	{
-	    if (!mask || (mask[i] & mask_bits))
-	    {
-		buffer[i] =
-		    bits_image_fetch_pixel_filtered (&image->bits, x, y);
-	    }
+	    buffer[i] = bits_image_fetch_pixel_filtered (
+		&image->bits, x, y, fetch_pixel_no_alpha);
+	}
+
+	x += ux;
+	y += uy;
+    }
+
+    return buffer;
+}
+
+/* General fetcher */
+static force_inline uint32_t
+fetch_pixel_general (bits_image_t *image, int x, int y, pixman_bool_t check_bounds)
+{
+    uint32_t pixel;
+
+    if (check_bounds &&
+	(x < 0 || x >= image->width || y < 0 || y >= image->height))
+    {
+	return 0;
+    }
+
+    pixel = image->fetch_pixel_32 (image, x, y);
+
+    if (image->common.alpha_map)
+    {
+	uint32_t pixel_a;
+
+	x -= image->common.alpha_origin_x;
+	y -= image->common.alpha_origin_y;
+
+	if (x < 0 || x >= image->common.alpha_map->width ||
+	    y < 0 || y >= image->common.alpha_map->height)
+	{
+	    pixel_a = 0;
+	}
+	else
+	{
+	    pixel_a = image->common.alpha_map->fetch_pixel_32 (
+		image->common.alpha_map, x, y);
 
-	    x += ux;
-	    y += uy;
+	    pixel_a = ALPHA_8 (pixel_a);
 	}
+
+	pixel &= 0x00ffffff;
+	pixel |= (pixel_a << 24);
+    }
+
+    return pixel;
+}
+
+static uint32_t *
+bits_image_fetch_general (pixman_iter_t  *iter,
+			  const uint32_t *mask)
+{
+    pixman_image_t *image  = iter->image;
+    int             offset = iter->x;
+    int             line   = iter->y++;
+    int             width  = iter->width;
+    uint32_t *      buffer = iter->buffer;
+
+    pixman_fixed_t x, y, w;
+    pixman_fixed_t ux, uy, uw;
+    pixman_vector_t v;
+    int i;
+
+    /* reference point is the center of the pixel */
+    v.vector[0] = pixman_int_to_fixed (offset) + pixman_fixed_1 / 2;
+    v.vector[1] = pixman_int_to_fixed (line) + pixman_fixed_1 / 2;
+    v.vector[2] = pixman_fixed_1;
+
+    if (image->common.transform)
+    {
+	if (!pixman_transform_point_3d (image->common.transform, &v))
+	    return buffer;
+
+	ux = image->common.transform->matrix[0][0];
+	uy = image->common.transform->matrix[1][0];
+	uw = image->common.transform->matrix[2][0];
     }
     else
     {
-	for (i = 0; i < width; ++i)
-	{
-	    pixman_fixed_t x0, y0;
+	ux = pixman_fixed_1;
+	uy = 0;
+	uw = 0;
+    }
+
+    x = v.vector[0];
+    y = v.vector[1];
+    w = v.vector[2];
+
+    for (i = 0; i < width; ++i)
+    {
+	pixman_fixed_t x0, y0;
 
-	    if (!mask || (mask[i] & mask_bits))
+	if (!mask || mask[i])
+	{
+	    if (w != 0)
 	    {
 		x0 = ((pixman_fixed_48_16_t)x << 16) / w;
 		y0 = ((pixman_fixed_48_16_t)y << 16) / w;
-
-		buffer[i] =
-		    bits_image_fetch_pixel_filtered (&image->bits, x0, y0);
+	    }
+	    else
+	    {
+		x0 = 0;
+		y0 = 0;
 	    }
 
-	    x += ux;
-	    y += uy;
-	    w += uw;
+	    buffer[i] = bits_image_fetch_pixel_filtered (
+		&image->bits, x0, y0, fetch_pixel_general);
 	}
+
+	x += ux;
+	y += uy;
+	w += uw;
     }
+
+    return buffer;
 }
 
 static void
-bits_image_fetch_solid_32 (pixman_image_t * image,
-                           int              x,
-                           int              y,
-                           int              width,
-                           uint32_t *       buffer,
-                           const uint32_t * mask,
-                           uint32_t         mask_bits)
+replicate_pixel_32 (bits_image_t *   bits,
+		    int              x,
+		    int              y,
+		    int              width,
+		    uint32_t *       buffer)
 {
     uint32_t color;
     uint32_t *end;
 
-    color = image->bits.fetch_pixel_raw_32 (&image->bits, 0, 0);
+    color = bits->fetch_pixel_32 (bits, x, y);
 
     end = buffer + width;
     while (buffer < end)
@@ -734,19 +523,17 @@ bits_image_fetch_solid_32 (pixman_image_t * image,
 }
 
 static void
-bits_image_fetch_solid_64 (pixman_image_t * image,
-                           int              x,
-                           int              y,
-                           int              width,
-                           uint32_t *       b,
-                           const uint32_t * unused,
-                           uint32_t         unused2)
+replicate_pixel_float (bits_image_t *   bits,
+		       int              x,
+		       int              y,
+		       int              width,
+		       uint32_t *       b)
 {
-    uint64_t color;
-    uint64_t *buffer = (uint64_t *)b;
-    uint64_t *end;
+    argb_t color;
+    argb_t *buffer = (argb_t *)b;
+    argb_t *end;
 
-    color = image->bits.fetch_pixel_raw_64 (&image->bits, 0, 0);
+    color = bits->fetch_pixel_float (bits, x, y);
 
     end = buffer + width;
     while (buffer < end)
@@ -765,7 +552,7 @@ bits_image_fetch_untransformed_repeat_none (bits_image_t *image,
 
     if (y < 0 || y >= image->height)
     {
-	memset (buffer, 0, width * (wide? 8 : 4));
+	memset (buffer, 0, width * (wide? sizeof (argb_t) : 4));
 	return;
     }
 
@@ -773,10 +560,10 @@ bits_image_fetch_untransformed_repeat_none (bits_image_t *image,
     {
 	w = MIN (width, -x);
 
-	memset (buffer, 0, w * (wide ? 8 : 4));
+	memset (buffer, 0, w * (wide ? sizeof (argb_t) : 4));
 
 	width -= w;
-	buffer += w * (wide? 2 : 1);
+	buffer += w * (wide? 4 : 1);
 	x += w;
     }
 
@@ -785,16 +572,16 @@ bits_image_fetch_untransformed_repeat_none (bits_image_t *image,
 	w = MIN (width, image->width - x);
 
 	if (wide)
-	    image->fetch_scanline_raw_64 ((pixman_image_t *)image, x, y, w, buffer, NULL, 0);
+	    image->fetch_scanline_float (image, x, y, w, buffer, NULL);
 	else
-	    image->fetch_scanline_raw_32 ((pixman_image_t *)image, x, y, w, buffer, NULL, 0);
+	    image->fetch_scanline_32 (image, x, y, w, buffer, NULL);
 
 	width -= w;
-	buffer += w * (wide? 2 : 1);
+	buffer += w * (wide? 4 : 1);
 	x += w;
     }
 
-    memset (buffer, 0, width * (wide ? 8 : 4));
+    memset (buffer, 0, width * (wide ? sizeof (argb_t) : 4));
 }
 
 static void
@@ -813,6 +600,16 @@ bits_image_fetch_untransformed_repeat_normal (bits_image_t *image,
     while (y >= image->height)
 	y -= image->height;
 
+    if (image->width == 1)
+    {
+	if (wide)
+	    replicate_pixel_float (image, 0, y, width, buffer);
+	else
+	    replicate_pixel_32 (image, 0, y, width, buffer);
+
+	return;
+    }
+
     while (width)
     {
 	while (x < 0)
@@ -823,25 +620,26 @@ bits_image_fetch_untransformed_repeat_normal (bits_image_t *image,
 	w = MIN (width, image->width - x);
 
 	if (wide)
-	    image->fetch_scanline_raw_64 ((pixman_image_t *)image, x, y, w, buffer, NULL, 0);
+	    image->fetch_scanline_float (image, x, y, w, buffer, NULL);
 	else
-	    image->fetch_scanline_raw_32 ((pixman_image_t *)image, x, y, w, buffer, NULL, 0);
+	    image->fetch_scanline_32 (image, x, y, w, buffer, NULL);
 
-	buffer += w * (wide? 2 : 1);
+	buffer += w * (wide? 4 : 1);
 	x += w;
 	width -= w;
     }
 }
 
-static void
-bits_image_fetch_untransformed_32 (pixman_image_t * image,
-                                   int              x,
-                                   int              y,
-                                   int              width,
-                                   uint32_t *       buffer,
-                                   const uint32_t * mask,
-                                   uint32_t         mask_bits)
+static uint32_t *
+bits_image_fetch_untransformed_32 (pixman_iter_t * iter,
+				   const uint32_t *mask)
 {
+    pixman_image_t *image  = iter->image;
+    int             x      = iter->x;
+    int             y      = iter->y;
+    int             width  = iter->width;
+    uint32_t *      buffer = iter->buffer;
+
     if (image->common.repeat == PIXMAN_REPEAT_NONE)
     {
 	bits_image_fetch_untransformed_repeat_none (
@@ -852,17 +650,21 @@ bits_image_fetch_untransformed_32 (pixman_image_t * image,
 	bits_image_fetch_untransformed_repeat_normal (
 	    &image->bits, FALSE, x, y, width, buffer);
     }
+
+    iter->y++;
+    return buffer;
 }
 
-static void
-bits_image_fetch_untransformed_64 (pixman_image_t * image,
-                                   int              x,
-                                   int              y,
-                                   int              width,
-                                   uint32_t *       buffer,
-                                   const uint32_t * unused,
-                                   uint32_t         unused2)
+static uint32_t *
+bits_image_fetch_untransformed_float (pixman_iter_t * iter,
+				      const uint32_t *mask)
 {
+    pixman_image_t *image  = iter->image;
+    int             x      = iter->x;
+    int             y      = iter->y;
+    int             width  = iter->width;
+    uint32_t *      buffer = iter->buffer;
+
     if (image->common.repeat == PIXMAN_REPEAT_NONE)
     {
 	bits_image_fetch_untransformed_repeat_none (
@@ -873,80 +675,234 @@ bits_image_fetch_untransformed_64 (pixman_image_t * image,
 	bits_image_fetch_untransformed_repeat_normal (
 	    &image->bits, TRUE, x, y, width, buffer);
     }
+
+    iter->y++;
+    return buffer;
 }
 
+typedef struct
+{
+    pixman_format_code_t	format;
+    uint32_t			flags;
+    pixman_iter_get_scanline_t	get_scanline_32;
+    pixman_iter_get_scanline_t  get_scanline_float;
+} fetcher_info_t;
+
+static const fetcher_info_t fetcher_info[] =
+{
+    { PIXMAN_any,
+      (FAST_PATH_NO_ALPHA_MAP			|
+       FAST_PATH_ID_TRANSFORM			|
+       FAST_PATH_NO_CONVOLUTION_FILTER		|
+       FAST_PATH_NO_PAD_REPEAT			|
+       FAST_PATH_NO_REFLECT_REPEAT),
+      bits_image_fetch_untransformed_32,
+      bits_image_fetch_untransformed_float
+    },
+
+    /* Affine, no alpha */
+    { PIXMAN_any,
+      (FAST_PATH_NO_ALPHA_MAP | FAST_PATH_HAS_TRANSFORM | FAST_PATH_AFFINE_TRANSFORM),
+      bits_image_fetch_affine_no_alpha,
+      _pixman_image_get_scanline_generic_float
+    },
+
+    /* General */
+    { PIXMAN_any,
+      0,
+      bits_image_fetch_general,
+      _pixman_image_get_scanline_generic_float
+    },
+
+    { PIXMAN_null },
+};
+
 static void
 bits_image_property_changed (pixman_image_t *image)
 {
-    bits_image_t *bits = (bits_image_t *)image;
+    _pixman_bits_image_setup_accessors (&image->bits);
+}
 
-    _pixman_bits_image_setup_raw_accessors (bits);
+void
+_pixman_bits_image_src_iter_init (pixman_image_t *image, pixman_iter_t *iter)
+{
+    pixman_format_code_t format = image->common.extended_format_code;
+    uint32_t flags = image->common.flags;
+    const fetcher_info_t *info;
 
-    image->bits.fetch_pixel_32 = image->bits.fetch_pixel_raw_32;
+    for (info = fetcher_info; info->format != PIXMAN_null; ++info)
+    {
+	if ((info->format == format || info->format == PIXMAN_any)	&&
+	    (info->flags & flags) == info->flags)
+	{
+	    if (iter->iter_flags & ITER_NARROW)
+	    {
+		iter->get_scanline = info->get_scanline_32;
+	    }
+	    else
+	    {
+		iter->data = info->get_scanline_32;
+		iter->get_scanline = info->get_scanline_float;
+	    }
+	    return;
+	}
+    }
 
-    if (bits->common.alpha_map)
+    /* Just in case we somehow didn't find a scanline function */
+    iter->get_scanline = _pixman_iter_get_scanline_noop;
+}
+
+static uint32_t *
+dest_get_scanline_narrow (pixman_iter_t *iter, const uint32_t *mask)
+{
+    pixman_image_t *image  = iter->image;
+    int             x      = iter->x;
+    int             y      = iter->y;
+    int             width  = iter->width;
+    uint32_t *	    buffer = iter->buffer;
+
+    image->bits.fetch_scanline_32 (&image->bits, x, y, width, buffer, mask);
+    if (image->common.alpha_map)
     {
-	image->common.get_scanline_64 =
-	    _pixman_image_get_scanline_generic_64;
-	image->common.get_scanline_32 =
-	    bits_image_fetch_transformed;
+	uint32_t *alpha;
+
+	if ((alpha = malloc (width * sizeof (uint32_t))))
+	{
+	    int i;
 
-	image->bits.fetch_pixel_32 = bits_image_fetch_pixel_alpha;
+	    x -= image->common.alpha_origin_x;
+	    y -= image->common.alpha_origin_y;
+
+	    image->common.alpha_map->fetch_scanline_32 (
+		image->common.alpha_map, x, y, width, alpha, mask);
+
+	    for (i = 0; i < width; ++i)
+	    {
+		buffer[i] &= ~0xff000000;
+		buffer[i] |= (alpha[i] & 0xff000000);
+	    }
+
+	    free (alpha);
+	}
     }
-    else if ((bits->common.repeat != PIXMAN_REPEAT_NONE) &&
-             bits->width == 1 &&
-             bits->height == 1)
+
+    return iter->buffer;
+}
+
+static uint32_t *
+dest_get_scanline_wide (pixman_iter_t *iter, const uint32_t *mask)
+{
+    bits_image_t *  image  = &iter->image->bits;
+    int             x      = iter->x;
+    int             y      = iter->y;
+    int             width  = iter->width;
+    argb_t *	    buffer = (argb_t *)iter->buffer;
+
+    image->fetch_scanline_float (
+	image, x, y, width, (uint32_t *)buffer, mask);
+    if (image->common.alpha_map)
     {
-	image->common.get_scanline_64 = bits_image_fetch_solid_64;
-	image->common.get_scanline_32 = bits_image_fetch_solid_32;
+	argb_t *alpha;
+
+	if ((alpha = malloc (width * sizeof (argb_t))))
+	{
+	    int i;
+
+	    x -= image->common.alpha_origin_x;
+	    y -= image->common.alpha_origin_y;
+
+	    image->common.alpha_map->fetch_scanline_float (
+		image->common.alpha_map, x, y, width, (uint32_t *)alpha, mask);
+
+	    for (i = 0; i < width; ++i)
+		buffer[i].a = alpha[i].a;
+
+	    free (alpha);
+	}
     }
-    else if (!bits->common.transform &&
-             bits->common.filter != PIXMAN_FILTER_CONVOLUTION &&
-             (bits->common.repeat == PIXMAN_REPEAT_NONE ||
-              bits->common.repeat == PIXMAN_REPEAT_NORMAL))
+
+    return iter->buffer;
+}
+
+static void
+dest_write_back_narrow (pixman_iter_t *iter)
+{
+    bits_image_t *  image  = &iter->image->bits;
+    int             x      = iter->x;
+    int             y      = iter->y;
+    int             width  = iter->width;
+    const uint32_t *buffer = iter->buffer;
+
+    image->store_scanline_32 (image, x, y, width, buffer);
+
+    if (image->common.alpha_map)
     {
-	image->common.get_scanline_64 = bits_image_fetch_untransformed_64;
-	image->common.get_scanline_32 = bits_image_fetch_untransformed_32;
+	x -= image->common.alpha_origin_x;
+	y -= image->common.alpha_origin_y;
+
+	image->common.alpha_map->store_scanline_32 (
+	    image->common.alpha_map, x, y, width, buffer);
     }
-    else if (bits->common.transform					&&
-	     bits->common.transform->matrix[2][0] == 0			&&
-	     bits->common.transform->matrix[2][1] == 0			&&
-	     bits->common.transform->matrix[2][2] == pixman_fixed_1	&&
-	     bits->common.transform->matrix[0][0] > 0			&&
-	     bits->common.transform->matrix[1][0] == 0			&&
-	     !bits->read_func						&&
-	     (bits->common.filter == PIXMAN_FILTER_BILINEAR ||
-	      bits->common.filter == PIXMAN_FILTER_GOOD	    ||
-	      bits->common.filter == PIXMAN_FILTER_BEST)		&&
-	     bits->common.repeat == PIXMAN_REPEAT_NONE			&&
-	     (bits->format == PIXMAN_a8r8g8b8	||
-	      bits->format == PIXMAN_x8r8g8b8))
+
+    iter->y++;
+}
+
+static void
+dest_write_back_wide (pixman_iter_t *iter)
+{
+    bits_image_t *  image  = &iter->image->bits;
+    int             x      = iter->x;
+    int             y      = iter->y;
+    int             width  = iter->width;
+    const uint32_t *buffer = iter->buffer;
+
+    image->store_scanline_float (image, x, y, width, buffer);
+
+    if (image->common.alpha_map)
     {
-	image->common.get_scanline_64 =
-	    _pixman_image_get_scanline_generic_64;
-	image->common.get_scanline_32 =
-	    bits_image_fetch_bilinear_no_repeat_8888;
+	x -= image->common.alpha_origin_x;
+	y -= image->common.alpha_origin_y;
+
+	image->common.alpha_map->store_scanline_float (
+	    image->common.alpha_map, x, y, width, buffer);
+    }
+
+    iter->y++;
+}
+
+void
+_pixman_bits_image_dest_iter_init (pixman_image_t *image, pixman_iter_t *iter)
+{
+    if (iter->iter_flags & ITER_NARROW)
+    {
+	if ((iter->iter_flags & (ITER_IGNORE_RGB | ITER_IGNORE_ALPHA)) ==
+	    (ITER_IGNORE_RGB | ITER_IGNORE_ALPHA))
+	{
+	    iter->get_scanline = _pixman_iter_get_scanline_noop;
+	}
+	else
+	{
+	    iter->get_scanline = dest_get_scanline_narrow;
+	}
+	
+	iter->write_back = dest_write_back_narrow;
     }
     else
     {
-	image->common.get_scanline_64 =
-	    _pixman_image_get_scanline_generic_64;
-	image->common.get_scanline_32 =
-	    bits_image_fetch_transformed;
+	iter->get_scanline = dest_get_scanline_wide;
+	iter->write_back = dest_write_back_wide;
     }
-
-    bits->store_scanline_64 = bits_image_store_scanline_64;
-    bits->store_scanline_32 = bits_image_store_scanline_32;
 }
 
 static uint32_t *
 create_bits (pixman_format_code_t format,
              int                  width,
              int                  height,
-             int *                rowstride_bytes)
+             int *		  rowstride_bytes,
+	     pixman_bool_t	  clear)
 {
     int stride;
-    int buf_size;
+    size_t buf_size;
     int bpp;
 
     /* what follows is a long-winded way, avoiding any possibility of integer
@@ -955,11 +911,11 @@ create_bits (pixman_format_code_t format,
      */
 
     bpp = PIXMAN_FORMAT_BPP (format);
-    if (pixman_multiply_overflows_int (width, bpp))
+    if (_pixman_multiply_overflows_int (width, bpp))
 	return NULL;
 
     stride = width * bpp;
-    if (pixman_addition_overflows_int (stride, 0x1f))
+    if (_pixman_addition_overflows_int (stride, 0x1f))
 	return NULL;
 
     stride += 0x1f;
@@ -967,51 +923,45 @@ create_bits (pixman_format_code_t format,
 
     stride *= sizeof (uint32_t);
 
-    if (pixman_multiply_overflows_int (height, stride))
+    if (_pixman_multiply_overflows_size (height, stride))
 	return NULL;
 
-    buf_size = height * stride;
+    buf_size = (size_t)height * stride;
 
     if (rowstride_bytes)
 	*rowstride_bytes = stride;
 
-    return calloc (buf_size, 1);
+    if (clear)
+	return calloc (buf_size, 1);
+    else
+	return malloc (buf_size);
 }
 
-PIXMAN_EXPORT pixman_image_t *
-pixman_image_create_bits (pixman_format_code_t format,
-                          int                  width,
-                          int                  height,
-                          uint32_t *           bits,
-                          int                  rowstride_bytes)
+pixman_bool_t
+_pixman_bits_image_init (pixman_image_t *     image,
+                         pixman_format_code_t format,
+                         int                  width,
+                         int                  height,
+                         uint32_t *           bits,
+                         int                  rowstride,
+			 pixman_bool_t	      clear)
 {
-    pixman_image_t *image;
     uint32_t *free_me = NULL;
 
-    /* must be a whole number of uint32_t's
-     */
-    return_val_if_fail (
-	bits == NULL || (rowstride_bytes % sizeof (uint32_t)) == 0, NULL);
-
-    return_val_if_fail (PIXMAN_FORMAT_BPP (format) >= PIXMAN_FORMAT_DEPTH (format), NULL);
-
     if (!bits && width && height)
     {
-	free_me = bits = create_bits (format, width, height, &rowstride_bytes);
-	if (!bits)
-	    return NULL;
-    }
+	int rowstride_bytes;
 
-    image = _pixman_image_allocate ();
+	free_me = bits = create_bits (format, width, height, &rowstride_bytes, clear);
 
-    if (!image)
-    {
-	if (free_me)
-	    free (free_me);
+	if (!bits)
+	    return FALSE;
 
-	return NULL;
+	rowstride = rowstride_bytes / (int) sizeof (uint32_t);
     }
 
+    _pixman_image_init (image);
+
     image->type = BITS;
     image->bits.format = format;
     image->bits.width = width;
@@ -1020,15 +970,70 @@ pixman_image_create_bits (pixman_format_code_t format,
     image->bits.free_me = free_me;
     image->bits.read_func = NULL;
     image->bits.write_func = NULL;
-
-    /* The rowstride is stored in number of uint32_t */
-    image->bits.rowstride = rowstride_bytes / (int) sizeof (uint32_t);
-
+    image->bits.rowstride = rowstride;
     image->bits.indexed = NULL;
 
     image->common.property_changed = bits_image_property_changed;
 
     _pixman_image_reset_clip_region (image);
 
+    return TRUE;
+}
+
+static pixman_image_t *
+create_bits_image_internal (pixman_format_code_t format,
+			    int                  width,
+			    int                  height,
+			    uint32_t *           bits,
+			    int                  rowstride_bytes,
+			    pixman_bool_t	 clear)
+{
+    pixman_image_t *image;
+
+    /* must be a whole number of uint32_t's
+     */
+    return_val_if_fail (
+	bits == NULL || (rowstride_bytes % sizeof (uint32_t)) == 0, NULL);
+
+    return_val_if_fail (PIXMAN_FORMAT_BPP (format) >= PIXMAN_FORMAT_DEPTH (format), NULL);
+
+    image = _pixman_image_allocate ();
+
+    if (!image)
+	return NULL;
+
+    if (!_pixman_bits_image_init (image, format, width, height, bits,
+				  rowstride_bytes / (int) sizeof (uint32_t),
+				  clear))
+    {
+	free (image);
+	return NULL;
+    }
+
     return image;
 }
+
+/* If bits is NULL, a buffer will be allocated and initialized to 0 */
+PIXMAN_EXPORT pixman_image_t *
+pixman_image_create_bits (pixman_format_code_t format,
+                          int                  width,
+                          int                  height,
+                          uint32_t *           bits,
+                          int                  rowstride_bytes)
+{
+    return create_bits_image_internal (
+	format, width, height, bits, rowstride_bytes, TRUE);
+}
+
+
+/* If bits is NULL, a buffer will be allocated and _not_ initialized */
+PIXMAN_EXPORT pixman_image_t *
+pixman_image_create_bits_no_clear (pixman_format_code_t format,
+				   int                  width,
+				   int                  height,
+				   uint32_t *           bits,
+				   int                  rowstride_bytes)
+{
+    return create_bits_image_internal (
+	format, width, height, bits, rowstride_bytes, FALSE);
+}
diff --git a/pixman/pixman/pixman-combine-float.c b/pixman/pixman/pixman-combine-float.c
new file mode 100644
index 000000000..5ea739f76
--- /dev/null
+++ b/pixman/pixman/pixman-combine-float.c
@@ -0,0 +1,1016 @@
+/* -*- Mode: c; c-basic-offset: 4; tab-width: 8; indent-tabs-mode: t; -*- */
+/*
+ * Copyright © 2010, 2012 Soren Sandmann Pedersen
+ * Copyright © 2010, 2012 Red Hat, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Author: Soren Sandmann Pedersen (sandmann@cs.au.dk)
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <math.h>
+#include <string.h>
+#include <float.h>
+
+#include "pixman-private.h"
+
+/* Workaround for http://gcc.gnu.org/PR54965 */
+/* GCC 4.6 has problems with force_inline, so just use normal inline instead */
+#if defined(__GNUC__) && (__GNUC__ == 4) && (__GNUC_MINOR__ == 6)
+#undef force_inline
+#define force_inline __inline__
+#endif
+
+typedef float (* combine_channel_t) (float sa, float s, float da, float d);
+
+static force_inline void
+combine_inner (pixman_bool_t component,
+	       float *dest, const float *src, const float *mask, int n_pixels,
+	       combine_channel_t combine_a, combine_channel_t combine_c)
+{
+    int i;
+
+    if (!mask)
+    {
+	for (i = 0; i < 4 * n_pixels; i += 4)
+	{
+	    float sa = src[i + 0];
+	    float sr = src[i + 1];
+	    float sg = src[i + 2];
+	    float sb = src[i + 3];
+	    
+	    float da = dest[i + 0];
+	    float dr = dest[i + 1];
+	    float dg = dest[i + 2];
+	    float db = dest[i + 3];					
+	    
+	    dest[i + 0] = combine_a (sa, sa, da, da);
+	    dest[i + 1] = combine_c (sa, sr, da, dr);
+	    dest[i + 2] = combine_c (sa, sg, da, dg);
+	    dest[i + 3] = combine_c (sa, sb, da, db);
+	}
+    }
+    else
+    {
+	for (i = 0; i < 4 * n_pixels; i += 4)
+	{
+	    float sa, sr, sg, sb;
+	    float ma, mr, mg, mb;
+	    float da, dr, dg, db;
+	    
+	    sa = src[i + 0];
+	    sr = src[i + 1];
+	    sg = src[i + 2];
+	    sb = src[i + 3];
+	    
+	    if (component)
+	    {
+		ma = mask[i + 0];
+		mr = mask[i + 1];
+		mg = mask[i + 2];
+		mb = mask[i + 3];
+
+		sr *= mr;
+		sg *= mg;
+		sb *= mb;
+
+		ma *= sa;
+		mr *= sa;
+		mg *= sa;
+		mb *= sa;
+		
+		sa = ma;
+	    }
+	    else
+	    {
+		ma = mask[i + 0];
+
+		sa *= ma;
+		sr *= ma;
+		sg *= ma;
+		sb *= ma;
+
+		ma = mr = mg = mb = sa;
+	    }
+	    
+	    da = dest[i + 0];
+	    dr = dest[i + 1];
+	    dg = dest[i + 2];
+	    db = dest[i + 3];
+	    
+	    dest[i + 0] = combine_a (ma, sa, da, da);
+	    dest[i + 1] = combine_c (mr, sr, da, dr);
+	    dest[i + 2] = combine_c (mg, sg, da, dg);
+	    dest[i + 3] = combine_c (mb, sb, da, db);
+	}
+    }
+}
+
+#define MAKE_COMBINER(name, component, combine_a, combine_c)		\
+    static void								\
+    combine_ ## name ## _float (pixman_implementation_t *imp,		\
+				pixman_op_t              op,		\
+				float                   *dest,		\
+				const float             *src,		\
+				const float             *mask,		\
+				int		         n_pixels)	\
+    {									\
+	combine_inner (component, dest, src, mask, n_pixels,		\
+		       combine_a, combine_c);				\
+    }
+
+#define MAKE_COMBINERS(name, combine_a, combine_c)			\
+    MAKE_COMBINER(name ## _ca, TRUE, combine_a, combine_c)		\
+    MAKE_COMBINER(name ## _u, FALSE, combine_a, combine_c)
+
+
+/*
+ * Porter/Duff operators
+ */
+typedef enum
+{
+    ZERO,
+    ONE,
+    SRC_ALPHA,
+    DEST_ALPHA,
+    INV_SA,
+    INV_DA,
+    SA_OVER_DA,
+    DA_OVER_SA,
+    INV_SA_OVER_DA,
+    INV_DA_OVER_SA,
+    ONE_MINUS_SA_OVER_DA,
+    ONE_MINUS_DA_OVER_SA,
+    ONE_MINUS_INV_DA_OVER_SA,
+    ONE_MINUS_INV_SA_OVER_DA
+} combine_factor_t;
+
+#define CLAMP(f)					\
+    (((f) < 0)? 0 : (((f) > 1.0) ? 1.0 : (f)))
+
+static force_inline float
+get_factor (combine_factor_t factor, float sa, float da)
+{
+    float f = -1;
+
+    switch (factor)
+    {
+    case ZERO:
+	f = 0.0f;
+	break;
+
+    case ONE:
+	f = 1.0f;
+	break;
+
+    case SRC_ALPHA:
+	f = sa;
+	break;
+
+    case DEST_ALPHA:
+	f = da;
+	break;
+
+    case INV_SA:
+	f = 1 - sa;
+	break;
+
+    case INV_DA:
+	f = 1 - da;
+	break;
+
+    case SA_OVER_DA:
+	if (FLOAT_IS_ZERO (da))
+	    f = 1.0f;
+	else
+	    f = CLAMP (sa / da);
+	break;
+
+    case DA_OVER_SA:
+	if (FLOAT_IS_ZERO (sa))
+	    f = 1.0f;
+	else
+	    f = CLAMP (da / sa);
+	break;
+
+    case INV_SA_OVER_DA:
+	if (FLOAT_IS_ZERO (da))
+	    f = 1.0f;
+	else
+	    f = CLAMP ((1.0f - sa) / da);
+	break;
+
+    case INV_DA_OVER_SA:
+	if (FLOAT_IS_ZERO (sa))
+	    f = 1.0f;
+	else
+	    f = CLAMP ((1.0f - da) / sa);
+	break;
+
+    case ONE_MINUS_SA_OVER_DA:
+	if (FLOAT_IS_ZERO (da))
+	    f = 0.0f;
+	else
+	    f = CLAMP (1.0f - sa / da);
+	break;
+
+    case ONE_MINUS_DA_OVER_SA:
+	if (FLOAT_IS_ZERO (sa))
+	    f = 0.0f;
+	else
+	    f = CLAMP (1.0f - da / sa);
+	break;
+
+    case ONE_MINUS_INV_DA_OVER_SA:
+	if (FLOAT_IS_ZERO (sa))
+	    f = 0.0f;
+	else
+	    f = CLAMP (1.0f - (1.0f - da) / sa);
+	break;
+
+    case ONE_MINUS_INV_SA_OVER_DA:
+	if (FLOAT_IS_ZERO (da))
+	    f = 0.0f;
+	else
+	    f = CLAMP (1.0f - (1.0f - sa) / da);
+	break;
+    }
+
+    return f;
+}
+
+#define MAKE_PD_COMBINERS(name, a, b)					\
+    static float force_inline						\
+    pd_combine_ ## name (float sa, float s, float da, float d)		\
+    {									\
+	const float fa = get_factor (a, sa, da);			\
+	const float fb = get_factor (b, sa, da);			\
+									\
+	return MIN (1.0f, s * fa + d * fb);				\
+    }									\
+    									\
+    MAKE_COMBINERS(name, pd_combine_ ## name, pd_combine_ ## name)
+
+MAKE_PD_COMBINERS (clear,			ZERO,				ZERO)
+MAKE_PD_COMBINERS (src,				ONE,				ZERO)
+MAKE_PD_COMBINERS (dst,				ZERO,				ONE)
+MAKE_PD_COMBINERS (over,			ONE,				INV_SA)
+MAKE_PD_COMBINERS (over_reverse,		INV_DA,				ONE)
+MAKE_PD_COMBINERS (in,				DEST_ALPHA,			ZERO)
+MAKE_PD_COMBINERS (in_reverse,			ZERO,				SRC_ALPHA)
+MAKE_PD_COMBINERS (out,				INV_DA,				ZERO)
+MAKE_PD_COMBINERS (out_reverse,			ZERO,				INV_SA)
+MAKE_PD_COMBINERS (atop,			DEST_ALPHA,			INV_SA)
+MAKE_PD_COMBINERS (atop_reverse,		INV_DA,				SRC_ALPHA)
+MAKE_PD_COMBINERS (xor,				INV_DA,				INV_SA)
+MAKE_PD_COMBINERS (add,				ONE,				ONE)
+
+MAKE_PD_COMBINERS (saturate,			INV_DA_OVER_SA,			ONE)
+
+MAKE_PD_COMBINERS (disjoint_clear,		ZERO,				ZERO)
+MAKE_PD_COMBINERS (disjoint_src,		ONE,				ZERO)
+MAKE_PD_COMBINERS (disjoint_dst,		ZERO,				ONE)
+MAKE_PD_COMBINERS (disjoint_over,		ONE,				INV_SA_OVER_DA)
+MAKE_PD_COMBINERS (disjoint_over_reverse,	INV_DA_OVER_SA,			ONE)
+MAKE_PD_COMBINERS (disjoint_in,			ONE_MINUS_INV_DA_OVER_SA,	ZERO)
+MAKE_PD_COMBINERS (disjoint_in_reverse,		ZERO,				ONE_MINUS_INV_SA_OVER_DA)
+MAKE_PD_COMBINERS (disjoint_out,		INV_DA_OVER_SA,			ZERO)
+MAKE_PD_COMBINERS (disjoint_out_reverse,	ZERO,				INV_SA_OVER_DA)
+MAKE_PD_COMBINERS (disjoint_atop,		ONE_MINUS_INV_DA_OVER_SA,	INV_SA_OVER_DA)
+MAKE_PD_COMBINERS (disjoint_atop_reverse,	INV_DA_OVER_SA,			ONE_MINUS_INV_SA_OVER_DA)
+MAKE_PD_COMBINERS (disjoint_xor,		INV_DA_OVER_SA,			INV_SA_OVER_DA)
+
+MAKE_PD_COMBINERS (conjoint_clear,		ZERO,				ZERO)
+MAKE_PD_COMBINERS (conjoint_src,		ONE,				ZERO)
+MAKE_PD_COMBINERS (conjoint_dst,		ZERO,				ONE)
+MAKE_PD_COMBINERS (conjoint_over,		ONE,				ONE_MINUS_SA_OVER_DA)
+MAKE_PD_COMBINERS (conjoint_over_reverse,	ONE_MINUS_DA_OVER_SA,		ONE)
+MAKE_PD_COMBINERS (conjoint_in,			DA_OVER_SA,			ZERO)
+MAKE_PD_COMBINERS (conjoint_in_reverse,		ZERO,				SA_OVER_DA)
+MAKE_PD_COMBINERS (conjoint_out,		ONE_MINUS_DA_OVER_SA,		ZERO)
+MAKE_PD_COMBINERS (conjoint_out_reverse,	ZERO,				ONE_MINUS_SA_OVER_DA)
+MAKE_PD_COMBINERS (conjoint_atop,		DA_OVER_SA,			ONE_MINUS_SA_OVER_DA)
+MAKE_PD_COMBINERS (conjoint_atop_reverse,	ONE_MINUS_DA_OVER_SA,		SA_OVER_DA)
+MAKE_PD_COMBINERS (conjoint_xor,		ONE_MINUS_DA_OVER_SA,		ONE_MINUS_SA_OVER_DA)
+
+/*
+ * PDF blend modes:
+ *
+ * The following blend modes have been taken from the PDF ISO 32000
+ * specification, which at this point in time is available from
+ * http://www.adobe.com/devnet/acrobat/pdfs/PDF32000_2008.pdf
+ * The relevant chapters are 11.3.5 and 11.3.6.
+ * The formula for computing the final pixel color given in 11.3.6 is:
+ * αr × Cr = (1 – αs) × αb × Cb + (1 – αb) × αs × Cs + αb × αs × B(Cb, Cs)
+ * with B() being the blend function.
+ * Note that OVER is a special case of this operation, using B(Cb, Cs) = Cs
+ *
+ * These blend modes should match the SVG filter draft specification, as
+ * it has been designed to mirror ISO 32000. Note that at the current point
+ * no released draft exists that shows this, as the formulas have not been
+ * updated yet after the release of ISO 32000.
+ *
+ * The default implementation here uses the PDF_SEPARABLE_BLEND_MODE and
+ * PDF_NON_SEPARABLE_BLEND_MODE macros, which take the blend function as an
+ * argument. Note that this implementation operates on premultiplied colors,
+ * while the PDF specification does not. Therefore the code uses the formula
+ * ar.Cra = (1 – as) . Dca + (1 – ad) . Sca + B(Dca, ad, Sca, as)
+ */
+
+#define MAKE_SEPARABLE_PDF_COMBINERS(name)				\
+    static force_inline float						\
+    combine_ ## name ## _a (float sa, float s, float da, float d)	\
+    {									\
+	return da + sa - da * sa;					\
+    }									\
+    									\
+    static force_inline float						\
+    combine_ ## name ## _c (float sa, float s, float da, float d)	\
+    {									\
+	float f = (1 - sa) * d + (1 - da) * s;				\
+									\
+	return f + blend_ ## name (sa, s, da, d);			\
+    }									\
+    									\
+    MAKE_COMBINERS (name, combine_ ## name ## _a, combine_ ## name ## _c)
+
+static force_inline float
+blend_multiply (float sa, float s, float da, float d)
+{
+    return d * s;
+}
+
+static force_inline float
+blend_screen (float sa, float s, float da, float d)
+{
+    return d * sa + s * da - s * d;
+}
+
+static force_inline float
+blend_overlay (float sa, float s, float da, float d)
+{
+    if (2 * d < da)
+	return 2 * s * d;
+    else
+	return sa * da - 2 * (da - d) * (sa - s);
+}
+
+static force_inline float
+blend_darken (float sa, float s, float da, float d)
+{
+    s = s * da;
+    d = d * sa;
+
+    if (s > d)
+	return d;
+    else
+	return s;
+}
+
+static force_inline float
+blend_lighten (float sa, float s, float da, float d)
+{
+    s = s * da;
+    d = d * sa;
+
+    if (s > d)
+	return s;
+    else
+	return d;
+}
+
+static force_inline float
+blend_color_dodge (float sa, float s, float da, float d)
+{
+    if (FLOAT_IS_ZERO (d))
+	return 0.0f;
+    else if (d * sa >= sa * da - s * da)
+	return sa * da;
+    else if (FLOAT_IS_ZERO (sa - s))
+	return sa * da;
+    else
+	return sa * sa * d / (sa - s);
+}
+
+static force_inline float
+blend_color_burn (float sa, float s, float da, float d)
+{
+    if (d >= da)
+	return sa * da;
+    else if (sa * (da - d) >= s * da)
+	return 0.0f;
+    else if (FLOAT_IS_ZERO (s))
+	return 0.0f;
+    else
+	return sa * (da - sa * (da - d) / s);
+}
+
+static force_inline float
+blend_hard_light (float sa, float s, float da, float d)
+{
+    if (2 * s < sa)
+	return 2 * s * d;
+    else
+	return sa * da - 2 * (da - d) * (sa - s);
+}
+
+static force_inline float
+blend_soft_light (float sa, float s, float da, float d)
+{
+    if (2 * s < sa)
+    {
+	if (FLOAT_IS_ZERO (da))
+	    return d * sa;
+	else
+	    return d * sa - d * (da - d) * (sa - 2 * s) / da;
+    }
+    else
+    {
+	if (FLOAT_IS_ZERO (da))
+	{
+	    return 0.0f;
+	}
+	else
+	{
+	    if (4 * d <= da)
+		return d * sa + (2 * s - sa) * d * ((16 * d / da - 12) * d / da + 3);
+	    else
+		return d * sa + (sqrtf (d * da) - d) * (2 * s - sa);
+	}
+    }
+}
+
+static force_inline float
+blend_difference (float sa, float s, float da, float d)
+{
+    float dsa = d * sa;
+    float sda = s * da;
+
+    if (sda < dsa)
+	return dsa - sda;
+    else
+	return sda - dsa;
+}
+
+static force_inline float
+blend_exclusion (float sa, float s, float da, float d)
+{
+    return s * da + d * sa - 2 * d * s;
+}
+
+MAKE_SEPARABLE_PDF_COMBINERS (multiply)
+MAKE_SEPARABLE_PDF_COMBINERS (screen)
+MAKE_SEPARABLE_PDF_COMBINERS (overlay)
+MAKE_SEPARABLE_PDF_COMBINERS (darken)
+MAKE_SEPARABLE_PDF_COMBINERS (lighten)
+MAKE_SEPARABLE_PDF_COMBINERS (color_dodge)
+MAKE_SEPARABLE_PDF_COMBINERS (color_burn)
+MAKE_SEPARABLE_PDF_COMBINERS (hard_light)
+MAKE_SEPARABLE_PDF_COMBINERS (soft_light)
+MAKE_SEPARABLE_PDF_COMBINERS (difference)
+MAKE_SEPARABLE_PDF_COMBINERS (exclusion)
+
+/*
+ * PDF nonseperable blend modes.
+ *
+ * These are implemented using the following functions to operate in Hsl
+ * space, with Cmax, Cmid, Cmin referring to the max, mid and min value
+ * of the red, green and blue components.
+ *
+ * LUM (C) = 0.3 × Cred + 0.59 × Cgreen + 0.11 × Cblue
+ *
+ * clip_color (C):
+ *   l = LUM (C)
+ *   min = Cmin
+ *   max = Cmax
+ *   if n < 0.0
+ *     C = l + (((C – l) × l) ⁄     (l – min))
+ *   if x > 1.0
+ *     C = l + (((C – l) × (1 – l)) (max – l))
+ *   return C
+ *
+ * set_lum (C, l):
+ *   d = l – LUM (C)
+ *   C += d
+ *   return clip_color (C)
+ *
+ * SAT (C) = CH_MAX (C) - CH_MIN (C)
+ *
+ * set_sat (C, s):
+ *  if Cmax > Cmin
+ *    Cmid = ( ( ( Cmid – Cmin ) × s ) ⁄ ( Cmax – Cmin ) )
+ *    Cmax = s
+ *  else
+ *    Cmid = Cmax = 0.0
+ *  Cmin = 0.0
+ *  return C
+ */
+
+/* For premultiplied colors, we need to know what happens when C is
+ * multiplied by a real number. LUM and SAT are linear:
+ *
+ *    LUM (r × C) = r × LUM (C)		SAT (r × C) = r × SAT (C)
+ *
+ * If we extend clip_color with an extra argument a and change
+ *
+ *        if x >= 1.0
+ *
+ * into
+ *
+ *        if x >= a
+ *
+ * then clip_color is also linear:
+ *
+ *     r * clip_color (C, a) = clip_color (r_c, ra);
+ *
+ * for positive r.
+ *
+ * Similarly, we can extend set_lum with an extra argument that is just passed
+ * on to clip_color:
+ *
+ *     r × set_lum ( C, l, a)
+ *
+ *   = r × clip_color ( C + l - LUM (C), a)
+ *
+ *   = clip_color ( r * C + r × l - LUM (r × C), r * a)
+ *
+ *   = set_lum ( r * C, r * l, r * a)
+ *
+ * Finally, set_sat:
+ *
+ *     r * set_sat (C, s) = set_sat (x * C, r * s)
+ *
+ * The above holds for all non-zero x because they x'es in the fraction for
+ * C_mid cancel out. Specifically, it holds for x = r:
+ *
+ *     r * set_sat (C, s) = set_sat (r_c, rs)
+ *
+ *
+ *
+ *
+ * So, for the non-separable PDF blend modes, we have (using s, d for
+ * non-premultiplied colors, and S, D for premultiplied:
+ *
+ *   Color:
+ *
+ *     a_s * a_d * B(s, d)
+ *   = a_s * a_d * set_lum (S/a_s, LUM (D/a_d), 1)
+ *   = set_lum (S * a_d, a_s * LUM (D), a_s * a_d)
+ *
+ *
+ *   Luminosity:
+ *
+ *     a_s * a_d * B(s, d)
+ *   = a_s * a_d * set_lum (D/a_d, LUM(S/a_s), 1)
+ *   = set_lum (a_s * D, a_d * LUM(S), a_s * a_d)
+ *
+ *
+ *   Saturation:
+ *
+ *     a_s * a_d * B(s, d)
+ *   = a_s * a_d * set_lum (set_sat (D/a_d, SAT (S/a_s)), LUM (D/a_d), 1)
+ *   = set_lum (a_s * a_d * set_sat (D/a_d, SAT (S/a_s)),
+ *                                        a_s * LUM (D), a_s * a_d)
+ *   = set_lum (set_sat (a_s * D, a_d * SAT (S), a_s * LUM (D), a_s * a_d))
+ *
+ *   Hue:
+ *
+ *     a_s * a_d * B(s, d)
+ *   = a_s * a_d * set_lum (set_sat (S/a_s, SAT (D/a_d)), LUM (D/a_d), 1)
+ *   = set_lum (set_sat (a_d * S, a_s * SAT (D)), a_s * LUM (D), a_s * a_d)
+ *
+ */
+
+typedef struct
+{
+    float	r;
+    float	g;
+    float	b;
+} rgb_t;
+
+static force_inline float
+minf (float a, float b)
+{
+    return a < b? a : b;
+}
+
+static force_inline float
+maxf (float a, float b)
+{
+    return a > b? a : b;
+}
+
+static force_inline float
+channel_min (const rgb_t *c)
+{
+    return minf (minf (c->r, c->g), c->b);
+}
+
+static force_inline float
+channel_max (const rgb_t *c)
+{
+    return maxf (maxf (c->r, c->g), c->b);
+}
+
+static force_inline float
+get_lum (const rgb_t *c)
+{
+    return c->r * 0.3f + c->g * 0.59f + c->b * 0.11f;
+}
+
+static force_inline float
+get_sat (const rgb_t *c)
+{
+    return channel_max (c) - channel_min (c);
+}
+
+static void
+clip_color (rgb_t *color, float a)
+{
+    float l = get_lum (color);
+    float n = channel_min (color);
+    float x = channel_max (color);
+    float t;
+
+    if (n < 0.0f)
+    {
+	t = l - n;
+	if (FLOAT_IS_ZERO (t))
+	{
+	    color->r = 0.0f;
+	    color->g = 0.0f;
+	    color->b = 0.0f;
+	}
+	else
+	{
+	    color->r = l + (((color->r - l) * l) / t);
+	    color->g = l + (((color->g - l) * l) / t);
+	    color->b = l + (((color->b - l) * l) / t);
+	}
+    }
+    if (x > a)
+    {
+	t = x - l;
+	if (FLOAT_IS_ZERO (t))
+	{
+	    color->r = a;
+	    color->g = a;
+	    color->b = a;
+	}
+	else
+	{
+	    color->r = l + (((color->r - l) * (a - l) / t));
+	    color->g = l + (((color->g - l) * (a - l) / t));
+	    color->b = l + (((color->b - l) * (a - l) / t));
+	}
+    }
+}
+
+static void
+set_lum (rgb_t *color, float sa, float l)
+{
+    float d = l - get_lum (color);
+
+    color->r = color->r + d;
+    color->g = color->g + d;
+    color->b = color->b + d;
+
+    clip_color (color, sa);
+}
+
+static void
+set_sat (rgb_t *src, float sat)
+{
+    float *max, *mid, *min;
+    float t;
+
+    if (src->r > src->g)
+    {
+	if (src->r > src->b)
+	{
+	    max = &(src->r);
+
+	    if (src->g > src->b)
+	    {
+		mid = &(src->g);
+		min = &(src->b);
+	    }
+	    else
+	    {
+		mid = &(src->b);
+		min = &(src->g);
+	    }
+	}
+	else
+	{
+	    max = &(src->b);
+	    mid = &(src->r);
+	    min = &(src->g);
+	}
+    }
+    else
+    {
+	if (src->r > src->b)
+	{
+	    max = &(src->g);
+	    mid = &(src->r);
+	    min = &(src->b);
+	}
+	else
+	{
+	    min = &(src->r);
+
+	    if (src->g > src->b)
+	    {
+		max = &(src->g);
+		mid = &(src->b);
+	    }
+	    else
+	    {
+		max = &(src->b);
+		mid = &(src->g);
+	    }
+	}
+    }
+
+    t = *max - *min;
+
+    if (FLOAT_IS_ZERO (t))
+    {
+	*mid = *max = 0.0f;
+    }
+    else
+    {
+	*mid = ((*mid - *min) * sat) / t;
+	*max = sat;
+    }
+
+    *min = 0.0f;
+}
+
+/*
+ * Hue:
+ * B(Cb, Cs) = set_lum (set_sat (Cs, SAT (Cb)), LUM (Cb))
+ */
+static force_inline void
+blend_hsl_hue (rgb_t *res,
+	       const rgb_t *dest, float da,
+	       const rgb_t *src, float sa)
+{
+    res->r = src->r * da;
+    res->g = src->g * da;
+    res->b = src->b * da;
+
+    set_sat (res, get_sat (dest) * sa);
+    set_lum (res, sa * da, get_lum (dest) * sa);
+}
+
+/*
+ * Saturation:
+ * B(Cb, Cs) = set_lum (set_sat (Cb, SAT (Cs)), LUM (Cb))
+ */
+static force_inline void
+blend_hsl_saturation (rgb_t *res,
+		      const rgb_t *dest, float da,
+		      const rgb_t *src, float sa)
+{
+    res->r = dest->r * sa;
+    res->g = dest->g * sa;
+    res->b = dest->b * sa;
+
+    set_sat (res, get_sat (src) * da);
+    set_lum (res, sa * da, get_lum (dest) * sa);
+}
+
+/*
+ * Color:
+ * B(Cb, Cs) = set_lum (Cs, LUM (Cb))
+ */
+static force_inline void
+blend_hsl_color (rgb_t *res,
+		 const rgb_t *dest, float da,
+		 const rgb_t *src, float sa)
+{
+    res->r = src->r * da;
+    res->g = src->g * da;
+    res->b = src->b * da;
+
+    set_lum (res, sa * da, get_lum (dest) * sa);
+}
+
+/*
+ * Luminosity:
+ * B(Cb, Cs) = set_lum (Cb, LUM (Cs))
+ */
+static force_inline void
+blend_hsl_luminosity (rgb_t *res,
+		      const rgb_t *dest, float da,
+		      const rgb_t *src, float sa)
+{
+    res->r = dest->r * sa;
+    res->g = dest->g * sa;
+    res->b = dest->b * sa;
+
+    set_lum (res, sa * da, get_lum (src) * da);
+}
+
+#define MAKE_NON_SEPARABLE_PDF_COMBINERS(name)				\
+    static void								\
+    combine_ ## name ## _u_float (pixman_implementation_t *imp,		\
+				  pixman_op_t              op,		\
+				  float                   *dest,	\
+				  const float             *src,		\
+				  const float             *mask,	\
+				  int		           n_pixels)	\
+    {									\
+    	int i;								\
+									\
+	for (i = 0; i < 4 * n_pixels; i += 4)				\
+	{								\
+	    float sa, da;						\
+	    rgb_t sc, dc, rc;						\
+									\
+	    sa = src[i + 0];						\
+	    sc.r = src[i + 1];						\
+	    sc.g = src[i + 2];						\
+	    sc.b = src[i + 3];						\
+									\
+	    da = dest[i + 0];						\
+	    dc.r = dest[i + 1];						\
+	    dc.g = dest[i + 2];						\
+	    dc.b = dest[i + 3];						\
+									\
+	    if (mask)							\
+	    {								\
+		float ma = mask[i + 0];					\
+									\
+		/* Component alpha is not supported for HSL modes */	\
+		sa *= ma;						\
+		sc.r *= ma;						\
+		sc.g *= ma;						\
+		sc.g *= ma;						\
+	    }								\
+									\
+	    blend_ ## name (&rc, &dc, da, &sc, sa);			\
+									\
+	    dest[i + 0] = sa + da - sa * da;				\
+	    dest[i + 1] = (1 - sa) * dc.r + (1 - da) * sc.r + rc.r;	\
+	    dest[i + 2] = (1 - sa) * dc.g + (1 - da) * sc.g + rc.g;	\
+	    dest[i + 3] = (1 - sa) * dc.b + (1 - da) * sc.b + rc.b;	\
+	}								\
+    }
+
+MAKE_NON_SEPARABLE_PDF_COMBINERS(hsl_hue)
+MAKE_NON_SEPARABLE_PDF_COMBINERS(hsl_saturation)
+MAKE_NON_SEPARABLE_PDF_COMBINERS(hsl_color)
+MAKE_NON_SEPARABLE_PDF_COMBINERS(hsl_luminosity)
+
+void
+_pixman_setup_combiner_functions_float (pixman_implementation_t *imp)
+{
+    /* Unified alpha */
+    imp->combine_float[PIXMAN_OP_CLEAR] = combine_clear_u_float;
+    imp->combine_float[PIXMAN_OP_SRC] = combine_src_u_float;
+    imp->combine_float[PIXMAN_OP_DST] = combine_dst_u_float;
+    imp->combine_float[PIXMAN_OP_OVER] = combine_over_u_float;
+    imp->combine_float[PIXMAN_OP_OVER_REVERSE] = combine_over_reverse_u_float;
+    imp->combine_float[PIXMAN_OP_IN] = combine_in_u_float;
+    imp->combine_float[PIXMAN_OP_IN_REVERSE] = combine_in_reverse_u_float;
+    imp->combine_float[PIXMAN_OP_OUT] = combine_out_u_float;
+    imp->combine_float[PIXMAN_OP_OUT_REVERSE] = combine_out_reverse_u_float;
+    imp->combine_float[PIXMAN_OP_ATOP] = combine_atop_u_float;
+    imp->combine_float[PIXMAN_OP_ATOP_REVERSE] = combine_atop_reverse_u_float;
+    imp->combine_float[PIXMAN_OP_XOR] = combine_xor_u_float;
+    imp->combine_float[PIXMAN_OP_ADD] = combine_add_u_float;
+    imp->combine_float[PIXMAN_OP_SATURATE] = combine_saturate_u_float;
+
+    /* Disjoint, unified */
+    imp->combine_float[PIXMAN_OP_DISJOINT_CLEAR] = combine_disjoint_clear_u_float;
+    imp->combine_float[PIXMAN_OP_DISJOINT_SRC] = combine_disjoint_src_u_float;
+    imp->combine_float[PIXMAN_OP_DISJOINT_DST] = combine_disjoint_dst_u_float;
+    imp->combine_float[PIXMAN_OP_DISJOINT_OVER] = combine_disjoint_over_u_float;
+    imp->combine_float[PIXMAN_OP_DISJOINT_OVER_REVERSE] = combine_disjoint_over_reverse_u_float;
+    imp->combine_float[PIXMAN_OP_DISJOINT_IN] = combine_disjoint_in_u_float;
+    imp->combine_float[PIXMAN_OP_DISJOINT_IN_REVERSE] = combine_disjoint_in_reverse_u_float;
+    imp->combine_float[PIXMAN_OP_DISJOINT_OUT] = combine_disjoint_out_u_float;
+    imp->combine_float[PIXMAN_OP_DISJOINT_OUT_REVERSE] = combine_disjoint_out_reverse_u_float;
+    imp->combine_float[PIXMAN_OP_DISJOINT_ATOP] = combine_disjoint_atop_u_float;
+    imp->combine_float[PIXMAN_OP_DISJOINT_ATOP_REVERSE] = combine_disjoint_atop_reverse_u_float;
+    imp->combine_float[PIXMAN_OP_DISJOINT_XOR] = combine_disjoint_xor_u_float;
+
+    /* Conjoint, unified */
+    imp->combine_float[PIXMAN_OP_CONJOINT_CLEAR] = combine_conjoint_clear_u_float;
+    imp->combine_float[PIXMAN_OP_CONJOINT_SRC] = combine_conjoint_src_u_float;
+    imp->combine_float[PIXMAN_OP_CONJOINT_DST] = combine_conjoint_dst_u_float;
+    imp->combine_float[PIXMAN_OP_CONJOINT_OVER] = combine_conjoint_over_u_float;
+    imp->combine_float[PIXMAN_OP_CONJOINT_OVER_REVERSE] = combine_conjoint_over_reverse_u_float;
+    imp->combine_float[PIXMAN_OP_CONJOINT_IN] = combine_conjoint_in_u_float;
+    imp->combine_float[PIXMAN_OP_CONJOINT_IN_REVERSE] = combine_conjoint_in_reverse_u_float;
+    imp->combine_float[PIXMAN_OP_CONJOINT_OUT] = combine_conjoint_out_u_float;
+    imp->combine_float[PIXMAN_OP_CONJOINT_OUT_REVERSE] = combine_conjoint_out_reverse_u_float;
+    imp->combine_float[PIXMAN_OP_CONJOINT_ATOP] = combine_conjoint_atop_u_float;
+    imp->combine_float[PIXMAN_OP_CONJOINT_ATOP_REVERSE] = combine_conjoint_atop_reverse_u_float;
+    imp->combine_float[PIXMAN_OP_CONJOINT_XOR] = combine_conjoint_xor_u_float;
+
+    /* PDF operators, unified */
+    imp->combine_float[PIXMAN_OP_MULTIPLY] = combine_multiply_u_float;
+    imp->combine_float[PIXMAN_OP_SCREEN] = combine_screen_u_float;
+    imp->combine_float[PIXMAN_OP_OVERLAY] = combine_overlay_u_float;
+    imp->combine_float[PIXMAN_OP_DARKEN] = combine_darken_u_float;
+    imp->combine_float[PIXMAN_OP_LIGHTEN] = combine_lighten_u_float;
+    imp->combine_float[PIXMAN_OP_COLOR_DODGE] = combine_color_dodge_u_float;
+    imp->combine_float[PIXMAN_OP_COLOR_BURN] = combine_color_burn_u_float;
+    imp->combine_float[PIXMAN_OP_HARD_LIGHT] = combine_hard_light_u_float;
+    imp->combine_float[PIXMAN_OP_SOFT_LIGHT] = combine_soft_light_u_float;
+    imp->combine_float[PIXMAN_OP_DIFFERENCE] = combine_difference_u_float;
+    imp->combine_float[PIXMAN_OP_EXCLUSION] = combine_exclusion_u_float;
+
+    imp->combine_float[PIXMAN_OP_HSL_HUE] = combine_hsl_hue_u_float;
+    imp->combine_float[PIXMAN_OP_HSL_SATURATION] = combine_hsl_saturation_u_float;
+    imp->combine_float[PIXMAN_OP_HSL_COLOR] = combine_hsl_color_u_float;
+    imp->combine_float[PIXMAN_OP_HSL_LUMINOSITY] = combine_hsl_luminosity_u_float;
+
+    /* Component alpha combiners */
+    imp->combine_float_ca[PIXMAN_OP_CLEAR] = combine_clear_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_SRC] = combine_src_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_DST] = combine_dst_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_OVER] = combine_over_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_OVER_REVERSE] = combine_over_reverse_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_IN] = combine_in_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_IN_REVERSE] = combine_in_reverse_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_OUT] = combine_out_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_OUT_REVERSE] = combine_out_reverse_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_ATOP] = combine_atop_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_ATOP_REVERSE] = combine_atop_reverse_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_XOR] = combine_xor_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_ADD] = combine_add_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_SATURATE] = combine_saturate_ca_float;
+
+    /* Disjoint CA */
+    imp->combine_float_ca[PIXMAN_OP_DISJOINT_CLEAR] = combine_disjoint_clear_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_DISJOINT_SRC] = combine_disjoint_src_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_DISJOINT_DST] = combine_disjoint_dst_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_DISJOINT_OVER] = combine_disjoint_over_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_DISJOINT_OVER_REVERSE] = combine_disjoint_over_reverse_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_DISJOINT_IN] = combine_disjoint_in_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_DISJOINT_IN_REVERSE] = combine_disjoint_in_reverse_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_DISJOINT_OUT] = combine_disjoint_out_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_DISJOINT_OUT_REVERSE] = combine_disjoint_out_reverse_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_DISJOINT_ATOP] = combine_disjoint_atop_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_DISJOINT_ATOP_REVERSE] = combine_disjoint_atop_reverse_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_DISJOINT_XOR] = combine_disjoint_xor_ca_float;
+
+    /* Conjoint CA */
+    imp->combine_float_ca[PIXMAN_OP_CONJOINT_CLEAR] = combine_conjoint_clear_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_CONJOINT_SRC] = combine_conjoint_src_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_CONJOINT_DST] = combine_conjoint_dst_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_CONJOINT_OVER] = combine_conjoint_over_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_CONJOINT_OVER_REVERSE] = combine_conjoint_over_reverse_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_CONJOINT_IN] = combine_conjoint_in_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_CONJOINT_IN_REVERSE] = combine_conjoint_in_reverse_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_CONJOINT_OUT] = combine_conjoint_out_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_CONJOINT_OUT_REVERSE] = combine_conjoint_out_reverse_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_CONJOINT_ATOP] = combine_conjoint_atop_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_CONJOINT_ATOP_REVERSE] = combine_conjoint_atop_reverse_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_CONJOINT_XOR] = combine_conjoint_xor_ca_float;
+
+    /* PDF operators CA */
+    imp->combine_float_ca[PIXMAN_OP_MULTIPLY] = combine_multiply_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_SCREEN] = combine_screen_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_OVERLAY] = combine_overlay_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_DARKEN] = combine_darken_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_LIGHTEN] = combine_lighten_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_COLOR_DODGE] = combine_color_dodge_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_COLOR_BURN] = combine_color_burn_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_HARD_LIGHT] = combine_hard_light_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_SOFT_LIGHT] = combine_soft_light_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_DIFFERENCE] = combine_difference_ca_float;
+    imp->combine_float_ca[PIXMAN_OP_EXCLUSION] = combine_exclusion_ca_float;
+
+    /* It is not clear that these make sense, so make them noops for now */
+    imp->combine_float_ca[PIXMAN_OP_HSL_HUE] = combine_dst_u_float;
+    imp->combine_float_ca[PIXMAN_OP_HSL_SATURATION] = combine_dst_u_float;
+    imp->combine_float_ca[PIXMAN_OP_HSL_COLOR] = combine_dst_u_float;
+    imp->combine_float_ca[PIXMAN_OP_HSL_LUMINOSITY] = combine_dst_u_float;
+}
diff --git a/pixman/pixman/pixman-combine.c.template b/pixman/pixman/pixman-combine.c.template
deleted file mode 100644
index c129980a8..000000000
--- a/pixman/pixman/pixman-combine.c.template
+++ /dev/null
@@ -1,2436 +0,0 @@
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#include <math.h>
-#include <string.h>
-
-#include "pixman-private.h"
-
-#include "pixman-combine.h"
-
-/*** per channel helper functions ***/
-
-static void
-combine_mask_ca (comp4_t *src, comp4_t *mask)
-{
-    comp4_t a = *mask;
-
-    comp4_t x;
-    comp2_t xa;
-
-    if (!a)
-    {
-	*(src) = 0;
-	return;
-    }
-
-    x = *(src);
-    if (a == ~0)
-    {
-	x = x >> A_SHIFT;
-	x |= x << G_SHIFT;
-	x |= x << R_SHIFT;
-	*(mask) = x;
-	return;
-    }
-
-    xa = x >> A_SHIFT;
-    UNcx4_MUL_UNcx4 (x, a);
-    *(src) = x;
-    
-    UNcx4_MUL_UNc (a, xa);
-    *(mask) = a;
-}
-
-static void
-combine_mask_value_ca (comp4_t *src, const comp4_t *mask)
-{
-    comp4_t a = *mask;
-    comp4_t x;
-
-    if (!a)
-    {
-	*(src) = 0;
-	return;
-    }
-
-    if (a == ~0)
-	return;
-
-    x = *(src);
-    UNcx4_MUL_UNcx4 (x, a);
-    *(src) = x;
-}
-
-static void
-combine_mask_alpha_ca (const comp4_t *src, comp4_t *mask)
-{
-    comp4_t a = *(mask);
-    comp4_t x;
-
-    if (!a)
-	return;
-
-    x = *(src) >> A_SHIFT;
-    if (x == MASK)
-	return;
-
-    if (a == ~0)
-    {
-	x |= x << G_SHIFT;
-	x |= x << R_SHIFT;
-	*(mask) = x;
-	return;
-    }
-
-    UNcx4_MUL_UNc (a, x);
-    *(mask) = a;
-}
-
-/*
- * There are two ways of handling alpha -- either as a single unified value or
- * a separate value for each component, hence each macro must have two
- * versions.  The unified alpha version has a 'U' at the end of the name,
- * the component version has a 'C'.  Similarly, functions which deal with
- * this difference will have two versions using the same convention.
- */
-
-/*
- * All of the composing functions
- */
-
-static force_inline comp4_t
-combine_mask (const comp4_t *src, const comp4_t *mask, int i)
-{
-    comp4_t s, m;
-
-    if (mask)
-    {
-	m = *(mask + i) >> A_SHIFT;
-
-	if (!m)
-	    return 0;
-    }
-
-    s = *(src + i);
-
-    if (mask)
-	UNcx4_MUL_UNc (s, m);
-
-    return s;
-}
-
-static void
-combine_clear (pixman_implementation_t *imp,
-               pixman_op_t              op,
-               comp4_t *                dest,
-               const comp4_t *          src,
-               const comp4_t *          mask,
-               int                      width)
-{
-    memset (dest, 0, width * sizeof(comp4_t));
-}
-
-static void
-combine_src_u (pixman_implementation_t *imp,
-               pixman_op_t              op,
-               comp4_t *                dest,
-               const comp4_t *          src,
-               const comp4_t *          mask,
-               int                      width)
-{
-    int i;
-
-    if (!mask)
-	memcpy (dest, src, width * sizeof (comp4_t));
-    else
-    {
-	for (i = 0; i < width; ++i)
-	{
-	    comp4_t s = combine_mask (src, mask, i);
-
-	    *(dest + i) = s;
-	}
-    }
-}
-
-/* if the Src is opaque, call combine_src_u */
-static void
-combine_over_u (pixman_implementation_t *imp,
-                pixman_op_t              op,
-                comp4_t *                dest,
-                const comp4_t *          src,
-                const comp4_t *          mask,
-                int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = combine_mask (src, mask, i);
-	comp4_t d = *(dest + i);
-	comp4_t ia = ALPHA_c (~s);
-
-	UNcx4_MUL_UNc_ADD_UNcx4 (d, ia, s);
-	*(dest + i) = d;
-    }
-}
-
-/* if the Dst is opaque, this is a noop */
-static void
-combine_over_reverse_u (pixman_implementation_t *imp,
-                        pixman_op_t              op,
-                        comp4_t *                dest,
-                        const comp4_t *          src,
-                        const comp4_t *          mask,
-                        int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = combine_mask (src, mask, i);
-	comp4_t d = *(dest + i);
-	comp4_t ia = ALPHA_c (~*(dest + i));
-	UNcx4_MUL_UNc_ADD_UNcx4 (s, ia, d);
-	*(dest + i) = s;
-    }
-}
-
-/* if the Dst is opaque, call combine_src_u */
-static void
-combine_in_u (pixman_implementation_t *imp,
-              pixman_op_t              op,
-              comp4_t *                dest,
-              const comp4_t *          src,
-              const comp4_t *          mask,
-              int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = combine_mask (src, mask, i);
-	comp4_t a = ALPHA_c (*(dest + i));
-	UNcx4_MUL_UNc (s, a);
-	*(dest + i) = s;
-    }
-}
-
-/* if the Src is opaque, this is a noop */
-static void
-combine_in_reverse_u (pixman_implementation_t *imp,
-                      pixman_op_t              op,
-                      comp4_t *                dest,
-                      const comp4_t *          src,
-                      const comp4_t *          mask,
-                      int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = combine_mask (src, mask, i);
-	comp4_t d = *(dest + i);
-	comp4_t a = ALPHA_c (s);
-	UNcx4_MUL_UNc (d, a);
-	*(dest + i) = d;
-    }
-}
-
-/* if the Dst is opaque, call combine_clear */
-static void
-combine_out_u (pixman_implementation_t *imp,
-               pixman_op_t              op,
-               comp4_t *                dest,
-               const comp4_t *          src,
-               const comp4_t *          mask,
-               int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = combine_mask (src, mask, i);
-	comp4_t a = ALPHA_c (~*(dest + i));
-	UNcx4_MUL_UNc (s, a);
-	*(dest + i) = s;
-    }
-}
-
-/* if the Src is opaque, call combine_clear */
-static void
-combine_out_reverse_u (pixman_implementation_t *imp,
-                       pixman_op_t              op,
-                       comp4_t *                dest,
-                       const comp4_t *          src,
-                       const comp4_t *          mask,
-                       int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = combine_mask (src, mask, i);
-	comp4_t d = *(dest + i);
-	comp4_t a = ALPHA_c (~s);
-	UNcx4_MUL_UNc (d, a);
-	*(dest + i) = d;
-    }
-}
-
-/* if the Src is opaque, call combine_in_u */
-/* if the Dst is opaque, call combine_over_u */
-/* if both the Src and Dst are opaque, call combine_src_u */
-static void
-combine_atop_u (pixman_implementation_t *imp,
-                pixman_op_t              op,
-                comp4_t *                dest,
-                const comp4_t *          src,
-                const comp4_t *          mask,
-                int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = combine_mask (src, mask, i);
-	comp4_t d = *(dest + i);
-	comp4_t dest_a = ALPHA_c (d);
-	comp4_t src_ia = ALPHA_c (~s);
-
-	UNcx4_MUL_UNc_ADD_UNcx4_MUL_UNc (s, dest_a, d, src_ia);
-	*(dest + i) = s;
-    }
-}
-
-/* if the Src is opaque, call combine_over_reverse_u */
-/* if the Dst is opaque, call combine_in_reverse_u */
-/* if both the Src and Dst are opaque, call combine_dst_u */
-static void
-combine_atop_reverse_u (pixman_implementation_t *imp,
-                        pixman_op_t              op,
-                        comp4_t *                dest,
-                        const comp4_t *          src,
-                        const comp4_t *          mask,
-                        int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = combine_mask (src, mask, i);
-	comp4_t d = *(dest + i);
-	comp4_t src_a = ALPHA_c (s);
-	comp4_t dest_ia = ALPHA_c (~d);
-
-	UNcx4_MUL_UNc_ADD_UNcx4_MUL_UNc (s, dest_ia, d, src_a);
-	*(dest + i) = s;
-    }
-}
-
-/* if the Src is opaque, call combine_over_u */
-/* if the Dst is opaque, call combine_over_reverse_u */
-/* if both the Src and Dst are opaque, call combine_clear */
-static void
-combine_xor_u (pixman_implementation_t *imp,
-               pixman_op_t              op,
-               comp4_t *                dest,
-               const comp4_t *          src,
-               const comp4_t *          mask,
-               int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = combine_mask (src, mask, i);
-	comp4_t d = *(dest + i);
-	comp4_t src_ia = ALPHA_c (~s);
-	comp4_t dest_ia = ALPHA_c (~d);
-
-	UNcx4_MUL_UNc_ADD_UNcx4_MUL_UNc (s, dest_ia, d, src_ia);
-	*(dest + i) = s;
-    }
-}
-
-static void
-combine_add_u (pixman_implementation_t *imp,
-               pixman_op_t              op,
-               comp4_t *                dest,
-               const comp4_t *          src,
-               const comp4_t *          mask,
-               int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = combine_mask (src, mask, i);
-	comp4_t d = *(dest + i);
-	UNcx4_ADD_UNcx4 (d, s);
-	*(dest + i) = d;
-    }
-}
-
-/* if the Src is opaque, call combine_add_u */
-/* if the Dst is opaque, call combine_add_u */
-/* if both the Src and Dst are opaque, call combine_add_u */
-static void
-combine_saturate_u (pixman_implementation_t *imp,
-                    pixman_op_t              op,
-                    comp4_t *                dest,
-                    const comp4_t *          src,
-                    const comp4_t *          mask,
-                    int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = combine_mask (src, mask, i);
-	comp4_t d = *(dest + i);
-	comp2_t sa, da;
-
-	sa = s >> A_SHIFT;
-	da = ~d >> A_SHIFT;
-	if (sa > da)
-	{
-	    sa = DIV_UNc (da, sa);
-	    UNcx4_MUL_UNc (s, sa);
-	}
-	;
-	UNcx4_ADD_UNcx4 (d, s);
-	*(dest + i) = d;
-    }
-}
-
-/*
- * PDF blend modes:
- * The following blend modes have been taken from the PDF ISO 32000
- * specification, which at this point in time is available from
- * http://www.adobe.com/devnet/acrobat/pdfs/PDF32000_2008.pdf
- * The relevant chapters are 11.3.5 and 11.3.6.
- * The formula for computing the final pixel color given in 11.3.6 is:
- * αr × Cr = (1 – αs) × αb × Cb + (1 – αb) × αs × Cs + αb × αs × B(Cb, Cs)
- * with B() being the blend function.
- * Note that OVER is a special case of this operation, using B(Cb, Cs) = Cs
- *
- * These blend modes should match the SVG filter draft specification, as
- * it has been designed to mirror ISO 32000. Note that at the current point
- * no released draft exists that shows this, as the formulas have not been
- * updated yet after the release of ISO 32000.
- *
- * The default implementation here uses the PDF_SEPARABLE_BLEND_MODE and
- * PDF_NON_SEPARABLE_BLEND_MODE macros, which take the blend function as an
- * argument. Note that this implementation operates on premultiplied colors,
- * while the PDF specification does not. Therefore the code uses the formula
- * ar.Cra = (1 – as) . Dca + (1 – ad) . Sca + B(Dca, ad, Sca, as)
- */
-
-/*
- * Multiply
- * B(Dca, ad, Sca, as) = Dca.Sca
- */
-
-static void
-combine_multiply_u (pixman_implementation_t *imp,
-                    pixman_op_t              op,
-                    comp4_t *                dest,
-                    const comp4_t *          src,
-                    const comp4_t *          mask,
-                    int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = combine_mask (src, mask, i);
-	comp4_t d = *(dest + i);
-	comp4_t ss = s;
-	comp4_t src_ia = ALPHA_c (~s);
-	comp4_t dest_ia = ALPHA_c (~d);
-
-	UNcx4_MUL_UNc_ADD_UNcx4_MUL_UNc (ss, dest_ia, d, src_ia);
-	UNcx4_MUL_UNcx4 (d, s);
-	UNcx4_ADD_UNcx4 (d, ss);
-
-	*(dest + i) = d;
-    }
-}
-
-static void
-combine_multiply_ca (pixman_implementation_t *imp,
-                     pixman_op_t              op,
-                     comp4_t *                dest,
-                     const comp4_t *          src,
-                     const comp4_t *          mask,
-                     int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t m = *(mask + i);
-	comp4_t s = *(src + i);
-	comp4_t d = *(dest + i);
-	comp4_t r = d;
-	comp4_t dest_ia = ALPHA_c (~d);
-
-	combine_mask_value_ca (&s, &m);
-
-	UNcx4_MUL_UNcx4_ADD_UNcx4_MUL_UNc (r, ~m, s, dest_ia);
-	UNcx4_MUL_UNcx4 (d, s);
-	UNcx4_ADD_UNcx4 (r, d);
-
-	*(dest + i) = r;
-    }
-}
-
-#define PDF_SEPARABLE_BLEND_MODE(name)					\
-    static void								\
-    combine_ ## name ## _u (pixman_implementation_t *imp,		\
-			    pixman_op_t              op,		\
-                            comp4_t *                dest,		\
-			    const comp4_t *          src,		\
-			    const comp4_t *          mask,		\
-			    int                      width)		\
-    {									\
-	int i;								\
-	for (i = 0; i < width; ++i) {					\
-	    comp4_t s = combine_mask (src, mask, i);			\
-	    comp4_t d = *(dest + i);					\
-	    comp1_t sa = ALPHA_c (s);					\
-	    comp1_t isa = ~sa;						\
-	    comp1_t da = ALPHA_c (d);					\
-	    comp1_t ida = ~da;						\
-	    comp4_t result;						\
-									\
-	    result = d;							\
-	    UNcx4_MUL_UNc_ADD_UNcx4_MUL_UNc (result, isa, s, ida);	\
-	    								\
-	    *(dest + i) = result +					\
-		(DIV_ONE_UNc (sa * da) << A_SHIFT) +			\
-		(blend_ ## name (RED_c (d), da, RED_c (s), sa) << R_SHIFT) + \
-		(blend_ ## name (GREEN_c (d), da, GREEN_c (s), sa) << G_SHIFT) + \
-		(blend_ ## name (BLUE_c (d), da, BLUE_c (s), sa));	\
-	}								\
-    }									\
-    									\
-    static void								\
-    combine_ ## name ## _ca (pixman_implementation_t *imp,		\
-			     pixman_op_t              op,		\
-                             comp4_t *                dest,		\
-			     const comp4_t *          src,		\
-			     const comp4_t *          mask,		\
-			     int                     width)		\
-    {									\
-	int i;								\
-	for (i = 0; i < width; ++i) {					\
-	    comp4_t m = *(mask + i);					\
-	    comp4_t s = *(src + i);					\
-	    comp4_t d = *(dest + i);					\
-	    comp1_t da = ALPHA_c (d);					\
-	    comp1_t ida = ~da;						\
-	    comp4_t result;						\
-            								\
-	    combine_mask_value_ca (&s, &m);				\
-            								\
-	    result = d;							\
-	    UNcx4_MUL_UNcx4_ADD_UNcx4_MUL_UNc (result, ~m, s, ida);     \
-            								\
-	    result +=							\
-	        (DIV_ONE_UNc (ALPHA_c (m) * da) << A_SHIFT) +		\
-	        (blend_ ## name (RED_c (d), da, RED_c (s), RED_c (m)) << R_SHIFT) + \
-	        (blend_ ## name (GREEN_c (d), da, GREEN_c (s), GREEN_c (m)) << G_SHIFT) + \
-	        (blend_ ## name (BLUE_c (d), da, BLUE_c (s), BLUE_c (m))); \
-	    								\
-	    *(dest + i) = result;					\
-	}								\
-    }
-
-/*
- * Screen
- * B(Dca, ad, Sca, as) = Dca.sa + Sca.da - Dca.Sca
- */
-static inline comp4_t
-blend_screen (comp4_t dca, comp4_t da, comp4_t sca, comp4_t sa)
-{
-    return DIV_ONE_UNc (sca * da + dca * sa - sca * dca);
-}
-
-PDF_SEPARABLE_BLEND_MODE (screen)
-
-/*
- * Overlay
- * B(Dca, Da, Sca, Sa) =
- *   if 2.Dca < Da
- *     2.Sca.Dca
- *   otherwise
- *     Sa.Da - 2.(Da - Dca).(Sa - Sca)
- */
-static inline comp4_t
-blend_overlay (comp4_t dca, comp4_t da, comp4_t sca, comp4_t sa)
-{
-    comp4_t rca;
-
-    if (2 * dca < da)
-	rca = 2 * sca * dca;
-    else
-	rca = sa * da - 2 * (da - dca) * (sa - sca);
-    return DIV_ONE_UNc (rca);
-}
-
-PDF_SEPARABLE_BLEND_MODE (overlay)
-
-/*
- * Darken
- * B(Dca, Da, Sca, Sa) = min (Sca.Da, Dca.Sa)
- */
-static inline comp4_t
-blend_darken (comp4_t dca, comp4_t da, comp4_t sca, comp4_t sa)
-{
-    comp4_t s, d;
-
-    s = sca * da;
-    d = dca * sa;
-    return DIV_ONE_UNc (s > d ? d : s);
-}
-
-PDF_SEPARABLE_BLEND_MODE (darken)
-
-/*
- * Lighten
- * B(Dca, Da, Sca, Sa) = max (Sca.Da, Dca.Sa)
- */
-static inline comp4_t
-blend_lighten (comp4_t dca, comp4_t da, comp4_t sca, comp4_t sa)
-{
-    comp4_t s, d;
-
-    s = sca * da;
-    d = dca * sa;
-    return DIV_ONE_UNc (s > d ? s : d);
-}
-
-PDF_SEPARABLE_BLEND_MODE (lighten)
-
-/*
- * Color dodge
- * B(Dca, Da, Sca, Sa) =
- *   if Dca == 0
- *     0
- *   if Sca == Sa
- *     Sa.Da
- *   otherwise
- *     Sa.Da. min (1, Dca / Da / (1 - Sca/Sa))
- */
-static inline comp4_t
-blend_color_dodge (comp4_t dca, comp4_t da, comp4_t sca, comp4_t sa)
-{
-    if (sca >= sa)
-    {
-	return dca == 0 ? 0 : DIV_ONE_UNc (sa * da);
-    }
-    else
-    {
-	comp4_t rca = dca * sa / (sa - sca);
-	return DIV_ONE_UNc (sa * MIN (rca, da));
-    }
-}
-
-PDF_SEPARABLE_BLEND_MODE (color_dodge)
-
-/*
- * Color burn
- * B(Dca, Da, Sca, Sa) =
- *   if Dca == Da
- *     Sa.Da
- *   if Sca == 0
- *     0
- *   otherwise
- *     Sa.Da.(1 - min (1, (1 - Dca/Da).Sa / Sca))
- */
-static inline comp4_t
-blend_color_burn (comp4_t dca, comp4_t da, comp4_t sca, comp4_t sa)
-{
-    if (sca == 0)
-    {
-	return dca < da ? 0 : DIV_ONE_UNc (sa * da);
-    }
-    else
-    {
-	comp4_t rca = (da - dca) * sa / sca;
-	return DIV_ONE_UNc (sa * (MAX (rca, da) - rca));
-    }
-}
-
-PDF_SEPARABLE_BLEND_MODE (color_burn)
-
-/*
- * Hard light
- * B(Dca, Da, Sca, Sa) =
- *   if 2.Sca < Sa
- *     2.Sca.Dca
- *   otherwise
- *     Sa.Da - 2.(Da - Dca).(Sa - Sca)
- */
-static inline comp4_t
-blend_hard_light (comp4_t dca, comp4_t da, comp4_t sca, comp4_t sa)
-{
-    if (2 * sca < sa)
-	return DIV_ONE_UNc (2 * sca * dca);
-    else
-	return DIV_ONE_UNc (sa * da - 2 * (da - dca) * (sa - sca));
-}
-
-PDF_SEPARABLE_BLEND_MODE (hard_light)
-
-/*
- * Soft light
- * B(Dca, Da, Sca, Sa) =
- *   if (2.Sca <= Sa)
- *     Dca.(Sa - (1 - Dca/Da).(2.Sca - Sa))
- *   otherwise if Dca.4 <= Da
- *     Dca.(Sa + (2.Sca - Sa).((16.Dca/Da - 12).Dca/Da + 3)
- *   otherwise
- *     (Dca.Sa + (SQRT (Dca/Da).Da - Dca).(2.Sca - Sa))
- */
-static inline comp4_t
-blend_soft_light (comp4_t dca_org,
-		  comp4_t da_org,
-		  comp4_t sca_org,
-		  comp4_t sa_org)
-{
-    double dca = dca_org * (1.0 / MASK);
-    double da = da_org * (1.0 / MASK);
-    double sca = sca_org * (1.0 / MASK);
-    double sa = sa_org * (1.0 / MASK);
-    double rca;
-
-    if (2 * sca < sa)
-    {
-	if (da == 0)
-	    rca = dca * sa;
-	else
-	    rca = dca * sa - dca * (da - dca) * (sa - 2 * sca) / da;
-    }
-    else if (da == 0)
-    {
-	rca = 0;
-    }
-    else if (4 * dca <= da)
-    {
-	rca = dca * sa +
-	    (2 * sca - sa) * dca * ((16 * dca / da - 12) * dca / da + 3);
-    }
-    else
-    {
-	rca = dca * sa + (sqrt (dca * da) - dca) * (2 * sca - sa);
-    }
-    return rca * MASK + 0.5;
-}
-
-PDF_SEPARABLE_BLEND_MODE (soft_light)
-
-/*
- * Difference
- * B(Dca, Da, Sca, Sa) = abs (Dca.Sa - Sca.Da)
- */
-static inline comp4_t
-blend_difference (comp4_t dca, comp4_t da, comp4_t sca, comp4_t sa)
-{
-    comp4_t dcasa = dca * sa;
-    comp4_t scada = sca * da;
-
-    if (scada < dcasa)
-	return DIV_ONE_UNc (dcasa - scada);
-    else
-	return DIV_ONE_UNc (scada - dcasa);
-}
-
-PDF_SEPARABLE_BLEND_MODE (difference)
-
-/*
- * Exclusion
- * B(Dca, Da, Sca, Sa) = (Sca.Da + Dca.Sa - 2.Sca.Dca)
- */
-
-/* This can be made faster by writing it directly and not using
- * PDF_SEPARABLE_BLEND_MODE, but that's a performance optimization */
-
-static inline comp4_t
-blend_exclusion (comp4_t dca, comp4_t da, comp4_t sca, comp4_t sa)
-{
-    return DIV_ONE_UNc (sca * da + dca * sa - 2 * dca * sca);
-}
-
-PDF_SEPARABLE_BLEND_MODE (exclusion)
-
-#undef PDF_SEPARABLE_BLEND_MODE
-
-/*
- * PDF nonseperable blend modes are implemented using the following functions
- * to operate in Hsl space, with Cmax, Cmid, Cmin referring to the max, mid
- * and min value of the red, green and blue components.
- *
- * LUM (C) = 0.3 × Cred + 0.59 × Cgreen + 0.11 × Cblue
- *
- * clip_color (C):
- *   l = LUM (C)
- *   min = Cmin
- *   max = Cmax
- *   if n < 0.0
- *     C = l + ( ( ( C – l ) × l ) ⁄ ( l – min ) )
- *   if x > 1.0
- *     C = l + ( ( ( C – l ) × ( 1 – l ) ) ⁄ ( max – l ) )
- *   return C
- *
- * set_lum (C, l):
- *   d = l – LUM (C)
- *   C += d
- *   return clip_color (C)
- *
- * SAT (C) = CH_MAX (C) - CH_MIN (C)
- *
- * set_sat (C, s):
- *  if Cmax > Cmin
- *    Cmid = ( ( ( Cmid – Cmin ) × s ) ⁄ ( Cmax – Cmin ) )
- *    Cmax = s
- *  else
- *    Cmid = Cmax = 0.0
- *  Cmin = 0.0
- *  return C
- */
-
-/* For premultiplied colors, we need to know what happens when C is
- * multiplied by a real number. LUM and SAT are linear:
- *
- *    LUM (r × C) = r × LUM (C)		SAT (r * C) = r * SAT (C)
- *
- * If we extend clip_color with an extra argument a and change
- *
- *        if x >= 1.0
- *
- * into
- *
- *        if x >= a
- *
- * then clip_color is also linear:
- *
- *    r * clip_color (C, a) = clip_color (r_c, ra);
- *
- * for positive r.
- *
- * Similarly, we can extend set_lum with an extra argument that is just passed
- * on to clip_color:
- *
- *   r * set_lum ( C, l, a)
- *
- *   = r × clip_color ( C + l - LUM (C), a)
- *
- *   = clip_color ( r * C + r × l - r * LUM (C), r * a)
- *
- *   = set_lum ( r * C, r * l, r * a)
- *
- * Finally, set_sat:
- *
- *    r * set_sat (C, s) = set_sat (x * C, r * s)
- *
- * The above holds for all non-zero x, because they x'es in the fraction for
- * C_mid cancel out. Specifically, it holds for x = r:
- *
- *    r * set_sat (C, s) = set_sat (r_c, rs)
- *
- */
-
-/* So, for the non-separable PDF blend modes, we have (using s, d for
- * non-premultiplied colors, and S, D for premultiplied:
- *
- *   Color:
- *
- *     a_s * a_d * B(s, d)
- *   = a_s * a_d * set_lum (S/a_s, LUM (D/a_d), 1)
- *   = set_lum (S * a_d, a_s * LUM (D), a_s * a_d)
- *
- *
- *   Luminosity:
- *
- *     a_s * a_d * B(s, d)
- *   = a_s * a_d * set_lum (D/a_d, LUM(S/a_s), 1)
- *   = set_lum (a_s * D, a_d * LUM(S), a_s * a_d)
- *
- *
- *   Saturation:
- *
- *     a_s * a_d * B(s, d)
- *   = a_s * a_d * set_lum (set_sat (D/a_d, SAT (S/a_s)), LUM (D/a_d), 1)
- *   = set_lum (a_s * a_d * set_sat (D/a_d, SAT (S/a_s)),
- *                                        a_s * LUM (D), a_s * a_d)
- *   = set_lum (set_sat (a_s * D, a_d * SAT (S), a_s * LUM (D), a_s * a_d))
- *
- *   Hue:
- *
- *     a_s * a_d * B(s, d)
- *   = a_s * a_d * set_lum (set_sat (S/a_s, SAT (D/a_d)), LUM (D/a_d), 1)
- *   = a_s * a_d * set_lum (set_sat (a_d * S, a_s * SAT (D)),
- *                                        a_s * LUM (D), a_s * a_d)
- *
- */
-
-#define CH_MIN(c) (c[0] < c[1] ? (c[0] < c[2] ? c[0] : c[2]) : (c[1] < c[2] ? c[1] : c[2]))
-#define CH_MAX(c) (c[0] > c[1] ? (c[0] > c[2] ? c[0] : c[2]) : (c[1] > c[2] ? c[1] : c[2]))
-#define LUM(c) ((c[0] * 30 + c[1] * 59 + c[2] * 11) / 100)
-#define SAT(c) (CH_MAX (c) - CH_MIN (c))
-
-#define PDF_NON_SEPARABLE_BLEND_MODE(name)				\
-    static void								\
-    combine_ ## name ## _u (pixman_implementation_t *imp,		\
-			    pixman_op_t op,				\
-                            comp4_t *dest,				\
-			    const comp4_t *src,				\
-			    const comp4_t *mask,			\
-			    int width)					\
-    {									\
-	int i;								\
-	for (i = 0; i < width; ++i)					\
-	{								\
-	    comp4_t s = combine_mask (src, mask, i);			\
-	    comp4_t d = *(dest + i);					\
-	    comp1_t sa = ALPHA_c (s);					\
-	    comp1_t isa = ~sa;						\
-	    comp1_t da = ALPHA_c (d);					\
-	    comp1_t ida = ~da;						\
-	    comp4_t result;						\
-	    comp4_t sc[3], dc[3], c[3];					\
-            								\
-	    result = d;							\
-	    UNcx4_MUL_UNc_ADD_UNcx4_MUL_UNc (result, isa, s, ida);	\
-	    dc[0] = RED_c (d);						\
-	    sc[0] = RED_c (s);						\
-	    dc[1] = GREEN_c (d);					\
-	    sc[1] = GREEN_c (s);					\
-	    dc[2] = BLUE_c (d);						\
-	    sc[2] = BLUE_c (s);						\
-	    blend_ ## name (c, dc, da, sc, sa);				\
-            								\
-	    *(dest + i) = result +					\
-		(DIV_ONE_UNc (sa * da) << A_SHIFT) +			\
-		(DIV_ONE_UNc (c[0]) << R_SHIFT) +			\
-		(DIV_ONE_UNc (c[1]) << G_SHIFT) +			\
-		(DIV_ONE_UNc (c[2]));					\
-	}								\
-    }
-
-static void
-set_lum (comp4_t dest[3], comp4_t src[3], comp4_t sa, comp4_t lum)
-{
-    double a, l, min, max;
-    double tmp[3];
-
-    a = sa * (1.0 / MASK);
-
-    l = lum * (1.0 / MASK);
-    tmp[0] = src[0] * (1.0 / MASK);
-    tmp[1] = src[1] * (1.0 / MASK);
-    tmp[2] = src[2] * (1.0 / MASK);
-
-    l = l - LUM (tmp);
-    tmp[0] += l;
-    tmp[1] += l;
-    tmp[2] += l;
-
-    /* clip_color */
-    l = LUM (tmp);
-    min = CH_MIN (tmp);
-    max = CH_MAX (tmp);
-
-    if (min < 0)
-    {
-	tmp[0] = l + (tmp[0] - l) * l / (l - min);
-	tmp[1] = l + (tmp[1] - l) * l / (l - min);
-	tmp[2] = l + (tmp[2] - l) * l / (l - min);
-    }
-    if (max > a)
-    {
-	tmp[0] = l + (tmp[0] - l) * (a - l) / (max - l);
-	tmp[1] = l + (tmp[1] - l) * (a - l) / (max - l);
-	tmp[2] = l + (tmp[2] - l) * (a - l) / (max - l);
-    }
-
-    dest[0] = tmp[0] * MASK + 0.5;
-    dest[1] = tmp[1] * MASK + 0.5;
-    dest[2] = tmp[2] * MASK + 0.5;
-}
-
-static void
-set_sat (comp4_t dest[3], comp4_t src[3], comp4_t sat)
-{
-    int id[3];
-    comp4_t min, max;
-
-    if (src[0] > src[1])
-    {
-	if (src[0] > src[2])
-	{
-	    id[0] = 0;
-	    if (src[1] > src[2])
-	    {
-		id[1] = 1;
-		id[2] = 2;
-	    }
-	    else
-	    {
-		id[1] = 2;
-		id[2] = 1;
-	    }
-	}
-	else
-	{
-	    id[0] = 2;
-	    id[1] = 0;
-	    id[2] = 1;
-	}
-    }
-    else
-    {
-	if (src[0] > src[2])
-	{
-	    id[0] = 1;
-	    id[1] = 0;
-	    id[2] = 2;
-	}
-	else
-	{
-	    id[2] = 0;
-	    if (src[1] > src[2])
-	    {
-		id[0] = 1;
-		id[1] = 2;
-	    }
-	    else
-	    {
-		id[0] = 2;
-		id[1] = 1;
-	    }
-	}
-    }
-
-    max = dest[id[0]];
-    min = dest[id[2]];
-    if (max > min)
-    {
-	dest[id[1]] = (dest[id[1]] - min) * sat / (max - min);
-	dest[id[0]] = sat;
-	dest[id[2]] = 0;
-    }
-    else
-    {
-	dest[0] = dest[1] = dest[2] = 0;
-    }
-}
-
-/*
- * Hue:
- * B(Cb, Cs) = set_lum (set_sat (Cs, SAT (Cb)), LUM (Cb))
- */
-static inline void
-blend_hsl_hue (comp4_t c[3],
-               comp4_t dc[3],
-               comp4_t da,
-               comp4_t sc[3],
-               comp4_t sa)
-{
-    c[0] = sc[0] * da;
-    c[1] = sc[1] * da;
-    c[2] = sc[2] * da;
-    set_sat (c, c, SAT (dc) * sa);
-    set_lum (c, c, sa * da, LUM (dc) * sa);
-}
-
-PDF_NON_SEPARABLE_BLEND_MODE (hsl_hue)
-
-/*
- * Saturation:
- * B(Cb, Cs) = set_lum (set_sat (Cb, SAT (Cs)), LUM (Cb))
- */
-static inline void
-blend_hsl_saturation (comp4_t c[3],
-                      comp4_t dc[3],
-                      comp4_t da,
-                      comp4_t sc[3],
-                      comp4_t sa)
-{
-    c[0] = dc[0] * sa;
-    c[1] = dc[1] * sa;
-    c[2] = dc[2] * sa;
-    set_sat (c, c, SAT (sc) * da);
-    set_lum (c, c, sa * da, LUM (dc) * sa);
-}
-
-PDF_NON_SEPARABLE_BLEND_MODE (hsl_saturation)
-
-/*
- * Color:
- * B(Cb, Cs) = set_lum (Cs, LUM (Cb))
- */
-static inline void
-blend_hsl_color (comp4_t c[3],
-                 comp4_t dc[3],
-                 comp4_t da,
-                 comp4_t sc[3],
-                 comp4_t sa)
-{
-    c[0] = sc[0] * da;
-    c[1] = sc[1] * da;
-    c[2] = sc[2] * da;
-    set_lum (c, c, sa * da, LUM (dc) * sa);
-}
-
-PDF_NON_SEPARABLE_BLEND_MODE (hsl_color)
-
-/*
- * Luminosity:
- * B(Cb, Cs) = set_lum (Cb, LUM (Cs))
- */
-static inline void
-blend_hsl_luminosity (comp4_t c[3],
-                      comp4_t dc[3],
-                      comp4_t da,
-                      comp4_t sc[3],
-                      comp4_t sa)
-{
-    c[0] = dc[0] * sa;
-    c[1] = dc[1] * sa;
-    c[2] = dc[2] * sa;
-    set_lum (c, c, sa * da, LUM (sc) * da);
-}
-
-PDF_NON_SEPARABLE_BLEND_MODE (hsl_luminosity)
-
-#undef SAT
-#undef LUM
-#undef CH_MAX
-#undef CH_MIN
-#undef PDF_NON_SEPARABLE_BLEND_MODE
-
-/* Overlay
- *
- * All of the disjoint composing functions
- *
- * The four entries in the first column indicate what source contributions
- * come from each of the four areas of the picture -- areas covered by neither
- * A nor B, areas covered only by A, areas covered only by B and finally
- * areas covered by both A and B.
- * 
- * Disjoint			Conjoint
- * Fa		Fb		Fa		Fb
- * (0,0,0,0)	0		0		0		0
- * (0,A,0,A)	1		0		1		0
- * (0,0,B,B)	0		1		0		1
- * (0,A,B,A)	1		min((1-a)/b,1)	1		max(1-a/b,0)
- * (0,A,B,B)	min((1-b)/a,1)	1		max(1-b/a,0)	1
- * (0,0,0,A)	max(1-(1-b)/a,0) 0		min(1,b/a)	0
- * (0,0,0,B)	0		max(1-(1-a)/b,0) 0		min(a/b,1)
- * (0,A,0,0)	min(1,(1-b)/a)	0		max(1-b/a,0)	0
- * (0,0,B,0)	0		min(1,(1-a)/b)	0		max(1-a/b,0)
- * (0,0,B,A)	max(1-(1-b)/a,0) min(1,(1-a)/b)	 min(1,b/a)	max(1-a/b,0)
- * (0,A,0,B)	min(1,(1-b)/a)	max(1-(1-a)/b,0) max(1-b/a,0)	min(1,a/b)
- * (0,A,B,0)	min(1,(1-b)/a)	min(1,(1-a)/b)	max(1-b/a,0)	max(1-a/b,0)
- */
-
-#define COMBINE_A_OUT 1
-#define COMBINE_A_IN  2
-#define COMBINE_B_OUT 4
-#define COMBINE_B_IN  8
-
-#define COMBINE_CLEAR   0
-#define COMBINE_A       (COMBINE_A_OUT | COMBINE_A_IN)
-#define COMBINE_B       (COMBINE_B_OUT | COMBINE_B_IN)
-#define COMBINE_A_OVER  (COMBINE_A_OUT | COMBINE_B_OUT | COMBINE_A_IN)
-#define COMBINE_B_OVER  (COMBINE_A_OUT | COMBINE_B_OUT | COMBINE_B_IN)
-#define COMBINE_A_ATOP  (COMBINE_B_OUT | COMBINE_A_IN)
-#define COMBINE_B_ATOP  (COMBINE_A_OUT | COMBINE_B_IN)
-#define COMBINE_XOR     (COMBINE_A_OUT | COMBINE_B_OUT)
-
-/* portion covered by a but not b */
-static comp1_t
-combine_disjoint_out_part (comp1_t a, comp1_t b)
-{
-    /* min (1, (1-b) / a) */
-
-    b = ~b;                 /* 1 - b */
-    if (b >= a)             /* 1 - b >= a -> (1-b)/a >= 1 */
-	return MASK;        /* 1 */
-    return DIV_UNc (b, a);     /* (1-b) / a */
-}
-
-/* portion covered by both a and b */
-static comp1_t
-combine_disjoint_in_part (comp1_t a, comp1_t b)
-{
-    /* max (1-(1-b)/a,0) */
-    /*  = - min ((1-b)/a - 1, 0) */
-    /*  = 1 - min (1, (1-b)/a) */
-
-    b = ~b;                 /* 1 - b */
-    if (b >= a)             /* 1 - b >= a -> (1-b)/a >= 1 */
-	return 0;           /* 1 - 1 */
-    return ~DIV_UNc(b, a);    /* 1 - (1-b) / a */
-}
-
-/* portion covered by a but not b */
-static comp1_t
-combine_conjoint_out_part (comp1_t a, comp1_t b)
-{
-    /* max (1-b/a,0) */
-    /* = 1-min(b/a,1) */
-
-    /* min (1, (1-b) / a) */
-
-    if (b >= a)             /* b >= a -> b/a >= 1 */
-	return 0x00;        /* 0 */
-    return ~DIV_UNc(b, a);    /* 1 - b/a */
-}
-
-/* portion covered by both a and b */
-static comp1_t
-combine_conjoint_in_part (comp1_t a, comp1_t b)
-{
-    /* min (1,b/a) */
-
-    if (b >= a)             /* b >= a -> b/a >= 1 */
-	return MASK;        /* 1 */
-    return DIV_UNc (b, a);     /* b/a */
-}
-
-#define GET_COMP(v, i)   ((comp2_t) (comp1_t) ((v) >> i))
-
-#define ADD(x, y, i, t)							\
-    ((t) = GET_COMP (x, i) + GET_COMP (y, i),				\
-     (comp4_t) ((comp1_t) ((t) | (0 - ((t) >> G_SHIFT)))) << (i))
-
-#define GENERIC(x, y, i, ax, ay, t, u, v)				\
-    ((t) = (MUL_UNc (GET_COMP (y, i), ay, (u)) +			\
-            MUL_UNc (GET_COMP (x, i), ax, (v))),			\
-     (comp4_t) ((comp1_t) ((t) |					\
-                           (0 - ((t) >> G_SHIFT)))) << (i))
-
-static void
-combine_disjoint_general_u (comp4_t *      dest,
-                            const comp4_t *src,
-                            const comp4_t *mask,
-                            int            width,
-                            comp1_t        combine)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = combine_mask (src, mask, i);
-	comp4_t d = *(dest + i);
-	comp4_t m, n, o, p;
-	comp2_t Fa, Fb, t, u, v;
-	comp1_t sa = s >> A_SHIFT;
-	comp1_t da = d >> A_SHIFT;
-
-	switch (combine & COMBINE_A)
-	{
-	default:
-	    Fa = 0;
-	    break;
-
-	case COMBINE_A_OUT:
-	    Fa = combine_disjoint_out_part (sa, da);
-	    break;
-
-	case COMBINE_A_IN:
-	    Fa = combine_disjoint_in_part (sa, da);
-	    break;
-
-	case COMBINE_A:
-	    Fa = MASK;
-	    break;
-	}
-
-	switch (combine & COMBINE_B)
-	{
-	default:
-	    Fb = 0;
-	    break;
-
-	case COMBINE_B_OUT:
-	    Fb = combine_disjoint_out_part (da, sa);
-	    break;
-
-	case COMBINE_B_IN:
-	    Fb = combine_disjoint_in_part (da, sa);
-	    break;
-
-	case COMBINE_B:
-	    Fb = MASK;
-	    break;
-	}
-	m = GENERIC (s, d, 0, Fa, Fb, t, u, v);
-	n = GENERIC (s, d, G_SHIFT, Fa, Fb, t, u, v);
-	o = GENERIC (s, d, R_SHIFT, Fa, Fb, t, u, v);
-	p = GENERIC (s, d, A_SHIFT, Fa, Fb, t, u, v);
-	s = m | n | o | p;
-	*(dest + i) = s;
-    }
-}
-
-static void
-combine_disjoint_over_u (pixman_implementation_t *imp,
-                         pixman_op_t              op,
-                         comp4_t *                dest,
-                         const comp4_t *          src,
-                         const comp4_t *          mask,
-                         int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = combine_mask (src, mask, i);
-	comp2_t a = s >> A_SHIFT;
-
-	if (a != 0x00)
-	{
-	    if (a != MASK)
-	    {
-		comp4_t d = *(dest + i);
-		a = combine_disjoint_out_part (d >> A_SHIFT, a);
-		UNcx4_MUL_UNc_ADD_UNcx4 (d, a, s);
-		s = d;
-	    }
-
-	    *(dest + i) = s;
-	}
-    }
-}
-
-static void
-combine_disjoint_in_u (pixman_implementation_t *imp,
-                       pixman_op_t              op,
-                       comp4_t *                dest,
-                       const comp4_t *          src,
-                       const comp4_t *          mask,
-                       int                      width)
-{
-    combine_disjoint_general_u (dest, src, mask, width, COMBINE_A_IN);
-}
-
-static void
-combine_disjoint_in_reverse_u (pixman_implementation_t *imp,
-                               pixman_op_t              op,
-                               comp4_t *                dest,
-                               const comp4_t *          src,
-                               const comp4_t *          mask,
-                               int                      width)
-{
-    combine_disjoint_general_u (dest, src, mask, width, COMBINE_B_IN);
-}
-
-static void
-combine_disjoint_out_u (pixman_implementation_t *imp,
-                        pixman_op_t              op,
-                        comp4_t *                dest,
-                        const comp4_t *          src,
-                        const comp4_t *          mask,
-                        int                      width)
-{
-    combine_disjoint_general_u (dest, src, mask, width, COMBINE_A_OUT);
-}
-
-static void
-combine_disjoint_out_reverse_u (pixman_implementation_t *imp,
-                                pixman_op_t              op,
-                                comp4_t *                dest,
-                                const comp4_t *          src,
-                                const comp4_t *          mask,
-                                int                      width)
-{
-    combine_disjoint_general_u (dest, src, mask, width, COMBINE_B_OUT);
-}
-
-static void
-combine_disjoint_atop_u (pixman_implementation_t *imp,
-                         pixman_op_t              op,
-                         comp4_t *                dest,
-                         const comp4_t *          src,
-                         const comp4_t *          mask,
-                         int                      width)
-{
-    combine_disjoint_general_u (dest, src, mask, width, COMBINE_A_ATOP);
-}
-
-static void
-combine_disjoint_atop_reverse_u (pixman_implementation_t *imp,
-                                 pixman_op_t              op,
-                                 comp4_t *                dest,
-                                 const comp4_t *          src,
-                                 const comp4_t *          mask,
-                                 int                      width)
-{
-    combine_disjoint_general_u (dest, src, mask, width, COMBINE_B_ATOP);
-}
-
-static void
-combine_disjoint_xor_u (pixman_implementation_t *imp,
-                        pixman_op_t              op,
-                        comp4_t *                dest,
-                        const comp4_t *          src,
-                        const comp4_t *          mask,
-                        int                      width)
-{
-    combine_disjoint_general_u (dest, src, mask, width, COMBINE_XOR);
-}
-
-static void
-combine_conjoint_general_u (comp4_t *      dest,
-                            const comp4_t *src,
-                            const comp4_t *mask,
-                            int            width,
-                            comp1_t        combine)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = combine_mask (src, mask, i);
-	comp4_t d = *(dest + i);
-	comp4_t m, n, o, p;
-	comp2_t Fa, Fb, t, u, v;
-	comp1_t sa = s >> A_SHIFT;
-	comp1_t da = d >> A_SHIFT;
-
-	switch (combine & COMBINE_A)
-	{
-	default:
-	    Fa = 0;
-	    break;
-
-	case COMBINE_A_OUT:
-	    Fa = combine_conjoint_out_part (sa, da);
-	    break;
-
-	case COMBINE_A_IN:
-	    Fa = combine_conjoint_in_part (sa, da);
-	    break;
-
-	case COMBINE_A:
-	    Fa = MASK;
-	    break;
-	}
-
-	switch (combine & COMBINE_B)
-	{
-	default:
-	    Fb = 0;
-	    break;
-
-	case COMBINE_B_OUT:
-	    Fb = combine_conjoint_out_part (da, sa);
-	    break;
-
-	case COMBINE_B_IN:
-	    Fb = combine_conjoint_in_part (da, sa);
-	    break;
-
-	case COMBINE_B:
-	    Fb = MASK;
-	    break;
-	}
-
-	m = GENERIC (s, d, 0, Fa, Fb, t, u, v);
-	n = GENERIC (s, d, G_SHIFT, Fa, Fb, t, u, v);
-	o = GENERIC (s, d, R_SHIFT, Fa, Fb, t, u, v);
-	p = GENERIC (s, d, A_SHIFT, Fa, Fb, t, u, v);
-
-	s = m | n | o | p;
-
-	*(dest + i) = s;
-    }
-}
-
-static void
-combine_conjoint_over_u (pixman_implementation_t *imp,
-                         pixman_op_t              op,
-                         comp4_t *                dest,
-                         const comp4_t *          src,
-                         const comp4_t *          mask,
-                         int                      width)
-{
-    combine_conjoint_general_u (dest, src, mask, width, COMBINE_A_OVER);
-}
-
-static void
-combine_conjoint_over_reverse_u (pixman_implementation_t *imp,
-                                 pixman_op_t              op,
-                                 comp4_t *                dest,
-                                 const comp4_t *          src,
-                                 const comp4_t *          mask,
-                                 int                      width)
-{
-    combine_conjoint_general_u (dest, src, mask, width, COMBINE_B_OVER);
-}
-
-static void
-combine_conjoint_in_u (pixman_implementation_t *imp,
-                       pixman_op_t              op,
-                       comp4_t *                dest,
-                       const comp4_t *          src,
-                       const comp4_t *          mask,
-                       int                      width)
-{
-    combine_conjoint_general_u (dest, src, mask, width, COMBINE_A_IN);
-}
-
-static void
-combine_conjoint_in_reverse_u (pixman_implementation_t *imp,
-                               pixman_op_t              op,
-                               comp4_t *                dest,
-                               const comp4_t *          src,
-                               const comp4_t *          mask,
-                               int                      width)
-{
-    combine_conjoint_general_u (dest, src, mask, width, COMBINE_B_IN);
-}
-
-static void
-combine_conjoint_out_u (pixman_implementation_t *imp,
-                        pixman_op_t              op,
-                        comp4_t *                dest,
-                        const comp4_t *          src,
-                        const comp4_t *          mask,
-                        int                      width)
-{
-    combine_conjoint_general_u (dest, src, mask, width, COMBINE_A_OUT);
-}
-
-static void
-combine_conjoint_out_reverse_u (pixman_implementation_t *imp,
-                                pixman_op_t              op,
-                                comp4_t *                dest,
-                                const comp4_t *          src,
-                                const comp4_t *          mask,
-                                int                      width)
-{
-    combine_conjoint_general_u (dest, src, mask, width, COMBINE_B_OUT);
-}
-
-static void
-combine_conjoint_atop_u (pixman_implementation_t *imp,
-                         pixman_op_t              op,
-                         comp4_t *                dest,
-                         const comp4_t *          src,
-                         const comp4_t *          mask,
-                         int                      width)
-{
-    combine_conjoint_general_u (dest, src, mask, width, COMBINE_A_ATOP);
-}
-
-static void
-combine_conjoint_atop_reverse_u (pixman_implementation_t *imp,
-                                 pixman_op_t              op,
-                                 comp4_t *                dest,
-                                 const comp4_t *          src,
-                                 const comp4_t *          mask,
-                                 int                      width)
-{
-    combine_conjoint_general_u (dest, src, mask, width, COMBINE_B_ATOP);
-}
-
-static void
-combine_conjoint_xor_u (pixman_implementation_t *imp,
-                        pixman_op_t              op,
-                        comp4_t *                dest,
-                        const comp4_t *          src,
-                        const comp4_t *          mask,
-                        int                      width)
-{
-    combine_conjoint_general_u (dest, src, mask, width, COMBINE_XOR);
-}
-
-/************************************************************************/
-/*********************** Per Channel functions **************************/
-/************************************************************************/
-
-static void
-combine_clear_ca (pixman_implementation_t *imp,
-                  pixman_op_t              op,
-                  comp4_t *                dest,
-                  const comp4_t *          src,
-                  const comp4_t *          mask,
-                  int                      width)
-{
-    memset (dest, 0, width * sizeof(comp4_t));
-}
-
-static void
-combine_src_ca (pixman_implementation_t *imp,
-                pixman_op_t              op,
-                comp4_t *                dest,
-                const comp4_t *          src,
-                const comp4_t *          mask,
-                int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = *(src + i);
-	comp4_t m = *(mask + i);
-
-	combine_mask_value_ca (&s, &m);
-
-	*(dest + i) = s;
-    }
-}
-
-static void
-combine_over_ca (pixman_implementation_t *imp,
-                 pixman_op_t              op,
-                 comp4_t *                dest,
-                 const comp4_t *          src,
-                 const comp4_t *          mask,
-                 int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = *(src + i);
-	comp4_t m = *(mask + i);
-	comp4_t a;
-
-	combine_mask_ca (&s, &m);
-
-	a = ~m;
-	if (a)
-	{
-	    comp4_t d = *(dest + i);
-	    UNcx4_MUL_UNcx4_ADD_UNcx4 (d, a, s);
-	    s = d;
-	}
-
-	*(dest + i) = s;
-    }
-}
-
-static void
-combine_over_reverse_ca (pixman_implementation_t *imp,
-                         pixman_op_t              op,
-                         comp4_t *                dest,
-                         const comp4_t *          src,
-                         const comp4_t *          mask,
-                         int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t d = *(dest + i);
-	comp4_t a = ~d >> A_SHIFT;
-
-	if (a)
-	{
-	    comp4_t s = *(src + i);
-	    comp4_t m = *(mask + i);
-
-	    UNcx4_MUL_UNcx4 (s, m);
-	    UNcx4_MUL_UNc_ADD_UNcx4 (s, a, d);
-
-	    *(dest + i) = s;
-	}
-    }
-}
-
-static void
-combine_in_ca (pixman_implementation_t *imp,
-               pixman_op_t              op,
-               comp4_t *                dest,
-               const comp4_t *          src,
-               const comp4_t *          mask,
-               int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t d = *(dest + i);
-	comp2_t a = d >> A_SHIFT;
-	comp4_t s = 0;
-
-	if (a)
-	{
-	    comp4_t m = *(mask + i);
-
-	    s = *(src + i);
-	    combine_mask_value_ca (&s, &m);
-
-	    if (a != MASK)
-		UNcx4_MUL_UNc (s, a);
-	}
-
-	*(dest + i) = s;
-    }
-}
-
-static void
-combine_in_reverse_ca (pixman_implementation_t *imp,
-                       pixman_op_t              op,
-                       comp4_t *                dest,
-                       const comp4_t *          src,
-                       const comp4_t *          mask,
-                       int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = *(src + i);
-	comp4_t m = *(mask + i);
-	comp4_t a;
-
-	combine_mask_alpha_ca (&s, &m);
-
-	a = m;
-	if (a != ~0)
-	{
-	    comp4_t d = 0;
-
-	    if (a)
-	    {
-		d = *(dest + i);
-		UNcx4_MUL_UNcx4 (d, a);
-	    }
-
-	    *(dest + i) = d;
-	}
-    }
-}
-
-static void
-combine_out_ca (pixman_implementation_t *imp,
-                pixman_op_t              op,
-                comp4_t *                dest,
-                const comp4_t *          src,
-                const comp4_t *          mask,
-                int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t d = *(dest + i);
-	comp2_t a = ~d >> A_SHIFT;
-	comp4_t s = 0;
-
-	if (a)
-	{
-	    comp4_t m = *(mask + i);
-
-	    s = *(src + i);
-	    combine_mask_value_ca (&s, &m);
-
-	    if (a != MASK)
-		UNcx4_MUL_UNc (s, a);
-	}
-
-	*(dest + i) = s;
-    }
-}
-
-static void
-combine_out_reverse_ca (pixman_implementation_t *imp,
-                        pixman_op_t              op,
-                        comp4_t *                dest,
-                        const comp4_t *          src,
-                        const comp4_t *          mask,
-                        int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = *(src + i);
-	comp4_t m = *(mask + i);
-	comp4_t a;
-
-	combine_mask_alpha_ca (&s, &m);
-
-	a = ~m;
-	if (a != ~0)
-	{
-	    comp4_t d = 0;
-
-	    if (a)
-	    {
-		d = *(dest + i);
-		UNcx4_MUL_UNcx4 (d, a);
-	    }
-
-	    *(dest + i) = d;
-	}
-    }
-}
-
-static void
-combine_atop_ca (pixman_implementation_t *imp,
-                 pixman_op_t              op,
-                 comp4_t *                dest,
-                 const comp4_t *          src,
-                 const comp4_t *          mask,
-                 int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t d = *(dest + i);
-	comp4_t s = *(src + i);
-	comp4_t m = *(mask + i);
-	comp4_t ad;
-	comp2_t as = d >> A_SHIFT;
-
-	combine_mask_ca (&s, &m);
-
-	ad = ~m;
-
-	UNcx4_MUL_UNcx4_ADD_UNcx4_MUL_UNc (d, ad, s, as);
-
-	*(dest + i) = d;
-    }
-}
-
-static void
-combine_atop_reverse_ca (pixman_implementation_t *imp,
-                         pixman_op_t              op,
-                         comp4_t *                dest,
-                         const comp4_t *          src,
-                         const comp4_t *          mask,
-                         int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t d = *(dest + i);
-	comp4_t s = *(src + i);
-	comp4_t m = *(mask + i);
-	comp4_t ad;
-	comp2_t as = ~d >> A_SHIFT;
-
-	combine_mask_ca (&s, &m);
-
-	ad = m;
-
-	UNcx4_MUL_UNcx4_ADD_UNcx4_MUL_UNc (d, ad, s, as);
-
-	*(dest + i) = d;
-    }
-}
-
-static void
-combine_xor_ca (pixman_implementation_t *imp,
-                pixman_op_t              op,
-                comp4_t *                dest,
-                const comp4_t *          src,
-                const comp4_t *          mask,
-                int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t d = *(dest + i);
-	comp4_t s = *(src + i);
-	comp4_t m = *(mask + i);
-	comp4_t ad;
-	comp2_t as = ~d >> A_SHIFT;
-
-	combine_mask_ca (&s, &m);
-
-	ad = ~m;
-
-	UNcx4_MUL_UNcx4_ADD_UNcx4_MUL_UNc (d, ad, s, as);
-
-	*(dest + i) = d;
-    }
-}
-
-static void
-combine_add_ca (pixman_implementation_t *imp,
-                pixman_op_t              op,
-                comp4_t *                dest,
-                const comp4_t *          src,
-                const comp4_t *          mask,
-                int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s = *(src + i);
-	comp4_t m = *(mask + i);
-	comp4_t d = *(dest + i);
-
-	combine_mask_value_ca (&s, &m);
-
-	UNcx4_ADD_UNcx4 (d, s);
-
-	*(dest + i) = d;
-    }
-}
-
-static void
-combine_saturate_ca (pixman_implementation_t *imp,
-                     pixman_op_t              op,
-                     comp4_t *                dest,
-                     const comp4_t *          src,
-                     const comp4_t *          mask,
-                     int                      width)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s, d;
-	comp2_t sa, sr, sg, sb, da;
-	comp2_t t, u, v;
-	comp4_t m, n, o, p;
-
-	d = *(dest + i);
-	s = *(src + i);
-	m = *(mask + i);
-
-	combine_mask_ca (&s, &m);
-
-	sa = (m >> A_SHIFT);
-	sr = (m >> R_SHIFT) & MASK;
-	sg = (m >> G_SHIFT) & MASK;
-	sb =  m             & MASK;
-	da = ~d >> A_SHIFT;
-
-	if (sb <= da)
-	    m = ADD (s, d, 0, t);
-	else
-	    m = GENERIC (s, d, 0, (da << G_SHIFT) / sb, MASK, t, u, v);
-
-	if (sg <= da)
-	    n = ADD (s, d, G_SHIFT, t);
-	else
-	    n = GENERIC (s, d, G_SHIFT, (da << G_SHIFT) / sg, MASK, t, u, v);
-
-	if (sr <= da)
-	    o = ADD (s, d, R_SHIFT, t);
-	else
-	    o = GENERIC (s, d, R_SHIFT, (da << G_SHIFT) / sr, MASK, t, u, v);
-
-	if (sa <= da)
-	    p = ADD (s, d, A_SHIFT, t);
-	else
-	    p = GENERIC (s, d, A_SHIFT, (da << G_SHIFT) / sa, MASK, t, u, v);
-
-	*(dest + i) = m | n | o | p;
-    }
-}
-
-static void
-combine_disjoint_general_ca (comp4_t *      dest,
-                             const comp4_t *src,
-                             const comp4_t *mask,
-                             int            width,
-                             comp1_t        combine)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s, d;
-	comp4_t m, n, o, p;
-	comp4_t Fa, Fb;
-	comp2_t t, u, v;
-	comp4_t sa;
-	comp1_t da;
-
-	s = *(src + i);
-	m = *(mask + i);
-	d = *(dest + i);
-	da = d >> A_SHIFT;
-
-	combine_mask_ca (&s, &m);
-
-	sa = m;
-
-	switch (combine & COMBINE_A)
-	{
-	default:
-	    Fa = 0;
-	    break;
-
-	case COMBINE_A_OUT:
-	    m = (comp4_t)combine_disjoint_out_part ((comp1_t) (sa >> 0), da);
-	    n = (comp4_t)combine_disjoint_out_part ((comp1_t) (sa >> G_SHIFT), da) << G_SHIFT;
-	    o = (comp4_t)combine_disjoint_out_part ((comp1_t) (sa >> R_SHIFT), da) << R_SHIFT;
-	    p = (comp4_t)combine_disjoint_out_part ((comp1_t) (sa >> A_SHIFT), da) << A_SHIFT;
-	    Fa = m | n | o | p;
-	    break;
-
-	case COMBINE_A_IN:
-	    m = (comp4_t)combine_disjoint_in_part ((comp1_t) (sa >> 0), da);
-	    n = (comp4_t)combine_disjoint_in_part ((comp1_t) (sa >> G_SHIFT), da) << G_SHIFT;
-	    o = (comp4_t)combine_disjoint_in_part ((comp1_t) (sa >> R_SHIFT), da) << R_SHIFT;
-	    p = (comp4_t)combine_disjoint_in_part ((comp1_t) (sa >> A_SHIFT), da) << A_SHIFT;
-	    Fa = m | n | o | p;
-	    break;
-
-	case COMBINE_A:
-	    Fa = ~0;
-	    break;
-	}
-
-	switch (combine & COMBINE_B)
-	{
-	default:
-	    Fb = 0;
-	    break;
-
-	case COMBINE_B_OUT:
-	    m = (comp4_t)combine_disjoint_out_part (da, (comp1_t) (sa >> 0));
-	    n = (comp4_t)combine_disjoint_out_part (da, (comp1_t) (sa >> G_SHIFT)) << G_SHIFT;
-	    o = (comp4_t)combine_disjoint_out_part (da, (comp1_t) (sa >> R_SHIFT)) << R_SHIFT;
-	    p = (comp4_t)combine_disjoint_out_part (da, (comp1_t) (sa >> A_SHIFT)) << A_SHIFT;
-	    Fb = m | n | o | p;
-	    break;
-
-	case COMBINE_B_IN:
-	    m = (comp4_t)combine_disjoint_in_part (da, (comp1_t) (sa >> 0));
-	    n = (comp4_t)combine_disjoint_in_part (da, (comp1_t) (sa >> G_SHIFT)) << G_SHIFT;
-	    o = (comp4_t)combine_disjoint_in_part (da, (comp1_t) (sa >> R_SHIFT)) << R_SHIFT;
-	    p = (comp4_t)combine_disjoint_in_part (da, (comp1_t) (sa >> A_SHIFT)) << A_SHIFT;
-	    Fb = m | n | o | p;
-	    break;
-
-	case COMBINE_B:
-	    Fb = ~0;
-	    break;
-	}
-	m = GENERIC (s, d, 0, GET_COMP (Fa, 0), GET_COMP (Fb, 0), t, u, v);
-	n = GENERIC (s, d, G_SHIFT, GET_COMP (Fa, G_SHIFT), GET_COMP (Fb, G_SHIFT), t, u, v);
-	o = GENERIC (s, d, R_SHIFT, GET_COMP (Fa, R_SHIFT), GET_COMP (Fb, R_SHIFT), t, u, v);
-	p = GENERIC (s, d, A_SHIFT, GET_COMP (Fa, A_SHIFT), GET_COMP (Fb, A_SHIFT), t, u, v);
-
-	s = m | n | o | p;
-
-	*(dest + i) = s;
-    }
-}
-
-static void
-combine_disjoint_over_ca (pixman_implementation_t *imp,
-                          pixman_op_t              op,
-                          comp4_t *                dest,
-                          const comp4_t *          src,
-                          const comp4_t *          mask,
-                          int                      width)
-{
-    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_A_OVER);
-}
-
-static void
-combine_disjoint_in_ca (pixman_implementation_t *imp,
-                        pixman_op_t              op,
-                        comp4_t *                dest,
-                        const comp4_t *          src,
-                        const comp4_t *          mask,
-                        int                      width)
-{
-    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_A_IN);
-}
-
-static void
-combine_disjoint_in_reverse_ca (pixman_implementation_t *imp,
-                                pixman_op_t              op,
-                                comp4_t *                dest,
-                                const comp4_t *          src,
-                                const comp4_t *          mask,
-                                int                      width)
-{
-    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_B_IN);
-}
-
-static void
-combine_disjoint_out_ca (pixman_implementation_t *imp,
-                         pixman_op_t              op,
-                         comp4_t *                dest,
-                         const comp4_t *          src,
-                         const comp4_t *          mask,
-                         int                      width)
-{
-    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_A_OUT);
-}
-
-static void
-combine_disjoint_out_reverse_ca (pixman_implementation_t *imp,
-                                 pixman_op_t              op,
-                                 comp4_t *                dest,
-                                 const comp4_t *          src,
-                                 const comp4_t *          mask,
-                                 int                      width)
-{
-    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_B_OUT);
-}
-
-static void
-combine_disjoint_atop_ca (pixman_implementation_t *imp,
-                          pixman_op_t              op,
-                          comp4_t *                dest,
-                          const comp4_t *          src,
-                          const comp4_t *          mask,
-                          int                      width)
-{
-    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_A_ATOP);
-}
-
-static void
-combine_disjoint_atop_reverse_ca (pixman_implementation_t *imp,
-                                  pixman_op_t              op,
-                                  comp4_t *                dest,
-                                  const comp4_t *          src,
-                                  const comp4_t *          mask,
-                                  int                      width)
-{
-    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_B_ATOP);
-}
-
-static void
-combine_disjoint_xor_ca (pixman_implementation_t *imp,
-                         pixman_op_t              op,
-                         comp4_t *                dest,
-                         const comp4_t *          src,
-                         const comp4_t *          mask,
-                         int                      width)
-{
-    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_XOR);
-}
-
-static void
-combine_conjoint_general_ca (comp4_t *      dest,
-                             const comp4_t *src,
-                             const comp4_t *mask,
-                             int            width,
-                             comp1_t        combine)
-{
-    int i;
-
-    for (i = 0; i < width; ++i)
-    {
-	comp4_t s, d;
-	comp4_t m, n, o, p;
-	comp4_t Fa, Fb;
-	comp2_t t, u, v;
-	comp4_t sa;
-	comp1_t da;
-
-	s = *(src + i);
-	m = *(mask + i);
-	d = *(dest + i);
-	da = d >> A_SHIFT;
-
-	combine_mask_ca (&s, &m);
-
-	sa = m;
-
-	switch (combine & COMBINE_A)
-	{
-	default:
-	    Fa = 0;
-	    break;
-
-	case COMBINE_A_OUT:
-	    m = (comp4_t)combine_conjoint_out_part ((comp1_t) (sa >> 0), da);
-	    n = (comp4_t)combine_conjoint_out_part ((comp1_t) (sa >> G_SHIFT), da) << G_SHIFT;
-	    o = (comp4_t)combine_conjoint_out_part ((comp1_t) (sa >> R_SHIFT), da) << R_SHIFT;
-	    p = (comp4_t)combine_conjoint_out_part ((comp1_t) (sa >> A_SHIFT), da) << A_SHIFT;
-	    Fa = m | n | o | p;
-	    break;
-
-	case COMBINE_A_IN:
-	    m = (comp4_t)combine_conjoint_in_part ((comp1_t) (sa >> 0), da);
-	    n = (comp4_t)combine_conjoint_in_part ((comp1_t) (sa >> G_SHIFT), da) << G_SHIFT;
-	    o = (comp4_t)combine_conjoint_in_part ((comp1_t) (sa >> R_SHIFT), da) << R_SHIFT;
-	    p = (comp4_t)combine_conjoint_in_part ((comp1_t) (sa >> A_SHIFT), da) << A_SHIFT;
-	    Fa = m | n | o | p;
-	    break;
-
-	case COMBINE_A:
-	    Fa = ~0;
-	    break;
-	}
-
-	switch (combine & COMBINE_B)
-	{
-	default:
-	    Fb = 0;
-	    break;
-
-	case COMBINE_B_OUT:
-	    m = (comp4_t)combine_conjoint_out_part (da, (comp1_t) (sa >> 0));
-	    n = (comp4_t)combine_conjoint_out_part (da, (comp1_t) (sa >> G_SHIFT)) << G_SHIFT;
-	    o = (comp4_t)combine_conjoint_out_part (da, (comp1_t) (sa >> R_SHIFT)) << R_SHIFT;
-	    p = (comp4_t)combine_conjoint_out_part (da, (comp1_t) (sa >> A_SHIFT)) << A_SHIFT;
-	    Fb = m | n | o | p;
-	    break;
-
-	case COMBINE_B_IN:
-	    m = (comp4_t)combine_conjoint_in_part (da, (comp1_t) (sa >> 0));
-	    n = (comp4_t)combine_conjoint_in_part (da, (comp1_t) (sa >> G_SHIFT)) << G_SHIFT;
-	    o = (comp4_t)combine_conjoint_in_part (da, (comp1_t) (sa >> R_SHIFT)) << R_SHIFT;
-	    p = (comp4_t)combine_conjoint_in_part (da, (comp1_t) (sa >> A_SHIFT)) << A_SHIFT;
-	    Fb = m | n | o | p;
-	    break;
-
-	case COMBINE_B:
-	    Fb = ~0;
-	    break;
-	}
-	m = GENERIC (s, d, 0, GET_COMP (Fa, 0), GET_COMP (Fb, 0), t, u, v);
-	n = GENERIC (s, d, G_SHIFT, GET_COMP (Fa, G_SHIFT), GET_COMP (Fb, G_SHIFT), t, u, v);
-	o = GENERIC (s, d, R_SHIFT, GET_COMP (Fa, R_SHIFT), GET_COMP (Fb, R_SHIFT), t, u, v);
-	p = GENERIC (s, d, A_SHIFT, GET_COMP (Fa, A_SHIFT), GET_COMP (Fb, A_SHIFT), t, u, v);
-
-	s = m | n | o | p;
-
-	*(dest + i) = s;
-    }
-}
-
-static void
-combine_conjoint_over_ca (pixman_implementation_t *imp,
-                          pixman_op_t              op,
-                          comp4_t *                dest,
-                          const comp4_t *          src,
-                          const comp4_t *          mask,
-                          int                      width)
-{
-    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_A_OVER);
-}
-
-static void
-combine_conjoint_over_reverse_ca (pixman_implementation_t *imp,
-                                  pixman_op_t              op,
-                                  comp4_t *                dest,
-                                  const comp4_t *          src,
-                                  const comp4_t *          mask,
-                                  int                      width)
-{
-    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_B_OVER);
-}
-
-static void
-combine_conjoint_in_ca (pixman_implementation_t *imp,
-                        pixman_op_t              op,
-                        comp4_t *                dest,
-                        const comp4_t *          src,
-                        const comp4_t *          mask,
-                        int                      width)
-{
-    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_A_IN);
-}
-
-static void
-combine_conjoint_in_reverse_ca (pixman_implementation_t *imp,
-                                pixman_op_t              op,
-                                comp4_t *                dest,
-                                const comp4_t *          src,
-                                const comp4_t *          mask,
-                                int                      width)
-{
-    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_B_IN);
-}
-
-static void
-combine_conjoint_out_ca (pixman_implementation_t *imp,
-                         pixman_op_t              op,
-                         comp4_t *                dest,
-                         const comp4_t *          src,
-                         const comp4_t *          mask,
-                         int                      width)
-{
-    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_A_OUT);
-}
-
-static void
-combine_conjoint_out_reverse_ca (pixman_implementation_t *imp,
-                                 pixman_op_t              op,
-                                 comp4_t *                dest,
-                                 const comp4_t *          src,
-                                 const comp4_t *          mask,
-                                 int                      width)
-{
-    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_B_OUT);
-}
-
-static void
-combine_conjoint_atop_ca (pixman_implementation_t *imp,
-                          pixman_op_t              op,
-                          comp4_t *                dest,
-                          const comp4_t *          src,
-                          const comp4_t *          mask,
-                          int                      width)
-{
-    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_A_ATOP);
-}
-
-static void
-combine_conjoint_atop_reverse_ca (pixman_implementation_t *imp,
-                                  pixman_op_t              op,
-                                  comp4_t *                dest,
-                                  const comp4_t *          src,
-                                  const comp4_t *          mask,
-                                  int                      width)
-{
-    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_B_ATOP);
-}
-
-static void
-combine_conjoint_xor_ca (pixman_implementation_t *imp,
-                         pixman_op_t              op,
-                         comp4_t *                dest,
-                         const comp4_t *          src,
-                         const comp4_t *          mask,
-                         int                      width)
-{
-    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_XOR);
-}
-
-void
-_pixman_setup_combiner_functions_width (pixman_implementation_t *imp)
-{
-    /* Unified alpha */
-    imp->combine_width[PIXMAN_OP_CLEAR] = combine_clear;
-    imp->combine_width[PIXMAN_OP_SRC] = combine_src_u;
-    /* dest */
-    imp->combine_width[PIXMAN_OP_OVER] = combine_over_u;
-    imp->combine_width[PIXMAN_OP_OVER_REVERSE] = combine_over_reverse_u;
-    imp->combine_width[PIXMAN_OP_IN] = combine_in_u;
-    imp->combine_width[PIXMAN_OP_IN_REVERSE] = combine_in_reverse_u;
-    imp->combine_width[PIXMAN_OP_OUT] = combine_out_u;
-    imp->combine_width[PIXMAN_OP_OUT_REVERSE] = combine_out_reverse_u;
-    imp->combine_width[PIXMAN_OP_ATOP] = combine_atop_u;
-    imp->combine_width[PIXMAN_OP_ATOP_REVERSE] = combine_atop_reverse_u;
-    imp->combine_width[PIXMAN_OP_XOR] = combine_xor_u;
-    imp->combine_width[PIXMAN_OP_ADD] = combine_add_u;
-    imp->combine_width[PIXMAN_OP_SATURATE] = combine_saturate_u;
-
-    /* Disjoint, unified */
-    imp->combine_width[PIXMAN_OP_DISJOINT_CLEAR] = combine_clear;
-    imp->combine_width[PIXMAN_OP_DISJOINT_SRC] = combine_src_u;
-    /* dest */
-    imp->combine_width[PIXMAN_OP_DISJOINT_OVER] = combine_disjoint_over_u;
-    imp->combine_width[PIXMAN_OP_DISJOINT_OVER_REVERSE] = combine_saturate_u;
-    imp->combine_width[PIXMAN_OP_DISJOINT_IN] = combine_disjoint_in_u;
-    imp->combine_width[PIXMAN_OP_DISJOINT_IN_REVERSE] = combine_disjoint_in_reverse_u;
-    imp->combine_width[PIXMAN_OP_DISJOINT_OUT] = combine_disjoint_out_u;
-    imp->combine_width[PIXMAN_OP_DISJOINT_OUT_REVERSE] = combine_disjoint_out_reverse_u;
-    imp->combine_width[PIXMAN_OP_DISJOINT_ATOP] = combine_disjoint_atop_u;
-    imp->combine_width[PIXMAN_OP_DISJOINT_ATOP_REVERSE] = combine_disjoint_atop_reverse_u;
-    imp->combine_width[PIXMAN_OP_DISJOINT_XOR] = combine_disjoint_xor_u;
-
-    /* Conjoint, unified */
-    imp->combine_width[PIXMAN_OP_CONJOINT_CLEAR] = combine_clear;
-    imp->combine_width[PIXMAN_OP_CONJOINT_SRC] = combine_src_u;
-    /* dest */
-    imp->combine_width[PIXMAN_OP_CONJOINT_OVER] = combine_conjoint_over_u;
-    imp->combine_width[PIXMAN_OP_CONJOINT_OVER_REVERSE] = combine_conjoint_over_reverse_u;
-    imp->combine_width[PIXMAN_OP_CONJOINT_IN] = combine_conjoint_in_u;
-    imp->combine_width[PIXMAN_OP_CONJOINT_IN_REVERSE] = combine_conjoint_in_reverse_u;
-    imp->combine_width[PIXMAN_OP_CONJOINT_OUT] = combine_conjoint_out_u;
-    imp->combine_width[PIXMAN_OP_CONJOINT_OUT_REVERSE] = combine_conjoint_out_reverse_u;
-    imp->combine_width[PIXMAN_OP_CONJOINT_ATOP] = combine_conjoint_atop_u;
-    imp->combine_width[PIXMAN_OP_CONJOINT_ATOP_REVERSE] = combine_conjoint_atop_reverse_u;
-    imp->combine_width[PIXMAN_OP_CONJOINT_XOR] = combine_conjoint_xor_u;
-
-    imp->combine_width[PIXMAN_OP_MULTIPLY] = combine_multiply_u;
-    imp->combine_width[PIXMAN_OP_SCREEN] = combine_screen_u;
-    imp->combine_width[PIXMAN_OP_OVERLAY] = combine_overlay_u;
-    imp->combine_width[PIXMAN_OP_DARKEN] = combine_darken_u;
-    imp->combine_width[PIXMAN_OP_LIGHTEN] = combine_lighten_u;
-    imp->combine_width[PIXMAN_OP_COLOR_DODGE] = combine_color_dodge_u;
-    imp->combine_width[PIXMAN_OP_COLOR_BURN] = combine_color_burn_u;
-    imp->combine_width[PIXMAN_OP_HARD_LIGHT] = combine_hard_light_u;
-    imp->combine_width[PIXMAN_OP_SOFT_LIGHT] = combine_soft_light_u;
-    imp->combine_width[PIXMAN_OP_DIFFERENCE] = combine_difference_u;
-    imp->combine_width[PIXMAN_OP_EXCLUSION] = combine_exclusion_u;
-    imp->combine_width[PIXMAN_OP_HSL_HUE] = combine_hsl_hue_u;
-    imp->combine_width[PIXMAN_OP_HSL_SATURATION] = combine_hsl_saturation_u;
-    imp->combine_width[PIXMAN_OP_HSL_COLOR] = combine_hsl_color_u;
-    imp->combine_width[PIXMAN_OP_HSL_LUMINOSITY] = combine_hsl_luminosity_u;
-
-    /* Component alpha combiners */
-    imp->combine_width_ca[PIXMAN_OP_CLEAR] = combine_clear_ca;
-    imp->combine_width_ca[PIXMAN_OP_SRC] = combine_src_ca;
-    /* dest */
-    imp->combine_width_ca[PIXMAN_OP_OVER] = combine_over_ca;
-    imp->combine_width_ca[PIXMAN_OP_OVER_REVERSE] = combine_over_reverse_ca;
-    imp->combine_width_ca[PIXMAN_OP_IN] = combine_in_ca;
-    imp->combine_width_ca[PIXMAN_OP_IN_REVERSE] = combine_in_reverse_ca;
-    imp->combine_width_ca[PIXMAN_OP_OUT] = combine_out_ca;
-    imp->combine_width_ca[PIXMAN_OP_OUT_REVERSE] = combine_out_reverse_ca;
-    imp->combine_width_ca[PIXMAN_OP_ATOP] = combine_atop_ca;
-    imp->combine_width_ca[PIXMAN_OP_ATOP_REVERSE] = combine_atop_reverse_ca;
-    imp->combine_width_ca[PIXMAN_OP_XOR] = combine_xor_ca;
-    imp->combine_width_ca[PIXMAN_OP_ADD] = combine_add_ca;
-    imp->combine_width_ca[PIXMAN_OP_SATURATE] = combine_saturate_ca;
-
-    /* Disjoint CA */
-    imp->combine_width_ca[PIXMAN_OP_DISJOINT_CLEAR] = combine_clear_ca;
-    imp->combine_width_ca[PIXMAN_OP_DISJOINT_SRC] = combine_src_ca;
-    /* dest */
-    imp->combine_width_ca[PIXMAN_OP_DISJOINT_OVER] = combine_disjoint_over_ca;
-    imp->combine_width_ca[PIXMAN_OP_DISJOINT_OVER_REVERSE] = combine_saturate_ca;
-    imp->combine_width_ca[PIXMAN_OP_DISJOINT_IN] = combine_disjoint_in_ca;
-    imp->combine_width_ca[PIXMAN_OP_DISJOINT_IN_REVERSE] = combine_disjoint_in_reverse_ca;
-    imp->combine_width_ca[PIXMAN_OP_DISJOINT_OUT] = combine_disjoint_out_ca;
-    imp->combine_width_ca[PIXMAN_OP_DISJOINT_OUT_REVERSE] = combine_disjoint_out_reverse_ca;
-    imp->combine_width_ca[PIXMAN_OP_DISJOINT_ATOP] = combine_disjoint_atop_ca;
-    imp->combine_width_ca[PIXMAN_OP_DISJOINT_ATOP_REVERSE] = combine_disjoint_atop_reverse_ca;
-    imp->combine_width_ca[PIXMAN_OP_DISJOINT_XOR] = combine_disjoint_xor_ca;
-
-    /* Conjoint CA */
-    imp->combine_width_ca[PIXMAN_OP_CONJOINT_CLEAR] = combine_clear_ca;
-    imp->combine_width_ca[PIXMAN_OP_CONJOINT_SRC] = combine_src_ca;
-    /* dest */
-    imp->combine_width_ca[PIXMAN_OP_CONJOINT_OVER] = combine_conjoint_over_ca;
-    imp->combine_width_ca[PIXMAN_OP_CONJOINT_OVER_REVERSE] = combine_conjoint_over_reverse_ca;
-    imp->combine_width_ca[PIXMAN_OP_CONJOINT_IN] = combine_conjoint_in_ca;
-    imp->combine_width_ca[PIXMAN_OP_CONJOINT_IN_REVERSE] = combine_conjoint_in_reverse_ca;
-    imp->combine_width_ca[PIXMAN_OP_CONJOINT_OUT] = combine_conjoint_out_ca;
-    imp->combine_width_ca[PIXMAN_OP_CONJOINT_OUT_REVERSE] = combine_conjoint_out_reverse_ca;
-    imp->combine_width_ca[PIXMAN_OP_CONJOINT_ATOP] = combine_conjoint_atop_ca;
-    imp->combine_width_ca[PIXMAN_OP_CONJOINT_ATOP_REVERSE] = combine_conjoint_atop_reverse_ca;
-    imp->combine_width_ca[PIXMAN_OP_CONJOINT_XOR] = combine_conjoint_xor_ca;
-
-    imp->combine_width_ca[PIXMAN_OP_MULTIPLY] = combine_multiply_ca;
-    imp->combine_width_ca[PIXMAN_OP_SCREEN] = combine_screen_ca;
-    imp->combine_width_ca[PIXMAN_OP_OVERLAY] = combine_overlay_ca;
-    imp->combine_width_ca[PIXMAN_OP_DARKEN] = combine_darken_ca;
-    imp->combine_width_ca[PIXMAN_OP_LIGHTEN] = combine_lighten_ca;
-    imp->combine_width_ca[PIXMAN_OP_COLOR_DODGE] = combine_color_dodge_ca;
-    imp->combine_width_ca[PIXMAN_OP_COLOR_BURN] = combine_color_burn_ca;
-    imp->combine_width_ca[PIXMAN_OP_HARD_LIGHT] = combine_hard_light_ca;
-    imp->combine_width_ca[PIXMAN_OP_SOFT_LIGHT] = combine_soft_light_ca;
-    imp->combine_width_ca[PIXMAN_OP_DIFFERENCE] = combine_difference_ca;
-    imp->combine_width_ca[PIXMAN_OP_EXCLUSION] = combine_exclusion_ca;
-
-    /* It is not clear that these make sense, so leave them out for now */
-    imp->combine_width_ca[PIXMAN_OP_HSL_HUE] = NULL;
-    imp->combine_width_ca[PIXMAN_OP_HSL_SATURATION] = NULL;
-    imp->combine_width_ca[PIXMAN_OP_HSL_COLOR] = NULL;
-    imp->combine_width_ca[PIXMAN_OP_HSL_LUMINOSITY] = NULL;
-}
-
diff --git a/pixman/pixman/pixman-combine.h.template b/pixman/pixman/pixman-combine.h.template
deleted file mode 100644
index 2f6392f96..000000000
--- a/pixman/pixman/pixman-combine.h.template
+++ /dev/null
@@ -1,226 +0,0 @@
-
-#define COMPONENT_SIZE
-#define MASK
-#define ONE_HALF
-
-#define A_SHIFT
-#define R_SHIFT
-#define G_SHIFT
-#define A_MASK
-#define R_MASK
-#define G_MASK
-
-#define RB_MASK
-#define AG_MASK
-#define RB_ONE_HALF
-#define RB_MASK_PLUS_ONE
-
-#define ALPHA_c(x) ((x) >> A_SHIFT)
-#define RED_c(x) (((x) >> R_SHIFT) & MASK)
-#define GREEN_c(x) (((x) >> G_SHIFT) & MASK)
-#define BLUE_c(x) ((x) & MASK)
-
-/*
- * Helper macros.
- */
-
-#define MUL_UNc(a, b, t)						\
-    ((t) = (a) * (b) + ONE_HALF, ((((t) >> G_SHIFT ) + (t) ) >> G_SHIFT ))
-
-#define DIV_UNc(a, b)							\
-    (((comp2_t) (a) * MASK) / (b))
-
-#define ADD_UNc(x, y, t)				     \
-    ((t) = x + y,					     \
-     (comp4_t) (comp1_t) ((t) | (0 - ((t) >> G_SHIFT))))
-
-#define DIV_ONE_UNc(x)							\
-    (((x) + ONE_HALF + (((x) + ONE_HALF) >> G_SHIFT)) >> G_SHIFT)
-
-/*
- * The methods below use some tricks to be able to do two color
- * components at the same time.
- */
-
-/*
- * x_rb = (x_rb * a) / 255
- */
-#define UNc_rb_MUL_UNc(x, a, t)						\
-    do									\
-    {									\
-	t  = ((x) & RB_MASK) * (a);					\
-	t += RB_ONE_HALF;						\
-	x = (t + ((t >> G_SHIFT) & RB_MASK)) >> G_SHIFT;		\
-	x &= RB_MASK;							\
-    } while (0)
-
-/*
- * x_rb = min (x_rb + y_rb, 255)
- */
-#define UNc_rb_ADD_UNc_rb(x, y, t)					\
-    do									\
-    {									\
-	t = ((x) + (y));						\
-	t |= RB_MASK_PLUS_ONE - ((t >> G_SHIFT) & RB_MASK);		\
-	x = (t & RB_MASK);						\
-    } while (0)
-
-/*
- * x_rb = (x_rb * a_rb) / 255
- */
-#define UNc_rb_MUL_UNc_rb(x, a, t)					\
-    do									\
-    {									\
-	t  = (x & MASK) * (a & MASK);					\
-	t |= (x & R_MASK) * ((a >> R_SHIFT) & MASK);			\
-	t += RB_ONE_HALF;						\
-	t = (t + ((t >> G_SHIFT) & RB_MASK)) >> G_SHIFT;		\
-	x = t & RB_MASK;						\
-    } while (0)
-
-/*
- * x_c = (x_c * a) / 255
- */
-#define UNcx4_MUL_UNc(x, a)						\
-    do									\
-    {									\
-	comp4_t r1, r2, t;						\
-									\
-	r1 = (x);							\
-	UNc_rb_MUL_UNc (r1, a, t);					\
-									\
-	r2 = (x) >> G_SHIFT;						\
-	UNc_rb_MUL_UNc (r2, a, t);					\
-									\
-	x = r1 | (r2 << G_SHIFT);					\
-    } while (0)
-
-/*
- * x_c = (x_c * a) / 255 + y_c
- */
-#define UNcx4_MUL_UNc_ADD_UNcx4(x, a, y)				\
-    do									\
-    {									\
-	comp4_t r1, r2, r3, t;						\
-									\
-	r1 = (x);							\
-	r2 = (y) & RB_MASK;						\
-	UNc_rb_MUL_UNc (r1, a, t);					\
-	UNc_rb_ADD_UNc_rb (r1, r2, t);					\
-									\
-	r2 = (x) >> G_SHIFT;						\
-	r3 = ((y) >> G_SHIFT) & RB_MASK;				\
-	UNc_rb_MUL_UNc (r2, a, t);					\
-	UNc_rb_ADD_UNc_rb (r2, r3, t);					\
-									\
-	x = r1 | (r2 << G_SHIFT);					\
-    } while (0)
-
-/*
- * x_c = (x_c * a + y_c * b) / 255
- */
-#define UNcx4_MUL_UNc_ADD_UNcx4_MUL_UNc(x, a, y, b)			\
-    do									\
-    {									\
-	comp4_t r1, r2, r3, t;						\
-									\
-	r1 = x;								\
-	r2 = y;								\
-	UNc_rb_MUL_UNc (r1, a, t);					\
-	UNc_rb_MUL_UNc (r2, b, t);					\
-	UNc_rb_ADD_UNc_rb (r1, r2, t);					\
-									\
-	r2 = (x >> G_SHIFT);						\
-	r3 = (y >> G_SHIFT);						\
-	UNc_rb_MUL_UNc (r2, a, t);					\
-	UNc_rb_MUL_UNc (r3, b, t);					\
-	UNc_rb_ADD_UNc_rb (r2, r3, t);					\
-									\
-	x = r1 | (r2 << G_SHIFT);					\
-    } while (0)
-
-/*
- * x_c = (x_c * a_c) / 255
- */
-#define UNcx4_MUL_UNcx4(x, a)						\
-    do									\
-    {									\
-	comp4_t r1, r2, r3, t;						\
-									\
-	r1 = x;								\
-	r2 = a;								\
-	UNc_rb_MUL_UNc_rb (r1, r2, t);					\
-									\
-	r2 = x >> G_SHIFT;						\
-	r3 = a >> G_SHIFT;						\
-	UNc_rb_MUL_UNc_rb (r2, r3, t);					\
-									\
-	x = r1 | (r2 << G_SHIFT);					\
-    } while (0)
-
-/*
- * x_c = (x_c * a_c) / 255 + y_c
- */
-#define UNcx4_MUL_UNcx4_ADD_UNcx4(x, a, y)				\
-    do									\
-    {									\
-	comp4_t r1, r2, r3, t;						\
-									\
-	r1 = x;								\
-	r2 = a;								\
-	UNc_rb_MUL_UNc_rb (r1, r2, t);					\
-	r2 = y & RB_MASK;						\
-	UNc_rb_ADD_UNc_rb (r1, r2, t);					\
-									\
-	r2 = (x >> G_SHIFT);						\
-	r3 = (a >> G_SHIFT);						\
-	UNc_rb_MUL_UNc_rb (r2, r3, t);					\
-	r3 = (y >> G_SHIFT) & RB_MASK;					\
-	UNc_rb_ADD_UNc_rb (r2, r3, t);					\
-									\
-	x = r1 | (r2 << G_SHIFT);					\
-    } while (0)
-
-/*
- * x_c = (x_c * a_c + y_c * b) / 255
- */
-#define UNcx4_MUL_UNcx4_ADD_UNcx4_MUL_UNc(x, a, y, b)			\
-    do									\
-    {									\
-	comp4_t r1, r2, r3, t;						\
-									\
-	r1 = x;								\
-	r2 = a;								\
-	UNc_rb_MUL_UNc_rb (r1, r2, t);					\
-	r2 = y;								\
-	UNc_rb_MUL_UNc (r2, b, t);					\
-	UNc_rb_ADD_UNc_rb (r1, r2, t);					\
-									\
-	r2 = x >> G_SHIFT;						\
-	r3 = a >> G_SHIFT;						\
-	UNc_rb_MUL_UNc_rb (r2, r3, t);					\
-	r3 = y >> G_SHIFT;						\
-	UNc_rb_MUL_UNc (r3, b, t);					\
-	UNc_rb_ADD_UNc_rb (r2, r3, t);					\
-									\
-	x = r1 | (r2 << G_SHIFT);					\
-    } while (0)
-
-/*
-   x_c = min(x_c + y_c, 255)
- */
-#define UNcx4_ADD_UNcx4(x, y)						\
-    do									\
-    {									\
-	comp4_t r1, r2, r3, t;						\
-									\
-	r1 = x & RB_MASK;						\
-	r2 = y & RB_MASK;						\
-	UNc_rb_ADD_UNc_rb (r1, r2, t);					\
-									\
-	r2 = (x >> G_SHIFT) & RB_MASK;					\
-	r3 = (y >> G_SHIFT) & RB_MASK;					\
-	UNc_rb_ADD_UNc_rb (r2, r3, t);					\
-									\
-	x = r1 | (r2 << G_SHIFT);					\
-    } while (0)
diff --git a/pixman/pixman/pixman-combine32.c b/pixman/pixman/pixman-combine32.c
new file mode 100644
index 000000000..450114a52
--- /dev/null
+++ b/pixman/pixman/pixman-combine32.c
@@ -0,0 +1,2581 @@
+/*
+ * Copyright © 2000 Keith Packard, member of The XFree86 Project, Inc.
+ *             2005 Lars Knoll & Zack Rusin, Trolltech
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Keith Packard not be used in
+ * advertising or publicity pertaining to distribution of the software without
+ * specific, written prior permission.  Keith Packard makes no
+ * representations about the suitability of this software for any purpose.  It
+ * is provided "as is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <math.h>
+#include <string.h>
+
+#include "pixman-private.h"
+#include "pixman-combine32.h"
+
+/* component alpha helper functions */
+
+static void
+combine_mask_ca (uint32_t *src, uint32_t *mask)
+{
+    uint32_t a = *mask;
+
+    uint32_t x;
+    uint16_t xa;
+
+    if (!a)
+    {
+	*(src) = 0;
+	return;
+    }
+
+    x = *(src);
+    if (a == ~0)
+    {
+	x = x >> A_SHIFT;
+	x |= x << G_SHIFT;
+	x |= x << R_SHIFT;
+	*(mask) = x;
+	return;
+    }
+
+    xa = x >> A_SHIFT;
+    UN8x4_MUL_UN8x4 (x, a);
+    *(src) = x;
+    
+    UN8x4_MUL_UN8 (a, xa);
+    *(mask) = a;
+}
+
+static void
+combine_mask_value_ca (uint32_t *src, const uint32_t *mask)
+{
+    uint32_t a = *mask;
+    uint32_t x;
+
+    if (!a)
+    {
+	*(src) = 0;
+	return;
+    }
+
+    if (a == ~0)
+	return;
+
+    x = *(src);
+    UN8x4_MUL_UN8x4 (x, a);
+    *(src) = x;
+}
+
+static void
+combine_mask_alpha_ca (const uint32_t *src, uint32_t *mask)
+{
+    uint32_t a = *(mask);
+    uint32_t x;
+
+    if (!a)
+	return;
+
+    x = *(src) >> A_SHIFT;
+    if (x == MASK)
+	return;
+
+    if (a == ~0)
+    {
+	x |= x << G_SHIFT;
+	x |= x << R_SHIFT;
+	*(mask) = x;
+	return;
+    }
+
+    UN8x4_MUL_UN8 (a, x);
+    *(mask) = a;
+}
+
+/*
+ * There are two ways of handling alpha -- either as a single unified value or
+ * a separate value for each component, hence each macro must have two
+ * versions.  The unified alpha version has a 'u' at the end of the name,
+ * the component version has a 'ca'.  Similarly, functions which deal with
+ * this difference will have two versions using the same convention.
+ */
+
+static force_inline uint32_t
+combine_mask (const uint32_t *src, const uint32_t *mask, int i)
+{
+    uint32_t s, m;
+
+    if (mask)
+    {
+	m = *(mask + i) >> A_SHIFT;
+
+	if (!m)
+	    return 0;
+    }
+
+    s = *(src + i);
+
+    if (mask)
+	UN8x4_MUL_UN8 (s, m);
+
+    return s;
+}
+
+static void
+combine_clear (pixman_implementation_t *imp,
+               pixman_op_t              op,
+               uint32_t *               dest,
+               const uint32_t *         src,
+               const uint32_t *         mask,
+               int                      width)
+{
+    memset (dest, 0, width * sizeof (uint32_t));
+}
+
+static void
+combine_dst (pixman_implementation_t *imp,
+	     pixman_op_t	      op,
+	     uint32_t *		      dest,
+	     const uint32_t *	      src,
+	     const uint32_t *         mask,
+	     int		      width)
+{
+    return;
+}
+
+static void
+combine_src_u (pixman_implementation_t *imp,
+               pixman_op_t              op,
+               uint32_t *               dest,
+               const uint32_t *         src,
+               const uint32_t *         mask,
+               int                      width)
+{
+    int i;
+
+    if (!mask)
+    {
+	memcpy (dest, src, width * sizeof (uint32_t));
+    }
+    else
+    {
+	for (i = 0; i < width; ++i)
+	{
+	    uint32_t s = combine_mask (src, mask, i);
+
+	    *(dest + i) = s;
+	}
+    }
+}
+
+static void
+combine_over_u (pixman_implementation_t *imp,
+                pixman_op_t              op,
+                uint32_t *               dest,
+                const uint32_t *         src,
+                const uint32_t *         mask,
+                int                      width)
+{
+    int i;
+
+    if (!mask)
+    {
+	for (i = 0; i < width; ++i)
+	{
+	    uint32_t s = *(src + i);
+	    uint32_t a = ALPHA_8 (s);
+	    if (a == 0xFF)
+	    {
+		*(dest + i) = s;
+	    }
+	    else if (s)
+	    {
+		uint32_t d = *(dest + i);
+		uint32_t ia = a ^ 0xFF;
+		UN8x4_MUL_UN8_ADD_UN8x4 (d, ia, s);
+		*(dest + i) = d;
+	    }
+	}
+    }
+    else
+    {
+	for (i = 0; i < width; ++i)
+	{
+	    uint32_t m = ALPHA_8 (*(mask + i));
+	    if (m == 0xFF)
+	    {
+		uint32_t s = *(src + i);
+		uint32_t a = ALPHA_8 (s);
+		if (a == 0xFF)
+		{
+		    *(dest + i) = s;
+		}
+		else if (s)
+		{
+		    uint32_t d = *(dest + i);
+		    uint32_t ia = a ^ 0xFF;
+		    UN8x4_MUL_UN8_ADD_UN8x4 (d, ia, s);
+		    *(dest + i) = d;
+		}
+	    }
+	    else if (m)
+	    {
+		uint32_t s = *(src + i);
+		if (s)
+		{
+		    uint32_t d = *(dest + i);
+		    UN8x4_MUL_UN8 (s, m);
+		    UN8x4_MUL_UN8_ADD_UN8x4 (d, ALPHA_8 (~s), s);
+		    *(dest + i) = d;
+		}
+	    }
+	}
+    }
+}
+
+static void
+combine_over_reverse_u (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        uint32_t *               dest,
+                        const uint32_t *         src,
+                        const uint32_t *         mask,
+                        int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = combine_mask (src, mask, i);
+	uint32_t d = *(dest + i);
+	uint32_t ia = ALPHA_8 (~*(dest + i));
+	UN8x4_MUL_UN8_ADD_UN8x4 (s, ia, d);
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_in_u (pixman_implementation_t *imp,
+              pixman_op_t              op,
+              uint32_t *               dest,
+              const uint32_t *         src,
+              const uint32_t *         mask,
+              int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = combine_mask (src, mask, i);
+	uint32_t a = ALPHA_8 (*(dest + i));
+	UN8x4_MUL_UN8 (s, a);
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_in_reverse_u (pixman_implementation_t *imp,
+                      pixman_op_t              op,
+                      uint32_t *               dest,
+                      const uint32_t *         src,
+                      const uint32_t *         mask,
+                      int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = combine_mask (src, mask, i);
+	uint32_t d = *(dest + i);
+	uint32_t a = ALPHA_8 (s);
+	UN8x4_MUL_UN8 (d, a);
+	*(dest + i) = d;
+    }
+}
+
+static void
+combine_out_u (pixman_implementation_t *imp,
+               pixman_op_t              op,
+               uint32_t *               dest,
+               const uint32_t *         src,
+               const uint32_t *         mask,
+               int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = combine_mask (src, mask, i);
+	uint32_t a = ALPHA_8 (~*(dest + i));
+	UN8x4_MUL_UN8 (s, a);
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_out_reverse_u (pixman_implementation_t *imp,
+                       pixman_op_t              op,
+                       uint32_t *               dest,
+                       const uint32_t *         src,
+                       const uint32_t *         mask,
+                       int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = combine_mask (src, mask, i);
+	uint32_t d = *(dest + i);
+	uint32_t a = ALPHA_8 (~s);
+	UN8x4_MUL_UN8 (d, a);
+	*(dest + i) = d;
+    }
+}
+
+static void
+combine_atop_u (pixman_implementation_t *imp,
+                pixman_op_t              op,
+                uint32_t *               dest,
+                const uint32_t *         src,
+                const uint32_t *         mask,
+                int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = combine_mask (src, mask, i);
+	uint32_t d = *(dest + i);
+	uint32_t dest_a = ALPHA_8 (d);
+	uint32_t src_ia = ALPHA_8 (~s);
+
+	UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8 (s, dest_a, d, src_ia);
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_atop_reverse_u (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        uint32_t *               dest,
+                        const uint32_t *         src,
+                        const uint32_t *         mask,
+                        int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = combine_mask (src, mask, i);
+	uint32_t d = *(dest + i);
+	uint32_t src_a = ALPHA_8 (s);
+	uint32_t dest_ia = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8 (s, dest_ia, d, src_a);
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_xor_u (pixman_implementation_t *imp,
+               pixman_op_t              op,
+               uint32_t *               dest,
+               const uint32_t *         src,
+               const uint32_t *         mask,
+               int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = combine_mask (src, mask, i);
+	uint32_t d = *(dest + i);
+	uint32_t src_ia = ALPHA_8 (~s);
+	uint32_t dest_ia = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8 (s, dest_ia, d, src_ia);
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_add_u (pixman_implementation_t *imp,
+               pixman_op_t              op,
+               uint32_t *               dest,
+               const uint32_t *         src,
+               const uint32_t *         mask,
+               int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = combine_mask (src, mask, i);
+	uint32_t d = *(dest + i);
+	UN8x4_ADD_UN8x4 (d, s);
+	*(dest + i) = d;
+    }
+}
+
+static void
+combine_saturate_u (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dest,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = combine_mask (src, mask, i);
+	uint32_t d = *(dest + i);
+	uint16_t sa, da;
+
+	sa = s >> A_SHIFT;
+	da = ~d >> A_SHIFT;
+	if (sa > da)
+	{
+	    sa = DIV_UN8 (da, sa);
+	    UN8x4_MUL_UN8 (s, sa);
+	}
+	;
+	UN8x4_ADD_UN8x4 (d, s);
+	*(dest + i) = d;
+    }
+}
+
+
+/*
+ * PDF blend modes:
+ *
+ * The following blend modes have been taken from the PDF ISO 32000
+ * specification, which at this point in time is available from
+ *
+ *     http://www.adobe.com/devnet/pdf/pdf_reference.html
+ *
+ * The specific documents of interest are the PDF spec itself:
+ *
+ *     http://wwwimages.adobe.com/www.adobe.com/content/dam/Adobe/en/devnet/pdf/pdfs/PDF32000_2008.pdf
+ *
+ * chapters 11.3.5 and 11.3.6 and a later supplement for Adobe Acrobat
+ * 9.1 and Reader 9.1:
+ *
+ *     http://wwwimages.adobe.com/www.adobe.com/content/dam/Adobe/en/devnet/pdf/pdfs/adobe_supplement_iso32000_1.pdf
+ *
+ * that clarifies the specifications for blend modes ColorDodge and
+ * ColorBurn.
+ *
+ * The formula for computing the final pixel color given in 11.3.6 is:
+ *
+ *     αr × Cr = (1 – αs) × αb × Cb + (1 – αb) × αs × Cs + αb × αs × B(Cb, Cs)
+ *
+ * with B() is the blend function. When B(Cb, Cs) = Cs, this formula
+ * reduces to the regular OVER operator.
+ *
+ * Cs and Cb are not premultiplied, so in our implementation we instead
+ * use:
+ *
+ *     cr = (1 – αs) × cb  +  (1 – αb) × cs  +  αb × αs × B (cb/αb, cs/αs)
+ *
+ * where cr, cs, and cb are premultiplied colors, and where the
+ *
+ *     αb × αs × B(cb/αb, cs/αs)
+ *
+ * part is first arithmetically simplified under the assumption that αb
+ * and αs are not 0, and then updated to produce a meaningful result when
+ * they are.
+ *
+ * For all the blend mode operators, the alpha channel is given by
+ *
+ *     αr = αs + αb + αb × αs
+ */
+
+/*
+ * Multiply
+ *
+ *      ad * as * B(d / ad, s / as)
+ *    = ad * as * d/ad * s/as
+ *    = d * s
+ *
+ */
+static void
+combine_multiply_u (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dest,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = combine_mask (src, mask, i);
+	uint32_t d = *(dest + i);
+	uint32_t ss = s;
+	uint32_t src_ia = ALPHA_8 (~s);
+	uint32_t dest_ia = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8 (ss, dest_ia, d, src_ia);
+	UN8x4_MUL_UN8x4 (d, s);
+	UN8x4_ADD_UN8x4 (d, ss);
+
+	*(dest + i) = d;
+    }
+}
+
+static void
+combine_multiply_ca (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     uint32_t *               dest,
+                     const uint32_t *         src,
+                     const uint32_t *         mask,
+                     int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t m = *(mask + i);
+	uint32_t s = *(src + i);
+	uint32_t d = *(dest + i);
+	uint32_t r = d;
+	uint32_t dest_ia = ALPHA_8 (~d);
+
+	combine_mask_ca (&s, &m);
+
+	UN8x4_MUL_UN8x4_ADD_UN8x4_MUL_UN8 (r, ~m, s, dest_ia);
+	UN8x4_MUL_UN8x4 (d, s);
+	UN8x4_ADD_UN8x4 (r, d);
+
+	*(dest + i) = r;
+    }
+}
+
+#define PDF_SEPARABLE_BLEND_MODE(name)					\
+    static void								\
+    combine_ ## name ## _u (pixman_implementation_t *imp,		\
+			    pixman_op_t              op,		\
+                            uint32_t *               dest,		\
+			    const uint32_t *         src,		\
+			    const uint32_t *         mask,		\
+			    int                      width)		\
+    {									\
+	int i;								\
+	for (i = 0; i < width; ++i)					\
+	{								\
+	    uint32_t s = combine_mask (src, mask, i);			\
+	    uint32_t d = *(dest + i);					\
+	    uint8_t sa = ALPHA_8 (s);					\
+	    uint8_t isa = ~sa;						\
+	    uint8_t da = ALPHA_8 (d);					\
+	    uint8_t ida = ~da;						\
+	    uint32_t result;						\
+									\
+	    result = d;							\
+	    UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8 (result, isa, s, ida);	\
+	    								\
+	    *(dest + i) = result +					\
+		(DIV_ONE_UN8 (sa * (uint32_t)da) << A_SHIFT) +		\
+		(blend_ ## name (RED_8 (d), da, RED_8 (s), sa) << R_SHIFT) + \
+		(blend_ ## name (GREEN_8 (d), da, GREEN_8 (s), sa) << G_SHIFT) + \
+		(blend_ ## name (BLUE_8 (d), da, BLUE_8 (s), sa));	\
+	}								\
+    }									\
+    									\
+    static void								\
+    combine_ ## name ## _ca (pixman_implementation_t *imp,		\
+			     pixman_op_t              op,		\
+                             uint32_t *               dest,		\
+			     const uint32_t *         src,		\
+			     const uint32_t *         mask,		\
+			     int                      width)		\
+    {									\
+	int i;								\
+	for (i = 0; i < width; ++i)					\
+	{								\
+	    uint32_t m = *(mask + i);					\
+	    uint32_t s = *(src + i);					\
+	    uint32_t d = *(dest + i);					\
+	    uint8_t da = ALPHA_8 (d);					\
+	    uint8_t ida = ~da;						\
+	    uint32_t result;						\
+            								\
+	    combine_mask_ca (&s, &m);					\
+            								\
+	    result = d;							\
+	    UN8x4_MUL_UN8x4_ADD_UN8x4_MUL_UN8 (result, ~m, s, ida);     \
+            								\
+	    result +=							\
+	        (DIV_ONE_UN8 (ALPHA_8 (m) * (uint32_t)da) << A_SHIFT) +	\
+	        (blend_ ## name (RED_8 (d), da, RED_8 (s), RED_8 (m)) << R_SHIFT) + \
+	        (blend_ ## name (GREEN_8 (d), da, GREEN_8 (s), GREEN_8 (m)) << G_SHIFT) + \
+	        (blend_ ## name (BLUE_8 (d), da, BLUE_8 (s), BLUE_8 (m))); \
+	    								\
+	    *(dest + i) = result;					\
+	}								\
+    }
+
+/*
+ * Screen
+ *
+ *      ad * as * B(d/ad, s/as)
+ *    = ad * as * (d/ad + s/as - s/as * d/ad)
+ *    = ad * s + as * d - s * d
+ */
+static inline uint32_t
+blend_screen (uint32_t d, uint32_t ad, uint32_t s, uint32_t as)
+{
+    return DIV_ONE_UN8 (s * ad + d * as - s * d);
+}
+
+PDF_SEPARABLE_BLEND_MODE (screen)
+
+/*
+ * Overlay
+ *
+ *     ad * as * B(d/ad, s/as)
+ *   = ad * as * Hardlight (s, d)
+ *   = if (d / ad < 0.5)
+ *         as * ad * Multiply (s/as, 2 * d/ad)
+ *     else
+ *         as * ad * Screen (s/as, 2 * d / ad - 1)
+ *   = if (d < 0.5 * ad)
+ *         as * ad * s/as * 2 * d /ad
+ *     else
+ *         as * ad * (s/as + 2 * d / ad - 1 - s / as * (2 * d / ad - 1))
+ *   = if (2 * d < ad)
+ *         2 * s * d
+ *     else
+ *         ad * s + 2 * as * d - as * ad - ad * s * (2 * d / ad - 1)
+ *   = if (2 * d < ad)
+ *         2 * s * d
+ *     else
+ *         as * ad - 2 * (ad - d) * (as - s)
+ */
+static inline uint32_t
+blend_overlay (uint32_t d, uint32_t ad, uint32_t s, uint32_t as)
+{
+    uint32_t r;
+
+    if (2 * d < ad)
+	r = 2 * s * d;
+    else
+	r = as * ad - 2 * (ad - d) * (as - s);
+
+    return DIV_ONE_UN8 (r);
+}
+
+PDF_SEPARABLE_BLEND_MODE (overlay)
+
+/*
+ * Darken
+ *
+ *     ad * as * B(d/ad, s/as)
+ *   = ad * as * MIN(d/ad, s/as)
+ *   = MIN (as * d, ad * s)
+ */
+static inline uint32_t
+blend_darken (uint32_t d, uint32_t ad, uint32_t s, uint32_t as)
+{
+    s = ad * s;
+    d = as * d;
+
+    return DIV_ONE_UN8 (s > d ? d : s);
+}
+
+PDF_SEPARABLE_BLEND_MODE (darken)
+
+/*
+ * Lighten
+ *
+ *     ad * as * B(d/ad, s/as)
+ *   = ad * as * MAX(d/ad, s/as)
+ *   = MAX (as * d, ad * s)
+ */
+static inline uint32_t
+blend_lighten (uint32_t d, uint32_t ad, uint32_t s, uint32_t as)
+{
+    s = ad * s;
+    d = as * d;
+    
+    return DIV_ONE_UN8 (s > d ? s : d);
+}
+
+PDF_SEPARABLE_BLEND_MODE (lighten)
+
+/*
+ * Color dodge
+ *
+ *     ad * as * B(d/ad, s/as)
+ *   = if d/ad = 0
+ *         ad * as * 0
+ *     else if (d/ad >= (1 - s/as)
+ *         ad * as * 1
+ *     else
+ *         ad * as * ((d/ad) / (1 - s/as))
+ *   = if d = 0
+ *         0
+ *     elif as * d >= ad * (as - s)
+ *         ad * as
+ *     else
+ *         as * (as * d / (as - s))
+ *
+ */
+static inline uint32_t
+blend_color_dodge (uint32_t d, uint32_t ad, uint32_t s, uint32_t as)
+{
+    if (d == 0)
+        return 0;
+    else if (as * d >= ad * (as - s))
+	return DIV_ONE_UN8 (as * ad);
+    else if (as - s == 0)
+        return DIV_ONE_UN8 (as * ad);
+    else
+        return DIV_ONE_UN8 (as * ((d * as) / ((as - s))));
+}
+
+PDF_SEPARABLE_BLEND_MODE (color_dodge)
+
+/*
+ * Color burn
+ *
+ * We modify the first clause "if d = 1" to "if d >= 1" since with
+ * premultiplied colors d > 1 can actually happen.
+ *
+ *     ad * as * B(d/ad, s/as)
+ *   = if d/ad >= 1
+ *         ad * as * 1
+ *     elif (1 - d/ad) >= s/as
+ *         ad * as * 0
+ *     else
+ *         ad * as * (1 - ((1 - d/ad) / (s/as)))
+ *   = if d >= ad
+ *         ad * as
+ *     elif as * ad - as * d >= ad * s
+ *         0
+ *     else
+ *         ad * as  - as * as * (ad - d) / s
+ */
+static inline uint32_t
+blend_color_burn (uint32_t d, uint32_t ad, uint32_t s, uint32_t as)
+{
+    if (d >= ad)
+	return DIV_ONE_UN8 (ad * as);
+    else if (as * ad - as * d >= ad * s)
+	return 0;
+    else if (s == 0)
+	return 0;
+    else
+	return DIV_ONE_UN8 (ad * as - (as * as * (ad - d)) / s);
+}
+
+PDF_SEPARABLE_BLEND_MODE (color_burn)
+
+/*
+ * Hard light
+ *
+ *     ad * as * B(d/ad, s/as)
+ *   = if (s/as <= 0.5)
+ *         ad * as * Multiply (d/ad, 2 * s/as)
+ *     else
+ *         ad * as * Screen (d/ad, 2 * s/as - 1)
+ *   = if 2 * s <= as
+ *         ad * as * d/ad * 2 * s / as
+ *     else
+ *         ad * as * (d/ad + (2 * s/as - 1) + d/ad * (2 * s/as - 1))
+ *   = if 2 * s <= as
+ *         2 * s * d
+ *     else
+ *         as * ad - 2 * (ad - d) * (as - s)
+ */
+static inline uint32_t
+blend_hard_light (uint32_t d, uint32_t ad, uint32_t s, uint32_t as)
+{
+    if (2 * s < as)
+	return DIV_ONE_UN8 (2 * s * d);
+    else
+	return DIV_ONE_UN8 (as * ad - 2 * (ad - d) * (as - s));
+}
+
+PDF_SEPARABLE_BLEND_MODE (hard_light)
+
+/*
+ * Soft light
+ *
+ *     ad * as * B(d/ad, s/as)
+ *   = if (s/as <= 0.5)
+ *         ad * as * (d/ad - (1 - 2 * s/as) * d/ad * (1 - d/ad))
+ *     else if (d/ad <= 0.25)
+ *         ad * as * (d/ad + (2 * s/as - 1) * ((((16 * d/ad - 12) * d/ad + 4) * d/ad) - d/ad))
+ *     else
+ *         ad * as * (d/ad + (2 * s/as - 1) * sqrt (d/ad))
+ *   = if (2 * s <= as)
+ *         d * as - d * (ad - d) * (as - 2 * s) / ad;
+ *     else if (4 * d <= ad)
+ *         (2 * s - as) * d * ((16 * d / ad - 12) * d / ad + 3);
+ *     else
+ *         d * as + (sqrt (d * ad) - d) * (2 * s - as);
+ */
+static inline uint32_t
+blend_soft_light (uint32_t d_org,
+		  uint32_t ad_org,
+		  uint32_t s_org,
+		  uint32_t as_org)
+{
+    double d = d_org * (1.0 / MASK);
+    double ad = ad_org * (1.0 / MASK);
+    double s = s_org * (1.0 / MASK);
+    double as = as_org * (1.0 / MASK);
+    double r;
+
+    if (2 * s < as)
+    {
+	if (ad == 0)
+	    r = d * as;
+	else
+	    r = d * as - d * (ad - d) * (as - 2 * s) / ad;
+    }
+    else if (ad == 0)
+    {
+	r = 0;
+    }
+    else if (4 * d <= ad)
+    {
+	r = d * as +
+	    (2 * s - as) * d * ((16 * d / ad - 12) * d / ad + 3);
+    }
+    else
+    {
+	r = d * as + (sqrt (d * ad) - d) * (2 * s - as);
+    }
+    return r * MASK + 0.5;
+}
+
+PDF_SEPARABLE_BLEND_MODE (soft_light)
+
+/*
+ * Difference
+ *
+ *     ad * as * B(s/as, d/ad)
+ *   = ad * as * abs (s/as - d/ad)
+ *   = if (s/as <= d/ad)
+ *         ad * as * (d/ad - s/as)
+ *     else
+ *         ad * as * (s/as - d/ad)
+ *   = if (ad * s <= as * d)
+ *        as * d - ad * s
+ *     else
+ *        ad * s - as * d
+ */
+static inline uint32_t
+blend_difference (uint32_t d, uint32_t ad, uint32_t s, uint32_t as)
+{
+    uint32_t das = d * as;
+    uint32_t sad = s * ad;
+
+    if (sad < das)
+	return DIV_ONE_UN8 (das - sad);
+    else
+	return DIV_ONE_UN8 (sad - das);
+}
+
+PDF_SEPARABLE_BLEND_MODE (difference)
+
+/*
+ * Exclusion
+ *
+ *     ad * as * B(s/as, d/ad)
+ *   = ad * as * (d/ad + s/as - 2 * d/ad * s/as)
+ *   = as * d + ad * s - 2 * s * d
+ */
+
+/* This can be made faster by writing it directly and not using
+ * PDF_SEPARABLE_BLEND_MODE, but that's a performance optimization */
+
+static inline uint32_t
+blend_exclusion (uint32_t d, uint32_t ad, uint32_t s, uint32_t as)
+{
+    return DIV_ONE_UN8 (s * ad + d * as - 2 * d * s);
+}
+
+PDF_SEPARABLE_BLEND_MODE (exclusion)
+
+#undef PDF_SEPARABLE_BLEND_MODE
+
+/*
+ * PDF nonseperable blend modes are implemented using the following functions
+ * to operate in Hsl space, with Cmax, Cmid, Cmin referring to the max, mid
+ * and min value of the red, green and blue components.
+ *
+ * LUM (C) = 0.3 × Cred + 0.59 × Cgreen + 0.11 × Cblue
+ *
+ * clip_color (C):
+ *     l = LUM (C)
+ *     min = Cmin
+ *     max = Cmax
+ *     if n < 0.0
+ *         C = l + (((C – l) × l) ⁄ (l – min))
+ *     if x > 1.0
+ *         C = l + (((C – l) × (1 – l) ) ⁄ (max – l))
+ *     return C
+ *
+ * set_lum (C, l):
+ *     d = l – LUM (C)
+ *     C += d
+ *     return clip_color (C)
+ *
+ * SAT (C) = CH_MAX (C) - CH_MIN (C)
+ *
+ * set_sat (C, s):
+ *     if Cmax > Cmin
+ *         Cmid = ( ( ( Cmid – Cmin ) × s ) ⁄ ( Cmax – Cmin ) )
+ *         Cmax = s
+ *     else
+ *         Cmid = Cmax = 0.0
+ *         Cmin = 0.0
+ *     return C
+ */
+
+/* For premultiplied colors, we need to know what happens when C is
+ * multiplied by a real number. LUM and SAT are linear:
+ *
+ *     LUM (r × C) = r × LUM (C)	SAT (r * C) = r * SAT (C)
+ *
+ * If we extend clip_color with an extra argument a and change
+ *
+ *     if x >= 1.0
+ *
+ * into
+ *
+ *     if x >= a
+ *
+ * then clip_color is also linear:
+ *
+ *     r * clip_color (C, a) = clip_color (r * C, r * a);
+ *
+ * for positive r.
+ *
+ * Similarly, we can extend set_lum with an extra argument that is just passed
+ * on to clip_color:
+ *
+ *       r * set_lum (C, l, a)
+ *
+ *     = r × clip_color (C + l - LUM (C), a)
+ *
+ *     = clip_color (r * C + r × l - r * LUM (C), r * a)
+ *
+ *     = set_lum (r * C, r * l, r * a)
+ *
+ * Finally, set_sat:
+ *
+ *       r * set_sat (C, s) = set_sat (x * C, r * s)
+ *
+ * The above holds for all non-zero x, because the x'es in the fraction for
+ * C_mid cancel out. Specifically, it holds for x = r:
+ *
+ *       r * set_sat (C, s) = set_sat (r * C, r * s)
+ *
+ */
+
+#define CH_MIN(c) (c[0] < c[1] ? (c[0] < c[2] ? c[0] : c[2]) : (c[1] < c[2] ? c[1] : c[2]))
+#define CH_MAX(c) (c[0] > c[1] ? (c[0] > c[2] ? c[0] : c[2]) : (c[1] > c[2] ? c[1] : c[2]))
+#define LUM(c) ((c[0] * 30 + c[1] * 59 + c[2] * 11) / 100)
+#define SAT(c) (CH_MAX (c) - CH_MIN (c))
+
+#define PDF_NON_SEPARABLE_BLEND_MODE(name)				\
+    static void								\
+    combine_ ## name ## _u (pixman_implementation_t *imp,		\
+			    pixman_op_t		     op,		\
+                            uint32_t *               dest,		\
+			    const uint32_t *         src,		\
+			    const uint32_t *         mask,		\
+			    int                      width)		\
+    {									\
+	int i;								\
+	for (i = 0; i < width; ++i)					\
+	{								\
+	    uint32_t s = combine_mask (src, mask, i);			\
+	    uint32_t d = *(dest + i);					\
+	    uint8_t sa = ALPHA_8 (s);					\
+	    uint8_t isa = ~sa;						\
+	    uint8_t da = ALPHA_8 (d);					\
+	    uint8_t ida = ~da;						\
+	    uint32_t result;						\
+	    uint32_t sc[3], dc[3], c[3];				\
+            								\
+	    result = d;							\
+	    UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8 (result, isa, s, ida);	\
+	    dc[0] = RED_8 (d);						\
+	    sc[0] = RED_8 (s);						\
+	    dc[1] = GREEN_8 (d);					\
+	    sc[1] = GREEN_8 (s);					\
+	    dc[2] = BLUE_8 (d);						\
+	    sc[2] = BLUE_8 (s);						\
+	    blend_ ## name (c, dc, da, sc, sa);				\
+            								\
+	    *(dest + i) = result +					\
+		(DIV_ONE_UN8 (sa * (uint32_t)da) << A_SHIFT) +		\
+		(DIV_ONE_UN8 (c[0]) << R_SHIFT) +			\
+		(DIV_ONE_UN8 (c[1]) << G_SHIFT) +			\
+		(DIV_ONE_UN8 (c[2]));					\
+	}								\
+    }
+
+static void
+set_lum (uint32_t dest[3], uint32_t src[3], uint32_t sa, uint32_t lum)
+{
+    double a, l, min, max;
+    double tmp[3];
+
+    a = sa * (1.0 / MASK);
+
+    l = lum * (1.0 / MASK);
+    tmp[0] = src[0] * (1.0 / MASK);
+    tmp[1] = src[1] * (1.0 / MASK);
+    tmp[2] = src[2] * (1.0 / MASK);
+
+    l = l - LUM (tmp);
+    tmp[0] += l;
+    tmp[1] += l;
+    tmp[2] += l;
+
+    /* clip_color */
+    l = LUM (tmp);
+    min = CH_MIN (tmp);
+    max = CH_MAX (tmp);
+
+    if (min < 0)
+    {
+	if (l - min == 0.0)
+	{
+	    tmp[0] = 0;
+	    tmp[1] = 0;
+	    tmp[2] = 0;
+	}
+	else
+	{
+	    tmp[0] = l + (tmp[0] - l) * l / (l - min);
+	    tmp[1] = l + (tmp[1] - l) * l / (l - min);
+	    tmp[2] = l + (tmp[2] - l) * l / (l - min);
+	}
+    }
+    if (max > a)
+    {
+	if (max - l == 0.0)
+	{
+	    tmp[0] = a;
+	    tmp[1] = a;
+	    tmp[2] = a;
+	}
+	else
+	{
+	    tmp[0] = l + (tmp[0] - l) * (a - l) / (max - l);
+	    tmp[1] = l + (tmp[1] - l) * (a - l) / (max - l);
+	    tmp[2] = l + (tmp[2] - l) * (a - l) / (max - l);
+	}
+    }
+
+    dest[0] = tmp[0] * MASK + 0.5;
+    dest[1] = tmp[1] * MASK + 0.5;
+    dest[2] = tmp[2] * MASK + 0.5;
+}
+
+static void
+set_sat (uint32_t dest[3], uint32_t src[3], uint32_t sat)
+{
+    int id[3];
+    uint32_t min, max;
+
+    if (src[0] > src[1])
+    {
+	if (src[0] > src[2])
+	{
+	    id[0] = 0;
+	    if (src[1] > src[2])
+	    {
+		id[1] = 1;
+		id[2] = 2;
+	    }
+	    else
+	    {
+		id[1] = 2;
+		id[2] = 1;
+	    }
+	}
+	else
+	{
+	    id[0] = 2;
+	    id[1] = 0;
+	    id[2] = 1;
+	}
+    }
+    else
+    {
+	if (src[0] > src[2])
+	{
+	    id[0] = 1;
+	    id[1] = 0;
+	    id[2] = 2;
+	}
+	else
+	{
+	    id[2] = 0;
+	    if (src[1] > src[2])
+	    {
+		id[0] = 1;
+		id[1] = 2;
+	    }
+	    else
+	    {
+		id[0] = 2;
+		id[1] = 1;
+	    }
+	}
+    }
+
+    max = dest[id[0]];
+    min = dest[id[2]];
+    if (max > min)
+    {
+	dest[id[1]] = (dest[id[1]] - min) * sat / (max - min);
+	dest[id[0]] = sat;
+	dest[id[2]] = 0;
+    }
+    else
+    {
+	dest[0] = dest[1] = dest[2] = 0;
+    }
+}
+
+/* Hue:
+ *
+ *       as * ad * B(s/as, d/as)
+ *     = as * ad * set_lum (set_sat (s/as, SAT (d/ad)), LUM (d/ad), 1)
+ *     = set_lum (set_sat (ad * s, as * SAT (d)), as * LUM (d), as * ad)
+ *
+ */
+static inline void
+blend_hsl_hue (uint32_t r[3],
+               uint32_t d[3],
+               uint32_t ad,
+               uint32_t s[3],
+               uint32_t as)
+{
+    r[0] = s[0] * ad;
+    r[1] = s[1] * ad;
+    r[2] = s[2] * ad;
+    set_sat (r, r, SAT (d) * as);
+    set_lum (r, r, as * ad, LUM (d) * as);
+}
+
+PDF_NON_SEPARABLE_BLEND_MODE (hsl_hue)
+
+/* 
+ * Saturation
+ *
+ *     as * ad * B(s/as, d/ad)
+ *   = as * ad * set_lum (set_sat (d/ad, SAT (s/as)), LUM (d/ad), 1)
+ *   = set_lum (as * ad * set_sat (d/ad, SAT (s/as)),
+ *                                       as * LUM (d), as * ad)
+ *   = set_lum (set_sat (as * d, ad * SAT (s), as * LUM (d), as * ad))
+ */
+static inline void
+blend_hsl_saturation (uint32_t r[3],
+                      uint32_t d[3],
+                      uint32_t ad,
+                      uint32_t s[3],
+                      uint32_t as)
+{
+    r[0] = d[0] * as;
+    r[1] = d[1] * as;
+    r[2] = d[2] * as;
+    set_sat (r, r, SAT (s) * ad);
+    set_lum (r, r, as * ad, LUM (d) * as);
+}
+
+PDF_NON_SEPARABLE_BLEND_MODE (hsl_saturation)
+
+/* 
+ * Color
+ *
+ *     as * ad * B(s/as, d/as)
+ *   = as * ad * set_lum (s/as, LUM (d/ad), 1)
+ *   = set_lum (s * ad, as * LUM (d), as * ad)
+ */
+static inline void
+blend_hsl_color (uint32_t r[3],
+                 uint32_t d[3],
+                 uint32_t ad,
+                 uint32_t s[3],
+                 uint32_t as)
+{
+    r[0] = s[0] * ad;
+    r[1] = s[1] * ad;
+    r[2] = s[2] * ad;
+    set_lum (r, r, as * ad, LUM (d) * as);
+}
+
+PDF_NON_SEPARABLE_BLEND_MODE (hsl_color)
+
+/*
+ * Luminosity
+ *
+ *     as * ad * B(s/as, d/ad)
+ *   = as * ad * set_lum (d/ad, LUM (s/as), 1)
+ *   = set_lum (as * d, ad * LUM (s), as * ad)
+ */
+static inline void
+blend_hsl_luminosity (uint32_t r[3],
+                      uint32_t d[3],
+                      uint32_t ad,
+                      uint32_t s[3],
+                      uint32_t as)
+{
+    r[0] = d[0] * as;
+    r[1] = d[1] * as;
+    r[2] = d[2] * as;
+    set_lum (r, r, as * ad, LUM (s) * ad);
+}
+
+PDF_NON_SEPARABLE_BLEND_MODE (hsl_luminosity)
+
+#undef SAT
+#undef LUM
+#undef CH_MAX
+#undef CH_MIN
+#undef PDF_NON_SEPARABLE_BLEND_MODE
+
+/* All of the disjoint/conjoint composing functions
+ *
+ * The four entries in the first column indicate what source contributions
+ * come from each of the four areas of the picture -- areas covered by neither
+ * A nor B, areas covered only by A, areas covered only by B and finally
+ * areas covered by both A and B.
+ *
+ * Disjoint			Conjoint
+ * Fa		Fb		Fa		Fb
+ * (0,0,0,0)	0		0		0		0
+ * (0,A,0,A)	1		0		1		0
+ * (0,0,B,B)	0		1		0		1
+ * (0,A,B,A)	1		min((1-a)/b,1)	1		max(1-a/b,0)
+ * (0,A,B,B)	min((1-b)/a,1)	1		max(1-b/a,0)	1
+ * (0,0,0,A)	max(1-(1-b)/a,0) 0		min(1,b/a)	0
+ * (0,0,0,B)	0		max(1-(1-a)/b,0) 0		min(a/b,1)
+ * (0,A,0,0)	min(1,(1-b)/a)	0		max(1-b/a,0)	0
+ * (0,0,B,0)	0		min(1,(1-a)/b)	0		max(1-a/b,0)
+ * (0,0,B,A)	max(1-(1-b)/a,0) min(1,(1-a)/b)	 min(1,b/a)	max(1-a/b,0)
+ * (0,A,0,B)	min(1,(1-b)/a)	max(1-(1-a)/b,0) max(1-b/a,0)	min(1,a/b)
+ * (0,A,B,0)	min(1,(1-b)/a)	min(1,(1-a)/b)	max(1-b/a,0)	max(1-a/b,0)
+ *
+ * See  http://marc.info/?l=xfree-render&m=99792000027857&w=2  for more
+ * information about these operators.
+ */
+
+#define COMBINE_A_OUT 1
+#define COMBINE_A_IN  2
+#define COMBINE_B_OUT 4
+#define COMBINE_B_IN  8
+
+#define COMBINE_CLEAR   0
+#define COMBINE_A       (COMBINE_A_OUT | COMBINE_A_IN)
+#define COMBINE_B       (COMBINE_B_OUT | COMBINE_B_IN)
+#define COMBINE_A_OVER  (COMBINE_A_OUT | COMBINE_B_OUT | COMBINE_A_IN)
+#define COMBINE_B_OVER  (COMBINE_A_OUT | COMBINE_B_OUT | COMBINE_B_IN)
+#define COMBINE_A_ATOP  (COMBINE_B_OUT | COMBINE_A_IN)
+#define COMBINE_B_ATOP  (COMBINE_A_OUT | COMBINE_B_IN)
+#define COMBINE_XOR     (COMBINE_A_OUT | COMBINE_B_OUT)
+
+/* portion covered by a but not b */
+static uint8_t
+combine_disjoint_out_part (uint8_t a, uint8_t b)
+{
+    /* min (1, (1-b) / a) */
+
+    b = ~b;                 /* 1 - b */
+    if (b >= a)             /* 1 - b >= a -> (1-b)/a >= 1 */
+	return MASK;        /* 1 */
+    return DIV_UN8 (b, a);     /* (1-b) / a */
+}
+
+/* portion covered by both a and b */
+static uint8_t
+combine_disjoint_in_part (uint8_t a, uint8_t b)
+{
+    /* max (1-(1-b)/a,0) */
+    /*  = - min ((1-b)/a - 1, 0) */
+    /*  = 1 - min (1, (1-b)/a) */
+
+    b = ~b;                 /* 1 - b */
+    if (b >= a)             /* 1 - b >= a -> (1-b)/a >= 1 */
+	return 0;           /* 1 - 1 */
+    return ~DIV_UN8(b, a);    /* 1 - (1-b) / a */
+}
+
+/* portion covered by a but not b */
+static uint8_t
+combine_conjoint_out_part (uint8_t a, uint8_t b)
+{
+    /* max (1-b/a,0) */
+    /* = 1-min(b/a,1) */
+
+    /* min (1, (1-b) / a) */
+
+    if (b >= a)             /* b >= a -> b/a >= 1 */
+	return 0x00;        /* 0 */
+    return ~DIV_UN8(b, a);    /* 1 - b/a */
+}
+
+/* portion covered by both a and b */
+static uint8_t
+combine_conjoint_in_part (uint8_t a, uint8_t b)
+{
+    /* min (1,b/a) */
+
+    if (b >= a)             /* b >= a -> b/a >= 1 */
+	return MASK;        /* 1 */
+    return DIV_UN8 (b, a);     /* b/a */
+}
+
+#define GET_COMP(v, i)   ((uint16_t) (uint8_t) ((v) >> i))
+
+#define ADD(x, y, i, t)							\
+    ((t) = GET_COMP (x, i) + GET_COMP (y, i),				\
+     (uint32_t) ((uint8_t) ((t) | (0 - ((t) >> G_SHIFT)))) << (i))
+
+#define GENERIC(x, y, i, ax, ay, t, u, v)				\
+    ((t) = (MUL_UN8 (GET_COMP (y, i), ay, (u)) +			\
+            MUL_UN8 (GET_COMP (x, i), ax, (v))),			\
+     (uint32_t) ((uint8_t) ((t) |					\
+                           (0 - ((t) >> G_SHIFT)))) << (i))
+
+static void
+combine_disjoint_general_u (uint32_t *      dest,
+                            const uint32_t *src,
+                            const uint32_t *mask,
+                            int            width,
+                            uint8_t        combine)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = combine_mask (src, mask, i);
+	uint32_t d = *(dest + i);
+	uint32_t m, n, o, p;
+	uint16_t Fa, Fb, t, u, v;
+	uint8_t sa = s >> A_SHIFT;
+	uint8_t da = d >> A_SHIFT;
+
+	switch (combine & COMBINE_A)
+	{
+	default:
+	    Fa = 0;
+	    break;
+
+	case COMBINE_A_OUT:
+	    Fa = combine_disjoint_out_part (sa, da);
+	    break;
+
+	case COMBINE_A_IN:
+	    Fa = combine_disjoint_in_part (sa, da);
+	    break;
+
+	case COMBINE_A:
+	    Fa = MASK;
+	    break;
+	}
+
+	switch (combine & COMBINE_B)
+	{
+	default:
+	    Fb = 0;
+	    break;
+
+	case COMBINE_B_OUT:
+	    Fb = combine_disjoint_out_part (da, sa);
+	    break;
+
+	case COMBINE_B_IN:
+	    Fb = combine_disjoint_in_part (da, sa);
+	    break;
+
+	case COMBINE_B:
+	    Fb = MASK;
+	    break;
+	}
+	m = GENERIC (s, d, 0, Fa, Fb, t, u, v);
+	n = GENERIC (s, d, G_SHIFT, Fa, Fb, t, u, v);
+	o = GENERIC (s, d, R_SHIFT, Fa, Fb, t, u, v);
+	p = GENERIC (s, d, A_SHIFT, Fa, Fb, t, u, v);
+	s = m | n | o | p;
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_disjoint_over_u (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         uint32_t *                dest,
+                         const uint32_t *          src,
+                         const uint32_t *          mask,
+                         int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = combine_mask (src, mask, i);
+	uint16_t a = s >> A_SHIFT;
+
+	if (s != 0x00)
+	{
+	    uint32_t d = *(dest + i);
+	    a = combine_disjoint_out_part (d >> A_SHIFT, a);
+	    UN8x4_MUL_UN8_ADD_UN8x4 (d, a, s);
+
+	    *(dest + i) = d;
+	}
+    }
+}
+
+static void
+combine_disjoint_in_u (pixman_implementation_t *imp,
+                       pixman_op_t              op,
+                       uint32_t *                dest,
+                       const uint32_t *          src,
+                       const uint32_t *          mask,
+                       int                      width)
+{
+    combine_disjoint_general_u (dest, src, mask, width, COMBINE_A_IN);
+}
+
+static void
+combine_disjoint_in_reverse_u (pixman_implementation_t *imp,
+                               pixman_op_t              op,
+                               uint32_t *                dest,
+                               const uint32_t *          src,
+                               const uint32_t *          mask,
+                               int                      width)
+{
+    combine_disjoint_general_u (dest, src, mask, width, COMBINE_B_IN);
+}
+
+static void
+combine_disjoint_out_u (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        uint32_t *                dest,
+                        const uint32_t *          src,
+                        const uint32_t *          mask,
+                        int                      width)
+{
+    combine_disjoint_general_u (dest, src, mask, width, COMBINE_A_OUT);
+}
+
+static void
+combine_disjoint_out_reverse_u (pixman_implementation_t *imp,
+                                pixman_op_t              op,
+                                uint32_t *                dest,
+                                const uint32_t *          src,
+                                const uint32_t *          mask,
+                                int                      width)
+{
+    combine_disjoint_general_u (dest, src, mask, width, COMBINE_B_OUT);
+}
+
+static void
+combine_disjoint_atop_u (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         uint32_t *                dest,
+                         const uint32_t *          src,
+                         const uint32_t *          mask,
+                         int                      width)
+{
+    combine_disjoint_general_u (dest, src, mask, width, COMBINE_A_ATOP);
+}
+
+static void
+combine_disjoint_atop_reverse_u (pixman_implementation_t *imp,
+                                 pixman_op_t              op,
+                                 uint32_t *                dest,
+                                 const uint32_t *          src,
+                                 const uint32_t *          mask,
+                                 int                      width)
+{
+    combine_disjoint_general_u (dest, src, mask, width, COMBINE_B_ATOP);
+}
+
+static void
+combine_disjoint_xor_u (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        uint32_t *                dest,
+                        const uint32_t *          src,
+                        const uint32_t *          mask,
+                        int                      width)
+{
+    combine_disjoint_general_u (dest, src, mask, width, COMBINE_XOR);
+}
+
+static void
+combine_conjoint_general_u (uint32_t *      dest,
+                            const uint32_t *src,
+                            const uint32_t *mask,
+                            int            width,
+                            uint8_t        combine)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = combine_mask (src, mask, i);
+	uint32_t d = *(dest + i);
+	uint32_t m, n, o, p;
+	uint16_t Fa, Fb, t, u, v;
+	uint8_t sa = s >> A_SHIFT;
+	uint8_t da = d >> A_SHIFT;
+
+	switch (combine & COMBINE_A)
+	{
+	default:
+	    Fa = 0;
+	    break;
+
+	case COMBINE_A_OUT:
+	    Fa = combine_conjoint_out_part (sa, da);
+	    break;
+
+	case COMBINE_A_IN:
+	    Fa = combine_conjoint_in_part (sa, da);
+	    break;
+
+	case COMBINE_A:
+	    Fa = MASK;
+	    break;
+	}
+
+	switch (combine & COMBINE_B)
+	{
+	default:
+	    Fb = 0;
+	    break;
+
+	case COMBINE_B_OUT:
+	    Fb = combine_conjoint_out_part (da, sa);
+	    break;
+
+	case COMBINE_B_IN:
+	    Fb = combine_conjoint_in_part (da, sa);
+	    break;
+
+	case COMBINE_B:
+	    Fb = MASK;
+	    break;
+	}
+
+	m = GENERIC (s, d, 0, Fa, Fb, t, u, v);
+	n = GENERIC (s, d, G_SHIFT, Fa, Fb, t, u, v);
+	o = GENERIC (s, d, R_SHIFT, Fa, Fb, t, u, v);
+	p = GENERIC (s, d, A_SHIFT, Fa, Fb, t, u, v);
+
+	s = m | n | o | p;
+
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_conjoint_over_u (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         uint32_t *                dest,
+                         const uint32_t *          src,
+                         const uint32_t *          mask,
+                         int                      width)
+{
+    combine_conjoint_general_u (dest, src, mask, width, COMBINE_A_OVER);
+}
+
+static void
+combine_conjoint_over_reverse_u (pixman_implementation_t *imp,
+                                 pixman_op_t              op,
+                                 uint32_t *                dest,
+                                 const uint32_t *          src,
+                                 const uint32_t *          mask,
+                                 int                      width)
+{
+    combine_conjoint_general_u (dest, src, mask, width, COMBINE_B_OVER);
+}
+
+static void
+combine_conjoint_in_u (pixman_implementation_t *imp,
+                       pixman_op_t              op,
+                       uint32_t *                dest,
+                       const uint32_t *          src,
+                       const uint32_t *          mask,
+                       int                      width)
+{
+    combine_conjoint_general_u (dest, src, mask, width, COMBINE_A_IN);
+}
+
+static void
+combine_conjoint_in_reverse_u (pixman_implementation_t *imp,
+                               pixman_op_t              op,
+                               uint32_t *                dest,
+                               const uint32_t *          src,
+                               const uint32_t *          mask,
+                               int                      width)
+{
+    combine_conjoint_general_u (dest, src, mask, width, COMBINE_B_IN);
+}
+
+static void
+combine_conjoint_out_u (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        uint32_t *                dest,
+                        const uint32_t *          src,
+                        const uint32_t *          mask,
+                        int                      width)
+{
+    combine_conjoint_general_u (dest, src, mask, width, COMBINE_A_OUT);
+}
+
+static void
+combine_conjoint_out_reverse_u (pixman_implementation_t *imp,
+                                pixman_op_t              op,
+                                uint32_t *                dest,
+                                const uint32_t *          src,
+                                const uint32_t *          mask,
+                                int                      width)
+{
+    combine_conjoint_general_u (dest, src, mask, width, COMBINE_B_OUT);
+}
+
+static void
+combine_conjoint_atop_u (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         uint32_t *                dest,
+                         const uint32_t *          src,
+                         const uint32_t *          mask,
+                         int                      width)
+{
+    combine_conjoint_general_u (dest, src, mask, width, COMBINE_A_ATOP);
+}
+
+static void
+combine_conjoint_atop_reverse_u (pixman_implementation_t *imp,
+                                 pixman_op_t              op,
+                                 uint32_t *                dest,
+                                 const uint32_t *          src,
+                                 const uint32_t *          mask,
+                                 int                      width)
+{
+    combine_conjoint_general_u (dest, src, mask, width, COMBINE_B_ATOP);
+}
+
+static void
+combine_conjoint_xor_u (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        uint32_t *                dest,
+                        const uint32_t *          src,
+                        const uint32_t *          mask,
+                        int                      width)
+{
+    combine_conjoint_general_u (dest, src, mask, width, COMBINE_XOR);
+}
+
+
+/* Component alpha combiners */
+
+static void
+combine_clear_ca (pixman_implementation_t *imp,
+                  pixman_op_t              op,
+                  uint32_t *                dest,
+                  const uint32_t *          src,
+                  const uint32_t *          mask,
+                  int                      width)
+{
+    memset (dest, 0, width * sizeof(uint32_t));
+}
+
+static void
+combine_src_ca (pixman_implementation_t *imp,
+                pixman_op_t              op,
+                uint32_t *                dest,
+                const uint32_t *          src,
+                const uint32_t *          mask,
+                int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = *(src + i);
+	uint32_t m = *(mask + i);
+
+	combine_mask_value_ca (&s, &m);
+
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_over_ca (pixman_implementation_t *imp,
+                 pixman_op_t              op,
+                 uint32_t *                dest,
+                 const uint32_t *          src,
+                 const uint32_t *          mask,
+                 int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = *(src + i);
+	uint32_t m = *(mask + i);
+	uint32_t a;
+
+	combine_mask_ca (&s, &m);
+
+	a = ~m;
+	if (a)
+	{
+	    uint32_t d = *(dest + i);
+	    UN8x4_MUL_UN8x4_ADD_UN8x4 (d, a, s);
+	    s = d;
+	}
+
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_over_reverse_ca (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         uint32_t *                dest,
+                         const uint32_t *          src,
+                         const uint32_t *          mask,
+                         int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t d = *(dest + i);
+	uint32_t a = ~d >> A_SHIFT;
+
+	if (a)
+	{
+	    uint32_t s = *(src + i);
+	    uint32_t m = *(mask + i);
+
+	    UN8x4_MUL_UN8x4 (s, m);
+	    UN8x4_MUL_UN8_ADD_UN8x4 (s, a, d);
+
+	    *(dest + i) = s;
+	}
+    }
+}
+
+static void
+combine_in_ca (pixman_implementation_t *imp,
+               pixman_op_t              op,
+               uint32_t *                dest,
+               const uint32_t *          src,
+               const uint32_t *          mask,
+               int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t d = *(dest + i);
+	uint16_t a = d >> A_SHIFT;
+	uint32_t s = 0;
+
+	if (a)
+	{
+	    uint32_t m = *(mask + i);
+
+	    s = *(src + i);
+	    combine_mask_value_ca (&s, &m);
+
+	    if (a != MASK)
+		UN8x4_MUL_UN8 (s, a);
+	}
+
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_in_reverse_ca (pixman_implementation_t *imp,
+                       pixman_op_t              op,
+                       uint32_t *                dest,
+                       const uint32_t *          src,
+                       const uint32_t *          mask,
+                       int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = *(src + i);
+	uint32_t m = *(mask + i);
+	uint32_t a;
+
+	combine_mask_alpha_ca (&s, &m);
+
+	a = m;
+	if (a != ~0)
+	{
+	    uint32_t d = 0;
+
+	    if (a)
+	    {
+		d = *(dest + i);
+		UN8x4_MUL_UN8x4 (d, a);
+	    }
+
+	    *(dest + i) = d;
+	}
+    }
+}
+
+static void
+combine_out_ca (pixman_implementation_t *imp,
+                pixman_op_t              op,
+                uint32_t *                dest,
+                const uint32_t *          src,
+                const uint32_t *          mask,
+                int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t d = *(dest + i);
+	uint16_t a = ~d >> A_SHIFT;
+	uint32_t s = 0;
+
+	if (a)
+	{
+	    uint32_t m = *(mask + i);
+
+	    s = *(src + i);
+	    combine_mask_value_ca (&s, &m);
+
+	    if (a != MASK)
+		UN8x4_MUL_UN8 (s, a);
+	}
+
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_out_reverse_ca (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        uint32_t *                dest,
+                        const uint32_t *          src,
+                        const uint32_t *          mask,
+                        int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = *(src + i);
+	uint32_t m = *(mask + i);
+	uint32_t a;
+
+	combine_mask_alpha_ca (&s, &m);
+
+	a = ~m;
+	if (a != ~0)
+	{
+	    uint32_t d = 0;
+
+	    if (a)
+	    {
+		d = *(dest + i);
+		UN8x4_MUL_UN8x4 (d, a);
+	    }
+
+	    *(dest + i) = d;
+	}
+    }
+}
+
+static void
+combine_atop_ca (pixman_implementation_t *imp,
+                 pixman_op_t              op,
+                 uint32_t *                dest,
+                 const uint32_t *          src,
+                 const uint32_t *          mask,
+                 int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t d = *(dest + i);
+	uint32_t s = *(src + i);
+	uint32_t m = *(mask + i);
+	uint32_t ad;
+	uint16_t as = d >> A_SHIFT;
+
+	combine_mask_ca (&s, &m);
+
+	ad = ~m;
+
+	UN8x4_MUL_UN8x4_ADD_UN8x4_MUL_UN8 (d, ad, s, as);
+
+	*(dest + i) = d;
+    }
+}
+
+static void
+combine_atop_reverse_ca (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         uint32_t *                dest,
+                         const uint32_t *          src,
+                         const uint32_t *          mask,
+                         int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t d = *(dest + i);
+	uint32_t s = *(src + i);
+	uint32_t m = *(mask + i);
+	uint32_t ad;
+	uint16_t as = ~d >> A_SHIFT;
+
+	combine_mask_ca (&s, &m);
+
+	ad = m;
+
+	UN8x4_MUL_UN8x4_ADD_UN8x4_MUL_UN8 (d, ad, s, as);
+
+	*(dest + i) = d;
+    }
+}
+
+static void
+combine_xor_ca (pixman_implementation_t *imp,
+                pixman_op_t              op,
+                uint32_t *                dest,
+                const uint32_t *          src,
+                const uint32_t *          mask,
+                int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t d = *(dest + i);
+	uint32_t s = *(src + i);
+	uint32_t m = *(mask + i);
+	uint32_t ad;
+	uint16_t as = ~d >> A_SHIFT;
+
+	combine_mask_ca (&s, &m);
+
+	ad = ~m;
+
+	UN8x4_MUL_UN8x4_ADD_UN8x4_MUL_UN8 (d, ad, s, as);
+
+	*(dest + i) = d;
+    }
+}
+
+static void
+combine_add_ca (pixman_implementation_t *imp,
+                pixman_op_t              op,
+                uint32_t *                dest,
+                const uint32_t *          src,
+                const uint32_t *          mask,
+                int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = *(src + i);
+	uint32_t m = *(mask + i);
+	uint32_t d = *(dest + i);
+
+	combine_mask_value_ca (&s, &m);
+
+	UN8x4_ADD_UN8x4 (d, s);
+
+	*(dest + i) = d;
+    }
+}
+
+static void
+combine_saturate_ca (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     uint32_t *                dest,
+                     const uint32_t *          src,
+                     const uint32_t *          mask,
+                     int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s, d;
+	uint16_t sa, sr, sg, sb, da;
+	uint16_t t, u, v;
+	uint32_t m, n, o, p;
+
+	d = *(dest + i);
+	s = *(src + i);
+	m = *(mask + i);
+
+	combine_mask_ca (&s, &m);
+
+	sa = (m >> A_SHIFT);
+	sr = (m >> R_SHIFT) & MASK;
+	sg = (m >> G_SHIFT) & MASK;
+	sb =  m             & MASK;
+	da = ~d >> A_SHIFT;
+
+	if (sb <= da)
+	    m = ADD (s, d, 0, t);
+	else
+	    m = GENERIC (s, d, 0, (da << G_SHIFT) / sb, MASK, t, u, v);
+
+	if (sg <= da)
+	    n = ADD (s, d, G_SHIFT, t);
+	else
+	    n = GENERIC (s, d, G_SHIFT, (da << G_SHIFT) / sg, MASK, t, u, v);
+
+	if (sr <= da)
+	    o = ADD (s, d, R_SHIFT, t);
+	else
+	    o = GENERIC (s, d, R_SHIFT, (da << G_SHIFT) / sr, MASK, t, u, v);
+
+	if (sa <= da)
+	    p = ADD (s, d, A_SHIFT, t);
+	else
+	    p = GENERIC (s, d, A_SHIFT, (da << G_SHIFT) / sa, MASK, t, u, v);
+
+	*(dest + i) = m | n | o | p;
+    }
+}
+
+static void
+combine_disjoint_general_ca (uint32_t *      dest,
+                             const uint32_t *src,
+                             const uint32_t *mask,
+                             int            width,
+                             uint8_t        combine)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s, d;
+	uint32_t m, n, o, p;
+	uint32_t Fa, Fb;
+	uint16_t t, u, v;
+	uint32_t sa;
+	uint8_t da;
+
+	s = *(src + i);
+	m = *(mask + i);
+	d = *(dest + i);
+	da = d >> A_SHIFT;
+
+	combine_mask_ca (&s, &m);
+
+	sa = m;
+
+	switch (combine & COMBINE_A)
+	{
+	default:
+	    Fa = 0;
+	    break;
+
+	case COMBINE_A_OUT:
+	    m = (uint32_t)combine_disjoint_out_part ((uint8_t) (sa >> 0), da);
+	    n = (uint32_t)combine_disjoint_out_part ((uint8_t) (sa >> G_SHIFT), da) << G_SHIFT;
+	    o = (uint32_t)combine_disjoint_out_part ((uint8_t) (sa >> R_SHIFT), da) << R_SHIFT;
+	    p = (uint32_t)combine_disjoint_out_part ((uint8_t) (sa >> A_SHIFT), da) << A_SHIFT;
+	    Fa = m | n | o | p;
+	    break;
+
+	case COMBINE_A_IN:
+	    m = (uint32_t)combine_disjoint_in_part ((uint8_t) (sa >> 0), da);
+	    n = (uint32_t)combine_disjoint_in_part ((uint8_t) (sa >> G_SHIFT), da) << G_SHIFT;
+	    o = (uint32_t)combine_disjoint_in_part ((uint8_t) (sa >> R_SHIFT), da) << R_SHIFT;
+	    p = (uint32_t)combine_disjoint_in_part ((uint8_t) (sa >> A_SHIFT), da) << A_SHIFT;
+	    Fa = m | n | o | p;
+	    break;
+
+	case COMBINE_A:
+	    Fa = ~0;
+	    break;
+	}
+
+	switch (combine & COMBINE_B)
+	{
+	default:
+	    Fb = 0;
+	    break;
+
+	case COMBINE_B_OUT:
+	    m = (uint32_t)combine_disjoint_out_part (da, (uint8_t) (sa >> 0));
+	    n = (uint32_t)combine_disjoint_out_part (da, (uint8_t) (sa >> G_SHIFT)) << G_SHIFT;
+	    o = (uint32_t)combine_disjoint_out_part (da, (uint8_t) (sa >> R_SHIFT)) << R_SHIFT;
+	    p = (uint32_t)combine_disjoint_out_part (da, (uint8_t) (sa >> A_SHIFT)) << A_SHIFT;
+	    Fb = m | n | o | p;
+	    break;
+
+	case COMBINE_B_IN:
+	    m = (uint32_t)combine_disjoint_in_part (da, (uint8_t) (sa >> 0));
+	    n = (uint32_t)combine_disjoint_in_part (da, (uint8_t) (sa >> G_SHIFT)) << G_SHIFT;
+	    o = (uint32_t)combine_disjoint_in_part (da, (uint8_t) (sa >> R_SHIFT)) << R_SHIFT;
+	    p = (uint32_t)combine_disjoint_in_part (da, (uint8_t) (sa >> A_SHIFT)) << A_SHIFT;
+	    Fb = m | n | o | p;
+	    break;
+
+	case COMBINE_B:
+	    Fb = ~0;
+	    break;
+	}
+	m = GENERIC (s, d, 0, GET_COMP (Fa, 0), GET_COMP (Fb, 0), t, u, v);
+	n = GENERIC (s, d, G_SHIFT, GET_COMP (Fa, G_SHIFT), GET_COMP (Fb, G_SHIFT), t, u, v);
+	o = GENERIC (s, d, R_SHIFT, GET_COMP (Fa, R_SHIFT), GET_COMP (Fb, R_SHIFT), t, u, v);
+	p = GENERIC (s, d, A_SHIFT, GET_COMP (Fa, A_SHIFT), GET_COMP (Fb, A_SHIFT), t, u, v);
+
+	s = m | n | o | p;
+
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_disjoint_over_ca (pixman_implementation_t *imp,
+                          pixman_op_t              op,
+                          uint32_t *                dest,
+                          const uint32_t *          src,
+                          const uint32_t *          mask,
+                          int                      width)
+{
+    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_A_OVER);
+}
+
+static void
+combine_disjoint_in_ca (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        uint32_t *                dest,
+                        const uint32_t *          src,
+                        const uint32_t *          mask,
+                        int                      width)
+{
+    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_A_IN);
+}
+
+static void
+combine_disjoint_in_reverse_ca (pixman_implementation_t *imp,
+                                pixman_op_t              op,
+                                uint32_t *                dest,
+                                const uint32_t *          src,
+                                const uint32_t *          mask,
+                                int                      width)
+{
+    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_B_IN);
+}
+
+static void
+combine_disjoint_out_ca (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         uint32_t *                dest,
+                         const uint32_t *          src,
+                         const uint32_t *          mask,
+                         int                      width)
+{
+    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_A_OUT);
+}
+
+static void
+combine_disjoint_out_reverse_ca (pixman_implementation_t *imp,
+                                 pixman_op_t              op,
+                                 uint32_t *                dest,
+                                 const uint32_t *          src,
+                                 const uint32_t *          mask,
+                                 int                      width)
+{
+    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_B_OUT);
+}
+
+static void
+combine_disjoint_atop_ca (pixman_implementation_t *imp,
+                          pixman_op_t              op,
+                          uint32_t *                dest,
+                          const uint32_t *          src,
+                          const uint32_t *          mask,
+                          int                      width)
+{
+    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_A_ATOP);
+}
+
+static void
+combine_disjoint_atop_reverse_ca (pixman_implementation_t *imp,
+                                  pixman_op_t              op,
+                                  uint32_t *                dest,
+                                  const uint32_t *          src,
+                                  const uint32_t *          mask,
+                                  int                      width)
+{
+    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_B_ATOP);
+}
+
+static void
+combine_disjoint_xor_ca (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         uint32_t *                dest,
+                         const uint32_t *          src,
+                         const uint32_t *          mask,
+                         int                      width)
+{
+    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_XOR);
+}
+
+static void
+combine_conjoint_general_ca (uint32_t *      dest,
+                             const uint32_t *src,
+                             const uint32_t *mask,
+                             int            width,
+                             uint8_t        combine)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s, d;
+	uint32_t m, n, o, p;
+	uint32_t Fa, Fb;
+	uint16_t t, u, v;
+	uint32_t sa;
+	uint8_t da;
+
+	s = *(src + i);
+	m = *(mask + i);
+	d = *(dest + i);
+	da = d >> A_SHIFT;
+
+	combine_mask_ca (&s, &m);
+
+	sa = m;
+
+	switch (combine & COMBINE_A)
+	{
+	default:
+	    Fa = 0;
+	    break;
+
+	case COMBINE_A_OUT:
+	    m = (uint32_t)combine_conjoint_out_part ((uint8_t) (sa >> 0), da);
+	    n = (uint32_t)combine_conjoint_out_part ((uint8_t) (sa >> G_SHIFT), da) << G_SHIFT;
+	    o = (uint32_t)combine_conjoint_out_part ((uint8_t) (sa >> R_SHIFT), da) << R_SHIFT;
+	    p = (uint32_t)combine_conjoint_out_part ((uint8_t) (sa >> A_SHIFT), da) << A_SHIFT;
+	    Fa = m | n | o | p;
+	    break;
+
+	case COMBINE_A_IN:
+	    m = (uint32_t)combine_conjoint_in_part ((uint8_t) (sa >> 0), da);
+	    n = (uint32_t)combine_conjoint_in_part ((uint8_t) (sa >> G_SHIFT), da) << G_SHIFT;
+	    o = (uint32_t)combine_conjoint_in_part ((uint8_t) (sa >> R_SHIFT), da) << R_SHIFT;
+	    p = (uint32_t)combine_conjoint_in_part ((uint8_t) (sa >> A_SHIFT), da) << A_SHIFT;
+	    Fa = m | n | o | p;
+	    break;
+
+	case COMBINE_A:
+	    Fa = ~0;
+	    break;
+	}
+
+	switch (combine & COMBINE_B)
+	{
+	default:
+	    Fb = 0;
+	    break;
+
+	case COMBINE_B_OUT:
+	    m = (uint32_t)combine_conjoint_out_part (da, (uint8_t) (sa >> 0));
+	    n = (uint32_t)combine_conjoint_out_part (da, (uint8_t) (sa >> G_SHIFT)) << G_SHIFT;
+	    o = (uint32_t)combine_conjoint_out_part (da, (uint8_t) (sa >> R_SHIFT)) << R_SHIFT;
+	    p = (uint32_t)combine_conjoint_out_part (da, (uint8_t) (sa >> A_SHIFT)) << A_SHIFT;
+	    Fb = m | n | o | p;
+	    break;
+
+	case COMBINE_B_IN:
+	    m = (uint32_t)combine_conjoint_in_part (da, (uint8_t) (sa >> 0));
+	    n = (uint32_t)combine_conjoint_in_part (da, (uint8_t) (sa >> G_SHIFT)) << G_SHIFT;
+	    o = (uint32_t)combine_conjoint_in_part (da, (uint8_t) (sa >> R_SHIFT)) << R_SHIFT;
+	    p = (uint32_t)combine_conjoint_in_part (da, (uint8_t) (sa >> A_SHIFT)) << A_SHIFT;
+	    Fb = m | n | o | p;
+	    break;
+
+	case COMBINE_B:
+	    Fb = ~0;
+	    break;
+	}
+	m = GENERIC (s, d, 0, GET_COMP (Fa, 0), GET_COMP (Fb, 0), t, u, v);
+	n = GENERIC (s, d, G_SHIFT, GET_COMP (Fa, G_SHIFT), GET_COMP (Fb, G_SHIFT), t, u, v);
+	o = GENERIC (s, d, R_SHIFT, GET_COMP (Fa, R_SHIFT), GET_COMP (Fb, R_SHIFT), t, u, v);
+	p = GENERIC (s, d, A_SHIFT, GET_COMP (Fa, A_SHIFT), GET_COMP (Fb, A_SHIFT), t, u, v);
+
+	s = m | n | o | p;
+
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_conjoint_over_ca (pixman_implementation_t *imp,
+                          pixman_op_t              op,
+                          uint32_t *                dest,
+                          const uint32_t *          src,
+                          const uint32_t *          mask,
+                          int                      width)
+{
+    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_A_OVER);
+}
+
+static void
+combine_conjoint_over_reverse_ca (pixman_implementation_t *imp,
+                                  pixman_op_t              op,
+                                  uint32_t *                dest,
+                                  const uint32_t *          src,
+                                  const uint32_t *          mask,
+                                  int                      width)
+{
+    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_B_OVER);
+}
+
+static void
+combine_conjoint_in_ca (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        uint32_t *                dest,
+                        const uint32_t *          src,
+                        const uint32_t *          mask,
+                        int                      width)
+{
+    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_A_IN);
+}
+
+static void
+combine_conjoint_in_reverse_ca (pixman_implementation_t *imp,
+                                pixman_op_t              op,
+                                uint32_t *                dest,
+                                const uint32_t *          src,
+                                const uint32_t *          mask,
+                                int                      width)
+{
+    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_B_IN);
+}
+
+static void
+combine_conjoint_out_ca (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         uint32_t *                dest,
+                         const uint32_t *          src,
+                         const uint32_t *          mask,
+                         int                      width)
+{
+    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_A_OUT);
+}
+
+static void
+combine_conjoint_out_reverse_ca (pixman_implementation_t *imp,
+                                 pixman_op_t              op,
+                                 uint32_t *                dest,
+                                 const uint32_t *          src,
+                                 const uint32_t *          mask,
+                                 int                      width)
+{
+    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_B_OUT);
+}
+
+static void
+combine_conjoint_atop_ca (pixman_implementation_t *imp,
+                          pixman_op_t              op,
+                          uint32_t *                dest,
+                          const uint32_t *          src,
+                          const uint32_t *          mask,
+                          int                      width)
+{
+    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_A_ATOP);
+}
+
+static void
+combine_conjoint_atop_reverse_ca (pixman_implementation_t *imp,
+                                  pixman_op_t              op,
+                                  uint32_t *                dest,
+                                  const uint32_t *          src,
+                                  const uint32_t *          mask,
+                                  int                      width)
+{
+    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_B_ATOP);
+}
+
+static void
+combine_conjoint_xor_ca (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         uint32_t *                dest,
+                         const uint32_t *          src,
+                         const uint32_t *          mask,
+                         int                      width)
+{
+    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_XOR);
+}
+
+void
+_pixman_setup_combiner_functions_32 (pixman_implementation_t *imp)
+{
+    /* Unified alpha */
+    imp->combine_32[PIXMAN_OP_CLEAR] = combine_clear;
+    imp->combine_32[PIXMAN_OP_SRC] = combine_src_u;
+    imp->combine_32[PIXMAN_OP_DST] = combine_dst;
+    imp->combine_32[PIXMAN_OP_OVER] = combine_over_u;
+    imp->combine_32[PIXMAN_OP_OVER_REVERSE] = combine_over_reverse_u;
+    imp->combine_32[PIXMAN_OP_IN] = combine_in_u;
+    imp->combine_32[PIXMAN_OP_IN_REVERSE] = combine_in_reverse_u;
+    imp->combine_32[PIXMAN_OP_OUT] = combine_out_u;
+    imp->combine_32[PIXMAN_OP_OUT_REVERSE] = combine_out_reverse_u;
+    imp->combine_32[PIXMAN_OP_ATOP] = combine_atop_u;
+    imp->combine_32[PIXMAN_OP_ATOP_REVERSE] = combine_atop_reverse_u;
+    imp->combine_32[PIXMAN_OP_XOR] = combine_xor_u;
+    imp->combine_32[PIXMAN_OP_ADD] = combine_add_u;
+    imp->combine_32[PIXMAN_OP_SATURATE] = combine_saturate_u;
+
+    /* Disjoint, unified */
+    imp->combine_32[PIXMAN_OP_DISJOINT_CLEAR] = combine_clear;
+    imp->combine_32[PIXMAN_OP_DISJOINT_SRC] = combine_src_u;
+    imp->combine_32[PIXMAN_OP_DISJOINT_DST] = combine_dst;
+    imp->combine_32[PIXMAN_OP_DISJOINT_OVER] = combine_disjoint_over_u;
+    imp->combine_32[PIXMAN_OP_DISJOINT_OVER_REVERSE] = combine_saturate_u;
+    imp->combine_32[PIXMAN_OP_DISJOINT_IN] = combine_disjoint_in_u;
+    imp->combine_32[PIXMAN_OP_DISJOINT_IN_REVERSE] = combine_disjoint_in_reverse_u;
+    imp->combine_32[PIXMAN_OP_DISJOINT_OUT] = combine_disjoint_out_u;
+    imp->combine_32[PIXMAN_OP_DISJOINT_OUT_REVERSE] = combine_disjoint_out_reverse_u;
+    imp->combine_32[PIXMAN_OP_DISJOINT_ATOP] = combine_disjoint_atop_u;
+    imp->combine_32[PIXMAN_OP_DISJOINT_ATOP_REVERSE] = combine_disjoint_atop_reverse_u;
+    imp->combine_32[PIXMAN_OP_DISJOINT_XOR] = combine_disjoint_xor_u;
+
+    /* Conjoint, unified */
+    imp->combine_32[PIXMAN_OP_CONJOINT_CLEAR] = combine_clear;
+    imp->combine_32[PIXMAN_OP_CONJOINT_SRC] = combine_src_u;
+    imp->combine_32[PIXMAN_OP_CONJOINT_DST] = combine_dst;
+    imp->combine_32[PIXMAN_OP_CONJOINT_OVER] = combine_conjoint_over_u;
+    imp->combine_32[PIXMAN_OP_CONJOINT_OVER_REVERSE] = combine_conjoint_over_reverse_u;
+    imp->combine_32[PIXMAN_OP_CONJOINT_IN] = combine_conjoint_in_u;
+    imp->combine_32[PIXMAN_OP_CONJOINT_IN_REVERSE] = combine_conjoint_in_reverse_u;
+    imp->combine_32[PIXMAN_OP_CONJOINT_OUT] = combine_conjoint_out_u;
+    imp->combine_32[PIXMAN_OP_CONJOINT_OUT_REVERSE] = combine_conjoint_out_reverse_u;
+    imp->combine_32[PIXMAN_OP_CONJOINT_ATOP] = combine_conjoint_atop_u;
+    imp->combine_32[PIXMAN_OP_CONJOINT_ATOP_REVERSE] = combine_conjoint_atop_reverse_u;
+    imp->combine_32[PIXMAN_OP_CONJOINT_XOR] = combine_conjoint_xor_u;
+
+    imp->combine_32[PIXMAN_OP_MULTIPLY] = combine_multiply_u;
+    imp->combine_32[PIXMAN_OP_SCREEN] = combine_screen_u;
+    imp->combine_32[PIXMAN_OP_OVERLAY] = combine_overlay_u;
+    imp->combine_32[PIXMAN_OP_DARKEN] = combine_darken_u;
+    imp->combine_32[PIXMAN_OP_LIGHTEN] = combine_lighten_u;
+    imp->combine_32[PIXMAN_OP_COLOR_DODGE] = combine_color_dodge_u;
+    imp->combine_32[PIXMAN_OP_COLOR_BURN] = combine_color_burn_u;
+    imp->combine_32[PIXMAN_OP_HARD_LIGHT] = combine_hard_light_u;
+    imp->combine_32[PIXMAN_OP_SOFT_LIGHT] = combine_soft_light_u;
+    imp->combine_32[PIXMAN_OP_DIFFERENCE] = combine_difference_u;
+    imp->combine_32[PIXMAN_OP_EXCLUSION] = combine_exclusion_u;
+    imp->combine_32[PIXMAN_OP_HSL_HUE] = combine_hsl_hue_u;
+    imp->combine_32[PIXMAN_OP_HSL_SATURATION] = combine_hsl_saturation_u;
+    imp->combine_32[PIXMAN_OP_HSL_COLOR] = combine_hsl_color_u;
+    imp->combine_32[PIXMAN_OP_HSL_LUMINOSITY] = combine_hsl_luminosity_u;
+
+    /* Component alpha combiners */
+    imp->combine_32_ca[PIXMAN_OP_CLEAR] = combine_clear_ca;
+    imp->combine_32_ca[PIXMAN_OP_SRC] = combine_src_ca;
+    /* dest */
+    imp->combine_32_ca[PIXMAN_OP_OVER] = combine_over_ca;
+    imp->combine_32_ca[PIXMAN_OP_OVER_REVERSE] = combine_over_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_IN] = combine_in_ca;
+    imp->combine_32_ca[PIXMAN_OP_IN_REVERSE] = combine_in_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_OUT] = combine_out_ca;
+    imp->combine_32_ca[PIXMAN_OP_OUT_REVERSE] = combine_out_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_ATOP] = combine_atop_ca;
+    imp->combine_32_ca[PIXMAN_OP_ATOP_REVERSE] = combine_atop_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_XOR] = combine_xor_ca;
+    imp->combine_32_ca[PIXMAN_OP_ADD] = combine_add_ca;
+    imp->combine_32_ca[PIXMAN_OP_SATURATE] = combine_saturate_ca;
+
+    /* Disjoint CA */
+    imp->combine_32_ca[PIXMAN_OP_DISJOINT_CLEAR] = combine_clear_ca;
+    imp->combine_32_ca[PIXMAN_OP_DISJOINT_SRC] = combine_src_ca;
+    imp->combine_32_ca[PIXMAN_OP_DISJOINT_DST] = combine_dst;
+    imp->combine_32_ca[PIXMAN_OP_DISJOINT_OVER] = combine_disjoint_over_ca;
+    imp->combine_32_ca[PIXMAN_OP_DISJOINT_OVER_REVERSE] = combine_saturate_ca;
+    imp->combine_32_ca[PIXMAN_OP_DISJOINT_IN] = combine_disjoint_in_ca;
+    imp->combine_32_ca[PIXMAN_OP_DISJOINT_IN_REVERSE] = combine_disjoint_in_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_DISJOINT_OUT] = combine_disjoint_out_ca;
+    imp->combine_32_ca[PIXMAN_OP_DISJOINT_OUT_REVERSE] = combine_disjoint_out_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_DISJOINT_ATOP] = combine_disjoint_atop_ca;
+    imp->combine_32_ca[PIXMAN_OP_DISJOINT_ATOP_REVERSE] = combine_disjoint_atop_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_DISJOINT_XOR] = combine_disjoint_xor_ca;
+
+    /* Conjoint CA */
+    imp->combine_32_ca[PIXMAN_OP_CONJOINT_CLEAR] = combine_clear_ca;
+    imp->combine_32_ca[PIXMAN_OP_CONJOINT_SRC] = combine_src_ca;
+    imp->combine_32_ca[PIXMAN_OP_CONJOINT_DST] = combine_dst;
+    imp->combine_32_ca[PIXMAN_OP_CONJOINT_OVER] = combine_conjoint_over_ca;
+    imp->combine_32_ca[PIXMAN_OP_CONJOINT_OVER_REVERSE] = combine_conjoint_over_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_CONJOINT_IN] = combine_conjoint_in_ca;
+    imp->combine_32_ca[PIXMAN_OP_CONJOINT_IN_REVERSE] = combine_conjoint_in_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_CONJOINT_OUT] = combine_conjoint_out_ca;
+    imp->combine_32_ca[PIXMAN_OP_CONJOINT_OUT_REVERSE] = combine_conjoint_out_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_CONJOINT_ATOP] = combine_conjoint_atop_ca;
+    imp->combine_32_ca[PIXMAN_OP_CONJOINT_ATOP_REVERSE] = combine_conjoint_atop_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_CONJOINT_XOR] = combine_conjoint_xor_ca;
+
+    imp->combine_32_ca[PIXMAN_OP_MULTIPLY] = combine_multiply_ca;
+    imp->combine_32_ca[PIXMAN_OP_SCREEN] = combine_screen_ca;
+    imp->combine_32_ca[PIXMAN_OP_OVERLAY] = combine_overlay_ca;
+    imp->combine_32_ca[PIXMAN_OP_DARKEN] = combine_darken_ca;
+    imp->combine_32_ca[PIXMAN_OP_LIGHTEN] = combine_lighten_ca;
+    imp->combine_32_ca[PIXMAN_OP_COLOR_DODGE] = combine_color_dodge_ca;
+    imp->combine_32_ca[PIXMAN_OP_COLOR_BURN] = combine_color_burn_ca;
+    imp->combine_32_ca[PIXMAN_OP_HARD_LIGHT] = combine_hard_light_ca;
+    imp->combine_32_ca[PIXMAN_OP_SOFT_LIGHT] = combine_soft_light_ca;
+    imp->combine_32_ca[PIXMAN_OP_DIFFERENCE] = combine_difference_ca;
+    imp->combine_32_ca[PIXMAN_OP_EXCLUSION] = combine_exclusion_ca;
+
+    /* It is not clear that these make sense, so make them noops for now */
+    imp->combine_32_ca[PIXMAN_OP_HSL_HUE] = combine_dst;
+    imp->combine_32_ca[PIXMAN_OP_HSL_SATURATION] = combine_dst;
+    imp->combine_32_ca[PIXMAN_OP_HSL_COLOR] = combine_dst;
+    imp->combine_32_ca[PIXMAN_OP_HSL_LUMINOSITY] = combine_dst;
+}
diff --git a/pixman/pixman/pixman-combine32.h b/pixman/pixman/pixman-combine32.h
new file mode 100644
index 000000000..cdd56a61a
--- /dev/null
+++ b/pixman/pixman/pixman-combine32.h
@@ -0,0 +1,272 @@
+#define COMPONENT_SIZE 8
+#define MASK 0xff
+#define ONE_HALF 0x80
+
+#define A_SHIFT 8 * 3
+#define R_SHIFT 8 * 2
+#define G_SHIFT 8
+#define A_MASK 0xff000000
+#define R_MASK 0xff0000
+#define G_MASK 0xff00
+
+#define RB_MASK 0xff00ff
+#define AG_MASK 0xff00ff00
+#define RB_ONE_HALF 0x800080
+#define RB_MASK_PLUS_ONE 0x10000100
+
+#define ALPHA_8(x) ((x) >> A_SHIFT)
+#define RED_8(x) (((x) >> R_SHIFT) & MASK)
+#define GREEN_8(x) (((x) >> G_SHIFT) & MASK)
+#define BLUE_8(x) ((x) & MASK)
+
+/*
+ * ARMv6 has UQADD8 instruction, which implements unsigned saturated
+ * addition for 8-bit values packed in 32-bit registers. It is very useful
+ * for UN8x4_ADD_UN8x4, UN8_rb_ADD_UN8_rb and ADD_UN8 macros (which would
+ * otherwise need a lot of arithmetic operations to simulate this operation).
+ * Since most of the major ARM linux distros are built for ARMv7, we are
+ * much less dependent on runtime CPU detection and can get practical
+ * benefits from conditional compilation here for a lot of users.
+ */
+
+#if defined(USE_GCC_INLINE_ASM) && defined(__arm__) && \
+    !defined(__aarch64__) && (!defined(__thumb__) || defined(__thumb2__))
+#if defined(__ARM_ARCH_6__)   || defined(__ARM_ARCH_6J__)  || \
+    defined(__ARM_ARCH_6K__)  || defined(__ARM_ARCH_6Z__)  || \
+    defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) || \
+    defined(__ARM_ARCH_6M__)  || defined(__ARM_ARCH_7__)   || \
+    defined(__ARM_ARCH_7A__)  || defined(__ARM_ARCH_7R__)  || \
+    defined(__ARM_ARCH_7M__)  || defined(__ARM_ARCH_7EM__)
+
+static force_inline uint32_t
+un8x4_add_un8x4 (uint32_t x, uint32_t y)
+{
+    uint32_t t;
+    asm ("uqadd8 %0, %1, %2" : "=r" (t) : "%r" (x), "r" (y));
+    return t;
+}
+
+#define UN8x4_ADD_UN8x4(x, y) \
+    ((x) = un8x4_add_un8x4 ((x), (y)))
+
+#define UN8_rb_ADD_UN8_rb(x, y, t) \
+    ((t) = un8x4_add_un8x4 ((x), (y)), (x) = (t))
+
+#define ADD_UN8(x, y, t) \
+    ((t) = (x), un8x4_add_un8x4 ((t), (y)))
+
+#endif
+#endif
+
+/*****************************************************************************/
+
+/*
+ * Helper macros.
+ */
+
+#define MUL_UN8(a, b, t)						\
+    ((t) = (a) * (uint16_t)(b) + ONE_HALF, ((((t) >> G_SHIFT ) + (t) ) >> G_SHIFT ))
+
+#define DIV_UN8(a, b)							\
+    (((uint16_t) (a) * MASK + ((b) / 2)) / (b))
+
+#ifndef ADD_UN8
+#define ADD_UN8(x, y, t)				     \
+    ((t) = (x) + (y),					     \
+     (uint32_t) (uint8_t) ((t) | (0 - ((t) >> G_SHIFT))))
+#endif
+
+#define DIV_ONE_UN8(x)							\
+    (((x) + ONE_HALF + (((x) + ONE_HALF) >> G_SHIFT)) >> G_SHIFT)
+
+/*
+ * The methods below use some tricks to be able to do two color
+ * components at the same time.
+ */
+
+/*
+ * x_rb = (x_rb * a) / 255
+ */
+#define UN8_rb_MUL_UN8(x, a, t)						\
+    do									\
+    {									\
+	t  = ((x) & RB_MASK) * (a);					\
+	t += RB_ONE_HALF;						\
+	x = (t + ((t >> G_SHIFT) & RB_MASK)) >> G_SHIFT;		\
+	x &= RB_MASK;							\
+    } while (0)
+
+/*
+ * x_rb = min (x_rb + y_rb, 255)
+ */
+#ifndef UN8_rb_ADD_UN8_rb
+#define UN8_rb_ADD_UN8_rb(x, y, t)					\
+    do									\
+    {									\
+	t = ((x) + (y));						\
+	t |= RB_MASK_PLUS_ONE - ((t >> G_SHIFT) & RB_MASK);		\
+	x = (t & RB_MASK);						\
+    } while (0)
+#endif
+
+/*
+ * x_rb = (x_rb * a_rb) / 255
+ */
+#define UN8_rb_MUL_UN8_rb(x, a, t)					\
+    do									\
+    {									\
+	t  = (x & MASK) * (a & MASK);					\
+	t |= (x & R_MASK) * ((a >> R_SHIFT) & MASK);			\
+	t += RB_ONE_HALF;						\
+	t = (t + ((t >> G_SHIFT) & RB_MASK)) >> G_SHIFT;		\
+	x = t & RB_MASK;						\
+    } while (0)
+
+/*
+ * x_c = (x_c * a) / 255
+ */
+#define UN8x4_MUL_UN8(x, a)						\
+    do									\
+    {									\
+	uint32_t r1__, r2__, t__;					\
+									\
+	r1__ = (x);							\
+	UN8_rb_MUL_UN8 (r1__, (a), t__);				\
+									\
+	r2__ = (x) >> G_SHIFT;						\
+	UN8_rb_MUL_UN8 (r2__, (a), t__);				\
+									\
+	(x) = r1__ | (r2__ << G_SHIFT);					\
+    } while (0)
+
+/*
+ * x_c = (x_c * a) / 255 + y_c
+ */
+#define UN8x4_MUL_UN8_ADD_UN8x4(x, a, y)				\
+    do									\
+    {									\
+	uint32_t r1__, r2__, r3__, t__;					\
+									\
+	r1__ = (x);							\
+	r2__ = (y) & RB_MASK;						\
+	UN8_rb_MUL_UN8 (r1__, (a), t__);				\
+	UN8_rb_ADD_UN8_rb (r1__, r2__, t__);				\
+									\
+	r2__ = (x) >> G_SHIFT;						\
+	r3__ = ((y) >> G_SHIFT) & RB_MASK;				\
+	UN8_rb_MUL_UN8 (r2__, (a), t__);				\
+	UN8_rb_ADD_UN8_rb (r2__, r3__, t__);				\
+									\
+	(x) = r1__ | (r2__ << G_SHIFT);					\
+    } while (0)
+
+/*
+ * x_c = (x_c * a + y_c * b) / 255
+ */
+#define UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8(x, a, y, b)			\
+    do									\
+    {									\
+	uint32_t r1__, r2__, r3__, t__;					\
+									\
+	r1__ = (x);							\
+	r2__ = (y);							\
+	UN8_rb_MUL_UN8 (r1__, (a), t__);				\
+	UN8_rb_MUL_UN8 (r2__, (b), t__);				\
+	UN8_rb_ADD_UN8_rb (r1__, r2__, t__);				\
+									\
+	r2__ = ((x) >> G_SHIFT);					\
+	r3__ = ((y) >> G_SHIFT);					\
+	UN8_rb_MUL_UN8 (r2__, (a), t__);				\
+	UN8_rb_MUL_UN8 (r3__, (b), t__);				\
+	UN8_rb_ADD_UN8_rb (r2__, r3__, t__);				\
+									\
+	(x) = r1__ | (r2__ << G_SHIFT);					\
+    } while (0)
+
+/*
+ * x_c = (x_c * a_c) / 255
+ */
+#define UN8x4_MUL_UN8x4(x, a)						\
+    do									\
+    {									\
+	uint32_t r1__, r2__, r3__, t__;					\
+									\
+	r1__ = (x);							\
+	r2__ = (a);							\
+	UN8_rb_MUL_UN8_rb (r1__, r2__, t__);				\
+									\
+	r2__ = (x) >> G_SHIFT;						\
+	r3__ = (a) >> G_SHIFT;						\
+	UN8_rb_MUL_UN8_rb (r2__, r3__, t__);				\
+									\
+	(x) = r1__ | (r2__ << G_SHIFT);					\
+    } while (0)
+
+/*
+ * x_c = (x_c * a_c) / 255 + y_c
+ */
+#define UN8x4_MUL_UN8x4_ADD_UN8x4(x, a, y)				\
+    do									\
+    {									\
+	uint32_t r1__, r2__, r3__, t__;					\
+									\
+	r1__ = (x);							\
+	r2__ = (a);							\
+	UN8_rb_MUL_UN8_rb (r1__, r2__, t__);				\
+	r2__ = (y) & RB_MASK;						\
+	UN8_rb_ADD_UN8_rb (r1__, r2__, t__);				\
+									\
+	r2__ = ((x) >> G_SHIFT);					\
+	r3__ = ((a) >> G_SHIFT);					\
+	UN8_rb_MUL_UN8_rb (r2__, r3__, t__);				\
+	r3__ = ((y) >> G_SHIFT) & RB_MASK;				\
+	UN8_rb_ADD_UN8_rb (r2__, r3__, t__);				\
+									\
+	(x) = r1__ | (r2__ << G_SHIFT);					\
+    } while (0)
+
+/*
+ * x_c = (x_c * a_c + y_c * b) / 255
+ */
+#define UN8x4_MUL_UN8x4_ADD_UN8x4_MUL_UN8(x, a, y, b)			\
+    do									\
+    {									\
+	uint32_t r1__, r2__, r3__, t__;					\
+									\
+	r1__ = (x);							\
+	r2__ = (a);							\
+	UN8_rb_MUL_UN8_rb (r1__, r2__, t__);				\
+	r2__ = (y);							\
+	UN8_rb_MUL_UN8 (r2__, (b), t__);				\
+	UN8_rb_ADD_UN8_rb (r1__, r2__, t__);				\
+									\
+	r2__ = (x) >> G_SHIFT;						\
+	r3__ = (a) >> G_SHIFT;						\
+	UN8_rb_MUL_UN8_rb (r2__, r3__, t__);				\
+	r3__ = (y) >> G_SHIFT;						\
+	UN8_rb_MUL_UN8 (r3__, (b), t__);				\
+	UN8_rb_ADD_UN8_rb (r2__, r3__, t__);				\
+									\
+	x = r1__ | (r2__ << G_SHIFT);					\
+    } while (0)
+
+/*
+  x_c = min(x_c + y_c, 255)
+*/
+#ifndef UN8x4_ADD_UN8x4
+#define UN8x4_ADD_UN8x4(x, y)						\
+    do									\
+    {									\
+	uint32_t r1__, r2__, r3__, t__;					\
+									\
+	r1__ = (x) & RB_MASK;						\
+	r2__ = (y) & RB_MASK;						\
+	UN8_rb_ADD_UN8_rb (r1__, r2__, t__);				\
+									\
+	r2__ = ((x) >> G_SHIFT) & RB_MASK;				\
+	r3__ = ((y) >> G_SHIFT) & RB_MASK;				\
+	UN8_rb_ADD_UN8_rb (r2__, r3__, t__);				\
+									\
+	x = r1__ | (r2__ << G_SHIFT);					\
+    } while (0)
+#endif
diff --git a/pixman/pixman/pixman-compiler.h b/pixman/pixman/pixman-compiler.h
index 26f7071c9..2489adc38 100644
--- a/pixman/pixman/pixman-compiler.h
+++ b/pixman/pixman/pixman-compiler.h
@@ -18,6 +18,18 @@
 #  define FUNC     ((const char*) ("???"))
 #endif
 
+#if defined (__GNUC__)
+#  define unlikely(expr) __builtin_expect ((expr), 0)
+#else
+#  define unlikely(expr)  (expr)
+#endif
+
+#if defined (__GNUC__)
+#  define MAYBE_UNUSED  __attribute__((unused))
+#else
+#  define MAYBE_UNUSED
+#endif
+
 #ifndef INT16_MIN
 # define INT16_MIN              (-32767-1)
 #endif
@@ -42,6 +54,19 @@
 # define UINT32_MAX             (4294967295U)
 #endif
 
+#ifndef INT64_MIN
+# define INT64_MIN              (-9223372036854775807-1)
+#endif
+
+#ifndef INT64_MAX
+# define INT64_MAX              (9223372036854775807)
+#endif
+
+#ifndef SIZE_MAX
+# define SIZE_MAX               ((size_t)-1)
+#endif
+
+
 #ifndef M_PI
 # define M_PI			3.14159265358979323846
 #endif
@@ -50,17 +75,22 @@
 /* 'inline' is available only in C++ in MSVC */
 #   define inline __inline
 #   define force_inline __forceinline
+#   define noinline __declspec(noinline)
 #elif defined __GNUC__ || (defined(__SUNPRO_C) && (__SUNPRO_C >= 0x590))
 #   define inline __inline__
 #   define force_inline __inline__ __attribute__ ((__always_inline__))
+#   define noinline __attribute__((noinline))
 #else
 #   ifndef force_inline
 #      define force_inline inline
 #   endif
+#   ifndef noinline
+#      define noinline
+#   endif
 #endif
 
 /* GCC visibility */
-#if defined(__GNUC__) && __GNUC__ >= 4
+#if defined(__GNUC__) && __GNUC__ >= 4 && !defined(_WIN32)
 #   define PIXMAN_EXPORT __attribute__ ((visibility("default")))
 /* Sun Studio 8 visibility */
 #elif defined(__SUNPRO_C) && (__SUNPRO_C >= 0x550)
@@ -69,30 +99,29 @@
 #   define PIXMAN_EXPORT
 #endif
 
+/* member offsets */
+#define CONTAINER_OF(type, member, data)				\
+    ((type *)(((uint8_t *)data) - offsetof (type, member)))
+
 /* TLS */
-#if defined(TOOLCHAIN_SUPPORTS__THREAD)
+#if defined(PIXMAN_NO_TLS)
 
 #   define PIXMAN_DEFINE_THREAD_LOCAL(type, name)			\
-    static __thread type name
+    static type name
 #   define PIXMAN_GET_THREAD_LOCAL(name)				\
     (&name)
 
-#elif defined(__MINGW32__) && !defined(__WIN64)
+#elif defined(TLS)
 
-/* We can't include <windows.h> as it causes carious clashes with
- * identifiers in pixman, sigh. So just declare the functions we need
- * here.
- */
-extern __stdcall long InterlockedCompareExchange(long volatile *, long, long);
-#define InterlockedCompareExchangePointer(d,e,c)			\
-    (void *)InterlockedCompareExchange((long volatile *)(d),(long)(e),(long)(c))
-extern __stdcall int TlsAlloc (void);
-extern __stdcall void *TlsGetValue (unsigned);
-extern __stdcall int TlsSetValue (unsigned, void *);
-extern __stdcall void *CreateMutexA(void *, int, char *);
-extern __stdcall int CloseHandle(void *);
-extern __stdcall unsigned WaitForSingleObject (void *, unsigned);
-extern __stdcall int ReleaseMutex (void *);
+#   define PIXMAN_DEFINE_THREAD_LOCAL(type, name)			\
+    static TLS type name
+#   define PIXMAN_GET_THREAD_LOCAL(name)				\
+    (&name)
+
+#elif defined(__MINGW32__)
+
+#   define _NO_W32_PSEUDO_MODIFIERS
+#   include <windows.h>
 
 #   define PIXMAN_DEFINE_THREAD_LOCAL(type, name)			\
     static volatile int tls_ ## name ## _initialized = 0;		\
@@ -149,7 +178,7 @@ extern __stdcall int ReleaseMutex (void *);
 #   define PIXMAN_GET_THREAD_LOCAL(name)				\
     (&name)
 
-#elif defined(HAVE_PTHREAD_SETSPECIFIC)
+#elif defined(HAVE_PTHREADS)
 
 #include <pthread.h>
 
@@ -198,6 +227,6 @@ extern __stdcall int ReleaseMutex (void *);
 
 #else
 
-#    error "Unknown thread local support for this system"
+#    error "Unknown thread local support for this system. Pixman will not work with multiple threads. Define PIXMAN_NO_TLS to acknowledge and accept this limitation and compile pixman without thread-safety support."
 
 #endif
diff --git a/pixman/pixman/pixman-conical-gradient.c b/pixman/pixman/pixman-conical-gradient.c
index 0341a8ebf..8bb46aecd 100644
--- a/pixman/pixman/pixman-conical-gradient.c
+++ b/pixman/pixman/pixman-conical-gradient.c
@@ -32,17 +32,34 @@
 #include <math.h>
 #include "pixman-private.h"
 
-static void
-conical_gradient_get_scanline_32 (pixman_image_t *image,
-                                  int             x,
-                                  int             y,
-                                  int             width,
-                                  uint32_t *      buffer,
-                                  const uint32_t *mask,
-                                  uint32_t        mask_bits)
+static force_inline double
+coordinates_to_parameter (double x, double y, double angle)
 {
-    source_image_t *source = (source_image_t *)image;
-    gradient_t *gradient = (gradient_t *)source;
+    double t;
+
+    t = atan2 (y, x) + angle;
+
+    while (t < 0)
+	t += 2 * M_PI;
+
+    while (t >= 2 * M_PI)
+	t -= 2 * M_PI;
+
+    return 1 - t * (1 / (2 * M_PI)); /* Scale t to [0, 1] and
+				      * make rotation CCW
+				      */
+}
+
+static uint32_t *
+conical_get_scanline_narrow (pixman_iter_t *iter, const uint32_t *mask)
+{
+    pixman_image_t *image = iter->image;
+    int x = iter->x;
+    int y = iter->y;
+    int width = iter->width;
+    uint32_t *buffer = iter->buffer;
+
+    gradient_t *gradient = (gradient_t *)image;
     conical_gradient_t *conical = (conical_gradient_t *)image;
     uint32_t       *end = buffer + width;
     pixman_gradient_walker_t walker;
@@ -53,11 +70,10 @@ conical_gradient_get_scanline_32 (pixman_image_t *image,
     double rx = x + 0.5;
     double ry = y + 0.5;
     double rz = 1.;
-    double a = (conical->angle * M_PI) / (180. * 65536);
 
-    _pixman_gradient_walker_init (&walker, gradient, source->common.repeat);
+    _pixman_gradient_walker_init (&walker, gradient, image->common.repeat);
 
-    if (source->common.transform)
+    if (image->common.transform)
     {
 	pixman_vector_t v;
 
@@ -66,19 +82,19 @@ conical_gradient_get_scanline_32 (pixman_image_t *image,
 	v.vector[1] = pixman_int_to_fixed (y) + pixman_fixed_1 / 2;
 	v.vector[2] = pixman_fixed_1;
 
-	if (!pixman_transform_point_3d (source->common.transform, &v))
-	    return;
+	if (!pixman_transform_point_3d (image->common.transform, &v))
+	    return iter->buffer;
+
+	cx = image->common.transform->matrix[0][0] / 65536.;
+	cy = image->common.transform->matrix[1][0] / 65536.;
+	cz = image->common.transform->matrix[2][0] / 65536.;
 
-	cx = source->common.transform->matrix[0][0] / 65536.;
-	cy = source->common.transform->matrix[1][0] / 65536.;
-	cz = source->common.transform->matrix[2][0] / 65536.;
-	
 	rx = v.vector[0] / 65536.;
 	ry = v.vector[1] / 65536.;
 	rz = v.vector[2] / 65536.;
-	
+
 	affine =
-	    source->common.transform->matrix[2][0] == 0 &&
+	    image->common.transform->matrix[2][0] == 0 &&
 	    v.vector[2] == pixman_fixed_1;
     }
 
@@ -89,20 +105,16 @@ conical_gradient_get_scanline_32 (pixman_image_t *image,
 
 	while (buffer < end)
 	{
-	    double angle;
-
-	    if (!mask || *mask++ & mask_bits)
+	    if (!mask || *mask++)
 	    {
-		pixman_fixed_48_16_t t;
-
-		angle = atan2 (ry, rx) + a;
-		t     = (pixman_fixed_48_16_t) (angle * (65536. / (2 * M_PI)));
+		double t = coordinates_to_parameter (rx, ry, conical->angle);
 
-		*buffer = _pixman_gradient_walker_pixel (&walker, t);
+		*buffer = _pixman_gradient_walker_pixel (
+		    &walker, (pixman_fixed_48_16_t)pixman_double_to_fixed (t));
 	    }
 
 	    ++buffer;
-	    
+
 	    rx += cx;
 	    ry += cy;
 	}
@@ -112,11 +124,10 @@ conical_gradient_get_scanline_32 (pixman_image_t *image,
 	while (buffer < end)
 	{
 	    double x, y;
-	    double angle;
 
-	    if (!mask || *mask++ & mask_bits)
+	    if (!mask || *mask++)
 	    {
-		pixman_fixed_48_16_t t;
+		double t;
 
 		if (rz != 0)
 		{
@@ -130,31 +141,47 @@ conical_gradient_get_scanline_32 (pixman_image_t *image,
 
 		x -= conical->center.x / 65536.;
 		y -= conical->center.y / 65536.;
-		
-		angle = atan2 (y, x) + a;
-		t     = (pixman_fixed_48_16_t) (angle * (65536. / (2 * M_PI)));
 
-		*buffer = _pixman_gradient_walker_pixel (&walker, t);
+		t = coordinates_to_parameter (x, y, conical->angle);
+
+		*buffer = _pixman_gradient_walker_pixel (
+		    &walker, (pixman_fixed_48_16_t)pixman_double_to_fixed (t));
 	    }
 
 	    ++buffer;
-	    
+
 	    rx += cx;
 	    ry += cy;
 	    rz += cz;
 	}
     }
+
+    iter->y++;
+    return iter->buffer;
+}
+
+static uint32_t *
+conical_get_scanline_wide (pixman_iter_t *iter, const uint32_t *mask)
+{
+    uint32_t *buffer = conical_get_scanline_narrow (iter, NULL);
+
+    pixman_expand_to_float (
+	(argb_t *)buffer, buffer, PIXMAN_a8r8g8b8, iter->width);
+
+    return buffer;
 }
 
-static void
-conical_gradient_property_changed (pixman_image_t *image)
+void
+_pixman_conical_gradient_iter_init (pixman_image_t *image, pixman_iter_t *iter)
 {
-    image->common.get_scanline_32 = conical_gradient_get_scanline_32;
-    image->common.get_scanline_64 = _pixman_image_get_scanline_generic_64;
+    if (iter->iter_flags & ITER_NARROW)
+	iter->get_scanline = conical_get_scanline_narrow;
+    else
+	iter->get_scanline = conical_get_scanline_wide;
 }
 
 PIXMAN_EXPORT pixman_image_t *
-pixman_image_create_conical_gradient (pixman_point_fixed_t *        center,
+pixman_image_create_conical_gradient (const pixman_point_fixed_t *  center,
                                       pixman_fixed_t                angle,
                                       const pixman_gradient_stop_t *stops,
                                       int                           n_stops)
@@ -173,11 +200,12 @@ pixman_image_create_conical_gradient (pixman_point_fixed_t *        center,
 	return NULL;
     }
 
+    angle = MOD (angle, pixman_int_to_fixed (360));
+
     image->type = CONICAL;
-    conical->center = *center;
-    conical->angle = angle;
 
-    image->common.property_changed = conical_gradient_property_changed;
+    conical->center = *center;
+    conical->angle = (pixman_fixed_to_double (angle) / 180.0) * M_PI;
 
     return image;
 }
diff --git a/pixman/pixman/pixman-cpu.c b/pixman/pixman/pixman-cpu.c
deleted file mode 100644
index e96b140bd..000000000
--- a/pixman/pixman/pixman-cpu.c
+++ /dev/null
@@ -1,575 +0,0 @@
-/*
- * Copyright © 2000 SuSE, Inc.
- * Copyright © 2007 Red Hat, Inc.
- *
- * Permission to use, copy, modify, distribute, and sell this software and its
- * documentation for any purpose is hereby granted without fee, provided that
- * the above copyright notice appear in all copies and that both that
- * copyright notice and this permission notice appear in supporting
- * documentation, and that the name of SuSE not be used in advertising or
- * publicity pertaining to distribution of the software without specific,
- * written prior permission.  SuSE makes no representations about the
- * suitability of this software for any purpose.  It is provided "as is"
- * without express or implied warranty.
- *
- * SuSE DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL SuSE
- * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
- * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
- * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
- * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
- */
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#include <string.h>
-
-#if defined(USE_ARM_SIMD) && defined(_MSC_VER)
-/* Needed for EXCEPTION_ILLEGAL_INSTRUCTION */
-#include <windows.h>
-#endif
-
-#include "pixman-private.h"
-
-#ifdef USE_VMX
-
-/* The CPU detection code needs to be in a file not compiled with
- * "-maltivec -mabi=altivec", as gcc would try to save vector register
- * across function calls causing SIGILL on cpus without Altivec/vmx.
- */
-static pixman_bool_t initialized = FALSE;
-static volatile pixman_bool_t have_vmx = TRUE;
-
-#ifdef __APPLE__
-#include <sys/sysctl.h>
-
-static pixman_bool_t
-pixman_have_vmx (void)
-{
-    if (!initialized)
-    {
-	size_t length = sizeof(have_vmx);
-	int error =
-	    sysctlbyname ("hw.optional.altivec", &have_vmx, &length, NULL, 0);
-
-	if (error)
-	    have_vmx = FALSE;
-
-	initialized = TRUE;
-    }
-    return have_vmx;
-}
-
-#elif defined (__linux__)
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <stdio.h>
-#include <linux/auxvec.h>
-#include <asm/cputable.h>
-
-static pixman_bool_t
-pixman_have_vmx (void)
-{
-    if (!initialized)
-    {
-	char fname[64];
-	unsigned long buf[64];
-	ssize_t count = 0;
-	pid_t pid;
-	int fd, i;
-
-	pid = getpid ();
-	snprintf (fname, sizeof(fname) - 1, "/proc/%d/auxv", pid);
-
-	fd = open (fname, O_RDONLY);
-	if (fd >= 0)
-	{
-	    for (i = 0; i <= (count / sizeof(unsigned long)); i += 2)
-	    {
-		/* Read more if buf is empty... */
-		if (i == (count / sizeof(unsigned long)))
-		{
-		    count = read (fd, buf, sizeof(buf));
-		    if (count <= 0)
-			break;
-		    i = 0;
-		}
-
-		if (buf[i] == AT_HWCAP)
-		{
-		    have_vmx = !!(buf[i + 1] & PPC_FEATURE_HAS_ALTIVEC);
-		    initialized = TRUE;
-		    break;
-		}
-		else if (buf[i] == AT_NULL)
-		{
-		    break;
-		}
-	    }
-	    close (fd);
-	}
-    }
-    if (!initialized)
-    {
-	/* Something went wrong. Assume 'no' rather than playing
-	   fragile tricks with catching SIGILL. */
-	have_vmx = FALSE;
-	initialized = TRUE;
-    }
-
-    return have_vmx;
-}
-
-#else /* !__APPLE__ && !__linux__ */
-#include <signal.h>
-#include <setjmp.h>
-
-static jmp_buf jump_env;
-
-static void
-vmx_test (int        sig,
-	  siginfo_t *si,
-	  void *     unused)
-{
-    longjmp (jump_env, 1);
-}
-
-static pixman_bool_t
-pixman_have_vmx (void)
-{
-    struct sigaction sa, osa;
-    int jmp_result;
-
-    if (!initialized)
-    {
-	sa.sa_flags = SA_SIGINFO;
-	sigemptyset (&sa.sa_mask);
-	sa.sa_sigaction = vmx_test;
-	sigaction (SIGILL, &sa, &osa);
-	jmp_result = setjmp (jump_env);
-	if (jmp_result == 0)
-	{
-	    asm volatile ( "vor 0, 0, 0" );
-	}
-	sigaction (SIGILL, &osa, NULL);
-	have_vmx = (jmp_result == 0);
-	initialized = TRUE;
-    }
-    return have_vmx;
-}
-
-#endif /* __APPLE__ */
-#endif /* USE_VMX */
-
-#if defined(USE_ARM_SIMD) || defined(USE_ARM_NEON)
-
-#if defined(_MSC_VER)
-
-#if defined(USE_ARM_SIMD)
-extern int pixman_msvc_try_arm_simd_op ();
-
-pixman_bool_t
-pixman_have_arm_simd (void)
-{
-    static pixman_bool_t initialized = FALSE;
-    static pixman_bool_t have_arm_simd = FALSE;
-
-    if (!initialized)
-    {
-	__try {
-	    pixman_msvc_try_arm_simd_op ();
-	    have_arm_simd = TRUE;
-	} __except (GetExceptionCode () == EXCEPTION_ILLEGAL_INSTRUCTION) {
-	    have_arm_simd = FALSE;
-	}
-	initialized = TRUE;
-    }
-
-    return have_arm_simd;
-}
-
-#endif /* USE_ARM_SIMD */
-
-#if defined(USE_ARM_NEON)
-extern int pixman_msvc_try_arm_neon_op ();
-
-pixman_bool_t
-pixman_have_arm_neon (void)
-{
-    static pixman_bool_t initialized = FALSE;
-    static pixman_bool_t have_arm_neon = FALSE;
-
-    if (!initialized)
-    {
-	__try
-	{
-	    pixman_msvc_try_arm_neon_op ();
-	    have_arm_neon = TRUE;
-	}
-	__except (GetExceptionCode () == EXCEPTION_ILLEGAL_INSTRUCTION)
-	{
-	    have_arm_neon = FALSE;
-	}
-	initialized = TRUE;
-    }
-
-    return have_arm_neon;
-}
-
-#endif /* USE_ARM_NEON */
-
-#else /* linux ELF */
-
-#include <stdlib.h>
-#include <unistd.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/mman.h>
-#include <fcntl.h>
-#include <string.h>
-#include <elf.h>
-
-static pixman_bool_t arm_has_v7 = FALSE;
-static pixman_bool_t arm_has_v6 = FALSE;
-static pixman_bool_t arm_has_vfp = FALSE;
-static pixman_bool_t arm_has_neon = FALSE;
-static pixman_bool_t arm_has_iwmmxt = FALSE;
-static pixman_bool_t arm_tests_initialized = FALSE;
-
-static void
-pixman_arm_read_auxv ()
-{
-    int fd;
-    Elf32_auxv_t aux;
-
-    fd = open ("/proc/self/auxv", O_RDONLY);
-    if (fd >= 0)
-    {
-	while (read (fd, &aux, sizeof(Elf32_auxv_t)) == sizeof(Elf32_auxv_t))
-	{
-	    if (aux.a_type == AT_HWCAP)
-	    {
-		uint32_t hwcap = aux.a_un.a_val;
-		/* hardcode these values to avoid depending on specific
-		 * versions of the hwcap header, e.g. HWCAP_NEON
-		 */
-		arm_has_vfp = (hwcap & 64) != 0;
-		arm_has_iwmmxt = (hwcap & 512) != 0;
-		/* this flag is only present on kernel 2.6.29 */
-		arm_has_neon = (hwcap & 4096) != 0;
-	    }
-	    else if (aux.a_type == AT_PLATFORM)
-	    {
-		const char *plat = (const char*) aux.a_un.a_val;
-		if (strncmp (plat, "v7l", 3) == 0)
-		{
-		    arm_has_v7 = TRUE;
-		    arm_has_v6 = TRUE;
-		}
-		else if (strncmp (plat, "v6l", 3) == 0)
-		{
-		    arm_has_v6 = TRUE;
-		}
-	    }
-	}
-	close (fd);
-    }
-
-    arm_tests_initialized = TRUE;
-}
-
-#if defined(USE_ARM_SIMD)
-pixman_bool_t
-pixman_have_arm_simd (void)
-{
-    if (!arm_tests_initialized)
-	pixman_arm_read_auxv ();
-
-    return arm_has_v6;
-}
-
-#endif /* USE_ARM_SIMD */
-
-#if defined(USE_ARM_NEON)
-pixman_bool_t
-pixman_have_arm_neon (void)
-{
-    if (!arm_tests_initialized)
-	pixman_arm_read_auxv ();
-
-    return arm_has_neon;
-}
-
-#endif /* USE_ARM_NEON */
-
-#endif /* linux */
-
-#endif /* USE_ARM_SIMD || USE_ARM_NEON */
-
-#if defined(USE_MMX) || defined(USE_SSE2)
-/* The CPU detection code needs to be in a file not compiled with
- * "-mmmx -msse", as gcc would generate CMOV instructions otherwise
- * that would lead to SIGILL instructions on old CPUs that don't have
- * it.
- */
-#if !defined(__amd64__) && !defined(__x86_64__) && !defined(_M_AMD64)
-
-#ifdef HAVE_GETISAX
-#include <sys/auxv.h>
-#endif
-
-typedef enum
-{
-    NO_FEATURES = 0,
-    MMX = 0x1,
-    MMX_EXTENSIONS = 0x2,
-    SSE = 0x6,
-    SSE2 = 0x8,
-    CMOV = 0x10
-} cpu_features_t;
-
-
-static unsigned int
-detect_cpu_features (void)
-{
-    unsigned int features = 0;
-    unsigned int result = 0;
-
-#ifdef HAVE_GETISAX
-    if (getisax (&result, 1))
-    {
-	if (result & AV_386_CMOV)
-	    features |= CMOV;
-	if (result & AV_386_MMX)
-	    features |= MMX;
-	if (result & AV_386_AMD_MMX)
-	    features |= MMX_EXTENSIONS;
-	if (result & AV_386_SSE)
-	    features |= SSE;
-	if (result & AV_386_SSE2)
-	    features |= SSE2;
-    }
-#else
-    char vendor[13];
-#ifdef _MSC_VER
-    int vendor0 = 0, vendor1, vendor2;
-#endif
-    vendor[0] = 0;
-    vendor[12] = 0;
-
-#ifdef __GNUC__
-    /* see p. 118 of amd64 instruction set manual Vol3 */
-    /* We need to be careful about the handling of %ebx and
-     * %esp here. We can't declare either one as clobbered
-     * since they are special registers (%ebx is the "PIC
-     * register" holding an offset to global data, %esp the
-     * stack pointer), so we need to make sure they have their
-     * original values when we access the output operands.
-     */
-    __asm__ (
-        "pushf\n"
-        "pop %%eax\n"
-        "mov %%eax, %%ecx\n"
-        "xor $0x00200000, %%eax\n"
-        "push %%eax\n"
-        "popf\n"
-        "pushf\n"
-        "pop %%eax\n"
-        "mov $0x0, %%edx\n"
-        "xor %%ecx, %%eax\n"
-        "jz 1f\n"
-
-        "mov $0x00000000, %%eax\n"
-        "push %%ebx\n"
-        "cpuid\n"
-        "mov %%ebx, %%eax\n"
-        "pop %%ebx\n"
-        "mov %%eax, %1\n"
-        "mov %%edx, %2\n"
-        "mov %%ecx, %3\n"
-        "mov $0x00000001, %%eax\n"
-        "push %%ebx\n"
-        "cpuid\n"
-        "pop %%ebx\n"
-        "1:\n"
-        "mov %%edx, %0\n"
-	: "=r" (result),
-        "=m" (vendor[0]),
-        "=m" (vendor[4]),
-        "=m" (vendor[8])
-	:
-	: "%eax", "%ecx", "%edx"
-        );
-
-#elif defined (_MSC_VER)
-
-    _asm {
-	pushfd
-	pop eax
-	mov ecx, eax
-	xor eax, 00200000h
-	push eax
-	popfd
-	pushfd
-	pop eax
-	mov edx, 0
-	xor eax, ecx
-	jz nocpuid
-
-	mov eax, 0
-	push ebx
-	cpuid
-	mov eax, ebx
-	pop ebx
-	mov vendor0, eax
-	mov vendor1, edx
-	mov vendor2, ecx
-	mov eax, 1
-	push ebx
-	cpuid
-	pop ebx
-    nocpuid:
-	mov result, edx
-    }
-    memmove (vendor + 0, &vendor0, 4);
-    memmove (vendor + 4, &vendor1, 4);
-    memmove (vendor + 8, &vendor2, 4);
-
-#else
-#   error unsupported compiler
-#endif
-
-    features = 0;
-    if (result)
-    {
-	/* result now contains the standard feature bits */
-	if (result & (1 << 15))
-	    features |= CMOV;
-	if (result & (1 << 23))
-	    features |= MMX;
-	if (result & (1 << 25))
-	    features |= SSE;
-	if (result & (1 << 26))
-	    features |= SSE2;
-	if ((features & MMX) && !(features & SSE) &&
-	    (strcmp (vendor, "AuthenticAMD") == 0 ||
-	     strcmp (vendor, "Geode by NSC") == 0))
-	{
-	    /* check for AMD MMX extensions */
-#ifdef __GNUC__
-	    __asm__ (
-	        "	push %%ebx\n"
-	        "	mov $0x80000000, %%eax\n"
-	        "	cpuid\n"
-	        "	xor %%edx, %%edx\n"
-	        "	cmp $0x1, %%eax\n"
-	        "	jge 2f\n"
-	        "	mov $0x80000001, %%eax\n"
-	        "	cpuid\n"
-	        "2:\n"
-	        "	pop %%ebx\n"
-	        "	mov %%edx, %0\n"
-		: "=r" (result)
-		:
-		: "%eax", "%ecx", "%edx"
-	        );
-#elif defined _MSC_VER
-	    _asm {
-		push ebx
-		mov eax, 80000000h
-		cpuid
-		xor edx, edx
-		cmp eax, 1
-		jge notamd
-		mov eax, 80000001h
-		cpuid
-	    notamd:
-		pop ebx
-		mov result, edx
-	    }
-#endif
-	    if (result & (1 << 22))
-		features |= MMX_EXTENSIONS;
-	}
-    }
-#endif /* HAVE_GETISAX */
-
-    return features;
-}
-
-static pixman_bool_t
-pixman_have_mmx (void)
-{
-    static pixman_bool_t initialized = FALSE;
-    static pixman_bool_t mmx_present;
-
-    if (!initialized)
-    {
-	unsigned int features = detect_cpu_features ();
-	mmx_present = (features & (MMX | MMX_EXTENSIONS)) == (MMX | MMX_EXTENSIONS);
-	initialized = TRUE;
-    }
-
-    return mmx_present;
-}
-
-#ifdef USE_SSE2
-static pixman_bool_t
-pixman_have_sse2 (void)
-{
-    static pixman_bool_t initialized = FALSE;
-    static pixman_bool_t sse2_present;
-
-    if (!initialized)
-    {
-	unsigned int features = detect_cpu_features ();
-	sse2_present = (features & (MMX | MMX_EXTENSIONS | SSE | SSE2)) == (MMX | MMX_EXTENSIONS | SSE | SSE2);
-	initialized = TRUE;
-    }
-
-    return sse2_present;
-}
-
-#endif
-
-#else /* __amd64__ */
-#ifdef USE_MMX
-#define pixman_have_mmx() TRUE
-#endif
-#ifdef USE_SSE2
-#define pixman_have_sse2() TRUE
-#endif
-#endif /* __amd64__ */
-#endif
-
-pixman_implementation_t *
-_pixman_choose_implementation (void)
-{
-#ifdef USE_SSE2
-    if (pixman_have_sse2 ())
-	return _pixman_implementation_create_sse2 ();
-#endif
-#ifdef USE_MMX
-    if (pixman_have_mmx ())
-	return _pixman_implementation_create_mmx ();
-#endif
-
-#ifdef USE_ARM_NEON
-    if (pixman_have_arm_neon ())
-	return _pixman_implementation_create_arm_neon ();
-#endif
-#ifdef USE_ARM_SIMD
-    if (pixman_have_arm_simd ())
-	return _pixman_implementation_create_arm_simd ();
-#endif
-#ifdef USE_VMX
-    if (pixman_have_vmx ())
-	return _pixman_implementation_create_vmx ();
-#endif
-
-    return _pixman_implementation_create_fast_path ();
-}
-
diff --git a/pixman/pixman/pixman-edge.c b/pixman/pixman/pixman-edge.c
index 8d498ab44..ad6dfc4cf 100644
--- a/pixman/pixman/pixman-edge.c
+++ b/pixman/pixman/pixman-edge.c
@@ -374,6 +374,7 @@ pixman_rasterize_edges (pixman_image_t *image,
                         pixman_fixed_t  b)
 {
     return_if_fail (image->type == BITS);
+    return_if_fail (PIXMAN_FORMAT_TYPE (image->bits.format) == PIXMAN_TYPE_A);
     
     if (image->bits.read_func || image->bits.write_func)
 	pixman_rasterize_edges_accessors (image, l, r, t, b);
diff --git a/pixman/pixman/pixman-fast-path.c b/pixman/pixman/pixman-fast-path.c
index bf5b298c8..c6e43de10 100644
--- a/pixman/pixman/pixman-fast-path.c
+++ b/pixman/pixman/pixman-fast-path.c
@@ -30,11 +30,12 @@
 #include <stdlib.h>
 #include "pixman-private.h"
 #include "pixman-combine32.h"
+#include "pixman-inlines.h"
 
 static force_inline uint32_t
 fetch_24 (uint8_t *a)
 {
-    if (((unsigned long)a) & 1)
+    if (((uintptr_t)a) & 1)
     {
 #ifdef WORDS_BIGENDIAN
 	return (*a << 16) | (*(uint16_t *)(a + 1));
@@ -56,7 +57,7 @@ static force_inline void
 store_24 (uint8_t *a,
           uint32_t v)
 {
-    if (((unsigned long)a) & 1)
+    if (((uintptr_t)a) & 1)
     {
 #ifdef WORDS_BIGENDIAN
 	*a = (uint8_t) (v >> 16);
@@ -89,7 +90,7 @@ over (uint32_t src,
     return dest;
 }
 
-static uint32_t
+static force_inline uint32_t
 in (uint32_t x,
     uint8_t  y)
 {
@@ -107,19 +108,9 @@ in (uint32_t x,
  */
 static void
 fast_composite_over_x888_8_8888 (pixman_implementation_t *imp,
-                                 pixman_op_t              op,
-                                 pixman_image_t *         src_image,
-                                 pixman_image_t *         mask_image,
-                                 pixman_image_t *         dst_image,
-                                 int32_t                  src_x,
-                                 int32_t                  src_y,
-                                 int32_t                  mask_x,
-                                 int32_t                  mask_y,
-                                 int32_t                  dest_x,
-                                 int32_t                  dest_y,
-                                 int32_t                  width,
-                                 int32_t                  height)
+                                 pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t    *src, *src_line;
     uint32_t    *dst, *dst_line;
     uint8_t     *mask, *mask_line;
@@ -128,7 +119,7 @@ fast_composite_over_x888_8_8888 (pixman_implementation_t *imp,
     uint32_t s, d;
     int32_t w;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
 
@@ -167,19 +158,9 @@ fast_composite_over_x888_8_8888 (pixman_implementation_t *imp,
 
 static void
 fast_composite_in_n_8_8 (pixman_implementation_t *imp,
-                         pixman_op_t              op,
-                         pixman_image_t *         src_image,
-                         pixman_image_t *         mask_image,
-                         pixman_image_t *         dest_image,
-                         int32_t                  src_x,
-                         int32_t                  src_y,
-                         int32_t                  mask_x,
-                         int32_t                  mask_y,
-                         int32_t                  dest_x,
-                         int32_t                  dest_y,
-                         int32_t                  width,
-                         int32_t                  height)
+                         pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t src, srca;
     uint8_t     *dst_line, *dst;
     uint8_t     *mask_line, *mask, m;
@@ -187,7 +168,7 @@ fast_composite_in_n_8_8 (pixman_implementation_t *imp,
     int32_t w;
     uint16_t t;
 
-    src = _pixman_image_get_solid (src_image, dest_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     srca = src >> 24;
 
@@ -245,19 +226,9 @@ fast_composite_in_n_8_8 (pixman_implementation_t *imp,
 
 static void
 fast_composite_in_8_8 (pixman_implementation_t *imp,
-                       pixman_op_t              op,
-                       pixman_image_t *         src_image,
-                       pixman_image_t *         mask_image,
-                       pixman_image_t *         dest_image,
-                       int32_t                  src_x,
-                       int32_t                  src_y,
-                       int32_t                  mask_x,
-                       int32_t                  mask_y,
-                       int32_t                  dest_x,
-                       int32_t                  dest_y,
-                       int32_t                  width,
-                       int32_t                  height)
+                       pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint8_t     *dst_line, *dst;
     uint8_t     *src_line, *src;
     int dst_stride, src_stride;
@@ -292,32 +263,22 @@ fast_composite_in_8_8 (pixman_implementation_t *imp,
 
 static void
 fast_composite_over_n_8_8888 (pixman_implementation_t *imp,
-                              pixman_op_t              op,
-                              pixman_image_t *         src_image,
-                              pixman_image_t *         mask_image,
-                              pixman_image_t *         dst_image,
-                              int32_t                  src_x,
-                              int32_t                  src_y,
-                              int32_t                  mask_x,
-                              int32_t                  mask_y,
-                              int32_t                  dest_x,
-                              int32_t                  dest_y,
-                              int32_t                  width,
-                              int32_t                  height)
+                              pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t src, srca;
     uint32_t    *dst_line, *dst, d;
     uint8_t     *mask_line, *mask, m;
     int dst_stride, mask_stride;
     int32_t w;
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     srca = src >> 24;
     if (src == 0)
 	return;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
 
     while (height--)
@@ -350,32 +311,21 @@ fast_composite_over_n_8_8888 (pixman_implementation_t *imp,
 
 static void
 fast_composite_add_n_8888_8888_ca (pixman_implementation_t *imp,
-				   pixman_op_t              op,
-				   pixman_image_t *         src_image,
-				   pixman_image_t *         mask_image,
-				   pixman_image_t *         dst_image,
-				   int32_t                  src_x,
-				   int32_t                  src_y,
-				   int32_t                  mask_x,
-				   int32_t                  mask_y,
-				   int32_t                  dest_x,
-				   int32_t                  dest_y,
-				   int32_t                  width,
-				   int32_t                  height)
+				   pixman_composite_info_t *info)
 {
-    uint32_t src, srca, s;
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint32_t src, s;
     uint32_t    *dst_line, *dst, d;
     uint32_t    *mask_line, *mask, ma;
     int dst_stride, mask_stride;
     int32_t w;
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
-    srca = src >> 24;
     if (src == 0)
 	return;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
 
     while (height--)
@@ -407,32 +357,22 @@ fast_composite_add_n_8888_8888_ca (pixman_implementation_t *imp,
 
 static void
 fast_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
-                                    pixman_op_t              op,
-                                    pixman_image_t *         src_image,
-                                    pixman_image_t *         mask_image,
-                                    pixman_image_t *         dst_image,
-                                    int32_t                  src_x,
-                                    int32_t                  src_y,
-                                    int32_t                  mask_x,
-                                    int32_t                  mask_y,
-                                    int32_t                  dest_x,
-                                    int32_t                  dest_y,
-                                    int32_t                  width,
-                                    int32_t                  height)
+                                    pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t src, srca, s;
     uint32_t    *dst_line, *dst, d;
     uint32_t    *mask_line, *mask, ma;
     int dst_stride, mask_stride;
     int32_t w;
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     srca = src >> 24;
     if (src == 0)
 	return;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
 
     while (height--)
@@ -473,19 +413,9 @@ fast_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
 
 static void
 fast_composite_over_n_8_0888 (pixman_implementation_t *imp,
-                              pixman_op_t              op,
-                              pixman_image_t *         src_image,
-                              pixman_image_t *         mask_image,
-                              pixman_image_t *         dst_image,
-                              int32_t                  src_x,
-                              int32_t                  src_y,
-                              int32_t                  mask_x,
-                              int32_t                  mask_y,
-                              int32_t                  dest_x,
-                              int32_t                  dest_y,
-                              int32_t                  width,
-                              int32_t                  height)
+                              pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t src, srca;
     uint8_t     *dst_line, *dst;
     uint32_t d;
@@ -493,13 +423,13 @@ fast_composite_over_n_8_0888 (pixman_implementation_t *imp,
     int dst_stride, mask_stride;
     int32_t w;
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     srca = src >> 24;
     if (src == 0)
 	return;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 3);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 3);
     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
 
     while (height--)
@@ -538,19 +468,9 @@ fast_composite_over_n_8_0888 (pixman_implementation_t *imp,
 
 static void
 fast_composite_over_n_8_0565 (pixman_implementation_t *imp,
-                              pixman_op_t              op,
-                              pixman_image_t *         src_image,
-                              pixman_image_t *         mask_image,
-                              pixman_image_t *         dst_image,
-                              int32_t                  src_x,
-                              int32_t                  src_y,
-                              int32_t                  mask_x,
-                              int32_t                  mask_y,
-                              int32_t                  dest_x,
-                              int32_t                  dest_y,
-                              int32_t                  width,
-                              int32_t                  height)
+                              pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t src, srca;
     uint16_t    *dst_line, *dst;
     uint32_t d;
@@ -558,13 +478,13 @@ fast_composite_over_n_8_0565 (pixman_implementation_t *imp,
     int dst_stride, mask_stride;
     int32_t w;
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     srca = src >> 24;
     if (src == 0)
 	return;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
 
     while (height--)
@@ -587,15 +507,15 @@ fast_composite_over_n_8_0565 (pixman_implementation_t *imp,
 		else
 		{
 		    d = *dst;
-		    d = over (src, CONVERT_0565_TO_0888 (d));
+		    d = over (src, convert_0565_to_0888 (d));
 		}
-		*dst = CONVERT_8888_TO_0565 (d);
+		*dst = convert_8888_to_0565 (d);
 	    }
 	    else if (m)
 	    {
 		d = *dst;
-		d = over (in (src, m), CONVERT_0565_TO_0888 (d));
-		*dst = CONVERT_8888_TO_0565 (d);
+		d = over (in (src, m), convert_0565_to_0888 (d));
+		*dst = convert_8888_to_0565 (d);
 	    }
 	    dst++;
 	}
@@ -604,19 +524,9 @@ fast_composite_over_n_8_0565 (pixman_implementation_t *imp,
 
 static void
 fast_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
-                                    pixman_op_t              op,
-                                    pixman_image_t *         src_image,
-                                    pixman_image_t *         mask_image,
-                                    pixman_image_t *         dst_image,
-                                    int32_t                  src_x,
-                                    int32_t                  src_y,
-                                    int32_t                  mask_x,
-                                    int32_t                  mask_y,
-                                    int32_t                  dest_x,
-                                    int32_t                  dest_y,
-                                    int32_t                  width,
-                                    int32_t                  height)
+                                    pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t  src, srca, s;
     uint16_t  src16;
     uint16_t *dst_line, *dst;
@@ -625,15 +535,15 @@ fast_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
     int dst_stride, mask_stride;
     int32_t w;
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     srca = src >> 24;
     if (src == 0)
 	return;
 
-    src16 = CONVERT_8888_TO_0565 (src);
+    src16 = convert_8888_to_0565 (src);
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
 
     while (height--)
@@ -656,14 +566,14 @@ fast_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
 		else
 		{
 		    d = *dst;
-		    d = over (src, CONVERT_0565_TO_0888 (d));
-		    *dst = CONVERT_8888_TO_0565 (d);
+		    d = over (src, convert_0565_to_0888 (d));
+		    *dst = convert_8888_to_0565 (d);
 		}
 	    }
 	    else if (ma)
 	    {
 		d = *dst;
-		d = CONVERT_0565_TO_0888 (d);
+		d = convert_0565_to_0888 (d);
 
 		s = src;
 
@@ -672,7 +582,7 @@ fast_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
 		ma = ~ma;
 		UN8x4_MUL_UN8x4_ADD_UN8x4 (d, ma, s);
 
-		*dst = CONVERT_8888_TO_0565 (d);
+		*dst = convert_8888_to_0565 (d);
 	    }
 	    dst++;
 	}
@@ -681,26 +591,16 @@ fast_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
 
 static void
 fast_composite_over_8888_8888 (pixman_implementation_t *imp,
-                               pixman_op_t              op,
-                               pixman_image_t *         src_image,
-                               pixman_image_t *         mask_image,
-                               pixman_image_t *         dst_image,
-                               int32_t                  src_x,
-                               int32_t                  src_y,
-                               int32_t                  mask_x,
-                               int32_t                  mask_y,
-                               int32_t                  dest_x,
-                               int32_t                  dest_y,
-                               int32_t                  width,
-                               int32_t                  height)
+                               pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t    *dst_line, *dst;
     uint32_t    *src_line, *src, s;
     int dst_stride, src_stride;
     uint8_t a;
     int32_t w;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
 
     while (height--)
@@ -724,22 +624,38 @@ fast_composite_over_8888_8888 (pixman_implementation_t *imp,
     }
 }
 
+static void
+fast_composite_src_x888_8888 (pixman_implementation_t *imp,
+			      pixman_composite_info_t *info)
+{
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint32_t    *dst_line, *dst;
+    uint32_t    *src_line, *src;
+    int dst_stride, src_stride;
+    int32_t w;
+
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w--)
+	    *dst++ = (*src++) | 0xff000000;
+    }
+}
+
 #if 0
 static void
 fast_composite_over_8888_0888 (pixman_implementation_t *imp,
-			       pixman_op_t              op,
-			       pixman_image_t *         src_image,
-			       pixman_image_t *         mask_image,
-			       pixman_image_t *         dst_image,
-			       int32_t                  src_x,
-			       int32_t                  src_y,
-			       int32_t                  mask_x,
-			       int32_t                  mask_y,
-			       int32_t                  dest_x,
-			       int32_t                  dest_y,
-			       int32_t                  width,
-			       int32_t                  height)
+			       pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint8_t     *dst_line, *dst;
     uint32_t d;
     uint32_t    *src_line, *src, s;
@@ -747,7 +663,7 @@ fast_composite_over_8888_0888 (pixman_implementation_t *imp,
     int dst_stride, src_stride;
     int32_t w;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 3);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 3);
     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
 
     while (height--)
@@ -779,19 +695,9 @@ fast_composite_over_8888_0888 (pixman_implementation_t *imp,
 
 static void
 fast_composite_over_8888_0565 (pixman_implementation_t *imp,
-                               pixman_op_t              op,
-                               pixman_image_t *         src_image,
-                               pixman_image_t *         mask_image,
-                               pixman_image_t *         dst_image,
-                               int32_t                  src_x,
-                               int32_t                  src_y,
-                               int32_t                  mask_x,
-                               int32_t                  mask_y,
-                               int32_t                  dest_x,
-                               int32_t                  dest_y,
-                               int32_t                  width,
-                               int32_t                  height)
+                               pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint16_t    *dst_line, *dst;
     uint32_t d;
     uint32_t    *src_line, *src, s;
@@ -800,7 +706,7 @@ fast_composite_over_8888_0565 (pixman_implementation_t *imp,
     int32_t w;
 
     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
 
     while (height--)
     {
@@ -823,9 +729,9 @@ fast_composite_over_8888_0565 (pixman_implementation_t *imp,
 		else
 		{
 		    d = *dst;
-		    d = over (s, CONVERT_0565_TO_0888 (d));
+		    d = over (s, convert_0565_to_0888 (d));
 		}
-		*dst = CONVERT_8888_TO_0565 (d);
+		*dst = convert_8888_to_0565 (d);
 	    }
 	    dst++;
 	}
@@ -833,27 +739,19 @@ fast_composite_over_8888_0565 (pixman_implementation_t *imp,
 }
 
 static void
-fast_composite_src_x888_0565 (pixman_implementation_t *imp,
-                              pixman_op_t              op,
-                              pixman_image_t *         src_image,
-                              pixman_image_t *         mask_image,
-                              pixman_image_t *         dst_image,
-                              int32_t                  src_x,
-                              int32_t                  src_y,
-                              int32_t                  mask_x,
-                              int32_t                  mask_y,
-                              int32_t                  dest_x,
-                              int32_t                  dest_y,
-                              int32_t                  width,
-                              int32_t                  height)
+fast_composite_add_8_8 (pixman_implementation_t *imp,
+			pixman_composite_info_t *info)
 {
-    uint16_t    *dst_line, *dst;
-    uint32_t    *src_line, *src, s;
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint8_t     *dst_line, *dst;
+    uint8_t     *src_line, *src;
     int dst_stride, src_stride;
     int32_t w;
+    uint8_t s, d;
+    uint16_t t;
 
-    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
 
     while (height--)
     {
@@ -866,36 +764,35 @@ fast_composite_src_x888_0565 (pixman_implementation_t *imp,
 	while (w--)
 	{
 	    s = *src++;
-	    *dst = CONVERT_8888_TO_0565 (s);
+	    if (s)
+	    {
+		if (s != 0xff)
+		{
+		    d = *dst;
+		    t = d + s;
+		    s = t | (0 - (t >> 8));
+		}
+		*dst = s;
+	    }
 	    dst++;
 	}
     }
 }
 
 static void
-fast_composite_add_8000_8000 (pixman_implementation_t *imp,
-                              pixman_op_t              op,
-                              pixman_image_t *         src_image,
-                              pixman_image_t *         mask_image,
-                              pixman_image_t *         dst_image,
-                              int32_t                  src_x,
-                              int32_t                  src_y,
-                              int32_t                  mask_x,
-                              int32_t                  mask_y,
-                              int32_t                  dest_x,
-                              int32_t                  dest_y,
-                              int32_t                  width,
-                              int32_t                  height)
+fast_composite_add_0565_0565 (pixman_implementation_t *imp,
+                              pixman_composite_info_t *info)
 {
-    uint8_t     *dst_line, *dst;
-    uint8_t     *src_line, *src;
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint16_t    *dst_line, *dst;
+    uint32_t	d;
+    uint16_t    *src_line, *src;
+    uint32_t	s;
     int dst_stride, src_stride;
     int32_t w;
-    uint8_t s, d;
-    uint16_t t;
 
-    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint16_t, src_stride, src_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
 
     while (height--)
     {
@@ -910,13 +807,14 @@ fast_composite_add_8000_8000 (pixman_implementation_t *imp,
 	    s = *src++;
 	    if (s)
 	    {
-		if (s != 0xff)
+		d = *dst;
+		s = convert_0565_to_8888 (s);
+		if (d)
 		{
-		    d = *dst;
-		    t = d + s;
-		    s = t | (0 - (t >> 8));
+		    d = convert_0565_to_8888 (d);
+		    UN8x4_ADD_UN8x4 (s, d);
 		}
-		*dst = s;
+		*dst = convert_8888_to_0565 (s);
 	    }
 	    dst++;
 	}
@@ -925,19 +823,9 @@ fast_composite_add_8000_8000 (pixman_implementation_t *imp,
 
 static void
 fast_composite_add_8888_8888 (pixman_implementation_t *imp,
-                              pixman_op_t              op,
-                              pixman_image_t *         src_image,
-                              pixman_image_t *         mask_image,
-                              pixman_image_t *         dst_image,
-                              int32_t                  src_x,
-                              int32_t                  src_y,
-                              int32_t                  mask_x,
-                              int32_t                  mask_y,
-                              int32_t                  dest_x,
-                              int32_t                  dest_y,
-                              int32_t                  width,
-                              int32_t                  height)
+                              pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t    *dst_line, *dst;
     uint32_t    *src_line, *src;
     int dst_stride, src_stride;
@@ -945,7 +833,7 @@ fast_composite_add_8888_8888 (pixman_implementation_t *imp,
     uint32_t s, d;
 
     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
 
     while (height--)
     {
@@ -975,19 +863,9 @@ fast_composite_add_8888_8888 (pixman_implementation_t *imp,
 
 static void
 fast_composite_add_n_8_8 (pixman_implementation_t *imp,
-			  pixman_op_t              op,
-			  pixman_image_t *         src_image,
-			  pixman_image_t *         mask_image,
-			  pixman_image_t *         dst_image,
-			  int32_t                  src_x,
-			  int32_t                  src_y,
-			  int32_t                  mask_x,
-			  int32_t                  mask_y,
-			  int32_t                  dest_x,
-			  int32_t                  dest_y,
-			  int32_t                  width,
-			  int32_t                  height)
+			  pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint8_t     *dst_line, *dst;
     uint8_t     *mask_line, *mask;
     int dst_stride, mask_stride;
@@ -995,9 +873,9 @@ fast_composite_add_n_8_8 (pixman_implementation_t *imp,
     uint32_t src;
     uint8_t sa;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
     sa = (src >> 24);
 
     while (height--)
@@ -1040,20 +918,10 @@ fast_composite_add_n_8_8 (pixman_implementation_t *imp,
     do { *((p) + ((n) >> 5)) |= CREATE_BITMASK ((n) & 31); } while (0);
 
 static void
-fast_composite_add_1000_1000 (pixman_implementation_t *imp,
-                              pixman_op_t              op,
-                              pixman_image_t *         src_image,
-                              pixman_image_t *         mask_image,
-                              pixman_image_t *         dst_image,
-                              int32_t                  src_x,
-                              int32_t                  src_y,
-                              int32_t                  mask_x,
-                              int32_t                  mask_y,
-                              int32_t                  dest_x,
-                              int32_t                  dest_y,
-                              int32_t                  width,
-                              int32_t                  height)
+fast_composite_add_1_1 (pixman_implementation_t *imp,
+			pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t     *dst_line, *dst;
     uint32_t     *src_line, *src;
     int           dst_stride, src_stride;
@@ -1061,7 +929,7 @@ fast_composite_add_1000_1000 (pixman_implementation_t *imp,
 
     PIXMAN_IMAGE_GET_LINE (src_image, 0, src_y, uint32_t,
                            src_stride, src_line, 1);
-    PIXMAN_IMAGE_GET_LINE (dst_image, 0, dest_y, uint32_t,
+    PIXMAN_IMAGE_GET_LINE (dest_image, 0, dest_y, uint32_t,
                            dst_stride, dst_line, 1);
 
     while (height--)
@@ -1086,19 +954,9 @@ fast_composite_add_1000_1000 (pixman_implementation_t *imp,
 
 static void
 fast_composite_over_n_1_8888 (pixman_implementation_t *imp,
-                              pixman_op_t              op,
-                              pixman_image_t *         src_image,
-                              pixman_image_t *         mask_image,
-                              pixman_image_t *         dst_image,
-                              int32_t                  src_x,
-                              int32_t                  src_y,
-                              int32_t                  mask_x,
-                              int32_t                  mask_y,
-                              int32_t                  dest_x,
-                              int32_t                  dest_y,
-                              int32_t                  width,
-                              int32_t                  height)
+                              pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t     src, srca;
     uint32_t    *dst, *dst_line;
     uint32_t    *mask, *mask_line;
@@ -1109,12 +967,12 @@ fast_composite_over_n_1_8888 (pixman_implementation_t *imp,
     if (width <= 0)
 	return;
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
     srca = src >> 24;
     if (src == 0)
 	return;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t,
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t,
                            dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (mask_image, 0, mask_y, uint32_t,
                            mask_stride, mask_line, 1);
@@ -1178,19 +1036,9 @@ fast_composite_over_n_1_8888 (pixman_implementation_t *imp,
 
 static void
 fast_composite_over_n_1_0565 (pixman_implementation_t *imp,
-                              pixman_op_t              op,
-                              pixman_image_t *         src_image,
-                              pixman_image_t *         mask_image,
-                              pixman_image_t *         dst_image,
-                              int32_t                  src_x,
-                              int32_t                  src_y,
-                              int32_t                  mask_x,
-                              int32_t                  mask_y,
-                              int32_t                  dest_x,
-                              int32_t                  dest_y,
-                              int32_t                  width,
-                              int32_t                  height)
+                              pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t     src, srca;
     uint16_t    *dst, *dst_line;
     uint32_t    *mask, *mask_line;
@@ -1203,12 +1051,12 @@ fast_composite_over_n_1_0565 (pixman_implementation_t *imp,
     if (width <= 0)
 	return;
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
     srca = src >> 24;
     if (src == 0)
 	return;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t,
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint16_t,
                            dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (mask_image, 0, mask_y, uint32_t,
                            mask_stride, mask_line, 1);
@@ -1216,7 +1064,7 @@ fast_composite_over_n_1_0565 (pixman_implementation_t *imp,
 
     if (srca == 0xff)
     {
-	src565 = CONVERT_8888_TO_0565 (src);
+	src565 = convert_8888_to_0565 (src);
 	while (height--)
 	{
 	    dst = dst_line;
@@ -1264,8 +1112,8 @@ fast_composite_over_n_1_0565 (pixman_implementation_t *imp,
 		}
 		if (bitcache & bitmask)
 		{
-		    d = over (src, CONVERT_0565_TO_0888 (*dst));
-		    *dst = CONVERT_8888_TO_0565 (d);
+		    d = over (src, convert_0565_to_0888 (*dst));
+		    *dst = convert_8888_to_0565 (d);
 		}
 		bitmask = UPDATE_BITMASK (bitmask);
 		dst++;
@@ -1280,62 +1128,50 @@ fast_composite_over_n_1_0565 (pixman_implementation_t *imp,
 
 static void
 fast_composite_solid_fill (pixman_implementation_t *imp,
-                           pixman_op_t              op,
-                           pixman_image_t *         src_image,
-                           pixman_image_t *         mask_image,
-                           pixman_image_t *         dst_image,
-                           int32_t                  src_x,
-                           int32_t                  src_y,
-                           int32_t                  mask_x,
-                           int32_t                  mask_y,
-                           int32_t                  dest_x,
-                           int32_t                  dest_y,
-                           int32_t                  width,
-                           int32_t                  height)
+                           pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t src;
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
-    if (dst_image->bits.format == PIXMAN_a8)
+    if (dest_image->bits.format == PIXMAN_a1)
+    {
+	src = src >> 31;
+    }
+    else if (dest_image->bits.format == PIXMAN_a8)
     {
 	src = src >> 24;
     }
-    else if (dst_image->bits.format == PIXMAN_r5g6b5 ||
-             dst_image->bits.format == PIXMAN_b5g6r5)
+    else if (dest_image->bits.format == PIXMAN_r5g6b5 ||
+             dest_image->bits.format == PIXMAN_b5g6r5)
     {
-	src = CONVERT_8888_TO_0565 (src);
+	src = convert_8888_to_0565 (src);
     }
 
-    pixman_fill (dst_image->bits.bits, dst_image->bits.rowstride,
-                 PIXMAN_FORMAT_BPP (dst_image->bits.format),
+    pixman_fill (dest_image->bits.bits, dest_image->bits.rowstride,
+                 PIXMAN_FORMAT_BPP (dest_image->bits.format),
                  dest_x, dest_y,
                  width, height,
                  src);
 }
 
 static void
-fast_composite_src_8888_x888 (pixman_implementation_t *imp,
-                              pixman_op_t              op,
-                              pixman_image_t *         src_image,
-                              pixman_image_t *         mask_image,
-                              pixman_image_t *         dst_image,
-                              int32_t                  src_x,
-                              int32_t                  src_y,
-                              int32_t                  mask_x,
-                              int32_t                  mask_y,
-                              int32_t                  dest_x,
-                              int32_t                  dest_y,
-                              int32_t                  width,
-                              int32_t                  height)
+fast_composite_src_memcpy (pixman_implementation_t *imp,
+			   pixman_composite_info_t *info)
 {
-    uint32_t    *dst;
-    uint32_t    *src;
+    PIXMAN_COMPOSITE_ARGS (info);
+    int bpp = PIXMAN_FORMAT_BPP (dest_image->bits.format) / 8;
+    uint32_t n_bytes = width * bpp;
     int dst_stride, src_stride;
-    uint32_t n_bytes = width * sizeof (uint32_t);
+    uint8_t    *dst;
+    uint8_t    *src;
 
-    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src, 1);
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst, 1);
+    src_stride = src_image->bits.rowstride * 4;
+    dst_stride = dest_image->bits.rowstride * 4;
+
+    src = (uint8_t *)src_image->bits.bits + src_y * src_stride + src_x * bpp;
+    dst = (uint8_t *)dest_image->bits.bits + dest_y * dst_stride + dest_x * bpp;
 
     while (height--)
     {
@@ -1346,247 +1182,239 @@ fast_composite_src_8888_x888 (pixman_implementation_t *imp,
     }
 }
 
-static force_inline pixman_bool_t
-repeat (pixman_repeat_t repeat, int *c, int size)
+FAST_NEAREST (8888_8888_cover, 8888, 8888, uint32_t, uint32_t, SRC, COVER)
+FAST_NEAREST (8888_8888_none, 8888, 8888, uint32_t, uint32_t, SRC, NONE)
+FAST_NEAREST (8888_8888_pad, 8888, 8888, uint32_t, uint32_t, SRC, PAD)
+FAST_NEAREST (8888_8888_normal, 8888, 8888, uint32_t, uint32_t, SRC, NORMAL)
+FAST_NEAREST (x888_8888_cover, x888, 8888, uint32_t, uint32_t, SRC, COVER)
+FAST_NEAREST (x888_8888_pad, x888, 8888, uint32_t, uint32_t, SRC, PAD)
+FAST_NEAREST (x888_8888_normal, x888, 8888, uint32_t, uint32_t, SRC, NORMAL)
+FAST_NEAREST (8888_8888_cover, 8888, 8888, uint32_t, uint32_t, OVER, COVER)
+FAST_NEAREST (8888_8888_none, 8888, 8888, uint32_t, uint32_t, OVER, NONE)
+FAST_NEAREST (8888_8888_pad, 8888, 8888, uint32_t, uint32_t, OVER, PAD)
+FAST_NEAREST (8888_8888_normal, 8888, 8888, uint32_t, uint32_t, OVER, NORMAL)
+FAST_NEAREST (8888_565_cover, 8888, 0565, uint32_t, uint16_t, SRC, COVER)
+FAST_NEAREST (8888_565_none, 8888, 0565, uint32_t, uint16_t, SRC, NONE)
+FAST_NEAREST (8888_565_pad, 8888, 0565, uint32_t, uint16_t, SRC, PAD)
+FAST_NEAREST (8888_565_normal, 8888, 0565, uint32_t, uint16_t, SRC, NORMAL)
+FAST_NEAREST (565_565_normal, 0565, 0565, uint16_t, uint16_t, SRC, NORMAL)
+FAST_NEAREST (8888_565_cover, 8888, 0565, uint32_t, uint16_t, OVER, COVER)
+FAST_NEAREST (8888_565_none, 8888, 0565, uint32_t, uint16_t, OVER, NONE)
+FAST_NEAREST (8888_565_pad, 8888, 0565, uint32_t, uint16_t, OVER, PAD)
+FAST_NEAREST (8888_565_normal, 8888, 0565, uint32_t, uint16_t, OVER, NORMAL)
+
+#define REPEAT_MIN_WIDTH    32
+
+static void
+fast_composite_tiled_repeat (pixman_implementation_t *imp,
+			     pixman_composite_info_t *info)
 {
-    if (repeat == PIXMAN_REPEAT_NONE)
+    PIXMAN_COMPOSITE_ARGS (info);
+    pixman_composite_func_t func;
+    pixman_format_code_t mask_format;
+    uint32_t src_flags, mask_flags;
+    int32_t sx, sy;
+    int32_t width_remain;
+    int32_t num_pixels;
+    int32_t src_width;
+    int32_t i, j;
+    pixman_image_t extended_src_image;
+    uint32_t extended_src[REPEAT_MIN_WIDTH * 2];
+    pixman_bool_t need_src_extension;
+    uint32_t *src_line;
+    int32_t src_stride;
+    int32_t src_bpp;
+    pixman_composite_info_t info2 = *info;
+
+    src_flags = (info->src_flags & ~FAST_PATH_NORMAL_REPEAT) |
+		    FAST_PATH_SAMPLES_COVER_CLIP_NEAREST;
+
+    if (mask_image)
     {
-	if (*c < 0 || *c >= size)
-	    return FALSE;
+	mask_format = mask_image->common.extended_format_code;
+	mask_flags = info->mask_flags;
     }
-    else if (repeat == PIXMAN_REPEAT_NORMAL)
+    else
     {
-	while (*c >= size)
-	    *c -= size;
-	while (*c < 0)
-	    *c += size;
+	mask_format = PIXMAN_null;
+	mask_flags = FAST_PATH_IS_OPAQUE;
     }
-    else if (repeat == PIXMAN_REPEAT_PAD)
+
+    _pixman_implementation_lookup_composite (
+	imp->toplevel, info->op,
+	src_image->common.extended_format_code, src_flags,
+	mask_format, mask_flags,
+	dest_image->common.extended_format_code, info->dest_flags,
+	&imp, &func);
+
+    src_bpp = PIXMAN_FORMAT_BPP (src_image->bits.format);
+
+    if (src_image->bits.width < REPEAT_MIN_WIDTH		&&
+	(src_bpp == 32 || src_bpp == 16 || src_bpp == 8)	&&
+	!src_image->bits.indexed)
     {
-	*c = CLIP (*c, 0, size - 1);
+	sx = src_x;
+	sx = MOD (sx, src_image->bits.width);
+	sx += width;
+	src_width = 0;
+
+	while (src_width < REPEAT_MIN_WIDTH && src_width <= sx)
+	    src_width += src_image->bits.width;
+
+	src_stride = (src_width * (src_bpp >> 3) + 3) / (int) sizeof (uint32_t);
+
+	/* Initialize/validate stack-allocated temporary image */
+	_pixman_bits_image_init (&extended_src_image, src_image->bits.format,
+				 src_width, 1, &extended_src[0], src_stride,
+				 FALSE);
+	_pixman_image_validate (&extended_src_image);
+
+	info2.src_image = &extended_src_image;
+	need_src_extension = TRUE;
     }
-    else /* REFLECT */
+    else
     {
-	*c = MOD (*c, size * 2);
-	if (*c >= size)
-	    *c = size * 2 - *c - 1;
+	src_width = src_image->bits.width;
+	need_src_extension = FALSE;
     }
-    return TRUE;
-}
 
-/* A macroified version of specialized nearest scalers for some
- * common 8888 and 565 formats. It supports SRC and OVER ops.
- *
- * There are two repeat versions, one that handles repeat normal,
- * and one without repeat handling that only works if the src region
- * used is completely covered by the pre-repeated source samples.
- *
- * The loops are unrolled to process two pixels per iteration for better
- * performance on most CPU architectures (superscalar processors
- * can issue several operations simultaneously, other processors can hide
- * instructions latencies by pipelining operations). Unrolling more
- * does not make much sense because the compiler will start running out
- * of spare registers soon.
- */
+    sx = src_x;
+    sy = src_y;
+
+    while (--height >= 0)
+    {
+	sx = MOD (sx, src_width);
+	sy = MOD (sy, src_image->bits.height);
+
+	if (need_src_extension)
+	{
+	    if (src_bpp == 32)
+	    {
+		PIXMAN_IMAGE_GET_LINE (src_image, 0, sy, uint32_t, src_stride, src_line, 1);
+
+		for (i = 0; i < src_width; )
+		{
+		    for (j = 0; j < src_image->bits.width; j++, i++)
+			extended_src[i] = src_line[j];
+		}
+	    }
+	    else if (src_bpp == 16)
+	    {
+		uint16_t *src_line_16;
+
+		PIXMAN_IMAGE_GET_LINE (src_image, 0, sy, uint16_t, src_stride,
+				       src_line_16, 1);
+		src_line = (uint32_t*)src_line_16;
+
+		for (i = 0; i < src_width; )
+		{
+		    for (j = 0; j < src_image->bits.width; j++, i++)
+			((uint16_t*)extended_src)[i] = ((uint16_t*)src_line)[j];
+		}
+	    }
+	    else if (src_bpp == 8)
+	    {
+		uint8_t *src_line_8;
+
+		PIXMAN_IMAGE_GET_LINE (src_image, 0, sy, uint8_t, src_stride,
+				       src_line_8, 1);
+		src_line = (uint32_t*)src_line_8;
+
+		for (i = 0; i < src_width; )
+		{
+		    for (j = 0; j < src_image->bits.width; j++, i++)
+			((uint8_t*)extended_src)[i] = ((uint8_t*)src_line)[j];
+		}
+	    }
+
+	    info2.src_y = 0;
+	}
+	else
+	{
+	    info2.src_y = sy;
+	}
+
+	width_remain = width;
 
-#define GET_8888_ALPHA(s) ((s) >> 24)
- /* This is not actually used since we don't have an OVER with
-    565 source, but it is needed to build. */
-#define GET_0565_ALPHA(s) 0xff
-
-#define FAST_NEAREST(scale_func_name, SRC_FORMAT, DST_FORMAT,					\
-		     src_type_t, dst_type_t, OP, do_repeat)					\
-static void											\
-fast_composite_scaled_nearest_ ## scale_func_name ## _ ## OP (pixman_implementation_t *imp,	\
-							      pixman_op_t              op,      \
-							      pixman_image_t *         src_image, \
-							      pixman_image_t *         mask_image, \
-							      pixman_image_t *         dst_image, \
-							      int32_t                  src_x,   \
-							      int32_t                  src_y,   \
-							      int32_t                  mask_x,  \
-							      int32_t                  mask_y,  \
-							      int32_t                  dst_x,   \
-							      int32_t                  dst_y,   \
-							      int32_t                  width,   \
-							      int32_t                  height)  \
-{												\
-    dst_type_t *dst_line;									\
-    src_type_t *src_first_line;									\
-    uint32_t   d;										\
-    src_type_t s1, s2;										\
-    uint8_t   a1, a2;										\
-    int       w;										\
-    int       x1, x2, y;									\
-    pixman_fixed_t orig_vx;									\
-    pixman_fixed_t max_vx, max_vy;								\
-    pixman_vector_t v;										\
-    pixman_fixed_t vx, vy;									\
-    pixman_fixed_t unit_x, unit_y;								\
-												\
-    src_type_t *src;										\
-    dst_type_t *dst;										\
-    int       src_stride, dst_stride;								\
-												\
-    if (PIXMAN_OP_ ## OP != PIXMAN_OP_SRC && PIXMAN_OP_ ## OP != PIXMAN_OP_OVER)		\
-	abort();										\
-												\
-    PIXMAN_IMAGE_GET_LINE (dst_image, dst_x, dst_y, dst_type_t, dst_stride, dst_line, 1);	\
-    /* pass in 0 instead of src_x and src_y because src_x and src_y need to be			\
-     * transformed from destination space to source space */					\
-    PIXMAN_IMAGE_GET_LINE (src_image, 0, 0, src_type_t, src_stride, src_first_line, 1);		\
-												\
-    /* reference point is the center of the pixel */						\
-    v.vector[0] = pixman_int_to_fixed (src_x) + pixman_fixed_1 / 2;				\
-    v.vector[1] = pixman_int_to_fixed (src_y) + pixman_fixed_1 / 2;				\
-    v.vector[2] = pixman_fixed_1;								\
-												\
-    if (!pixman_transform_point_3d (src_image->common.transform, &v))				\
-	return;											\
-												\
-    unit_x = src_image->common.transform->matrix[0][0];						\
-    unit_y = src_image->common.transform->matrix[1][1];						\
-												\
-    /* Round down to closest integer, ensuring that 0.5 rounds to 0, not 1 */			\
-    v.vector[0] -= pixman_fixed_e;								\
-    v.vector[1] -= pixman_fixed_e;								\
-												\
-    vx = v.vector[0];										\
-    vy = v.vector[1];										\
-												\
-    if (do_repeat)										\
-    {												\
-	/* Clamp repeating positions inside the actual samples */				\
-	max_vx = src_image->bits.width << 16;							\
-	max_vy = src_image->bits.height << 16;							\
-												\
-	repeat (PIXMAN_REPEAT_NORMAL, &vx, max_vx);						\
-	repeat (PIXMAN_REPEAT_NORMAL, &vy, max_vy);						\
-    }												\
-												\
-    orig_vx = vx;										\
-												\
-    while (--height >= 0)									\
-    {												\
-	dst = dst_line;										\
-	dst_line += dst_stride;									\
-												\
-	y = vy >> 16;										\
-	vy += unit_y;										\
-	if (do_repeat)										\
-	    repeat (PIXMAN_REPEAT_NORMAL, &vy, max_vy);						\
-												\
-	src = src_first_line + src_stride * y;							\
-												\
-	w = width;										\
-	vx = orig_vx;										\
-	while ((w -= 2) >= 0)									\
-	{											\
-	    x1 = vx >> 16;									\
-	    vx += unit_x;									\
-	    if (do_repeat)									\
-	    {											\
-		/* This works because we know that unit_x is positive */			\
-		while (vx >= max_vx)								\
-		    vx -= max_vx;								\
-	    }											\
-	    s1 = src[x1];									\
-												\
-	    x2 = vx >> 16;									\
-	    vx += unit_x;									\
-	    if (do_repeat)									\
-	    {											\
-		/* This works because we know that unit_x is positive */			\
-		while (vx >= max_vx)								\
-		    vx -= max_vx;								\
-	    }											\
-	    s2 = src[x2];									\
-												\
-	    if (PIXMAN_OP_ ## OP == PIXMAN_OP_OVER)						\
-	    {											\
-		a1 = GET_ ## SRC_FORMAT ## _ALPHA(s1);						\
-		a2 = GET_ ## SRC_FORMAT ## _ALPHA(s2);						\
-												\
-		if (a1 == 0xff)									\
-		{										\
-		    *dst = CONVERT_ ## SRC_FORMAT ## _TO_ ## DST_FORMAT (s1);			\
-		}										\
-		else if (s1)									\
-		{										\
-		    d = CONVERT_## DST_FORMAT ## _TO_8888 (*dst);				\
-		    a1 ^= 0xff;									\
-		    UN8x4_MUL_UN8_ADD_UN8x4 (d, a1, s1);					\
-		    *dst = CONVERT_8888_TO_ ## DST_FORMAT (d);					\
-		}										\
-		dst++;										\
-												\
-		if (a2 == 0xff)									\
-		{										\
-		    *dst = CONVERT_ ## SRC_FORMAT ## _TO_ ## DST_FORMAT (s2);			\
-		}										\
-		else if (s2)									\
-		{										\
-		    d = CONVERT_## DST_FORMAT ## _TO_8888 (*dst);				\
-		    a2 ^= 0xff;									\
-		    UN8x4_MUL_UN8_ADD_UN8x4 (d, a2, s2);					\
-		    *dst = CONVERT_8888_TO_ ## DST_FORMAT (d);					\
-		}										\
-		dst++;										\
-	    }											\
-	    else /* PIXMAN_OP_SRC */								\
-	    {											\
-		    *dst++ = CONVERT_ ## SRC_FORMAT ## _TO_ ## DST_FORMAT (s1);			\
-		    *dst++ = CONVERT_ ## SRC_FORMAT ## _TO_ ## DST_FORMAT (s2);			\
-	    }											\
-	}											\
-												\
-	if (w & 1)										\
-	{											\
-	    x1 = vx >> 16;									\
-	    vx += unit_x;									\
-	    if (do_repeat)									\
-	    {											\
-		/* This works because we know that unit_x is positive */			\
-		while (vx >= max_vx)								\
-		    vx -= max_vx;								\
-	    }											\
-	    s1 = src[x1];									\
-												\
-	    if (PIXMAN_OP_ ## OP == PIXMAN_OP_OVER)						\
-	    {											\
-		a1 = GET_ ## SRC_FORMAT ## _ALPHA(s1);						\
-												\
-		if (a1 == 0xff)									\
-		{										\
-		    *dst = CONVERT_ ## SRC_FORMAT ## _TO_ ## DST_FORMAT (s1);			\
-		}										\
-		else if (s1)									\
-		{										\
-		    d = CONVERT_## DST_FORMAT ## _TO_8888 (*dst);				\
-		    a1 ^= 0xff;									\
-		    UN8x4_MUL_UN8_ADD_UN8x4 (d, a1, s1);					\
-		    *dst = CONVERT_8888_TO_ ## DST_FORMAT (d);					\
-		}										\
-		dst++;										\
-	    }											\
-	    else /* PIXMAN_OP_SRC */								\
-	    {											\
-		    *dst++ = CONVERT_ ## SRC_FORMAT ## _TO_ ## DST_FORMAT (s1);			\
-	    }											\
-	}											\
-    }												\
+	while (width_remain > 0)
+	{
+	    num_pixels = src_width - sx;
+
+	    if (num_pixels > width_remain)
+		num_pixels = width_remain;
+
+	    info2.src_x = sx;
+	    info2.width = num_pixels;
+	    info2.height = 1;
+
+	    func (imp, &info2);
+
+	    width_remain -= num_pixels;
+	    info2.mask_x += num_pixels;
+	    info2.dest_x += num_pixels;
+	    sx = 0;
+	}
+
+	sx = src_x;
+	sy++;
+	info2.mask_x = info->mask_x;
+	info2.mask_y++;
+	info2.dest_x = info->dest_x;
+	info2.dest_y++;
+    }
+
+    if (need_src_extension)
+	_pixman_image_fini (&extended_src_image);
 }
 
-FAST_NEAREST(x888_x888_none, 8888, 8888, uint32_t, uint32_t, SRC, /*repeat: */ 0);
-FAST_NEAREST(x888_x888_normal, 8888, 8888, uint32_t, uint32_t, SRC, /*repeat: */ 1);
-FAST_NEAREST(x888_x888_none, 8888, 8888, uint32_t, uint32_t, OVER, /*repeat: */ 0);
-FAST_NEAREST(x888_x888_normal, 8888, 8888, uint32_t, uint32_t, OVER, /*repeat: */ 1);
-FAST_NEAREST(x888_565_none, 8888, 0565, uint32_t, uint16_t, SRC, /*repeat: */ 0);
-FAST_NEAREST(x888_565_normal, 8888, 0565, uint32_t, uint16_t, SRC, /*repeat: */ 1);
-FAST_NEAREST(565_565_none, 0565, 0565, uint16_t, uint16_t, SRC, /*repeat: */ 0);
-FAST_NEAREST(565_565_normal, 0565, 0565, uint16_t, uint16_t, SRC, /*repeat: */ 1);
-FAST_NEAREST(8888_565_none, 8888, 0565, uint32_t, uint16_t, OVER, /*repeat: */ 0);
-FAST_NEAREST(8888_565_normal, 8888, 0565, uint32_t, uint16_t, OVER, /*repeat: */ 1);
+/* Use more unrolling for src_0565_0565 because it is typically CPU bound */
+static force_inline void
+scaled_nearest_scanline_565_565_SRC (uint16_t *       dst,
+				     const uint16_t * src,
+				     int32_t          w,
+				     pixman_fixed_t   vx,
+				     pixman_fixed_t   unit_x,
+				     pixman_fixed_t   max_vx,
+				     pixman_bool_t    fully_transparent_src)
+{
+    uint16_t tmp1, tmp2, tmp3, tmp4;
+    while ((w -= 4) >= 0)
+    {
+	tmp1 = *(src + pixman_fixed_to_int (vx));
+	vx += unit_x;
+	tmp2 = *(src + pixman_fixed_to_int (vx));
+	vx += unit_x;
+	tmp3 = *(src + pixman_fixed_to_int (vx));
+	vx += unit_x;
+	tmp4 = *(src + pixman_fixed_to_int (vx));
+	vx += unit_x;
+	*dst++ = tmp1;
+	*dst++ = tmp2;
+	*dst++ = tmp3;
+	*dst++ = tmp4;
+    }
+    if (w & 2)
+    {
+	tmp1 = *(src + pixman_fixed_to_int (vx));
+	vx += unit_x;
+	tmp2 = *(src + pixman_fixed_to_int (vx));
+	vx += unit_x;
+	*dst++ = tmp1;
+	*dst++ = tmp2;
+    }
+    if (w & 1)
+	*dst = *(src + pixman_fixed_to_int (vx));
+}
+
+FAST_NEAREST_MAINLOOP (565_565_cover_SRC,
+		       scaled_nearest_scanline_565_565_SRC,
+		       uint16_t, uint16_t, COVER)
+FAST_NEAREST_MAINLOOP (565_565_none_SRC,
+		       scaled_nearest_scanline_565_565_SRC,
+		       uint16_t, uint16_t, NONE)
+FAST_NEAREST_MAINLOOP (565_565_pad_SRC,
+		       scaled_nearest_scanline_565_565_SRC,
+		       uint16_t, uint16_t, PAD)
 
 static force_inline uint32_t
 fetch_nearest (pixman_repeat_t src_repeat,
@@ -1595,7 +1423,7 @@ fetch_nearest (pixman_repeat_t src_repeat,
 {
     if (repeat (src_repeat, &x, src_width))
     {
-	if (format == PIXMAN_x8r8g8b8)
+	if (format == PIXMAN_x8r8g8b8 || format == PIXMAN_x8b8g8r8)
 	    return *(src + x) | 0xff000000;
 	else
 	    return *(src + x);
@@ -1628,19 +1456,9 @@ combine_src (uint32_t s, uint32_t *dst)
 
 static void
 fast_composite_scaled_nearest (pixman_implementation_t *imp,
-			       pixman_op_t              op,
-			       pixman_image_t *         src_image,
-			       pixman_image_t *         mask_image,
-			       pixman_image_t *         dst_image,
-			       int32_t                  src_x,
-			       int32_t                  src_y,
-			       int32_t                  mask_x,
-			       int32_t                  mask_y,
-			       int32_t                  dest_x,
-			       int32_t                  dest_y,
-			       int32_t                  width,
-			       int32_t                  height)
+			       pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t       *dst_line;
     uint32_t       *src_line;
     int             dst_stride, src_stride;
@@ -1651,7 +1469,7 @@ fast_composite_scaled_nearest (pixman_implementation_t *imp,
     pixman_vector_t v;
     pixman_fixed_t vy;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     /* pass in 0 instead of src_x and src_y because src_x and src_y need to be
      * transformed from destination space to source space
      */
@@ -1748,6 +1566,252 @@ fast_composite_scaled_nearest (pixman_implementation_t *imp,
     }
 }
 
+#define CACHE_LINE_SIZE 64
+
+#define FAST_SIMPLE_ROTATE(suffix, pix_type)                                  \
+                                                                              \
+static void                                                                   \
+blt_rotated_90_trivial_##suffix (pix_type       *dst,                         \
+				 int             dst_stride,                  \
+				 const pix_type *src,                         \
+				 int             src_stride,                  \
+				 int             w,                           \
+				 int             h)                           \
+{                                                                             \
+    int x, y;                                                                 \
+    for (y = 0; y < h; y++)                                                   \
+    {                                                                         \
+	const pix_type *s = src + (h - y - 1);                                \
+	pix_type *d = dst + dst_stride * y;                                   \
+	for (x = 0; x < w; x++)                                               \
+	{                                                                     \
+	    *d++ = *s;                                                        \
+	    s += src_stride;                                                  \
+	}                                                                     \
+    }                                                                         \
+}                                                                             \
+                                                                              \
+static void                                                                   \
+blt_rotated_270_trivial_##suffix (pix_type       *dst,                        \
+				  int             dst_stride,                 \
+				  const pix_type *src,                        \
+				  int             src_stride,                 \
+				  int             w,                          \
+				  int             h)                          \
+{                                                                             \
+    int x, y;                                                                 \
+    for (y = 0; y < h; y++)                                                   \
+    {                                                                         \
+	const pix_type *s = src + src_stride * (w - 1) + y;                   \
+	pix_type *d = dst + dst_stride * y;                                   \
+	for (x = 0; x < w; x++)                                               \
+	{                                                                     \
+	    *d++ = *s;                                                        \
+	    s -= src_stride;                                                  \
+	}                                                                     \
+    }                                                                         \
+}                                                                             \
+                                                                              \
+static void                                                                   \
+blt_rotated_90_##suffix (pix_type       *dst,                                 \
+			 int             dst_stride,                          \
+			 const pix_type *src,                                 \
+			 int             src_stride,                          \
+			 int             W,                                   \
+			 int             H)                                   \
+{                                                                             \
+    int x;                                                                    \
+    int leading_pixels = 0, trailing_pixels = 0;                              \
+    const int TILE_SIZE = CACHE_LINE_SIZE / sizeof(pix_type);                 \
+                                                                              \
+    /*                                                                        \
+     * split processing into handling destination as TILE_SIZExH cache line   \
+     * aligned vertical stripes (optimistically assuming that destination     \
+     * stride is a multiple of cache line, if not - it will be just a bit     \
+     * slower)                                                                \
+     */                                                                       \
+                                                                              \
+    if ((uintptr_t)dst & (CACHE_LINE_SIZE - 1))                               \
+    {                                                                         \
+	leading_pixels = TILE_SIZE - (((uintptr_t)dst &                       \
+			    (CACHE_LINE_SIZE - 1)) / sizeof(pix_type));       \
+	if (leading_pixels > W)                                               \
+	    leading_pixels = W;                                               \
+                                                                              \
+	/* unaligned leading part NxH (where N < TILE_SIZE) */                \
+	blt_rotated_90_trivial_##suffix (                                     \
+	    dst,                                                              \
+	    dst_stride,                                                       \
+	    src,                                                              \
+	    src_stride,                                                       \
+	    leading_pixels,                                                   \
+	    H);                                                               \
+	                                                                      \
+	dst += leading_pixels;                                                \
+	src += leading_pixels * src_stride;                                   \
+	W -= leading_pixels;                                                  \
+    }                                                                         \
+                                                                              \
+    if ((uintptr_t)(dst + W) & (CACHE_LINE_SIZE - 1))                         \
+    {                                                                         \
+	trailing_pixels = (((uintptr_t)(dst + W) &                            \
+			    (CACHE_LINE_SIZE - 1)) / sizeof(pix_type));       \
+	if (trailing_pixels > W)                                              \
+	    trailing_pixels = W;                                              \
+	W -= trailing_pixels;                                                 \
+    }                                                                         \
+                                                                              \
+    for (x = 0; x < W; x += TILE_SIZE)                                        \
+    {                                                                         \
+	/* aligned middle part TILE_SIZExH */                                 \
+	blt_rotated_90_trivial_##suffix (                                     \
+	    dst + x,                                                          \
+	    dst_stride,                                                       \
+	    src + src_stride * x,                                             \
+	    src_stride,                                                       \
+	    TILE_SIZE,                                                        \
+	    H);                                                               \
+    }                                                                         \
+                                                                              \
+    if (trailing_pixels)                                                      \
+    {                                                                         \
+	/* unaligned trailing part NxH (where N < TILE_SIZE) */               \
+	blt_rotated_90_trivial_##suffix (                                     \
+	    dst + W,                                                          \
+	    dst_stride,                                                       \
+	    src + W * src_stride,                                             \
+	    src_stride,                                                       \
+	    trailing_pixels,                                                  \
+	    H);                                                               \
+    }                                                                         \
+}                                                                             \
+                                                                              \
+static void                                                                   \
+blt_rotated_270_##suffix (pix_type       *dst,                                \
+			  int             dst_stride,                         \
+			  const pix_type *src,                                \
+			  int             src_stride,                         \
+			  int             W,                                  \
+			  int             H)                                  \
+{                                                                             \
+    int x;                                                                    \
+    int leading_pixels = 0, trailing_pixels = 0;                              \
+    const int TILE_SIZE = CACHE_LINE_SIZE / sizeof(pix_type);                 \
+                                                                              \
+    /*                                                                        \
+     * split processing into handling destination as TILE_SIZExH cache line   \
+     * aligned vertical stripes (optimistically assuming that destination     \
+     * stride is a multiple of cache line, if not - it will be just a bit     \
+     * slower)                                                                \
+     */                                                                       \
+                                                                              \
+    if ((uintptr_t)dst & (CACHE_LINE_SIZE - 1))                               \
+    {                                                                         \
+	leading_pixels = TILE_SIZE - (((uintptr_t)dst &                       \
+			    (CACHE_LINE_SIZE - 1)) / sizeof(pix_type));       \
+	if (leading_pixels > W)                                               \
+	    leading_pixels = W;                                               \
+                                                                              \
+	/* unaligned leading part NxH (where N < TILE_SIZE) */                \
+	blt_rotated_270_trivial_##suffix (                                    \
+	    dst,                                                              \
+	    dst_stride,                                                       \
+	    src + src_stride * (W - leading_pixels),                          \
+	    src_stride,                                                       \
+	    leading_pixels,                                                   \
+	    H);                                                               \
+	                                                                      \
+	dst += leading_pixels;                                                \
+	W -= leading_pixels;                                                  \
+    }                                                                         \
+                                                                              \
+    if ((uintptr_t)(dst + W) & (CACHE_LINE_SIZE - 1))                         \
+    {                                                                         \
+	trailing_pixels = (((uintptr_t)(dst + W) &                            \
+			    (CACHE_LINE_SIZE - 1)) / sizeof(pix_type));       \
+	if (trailing_pixels > W)                                              \
+	    trailing_pixels = W;                                              \
+	W -= trailing_pixels;                                                 \
+	src += trailing_pixels * src_stride;                                  \
+    }                                                                         \
+                                                                              \
+    for (x = 0; x < W; x += TILE_SIZE)                                        \
+    {                                                                         \
+	/* aligned middle part TILE_SIZExH */                                 \
+	blt_rotated_270_trivial_##suffix (                                    \
+	    dst + x,                                                          \
+	    dst_stride,                                                       \
+	    src + src_stride * (W - x - TILE_SIZE),                           \
+	    src_stride,                                                       \
+	    TILE_SIZE,                                                        \
+	    H);                                                               \
+    }                                                                         \
+                                                                              \
+    if (trailing_pixels)                                                      \
+    {                                                                         \
+	/* unaligned trailing part NxH (where N < TILE_SIZE) */               \
+	blt_rotated_270_trivial_##suffix (                                    \
+	    dst + W,                                                          \
+	    dst_stride,                                                       \
+	    src - trailing_pixels * src_stride,                               \
+	    src_stride,                                                       \
+	    trailing_pixels,                                                  \
+	    H);                                                               \
+    }                                                                         \
+}                                                                             \
+                                                                              \
+static void                                                                   \
+fast_composite_rotate_90_##suffix (pixman_implementation_t *imp,              \
+				   pixman_composite_info_t *info)	      \
+{									      \
+    PIXMAN_COMPOSITE_ARGS (info);					      \
+    pix_type       *dst_line;						      \
+    pix_type       *src_line;                                                 \
+    int             dst_stride, src_stride;                                   \
+    int             src_x_t, src_y_t;                                         \
+                                                                              \
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, pix_type,              \
+			   dst_stride, dst_line, 1);                          \
+    src_x_t = -src_y + pixman_fixed_to_int (                                  \
+				src_image->common.transform->matrix[0][2] +   \
+				pixman_fixed_1 / 2 - pixman_fixed_e) - height;\
+    src_y_t = src_x + pixman_fixed_to_int (                                   \
+				src_image->common.transform->matrix[1][2] +   \
+				pixman_fixed_1 / 2 - pixman_fixed_e);         \
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x_t, src_y_t, pix_type,             \
+			   src_stride, src_line, 1);                          \
+    blt_rotated_90_##suffix (dst_line, dst_stride, src_line, src_stride,      \
+			     width, height);                                  \
+}                                                                             \
+                                                                              \
+static void                                                                   \
+fast_composite_rotate_270_##suffix (pixman_implementation_t *imp,             \
+				    pixman_composite_info_t *info)            \
+{                                                                             \
+    PIXMAN_COMPOSITE_ARGS (info);					      \
+    pix_type       *dst_line;						      \
+    pix_type       *src_line;                                                 \
+    int             dst_stride, src_stride;                                   \
+    int             src_x_t, src_y_t;                                         \
+                                                                              \
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, pix_type,              \
+			   dst_stride, dst_line, 1);                          \
+    src_x_t = src_y + pixman_fixed_to_int (                                   \
+				src_image->common.transform->matrix[0][2] +   \
+				pixman_fixed_1 / 2 - pixman_fixed_e);         \
+    src_y_t = -src_x + pixman_fixed_to_int (                                  \
+				src_image->common.transform->matrix[1][2] +   \
+				pixman_fixed_1 / 2 - pixman_fixed_e) - width; \
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x_t, src_y_t, pix_type,             \
+			   src_stride, src_line, 1);                          \
+    blt_rotated_270_##suffix (dst_line, dst_stride, src_line, src_stride,     \
+			      width, height);                                 \
+}
+
+FAST_SIMPLE_ROTATE (8, uint8_t)
+FAST_SIMPLE_ROTATE (565, uint16_t)
+FAST_SIMPLE_ROTATE (8888, uint32_t)
+
 static const pixman_fast_path_t c_fast_paths[] =
 {
     PIXMAN_STD_FAST_PATH (OVER, solid, a8, r5g6b5, fast_composite_over_n_8_0565),
@@ -1780,73 +1844,66 @@ static const pixman_fast_path_t c_fast_paths[] =
     PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, a8b8g8r8, fast_composite_over_8888_8888),
     PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, x8b8g8r8, fast_composite_over_8888_8888),
     PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, b5g6r5, fast_composite_over_8888_0565),
+    PIXMAN_STD_FAST_PATH (ADD, r5g6b5, null, r5g6b5, fast_composite_add_0565_0565),
+    PIXMAN_STD_FAST_PATH (ADD, b5g6r5, null, b5g6r5, fast_composite_add_0565_0565),
     PIXMAN_STD_FAST_PATH (ADD, a8r8g8b8, null, a8r8g8b8, fast_composite_add_8888_8888),
     PIXMAN_STD_FAST_PATH (ADD, a8b8g8r8, null, a8b8g8r8, fast_composite_add_8888_8888),
-    PIXMAN_STD_FAST_PATH (ADD, a8, null, a8, fast_composite_add_8000_8000),
-    PIXMAN_STD_FAST_PATH (ADD, a1, null, a1, fast_composite_add_1000_1000),
+    PIXMAN_STD_FAST_PATH (ADD, a8, null, a8, fast_composite_add_8_8),
+    PIXMAN_STD_FAST_PATH (ADD, a1, null, a1, fast_composite_add_1_1),
     PIXMAN_STD_FAST_PATH_CA (ADD, solid, a8r8g8b8, a8r8g8b8, fast_composite_add_n_8888_8888_ca),
     PIXMAN_STD_FAST_PATH (ADD, solid, a8, a8, fast_composite_add_n_8_8),
     PIXMAN_STD_FAST_PATH (SRC, solid, null, a8r8g8b8, fast_composite_solid_fill),
     PIXMAN_STD_FAST_PATH (SRC, solid, null, x8r8g8b8, fast_composite_solid_fill),
     PIXMAN_STD_FAST_PATH (SRC, solid, null, a8b8g8r8, fast_composite_solid_fill),
     PIXMAN_STD_FAST_PATH (SRC, solid, null, x8b8g8r8, fast_composite_solid_fill),
+    PIXMAN_STD_FAST_PATH (SRC, solid, null, a1, fast_composite_solid_fill),
     PIXMAN_STD_FAST_PATH (SRC, solid, null, a8, fast_composite_solid_fill),
     PIXMAN_STD_FAST_PATH (SRC, solid, null, r5g6b5, fast_composite_solid_fill),
-    PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, x8r8g8b8, fast_composite_src_8888_x888),
-    PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, x8r8g8b8, fast_composite_src_8888_x888),
-    PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, x8b8g8r8, fast_composite_src_8888_x888),
-    PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, x8b8g8r8, fast_composite_src_8888_x888),
-    PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, r5g6b5, fast_composite_src_x888_0565),
-    PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, r5g6b5, fast_composite_src_x888_0565),
-    PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, b5g6r5, fast_composite_src_x888_0565),
-    PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, b5g6r5, fast_composite_src_x888_0565),
+    PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, a8r8g8b8, fast_composite_src_x888_8888),
+    PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, a8b8g8r8, fast_composite_src_x888_8888),
+    PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, x8r8g8b8, fast_composite_src_memcpy),
+    PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, a8r8g8b8, fast_composite_src_memcpy),
+    PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, x8r8g8b8, fast_composite_src_memcpy),
+    PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, x8b8g8r8, fast_composite_src_memcpy),
+    PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, a8b8g8r8, fast_composite_src_memcpy),
+    PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, x8b8g8r8, fast_composite_src_memcpy),
+    PIXMAN_STD_FAST_PATH (SRC, b8g8r8a8, null, b8g8r8x8, fast_composite_src_memcpy),
+    PIXMAN_STD_FAST_PATH (SRC, b8g8r8a8, null, b8g8r8a8, fast_composite_src_memcpy),
+    PIXMAN_STD_FAST_PATH (SRC, b8g8r8x8, null, b8g8r8x8, fast_composite_src_memcpy),
+    PIXMAN_STD_FAST_PATH (SRC, r5g6b5, null, r5g6b5, fast_composite_src_memcpy),
+    PIXMAN_STD_FAST_PATH (SRC, b5g6r5, null, b5g6r5, fast_composite_src_memcpy),
+    PIXMAN_STD_FAST_PATH (SRC, r8g8b8, null, r8g8b8, fast_composite_src_memcpy),
+    PIXMAN_STD_FAST_PATH (SRC, b8g8r8, null, b8g8r8, fast_composite_src_memcpy),
+    PIXMAN_STD_FAST_PATH (SRC, x1r5g5b5, null, x1r5g5b5, fast_composite_src_memcpy),
+    PIXMAN_STD_FAST_PATH (SRC, a1r5g5b5, null, x1r5g5b5, fast_composite_src_memcpy),
+    PIXMAN_STD_FAST_PATH (SRC, a8, null, a8, fast_composite_src_memcpy),
     PIXMAN_STD_FAST_PATH (IN, a8, null, a8, fast_composite_in_8_8),
     PIXMAN_STD_FAST_PATH (IN, solid, a8, a8, fast_composite_in_n_8_8),
 
-#define SCALED_NEAREST_FLAGS						\
-    (FAST_PATH_SCALE_TRANSFORM	|					\
-     FAST_PATH_NO_ALPHA_MAP	|					\
-     FAST_PATH_NEAREST_FILTER	|					\
-     FAST_PATH_NO_ACCESSORS	|					\
-     FAST_PATH_NO_WIDE_FORMAT)
-
-#define HAS_NORMAL_REPEAT_FLAGS						\
-    (FAST_PATH_NO_REFLECT_REPEAT |					\
-     FAST_PATH_NO_PAD_REPEAT     |					\
-     FAST_PATH_NO_NONE_REPEAT)
-
-#define SIMPLE_NEAREST_FAST_PATH(op,s,d,func)				\
-    {   PIXMAN_OP_ ## op,						\
-	PIXMAN_ ## s,							\
-	SCALED_NEAREST_FLAGS | HAS_NORMAL_REPEAT_FLAGS | FAST_PATH_16BIT_SAFE | FAST_PATH_X_UNIT_POSITIVE, \
-	PIXMAN_null, 0,							\
-	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
-	fast_composite_scaled_nearest_ ## func ## _normal ## _ ## op,	\
-    },									\
-    {   PIXMAN_OP_ ## op,						\
-	PIXMAN_ ## s,							\
-	SCALED_NEAREST_FLAGS | FAST_PATH_SAMPLES_COVER_CLIP,		\
-	PIXMAN_null, 0,							\
-	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
-	fast_composite_scaled_nearest_ ## func ## _none ## _ ## op,	\
-    }
-    SIMPLE_NEAREST_FAST_PATH (SRC, x8r8g8b8, x8r8g8b8, x888_x888),
-    SIMPLE_NEAREST_FAST_PATH (SRC, a8r8g8b8, x8r8g8b8, x888_x888),
-    SIMPLE_NEAREST_FAST_PATH (SRC, x8b8g8r8, x8b8g8r8, x888_x888),
-    SIMPLE_NEAREST_FAST_PATH (SRC, a8b8g8r8, x8b8g8r8, x888_x888),
-
-    SIMPLE_NEAREST_FAST_PATH (SRC, a8r8g8b8, a8r8g8b8, x888_x888),
-    SIMPLE_NEAREST_FAST_PATH (SRC, a8b8g8r8, a8b8g8r8, x888_x888),
-
-    SIMPLE_NEAREST_FAST_PATH (SRC, x8r8g8b8, r5g6b5, x888_565),
-    SIMPLE_NEAREST_FAST_PATH (SRC, a8r8g8b8, r5g6b5, x888_565),
+    SIMPLE_NEAREST_FAST_PATH (SRC, x8r8g8b8, x8r8g8b8, 8888_8888),
+    SIMPLE_NEAREST_FAST_PATH (SRC, a8r8g8b8, x8r8g8b8, 8888_8888),
+    SIMPLE_NEAREST_FAST_PATH (SRC, x8b8g8r8, x8b8g8r8, 8888_8888),
+    SIMPLE_NEAREST_FAST_PATH (SRC, a8b8g8r8, x8b8g8r8, 8888_8888),
+
+    SIMPLE_NEAREST_FAST_PATH (SRC, a8r8g8b8, a8r8g8b8, 8888_8888),
+    SIMPLE_NEAREST_FAST_PATH (SRC, a8b8g8r8, a8b8g8r8, 8888_8888),
+
+    SIMPLE_NEAREST_FAST_PATH (SRC, x8r8g8b8, r5g6b5, 8888_565),
+    SIMPLE_NEAREST_FAST_PATH (SRC, a8r8g8b8, r5g6b5, 8888_565),
 
     SIMPLE_NEAREST_FAST_PATH (SRC, r5g6b5, r5g6b5, 565_565),
 
-    SIMPLE_NEAREST_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, x888_x888),
-    SIMPLE_NEAREST_FAST_PATH (OVER, a8b8g8r8, x8b8g8r8, x888_x888),
-    SIMPLE_NEAREST_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, x888_x888),
-    SIMPLE_NEAREST_FAST_PATH (OVER, a8b8g8r8, a8b8g8r8, x888_x888),
+    SIMPLE_NEAREST_FAST_PATH_COVER (SRC, x8r8g8b8, a8r8g8b8, x888_8888),
+    SIMPLE_NEAREST_FAST_PATH_COVER (SRC, x8b8g8r8, a8b8g8r8, x888_8888),
+    SIMPLE_NEAREST_FAST_PATH_PAD (SRC, x8r8g8b8, a8r8g8b8, x888_8888),
+    SIMPLE_NEAREST_FAST_PATH_PAD (SRC, x8b8g8r8, a8b8g8r8, x888_8888),
+    SIMPLE_NEAREST_FAST_PATH_NORMAL (SRC, x8r8g8b8, a8r8g8b8, x888_8888),
+    SIMPLE_NEAREST_FAST_PATH_NORMAL (SRC, x8b8g8r8, a8b8g8r8, x888_8888),
+
+    SIMPLE_NEAREST_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, 8888_8888),
+    SIMPLE_NEAREST_FAST_PATH (OVER, a8b8g8r8, x8b8g8r8, 8888_8888),
+    SIMPLE_NEAREST_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, 8888_8888),
+    SIMPLE_NEAREST_FAST_PATH (OVER, a8b8g8r8, a8b8g8r8, 8888_8888),
 
     SIMPLE_NEAREST_FAST_PATH (OVER, a8r8g8b8, r5g6b5, 8888_565),
 
@@ -1878,9 +1935,121 @@ static const pixman_fast_path_t c_fast_paths[] =
     NEAREST_FAST_PATH (OVER, x8b8g8r8, a8b8g8r8),
     NEAREST_FAST_PATH (OVER, a8b8g8r8, a8b8g8r8),
 
+#define SIMPLE_ROTATE_FLAGS(angle)					  \
+    (FAST_PATH_ROTATE_ ## angle ## _TRANSFORM	|			  \
+     FAST_PATH_NEAREST_FILTER			|			  \
+     FAST_PATH_SAMPLES_COVER_CLIP_NEAREST	|			  \
+     FAST_PATH_STANDARD_FLAGS)
+
+#define SIMPLE_ROTATE_FAST_PATH(op,s,d,suffix)				  \
+    {   PIXMAN_OP_ ## op,						  \
+	PIXMAN_ ## s, SIMPLE_ROTATE_FLAGS (90),				  \
+	PIXMAN_null, 0,							  \
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				  \
+	fast_composite_rotate_90_##suffix,				  \
+    },									  \
+    {   PIXMAN_OP_ ## op,						  \
+	PIXMAN_ ## s, SIMPLE_ROTATE_FLAGS (270),			  \
+	PIXMAN_null, 0,							  \
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				  \
+	fast_composite_rotate_270_##suffix,				  \
+    }
+
+    SIMPLE_ROTATE_FAST_PATH (SRC, a8r8g8b8, a8r8g8b8, 8888),
+    SIMPLE_ROTATE_FAST_PATH (SRC, a8r8g8b8, x8r8g8b8, 8888),
+    SIMPLE_ROTATE_FAST_PATH (SRC, x8r8g8b8, x8r8g8b8, 8888),
+    SIMPLE_ROTATE_FAST_PATH (SRC, r5g6b5, r5g6b5, 565),
+    SIMPLE_ROTATE_FAST_PATH (SRC, a8, a8, 8),
+
+    /* Simple repeat fast path entry. */
+    {	PIXMAN_OP_any,
+	PIXMAN_any,
+	(FAST_PATH_STANDARD_FLAGS | FAST_PATH_ID_TRANSFORM | FAST_PATH_BITS_IMAGE |
+	 FAST_PATH_NORMAL_REPEAT),
+	PIXMAN_any, 0,
+	PIXMAN_any, FAST_PATH_STD_DEST_FLAGS,
+	fast_composite_tiled_repeat
+    },
+
     {   PIXMAN_OP_NONE	},
 };
 
+#ifdef WORDS_BIGENDIAN
+#define A1_FILL_MASK(n, offs) (((1U << (n)) - 1) << (32 - (offs) - (n)))
+#else
+#define A1_FILL_MASK(n, offs) (((1U << (n)) - 1) << (offs))
+#endif
+
+static force_inline void
+pixman_fill1_line (uint32_t *dst, int offs, int width, int v)
+{
+    if (offs)
+    {
+	int leading_pixels = 32 - offs;
+	if (leading_pixels >= width)
+	{
+	    if (v)
+		*dst |= A1_FILL_MASK (width, offs);
+	    else
+		*dst &= ~A1_FILL_MASK (width, offs);
+	    return;
+	}
+	else
+	{
+	    if (v)
+		*dst++ |= A1_FILL_MASK (leading_pixels, offs);
+	    else
+		*dst++ &= ~A1_FILL_MASK (leading_pixels, offs);
+	    width -= leading_pixels;
+	}
+    }
+    while (width >= 32)
+    {
+	if (v)
+	    *dst++ = 0xFFFFFFFF;
+	else
+	    *dst++ = 0;
+	width -= 32;
+    }
+    if (width > 0)
+    {
+	if (v)
+	    *dst |= A1_FILL_MASK (width, 0);
+	else
+	    *dst &= ~A1_FILL_MASK (width, 0);
+    }
+}
+
+static void
+pixman_fill1 (uint32_t *bits,
+              int       stride,
+              int       x,
+              int       y,
+              int       width,
+              int       height,
+              uint32_t  filler)
+{
+    uint32_t *dst = bits + y * stride + (x >> 5);
+    int offs = x & 31;
+
+    if (filler & 1)
+    {
+	while (height--)
+	{
+	    pixman_fill1_line (dst, offs, width, 1);
+	    dst += stride;
+	}
+    }
+    else
+    {
+	while (height--)
+	{
+	    pixman_fill1_line (dst, offs, width, 0);
+	    dst += stride;
+	}
+    }
+}
+
 static void
 pixman_fill8 (uint32_t *bits,
               int       stride,
@@ -1888,11 +2057,11 @@ pixman_fill8 (uint32_t *bits,
               int       y,
               int       width,
               int       height,
-              uint32_t xor)
+              uint32_t  filler)
 {
     int byte_stride = stride * (int) sizeof (uint32_t);
     uint8_t *dst = (uint8_t *) bits;
-    uint8_t v = xor & 0xff;
+    uint8_t v = filler & 0xff;
     int i;
 
     dst = dst + y * byte_stride + x;
@@ -1913,12 +2082,12 @@ pixman_fill16 (uint32_t *bits,
                int       y,
                int       width,
                int       height,
-               uint32_t xor)
+               uint32_t  filler)
 {
     int short_stride =
 	(stride * (int)sizeof (uint32_t)) / (int)sizeof (uint16_t);
     uint16_t *dst = (uint16_t *)bits;
-    uint16_t v = xor & 0xffff;
+    uint16_t v = filler & 0xffff;
     int i;
 
     dst = dst + y * short_stride + x;
@@ -1939,7 +2108,7 @@ pixman_fill32 (uint32_t *bits,
                int       y,
                int       width,
                int       height,
-               uint32_t  xor)
+               uint32_t  filler)
 {
     int i;
 
@@ -1948,7 +2117,7 @@ pixman_fill32 (uint32_t *bits,
     while (height--)
     {
 	for (i = 0; i < width; ++i)
-	    bits[i] = xor;
+	    bits[i] = filler;
 
 	bits += stride;
     }
@@ -1963,38 +2132,1161 @@ fast_path_fill (pixman_implementation_t *imp,
                 int                      y,
                 int                      width,
                 int                      height,
-                uint32_t		 xor)
+                uint32_t		 filler)
 {
     switch (bpp)
     {
+    case 1:
+	pixman_fill1 (bits, stride, x, y, width, height, filler);
+	break;
+
     case 8:
-	pixman_fill8 (bits, stride, x, y, width, height, xor);
+	pixman_fill8 (bits, stride, x, y, width, height, filler);
 	break;
 
     case 16:
-	pixman_fill16 (bits, stride, x, y, width, height, xor);
+	pixman_fill16 (bits, stride, x, y, width, height, filler);
 	break;
 
     case 32:
-	pixman_fill32 (bits, stride, x, y, width, height, xor);
+	pixman_fill32 (bits, stride, x, y, width, height, filler);
 	break;
 
     default:
-	return _pixman_implementation_fill (
-	    imp->delegate, bits, stride, bpp, x, y, width, height, xor);
-	break;
+	return FALSE;
     }
 
     return TRUE;
 }
 
+/*****************************************************************************/
+
+static uint32_t *
+fast_fetch_r5g6b5 (pixman_iter_t *iter, const uint32_t *mask)
+{
+    int32_t w = iter->width;
+    uint32_t *dst = iter->buffer;
+    const uint16_t *src = (const uint16_t *)iter->bits;
+
+    iter->bits += iter->stride;
+
+    /* Align the source buffer at 4 bytes boundary */
+    if (w > 0 && ((uintptr_t)src & 3))
+    {
+	*dst++ = convert_0565_to_8888 (*src++);
+	w--;
+    }
+    /* Process two pixels per iteration */
+    while ((w -= 2) >= 0)
+    {
+	uint32_t sr, sb, sg, t0, t1;
+	uint32_t s = *(const uint32_t *)src;
+	src += 2;
+	sr = (s >> 8) & 0x00F800F8;
+	sb = (s << 3) & 0x00F800F8;
+	sg = (s >> 3) & 0x00FC00FC;
+	sr |= sr >> 5;
+	sb |= sb >> 5;
+	sg |= sg >> 6;
+	t0 = ((sr << 16) & 0x00FF0000) | ((sg << 8) & 0x0000FF00) |
+	     (sb & 0xFF) | 0xFF000000;
+	t1 = (sr & 0x00FF0000) | ((sg >> 8) & 0x0000FF00) |
+	     (sb >> 16) | 0xFF000000;
+#ifdef WORDS_BIGENDIAN
+	*dst++ = t1;
+	*dst++ = t0;
+#else
+	*dst++ = t0;
+	*dst++ = t1;
+#endif
+    }
+    if (w & 1)
+    {
+	*dst = convert_0565_to_8888 (*src);
+    }
+
+    return iter->buffer;
+}
+
+static uint32_t *
+fast_dest_fetch_noop (pixman_iter_t *iter, const uint32_t *mask)
+{
+    iter->bits += iter->stride;
+    return iter->buffer;
+}
+
+/* Helper function for a workaround, which tries to ensure that 0x1F001F
+ * constant is always allocated in a register on RISC architectures.
+ */
+static force_inline uint32_t
+convert_8888_to_0565_workaround (uint32_t s, uint32_t x1F001F)
+{
+    uint32_t a, b;
+    a = (s >> 3) & x1F001F;
+    b = s & 0xFC00;
+    a |= a >> 5;
+    a |= b >> 5;
+    return a;
+}
+
+static void
+fast_write_back_r5g6b5 (pixman_iter_t *iter)
+{
+    int32_t w = iter->width;
+    uint16_t *dst = (uint16_t *)(iter->bits - iter->stride);
+    const uint32_t *src = iter->buffer;
+    /* Workaround to ensure that x1F001F variable is allocated in a register */
+    static volatile uint32_t volatile_x1F001F = 0x1F001F;
+    uint32_t x1F001F = volatile_x1F001F;
+
+    while ((w -= 4) >= 0)
+    {
+	uint32_t s1 = *src++;
+	uint32_t s2 = *src++;
+	uint32_t s3 = *src++;
+	uint32_t s4 = *src++;
+	*dst++ = convert_8888_to_0565_workaround (s1, x1F001F);
+	*dst++ = convert_8888_to_0565_workaround (s2, x1F001F);
+	*dst++ = convert_8888_to_0565_workaround (s3, x1F001F);
+	*dst++ = convert_8888_to_0565_workaround (s4, x1F001F);
+    }
+    if (w & 2)
+    {
+	*dst++ = convert_8888_to_0565_workaround (*src++, x1F001F);
+	*dst++ = convert_8888_to_0565_workaround (*src++, x1F001F);
+    }
+    if (w & 1)
+    {
+	*dst = convert_8888_to_0565_workaround (*src, x1F001F);
+    }
+}
+
+typedef struct
+{
+    int		y;
+    uint64_t *	buffer;
+} line_t;
+
+typedef struct
+{
+    line_t		lines[2];
+    pixman_fixed_t	y;
+    pixman_fixed_t	x;
+    uint64_t		data[1];
+} bilinear_info_t;
+
+static void
+fetch_horizontal (bits_image_t *image, line_t *line,
+		  int y, pixman_fixed_t x, pixman_fixed_t ux, int n)
+{
+    uint32_t *bits = image->bits + y * image->rowstride;
+    int i;
+
+    for (i = 0; i < n; ++i)
+    {
+	int x0 = pixman_fixed_to_int (x);
+	int x1 = x0 + 1;
+	int32_t dist_x;
+
+	uint32_t left = *(bits + x0);
+	uint32_t right = *(bits + x1);
+
+	dist_x = pixman_fixed_to_bilinear_weight (x);
+	dist_x <<= (8 - BILINEAR_INTERPOLATION_BITS);
+
+#if SIZEOF_LONG <= 4
+	{
+	    uint32_t lag, rag, ag;
+	    uint32_t lrb, rrb, rb;
+
+	    lag = (left & 0xff00ff00) >> 8;
+	    rag = (right & 0xff00ff00) >> 8;
+	    ag = (lag << 8) + dist_x * (rag - lag);
+
+	    lrb = (left & 0x00ff00ff);
+	    rrb = (right & 0x00ff00ff);
+	    rb = (lrb << 8) + dist_x * (rrb - lrb);
+
+	    *((uint32_t *)(line->buffer + i)) = ag;
+	    *((uint32_t *)(line->buffer + i) + 1) = rb;
+	}
+#else
+	{
+	    uint64_t lagrb, ragrb;
+	    uint32_t lag, rag;
+	    uint32_t lrb, rrb;
+
+	    lag = (left & 0xff00ff00);
+	    lrb = (left & 0x00ff00ff);
+	    rag = (right & 0xff00ff00);
+	    rrb = (right & 0x00ff00ff);
+	    lagrb = (((uint64_t)lag) << 24) | lrb;
+	    ragrb = (((uint64_t)rag) << 24) | rrb;
+
+	    line->buffer[i] = (lagrb << 8) + dist_x * (ragrb - lagrb);
+	}
+#endif
+
+	x += ux;
+    }
+
+    line->y = y;
+}
+
+static uint32_t *
+fast_fetch_bilinear_cover (pixman_iter_t *iter, const uint32_t *mask)
+{
+    pixman_fixed_t fx, ux;
+    bilinear_info_t *info = iter->data;
+    line_t *line0, *line1;
+    int y0, y1;
+    int32_t dist_y;
+    int i;
+
+    fx = info->x;
+    ux = iter->image->common.transform->matrix[0][0];
+
+    y0 = pixman_fixed_to_int (info->y);
+    y1 = y0 + 1;
+    dist_y = pixman_fixed_to_bilinear_weight (info->y);
+    dist_y <<= (8 - BILINEAR_INTERPOLATION_BITS);
+
+    line0 = &info->lines[y0 & 0x01];
+    line1 = &info->lines[y1 & 0x01];
+
+    if (line0->y != y0)
+    {
+	fetch_horizontal (
+	    &iter->image->bits, line0, y0, fx, ux, iter->width);
+    }
+
+    if (line1->y != y1)
+    {
+	fetch_horizontal (
+	    &iter->image->bits, line1, y1, fx, ux, iter->width);
+    }
+
+    for (i = 0; i < iter->width; ++i)
+    {
+#if SIZEOF_LONG <= 4
+	uint32_t ta, tr, tg, tb;
+	uint32_t ba, br, bg, bb;
+	uint32_t tag, trb;
+	uint32_t bag, brb;
+	uint32_t a, r, g, b;
+
+	tag = *((uint32_t *)(line0->buffer + i));
+	trb = *((uint32_t *)(line0->buffer + i) + 1);
+	bag = *((uint32_t *)(line1->buffer + i));
+	brb = *((uint32_t *)(line1->buffer + i) + 1);
+
+	ta = tag >> 16;
+	ba = bag >> 16;
+	a = (ta << 8) + dist_y * (ba - ta);
+
+	tr = trb >> 16;
+	br = brb >> 16;
+	r = (tr << 8) + dist_y * (br - tr);
+
+	tg = tag & 0xffff;
+	bg = bag & 0xffff;
+	g = (tg << 8) + dist_y * (bg - tg);
+	
+	tb = trb & 0xffff;
+	bb = brb & 0xffff;
+	b = (tb << 8) + dist_y * (bb - tb);
+
+	a = (a <<  8) & 0xff000000;
+	r = (r <<  0) & 0x00ff0000;
+	g = (g >>  8) & 0x0000ff00;
+	b = (b >> 16) & 0x000000ff;
+#else
+	uint64_t top = line0->buffer[i];
+	uint64_t bot = line1->buffer[i];
+	uint64_t tar = (top & 0xffff0000ffff0000ULL) >> 16;
+	uint64_t bar = (bot & 0xffff0000ffff0000ULL) >> 16;
+	uint64_t tgb = (top & 0x0000ffff0000ffffULL);
+	uint64_t bgb = (bot & 0x0000ffff0000ffffULL);
+	uint64_t ar, gb;
+	uint32_t a, r, g, b;
+
+	ar = (tar << 8) + dist_y * (bar - tar);
+	gb = (tgb << 8) + dist_y * (bgb - tgb);
+
+	a = ((ar >> 24) & 0xff000000);
+	r = ((ar >>  0) & 0x00ff0000);
+	g = ((gb >> 40) & 0x0000ff00);
+	b = ((gb >> 16) & 0x000000ff);
+#endif
+
+	iter->buffer[i] = a | r | g | b;
+    }
+
+    info->y += iter->image->common.transform->matrix[1][1];
+
+    return iter->buffer;
+}
+
+static void
+bilinear_cover_iter_fini (pixman_iter_t *iter)
+{
+    free (iter->data);
+}
+
+static void
+fast_bilinear_cover_iter_init (pixman_iter_t *iter, const pixman_iter_info_t *iter_info)
+{
+    int width = iter->width;
+    bilinear_info_t *info;
+    pixman_vector_t v;
+
+    /* Reference point is the center of the pixel */
+    v.vector[0] = pixman_int_to_fixed (iter->x) + pixman_fixed_1 / 2;
+    v.vector[1] = pixman_int_to_fixed (iter->y) + pixman_fixed_1 / 2;
+    v.vector[2] = pixman_fixed_1;
+
+    if (!pixman_transform_point_3d (iter->image->common.transform, &v))
+	goto fail;
+
+    info = malloc (sizeof (*info) + (2 * width - 1) * sizeof (uint64_t));
+    if (!info)
+	goto fail;
+
+    info->x = v.vector[0] - pixman_fixed_1 / 2;
+    info->y = v.vector[1] - pixman_fixed_1 / 2;
+
+    /* It is safe to set the y coordinates to -1 initially
+     * because COVER_CLIP_BILINEAR ensures that we will only
+     * be asked to fetch lines in the [0, height) interval
+     */
+    info->lines[0].y = -1;
+    info->lines[0].buffer = &(info->data[0]);
+    info->lines[1].y = -1;
+    info->lines[1].buffer = &(info->data[width]);
+
+    iter->get_scanline = fast_fetch_bilinear_cover;
+    iter->fini = bilinear_cover_iter_fini;
+
+    iter->data = info;
+    return;
+
+fail:
+    /* Something went wrong, either a bad matrix or OOM; in such cases,
+     * we don't guarantee any particular rendering.
+     */
+    _pixman_log_error (
+	FUNC, "Allocation failure or bad matrix, skipping rendering\n");
+    
+    iter->get_scanline = _pixman_iter_get_scanline_noop;
+    iter->fini = NULL;
+}
+
+static uint32_t *
+bits_image_fetch_bilinear_no_repeat_8888 (pixman_iter_t *iter,
+					  const uint32_t *mask)
+{
+
+    pixman_image_t * ima = iter->image;
+    int              offset = iter->x;
+    int              line = iter->y++;
+    int              width = iter->width;
+    uint32_t *       buffer = iter->buffer;
+
+    bits_image_t *bits = &ima->bits;
+    pixman_fixed_t x_top, x_bottom, x;
+    pixman_fixed_t ux_top, ux_bottom, ux;
+    pixman_vector_t v;
+    uint32_t top_mask, bottom_mask;
+    uint32_t *top_row;
+    uint32_t *bottom_row;
+    uint32_t *end;
+    uint32_t zero[2] = { 0, 0 };
+    uint32_t one = 1;
+    int y, y1, y2;
+    int disty;
+    int mask_inc;
+    int w;
+
+    /* reference point is the center of the pixel */
+    v.vector[0] = pixman_int_to_fixed (offset) + pixman_fixed_1 / 2;
+    v.vector[1] = pixman_int_to_fixed (line) + pixman_fixed_1 / 2;
+    v.vector[2] = pixman_fixed_1;
+
+    if (!pixman_transform_point_3d (bits->common.transform, &v))
+	return iter->buffer;
+
+    ux = ux_top = ux_bottom = bits->common.transform->matrix[0][0];
+    x = x_top = x_bottom = v.vector[0] - pixman_fixed_1/2;
+
+    y = v.vector[1] - pixman_fixed_1/2;
+    disty = pixman_fixed_to_bilinear_weight (y);
+
+    /* Load the pointers to the first and second lines from the source
+     * image that bilinear code must read.
+     *
+     * The main trick in this code is about the check if any line are
+     * outside of the image;
+     *
+     * When I realize that a line (any one) is outside, I change
+     * the pointer to a dummy area with zeros. Once I change this, I
+     * must be sure the pointer will not change, so I set the
+     * variables to each pointer increments inside the loop.
+     */
+    y1 = pixman_fixed_to_int (y);
+    y2 = y1 + 1;
+
+    if (y1 < 0 || y1 >= bits->height)
+    {
+	top_row = zero;
+	x_top = 0;
+	ux_top = 0;
+    }
+    else
+    {
+	top_row = bits->bits + y1 * bits->rowstride;
+	x_top = x;
+	ux_top = ux;
+    }
+
+    if (y2 < 0 || y2 >= bits->height)
+    {
+	bottom_row = zero;
+	x_bottom = 0;
+	ux_bottom = 0;
+    }
+    else
+    {
+	bottom_row = bits->bits + y2 * bits->rowstride;
+	x_bottom = x;
+	ux_bottom = ux;
+    }
+
+    /* Instead of checking whether the operation uses the mast in
+     * each loop iteration, verify this only once and prepare the
+     * variables to make the code smaller inside the loop.
+     */
+    if (!mask)
+    {
+        mask_inc = 0;
+        mask = &one;
+    }
+    else
+    {
+        /* If have a mask, prepare the variables to check it */
+        mask_inc = 1;
+    }
+
+    /* If both are zero, then the whole thing is zero */
+    if (top_row == zero && bottom_row == zero)
+    {
+	memset (buffer, 0, width * sizeof (uint32_t));
+	return iter->buffer;
+    }
+    else if (bits->format == PIXMAN_x8r8g8b8)
+    {
+	if (top_row == zero)
+	{
+	    top_mask = 0;
+	    bottom_mask = 0xff000000;
+	}
+	else if (bottom_row == zero)
+	{
+	    top_mask = 0xff000000;
+	    bottom_mask = 0;
+	}
+	else
+	{
+	    top_mask = 0xff000000;
+	    bottom_mask = 0xff000000;
+	}
+    }
+    else
+    {
+	top_mask = 0;
+	bottom_mask = 0;
+    }
+
+    end = buffer + width;
+
+    /* Zero fill to the left of the image */
+    while (buffer < end && x < pixman_fixed_minus_1)
+    {
+	*buffer++ = 0;
+	x += ux;
+	x_top += ux_top;
+	x_bottom += ux_bottom;
+	mask += mask_inc;
+    }
+
+    /* Left edge
+     */
+    while (buffer < end && x < 0)
+    {
+	uint32_t tr, br;
+	int32_t distx;
+
+	tr = top_row[pixman_fixed_to_int (x_top) + 1] | top_mask;
+	br = bottom_row[pixman_fixed_to_int (x_bottom) + 1] | bottom_mask;
+
+	distx = pixman_fixed_to_bilinear_weight (x);
+
+	*buffer++ = bilinear_interpolation (0, tr, 0, br, distx, disty);
+
+	x += ux;
+	x_top += ux_top;
+	x_bottom += ux_bottom;
+	mask += mask_inc;
+    }
+
+    /* Main part */
+    w = pixman_int_to_fixed (bits->width - 1);
+
+    while (buffer < end  &&  x < w)
+    {
+	if (*mask)
+	{
+	    uint32_t tl, tr, bl, br;
+	    int32_t distx;
+
+	    tl = top_row [pixman_fixed_to_int (x_top)] | top_mask;
+	    tr = top_row [pixman_fixed_to_int (x_top) + 1] | top_mask;
+	    bl = bottom_row [pixman_fixed_to_int (x_bottom)] | bottom_mask;
+	    br = bottom_row [pixman_fixed_to_int (x_bottom) + 1] | bottom_mask;
+
+	    distx = pixman_fixed_to_bilinear_weight (x);
+
+	    *buffer = bilinear_interpolation (tl, tr, bl, br, distx, disty);
+	}
+
+	buffer++;
+	x += ux;
+	x_top += ux_top;
+	x_bottom += ux_bottom;
+	mask += mask_inc;
+    }
+
+    /* Right Edge */
+    w = pixman_int_to_fixed (bits->width);
+    while (buffer < end  &&  x < w)
+    {
+	if (*mask)
+	{
+	    uint32_t tl, bl;
+	    int32_t distx;
+
+	    tl = top_row [pixman_fixed_to_int (x_top)] | top_mask;
+	    bl = bottom_row [pixman_fixed_to_int (x_bottom)] | bottom_mask;
+
+	    distx = pixman_fixed_to_bilinear_weight (x);
+
+	    *buffer = bilinear_interpolation (tl, 0, bl, 0, distx, disty);
+	}
+
+	buffer++;
+	x += ux;
+	x_top += ux_top;
+	x_bottom += ux_bottom;
+	mask += mask_inc;
+    }
+
+    /* Zero fill to the left of the image */
+    while (buffer < end)
+	*buffer++ = 0;
+
+    return iter->buffer;
+}
+
+typedef uint32_t (* convert_pixel_t) (const uint8_t *row, int x);
+
+static force_inline void
+bits_image_fetch_separable_convolution_affine (pixman_image_t * image,
+					       int              offset,
+					       int              line,
+					       int              width,
+					       uint32_t *       buffer,
+					       const uint32_t * mask,
+
+					       convert_pixel_t	convert_pixel,
+					       pixman_format_code_t	format,
+					       pixman_repeat_t	repeat_mode)
+{
+    bits_image_t *bits = &image->bits;
+    pixman_fixed_t *params = image->common.filter_params;
+    int cwidth = pixman_fixed_to_int (params[0]);
+    int cheight = pixman_fixed_to_int (params[1]);
+    int x_off = ((cwidth << 16) - pixman_fixed_1) >> 1;
+    int y_off = ((cheight << 16) - pixman_fixed_1) >> 1;
+    int x_phase_bits = pixman_fixed_to_int (params[2]);
+    int y_phase_bits = pixman_fixed_to_int (params[3]);
+    int x_phase_shift = 16 - x_phase_bits;
+    int y_phase_shift = 16 - y_phase_bits;
+    pixman_fixed_t vx, vy;
+    pixman_fixed_t ux, uy;
+    pixman_vector_t v;
+    int k;
+
+    /* reference point is the center of the pixel */
+    v.vector[0] = pixman_int_to_fixed (offset) + pixman_fixed_1 / 2;
+    v.vector[1] = pixman_int_to_fixed (line) + pixman_fixed_1 / 2;
+    v.vector[2] = pixman_fixed_1;
+
+    if (!pixman_transform_point_3d (image->common.transform, &v))
+	return;
+
+    ux = image->common.transform->matrix[0][0];
+    uy = image->common.transform->matrix[1][0];
+
+    vx = v.vector[0];
+    vy = v.vector[1];
+
+    for (k = 0; k < width; ++k)
+    {
+	pixman_fixed_t *y_params;
+	int satot, srtot, sgtot, sbtot;
+	pixman_fixed_t x, y;
+	int32_t x1, x2, y1, y2;
+	int32_t px, py;
+	int i, j;
+
+	if (mask && !mask[k])
+	    goto next;
+
+	/* Round x and y to the middle of the closest phase before continuing. This
+	 * ensures that the convolution matrix is aligned right, since it was
+	 * positioned relative to a particular phase (and not relative to whatever
+	 * exact fraction we happen to get here).
+	 */
+	x = ((vx >> x_phase_shift) << x_phase_shift) + ((1 << x_phase_shift) >> 1);
+	y = ((vy >> y_phase_shift) << y_phase_shift) + ((1 << y_phase_shift) >> 1);
+
+	px = (x & 0xffff) >> x_phase_shift;
+	py = (y & 0xffff) >> y_phase_shift;
+
+	x1 = pixman_fixed_to_int (x - pixman_fixed_e - x_off);
+	y1 = pixman_fixed_to_int (y - pixman_fixed_e - y_off);
+	x2 = x1 + cwidth;
+	y2 = y1 + cheight;
+
+	satot = srtot = sgtot = sbtot = 0;
+
+	y_params = params + 4 + (1 << x_phase_bits) * cwidth + py * cheight;
+
+	for (i = y1; i < y2; ++i)
+	{
+	    pixman_fixed_t fy = *y_params++;
+
+	    if (fy)
+	    {
+		pixman_fixed_t *x_params = params + 4 + px * cwidth;
+
+		for (j = x1; j < x2; ++j)
+		{
+		    pixman_fixed_t fx = *x_params++;
+		    int rx = j;
+		    int ry = i;
+		    
+		    if (fx)
+		    {
+			pixman_fixed_t f;
+			uint32_t pixel, mask;
+			uint8_t *row;
+
+			mask = PIXMAN_FORMAT_A (format)? 0 : 0xff000000;
+
+			if (repeat_mode != PIXMAN_REPEAT_NONE)
+			{
+			    repeat (repeat_mode, &rx, bits->width);
+			    repeat (repeat_mode, &ry, bits->height);
+
+			    row = (uint8_t *)bits->bits + bits->rowstride * 4 * ry;
+			    pixel = convert_pixel (row, rx) | mask;
+			}
+			else
+			{
+			    if (rx < 0 || ry < 0 || rx >= bits->width || ry >= bits->height)
+			    {
+				pixel = 0;
+			    }
+			    else
+			    {
+				row = (uint8_t *)bits->bits + bits->rowstride * 4 * ry;
+				pixel = convert_pixel (row, rx) | mask;
+			    }
+			}
+
+			f = ((pixman_fixed_32_32_t)fx * fy + 0x8000) >> 16;
+			srtot += (int)RED_8 (pixel) * f;
+			sgtot += (int)GREEN_8 (pixel) * f;
+			sbtot += (int)BLUE_8 (pixel) * f;
+			satot += (int)ALPHA_8 (pixel) * f;
+		    }
+		}
+	    }
+	}
+
+	satot = (satot + 0x8000) >> 16;
+	srtot = (srtot + 0x8000) >> 16;
+	sgtot = (sgtot + 0x8000) >> 16;
+	sbtot = (sbtot + 0x8000) >> 16;
+
+	satot = CLIP (satot, 0, 0xff);
+	srtot = CLIP (srtot, 0, 0xff);
+	sgtot = CLIP (sgtot, 0, 0xff);
+	sbtot = CLIP (sbtot, 0, 0xff);
+
+	buffer[k] = (satot << 24) | (srtot << 16) | (sgtot << 8) | (sbtot << 0);
+
+    next:
+	vx += ux;
+	vy += uy;
+    }
+}
+
+static const uint8_t zero[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
+
+static force_inline void
+bits_image_fetch_bilinear_affine (pixman_image_t * image,
+				  int              offset,
+				  int              line,
+				  int              width,
+				  uint32_t *       buffer,
+				  const uint32_t * mask,
+
+				  convert_pixel_t	convert_pixel,
+				  pixman_format_code_t	format,
+				  pixman_repeat_t	repeat_mode)
+{
+    pixman_fixed_t x, y;
+    pixman_fixed_t ux, uy;
+    pixman_vector_t v;
+    bits_image_t *bits = &image->bits;
+    int i;
+
+    /* reference point is the center of the pixel */
+    v.vector[0] = pixman_int_to_fixed (offset) + pixman_fixed_1 / 2;
+    v.vector[1] = pixman_int_to_fixed (line) + pixman_fixed_1 / 2;
+    v.vector[2] = pixman_fixed_1;
+
+    if (!pixman_transform_point_3d (image->common.transform, &v))
+	return;
+
+    ux = image->common.transform->matrix[0][0];
+    uy = image->common.transform->matrix[1][0];
+
+    x = v.vector[0];
+    y = v.vector[1];
+
+    for (i = 0; i < width; ++i)
+    {
+	int x1, y1, x2, y2;
+	uint32_t tl, tr, bl, br;
+	int32_t distx, disty;
+	int width = image->bits.width;
+	int height = image->bits.height;
+	const uint8_t *row1;
+	const uint8_t *row2;
+
+	if (mask && !mask[i])
+	    goto next;
+
+	x1 = x - pixman_fixed_1 / 2;
+	y1 = y - pixman_fixed_1 / 2;
+
+	distx = pixman_fixed_to_bilinear_weight (x1);
+	disty = pixman_fixed_to_bilinear_weight (y1);
+
+	y1 = pixman_fixed_to_int (y1);
+	y2 = y1 + 1;
+	x1 = pixman_fixed_to_int (x1);
+	x2 = x1 + 1;
+
+	if (repeat_mode != PIXMAN_REPEAT_NONE)
+	{
+	    uint32_t mask;
+
+	    mask = PIXMAN_FORMAT_A (format)? 0 : 0xff000000;
+
+	    repeat (repeat_mode, &x1, width);
+	    repeat (repeat_mode, &y1, height);
+	    repeat (repeat_mode, &x2, width);
+	    repeat (repeat_mode, &y2, height);
+
+	    row1 = (uint8_t *)bits->bits + bits->rowstride * 4 * y1;
+	    row2 = (uint8_t *)bits->bits + bits->rowstride * 4 * y2;
+
+	    tl = convert_pixel (row1, x1) | mask;
+	    tr = convert_pixel (row1, x2) | mask;
+	    bl = convert_pixel (row2, x1) | mask;
+	    br = convert_pixel (row2, x2) | mask;
+	}
+	else
+	{
+	    uint32_t mask1, mask2;
+	    int bpp;
+
+	    /* Note: PIXMAN_FORMAT_BPP() returns an unsigned value,
+	     * which means if you use it in expressions, those
+	     * expressions become unsigned themselves. Since
+	     * the variables below can be negative in some cases,
+	     * that will lead to crashes on 64 bit architectures.
+	     *
+	     * So this line makes sure bpp is signed
+	     */
+	    bpp = PIXMAN_FORMAT_BPP (format);
+
+	    if (x1 >= width || x2 < 0 || y1 >= height || y2 < 0)
+	    {
+		buffer[i] = 0;
+		goto next;
+	    }
+
+	    if (y2 == 0)
+	    {
+		row1 = zero;
+		mask1 = 0;
+	    }
+	    else
+	    {
+		row1 = (uint8_t *)bits->bits + bits->rowstride * 4 * y1;
+		row1 += bpp / 8 * x1;
+
+		mask1 = PIXMAN_FORMAT_A (format)? 0 : 0xff000000;
+	    }
+
+	    if (y1 == height - 1)
+	    {
+		row2 = zero;
+		mask2 = 0;
+	    }
+	    else
+	    {
+		row2 = (uint8_t *)bits->bits + bits->rowstride * 4 * y2;
+		row2 += bpp / 8 * x1;
+
+		mask2 = PIXMAN_FORMAT_A (format)? 0 : 0xff000000;
+	    }
+
+	    if (x2 == 0)
+	    {
+		tl = 0;
+		bl = 0;
+	    }
+	    else
+	    {
+		tl = convert_pixel (row1, 0) | mask1;
+		bl = convert_pixel (row2, 0) | mask2;
+	    }
+
+	    if (x1 == width - 1)
+	    {
+		tr = 0;
+		br = 0;
+	    }
+	    else
+	    {
+		tr = convert_pixel (row1, 1) | mask1;
+		br = convert_pixel (row2, 1) | mask2;
+	    }
+	}
+
+	buffer[i] = bilinear_interpolation (
+	    tl, tr, bl, br, distx, disty);
+
+    next:
+	x += ux;
+	y += uy;
+    }
+}
+
+static force_inline void
+bits_image_fetch_nearest_affine (pixman_image_t * image,
+				 int              offset,
+				 int              line,
+				 int              width,
+				 uint32_t *       buffer,
+				 const uint32_t * mask,
+				 
+				 convert_pixel_t	convert_pixel,
+				 pixman_format_code_t	format,
+				 pixman_repeat_t	repeat_mode)
+{
+    pixman_fixed_t x, y;
+    pixman_fixed_t ux, uy;
+    pixman_vector_t v;
+    bits_image_t *bits = &image->bits;
+    int i;
+
+    /* reference point is the center of the pixel */
+    v.vector[0] = pixman_int_to_fixed (offset) + pixman_fixed_1 / 2;
+    v.vector[1] = pixman_int_to_fixed (line) + pixman_fixed_1 / 2;
+    v.vector[2] = pixman_fixed_1;
+
+    if (!pixman_transform_point_3d (image->common.transform, &v))
+	return;
+
+    ux = image->common.transform->matrix[0][0];
+    uy = image->common.transform->matrix[1][0];
+
+    x = v.vector[0];
+    y = v.vector[1];
+
+    for (i = 0; i < width; ++i)
+    {
+	int width, height, x0, y0;
+	const uint8_t *row;
+
+	if (mask && !mask[i])
+	    goto next;
+	
+	width = image->bits.width;
+	height = image->bits.height;
+	x0 = pixman_fixed_to_int (x - pixman_fixed_e);
+	y0 = pixman_fixed_to_int (y - pixman_fixed_e);
+
+	if (repeat_mode == PIXMAN_REPEAT_NONE &&
+	    (y0 < 0 || y0 >= height || x0 < 0 || x0 >= width))
+	{
+	    buffer[i] = 0;
+	}
+	else
+	{
+	    uint32_t mask = PIXMAN_FORMAT_A (format)? 0 : 0xff000000;
+
+	    if (repeat_mode != PIXMAN_REPEAT_NONE)
+	    {
+		repeat (repeat_mode, &x0, width);
+		repeat (repeat_mode, &y0, height);
+	    }
+
+	    row = (uint8_t *)bits->bits + bits->rowstride * 4 * y0;
+
+	    buffer[i] = convert_pixel (row, x0) | mask;
+	}
+
+    next:
+	x += ux;
+	y += uy;
+    }
+}
+
+static force_inline uint32_t
+convert_a8r8g8b8 (const uint8_t *row, int x)
+{
+    return *(((uint32_t *)row) + x);
+}
+
+static force_inline uint32_t
+convert_x8r8g8b8 (const uint8_t *row, int x)
+{
+    return *(((uint32_t *)row) + x);
+}
+
+static force_inline uint32_t
+convert_a8 (const uint8_t *row, int x)
+{
+    return *(row + x) << 24;
+}
+
+static force_inline uint32_t
+convert_r5g6b5 (const uint8_t *row, int x)
+{
+    return convert_0565_to_0888 (*((uint16_t *)row + x));
+}
+
+#define MAKE_SEPARABLE_CONVOLUTION_FETCHER(name, format, repeat_mode)  \
+    static uint32_t *							\
+    bits_image_fetch_separable_convolution_affine_ ## name (pixman_iter_t   *iter, \
+							    const uint32_t * mask) \
+    {									\
+	bits_image_fetch_separable_convolution_affine (                 \
+	    iter->image,                                                \
+	    iter->x, iter->y++,                                         \
+	    iter->width,                                                \
+	    iter->buffer, mask,                                         \
+	    convert_ ## format,                                         \
+	    PIXMAN_ ## format,                                          \
+	    repeat_mode);                                               \
+									\
+	return iter->buffer;                                            \
+    }
+
+#define MAKE_BILINEAR_FETCHER(name, format, repeat_mode)		\
+    static uint32_t *							\
+    bits_image_fetch_bilinear_affine_ ## name (pixman_iter_t   *iter,	\
+					       const uint32_t * mask)	\
+    {									\
+	bits_image_fetch_bilinear_affine (iter->image,			\
+					  iter->x, iter->y++,		\
+					  iter->width,			\
+					  iter->buffer, mask,		\
+					  convert_ ## format,		\
+					  PIXMAN_ ## format,		\
+					  repeat_mode);			\
+	return iter->buffer;						\
+    }
+
+#define MAKE_NEAREST_FETCHER(name, format, repeat_mode)			\
+    static uint32_t *							\
+    bits_image_fetch_nearest_affine_ ## name (pixman_iter_t   *iter,	\
+					      const uint32_t * mask)	\
+    {									\
+	bits_image_fetch_nearest_affine (iter->image,			\
+					 iter->x, iter->y++,		\
+					 iter->width,			\
+					 iter->buffer, mask,		\
+					 convert_ ## format,		\
+					 PIXMAN_ ## format,		\
+					 repeat_mode);			\
+	return iter->buffer;						\
+    }
+
+#define MAKE_FETCHERS(name, format, repeat_mode)			\
+    MAKE_NEAREST_FETCHER (name, format, repeat_mode)			\
+    MAKE_BILINEAR_FETCHER (name, format, repeat_mode)			\
+    MAKE_SEPARABLE_CONVOLUTION_FETCHER (name, format, repeat_mode)
+
+MAKE_FETCHERS (pad_a8r8g8b8,     a8r8g8b8, PIXMAN_REPEAT_PAD)
+MAKE_FETCHERS (none_a8r8g8b8,    a8r8g8b8, PIXMAN_REPEAT_NONE)
+MAKE_FETCHERS (reflect_a8r8g8b8, a8r8g8b8, PIXMAN_REPEAT_REFLECT)
+MAKE_FETCHERS (normal_a8r8g8b8,  a8r8g8b8, PIXMAN_REPEAT_NORMAL)
+MAKE_FETCHERS (pad_x8r8g8b8,     x8r8g8b8, PIXMAN_REPEAT_PAD)
+MAKE_FETCHERS (none_x8r8g8b8,    x8r8g8b8, PIXMAN_REPEAT_NONE)
+MAKE_FETCHERS (reflect_x8r8g8b8, x8r8g8b8, PIXMAN_REPEAT_REFLECT)
+MAKE_FETCHERS (normal_x8r8g8b8,  x8r8g8b8, PIXMAN_REPEAT_NORMAL)
+MAKE_FETCHERS (pad_a8,           a8,       PIXMAN_REPEAT_PAD)
+MAKE_FETCHERS (none_a8,          a8,       PIXMAN_REPEAT_NONE)
+MAKE_FETCHERS (reflect_a8,	 a8,       PIXMAN_REPEAT_REFLECT)
+MAKE_FETCHERS (normal_a8,	 a8,       PIXMAN_REPEAT_NORMAL)
+MAKE_FETCHERS (pad_r5g6b5,       r5g6b5,   PIXMAN_REPEAT_PAD)
+MAKE_FETCHERS (none_r5g6b5,      r5g6b5,   PIXMAN_REPEAT_NONE)
+MAKE_FETCHERS (reflect_r5g6b5,   r5g6b5,   PIXMAN_REPEAT_REFLECT)
+MAKE_FETCHERS (normal_r5g6b5,    r5g6b5,   PIXMAN_REPEAT_NORMAL)
+
+#define IMAGE_FLAGS							\
+    (FAST_PATH_STANDARD_FLAGS | FAST_PATH_ID_TRANSFORM |		\
+     FAST_PATH_BITS_IMAGE | FAST_PATH_SAMPLES_COVER_CLIP_NEAREST)
+
+static const pixman_iter_info_t fast_iters[] = 
+{
+    { PIXMAN_r5g6b5, IMAGE_FLAGS, ITER_NARROW | ITER_SRC,
+      _pixman_iter_init_bits_stride, fast_fetch_r5g6b5, NULL },
+
+    { PIXMAN_r5g6b5, FAST_PATH_STD_DEST_FLAGS,
+      ITER_NARROW | ITER_DEST,
+      _pixman_iter_init_bits_stride,
+      fast_fetch_r5g6b5, fast_write_back_r5g6b5 },
+    
+    { PIXMAN_r5g6b5, FAST_PATH_STD_DEST_FLAGS,
+      ITER_NARROW | ITER_DEST | ITER_IGNORE_RGB | ITER_IGNORE_ALPHA,
+      _pixman_iter_init_bits_stride,
+      fast_dest_fetch_noop, fast_write_back_r5g6b5 },
+
+    { PIXMAN_a8r8g8b8,
+      (FAST_PATH_STANDARD_FLAGS			|
+       FAST_PATH_SCALE_TRANSFORM		|
+       FAST_PATH_BILINEAR_FILTER		|
+       FAST_PATH_SAMPLES_COVER_CLIP_BILINEAR),
+      ITER_NARROW | ITER_SRC,
+      fast_bilinear_cover_iter_init,
+      NULL, NULL
+    },
+
+#define FAST_BILINEAR_FLAGS						\
+    (FAST_PATH_NO_ALPHA_MAP		|				\
+     FAST_PATH_NO_ACCESSORS		|				\
+     FAST_PATH_HAS_TRANSFORM		|				\
+     FAST_PATH_AFFINE_TRANSFORM		|				\
+     FAST_PATH_X_UNIT_POSITIVE		|				\
+     FAST_PATH_Y_UNIT_ZERO		|				\
+     FAST_PATH_NONE_REPEAT		|				\
+     FAST_PATH_BILINEAR_FILTER)
+
+    { PIXMAN_a8r8g8b8,
+      FAST_BILINEAR_FLAGS,
+      ITER_NARROW | ITER_SRC,
+      NULL, bits_image_fetch_bilinear_no_repeat_8888, NULL
+    },
+
+    { PIXMAN_x8r8g8b8,
+      FAST_BILINEAR_FLAGS,
+      ITER_NARROW | ITER_SRC,
+      NULL, bits_image_fetch_bilinear_no_repeat_8888, NULL
+    },
+
+#define GENERAL_BILINEAR_FLAGS						\
+    (FAST_PATH_NO_ALPHA_MAP		|				\
+     FAST_PATH_NO_ACCESSORS		|				\
+     FAST_PATH_HAS_TRANSFORM		|				\
+     FAST_PATH_AFFINE_TRANSFORM		|				\
+     FAST_PATH_BILINEAR_FILTER)
+
+#define GENERAL_NEAREST_FLAGS						\
+    (FAST_PATH_NO_ALPHA_MAP		|				\
+     FAST_PATH_NO_ACCESSORS		|				\
+     FAST_PATH_HAS_TRANSFORM		|				\
+     FAST_PATH_AFFINE_TRANSFORM		|				\
+     FAST_PATH_NEAREST_FILTER)
+
+#define GENERAL_SEPARABLE_CONVOLUTION_FLAGS				\
+    (FAST_PATH_NO_ALPHA_MAP            |				\
+     FAST_PATH_NO_ACCESSORS            |				\
+     FAST_PATH_HAS_TRANSFORM           |				\
+     FAST_PATH_AFFINE_TRANSFORM        |				\
+     FAST_PATH_SEPARABLE_CONVOLUTION_FILTER)
+    
+#define SEPARABLE_CONVOLUTION_AFFINE_FAST_PATH(name, format, repeat)   \
+    { PIXMAN_ ## format,						\
+      GENERAL_SEPARABLE_CONVOLUTION_FLAGS | FAST_PATH_ ## repeat ## _REPEAT, \
+      ITER_NARROW | ITER_SRC,						\
+      NULL, bits_image_fetch_separable_convolution_affine_ ## name, NULL \
+    },
+
+#define BILINEAR_AFFINE_FAST_PATH(name, format, repeat)			\
+    { PIXMAN_ ## format,						\
+      GENERAL_BILINEAR_FLAGS | FAST_PATH_ ## repeat ## _REPEAT,		\
+      ITER_NARROW | ITER_SRC,						\
+      NULL, bits_image_fetch_bilinear_affine_ ## name, NULL,		\
+    },
+
+#define NEAREST_AFFINE_FAST_PATH(name, format, repeat)			\
+    { PIXMAN_ ## format,						\
+      GENERAL_NEAREST_FLAGS | FAST_PATH_ ## repeat ## _REPEAT,		\
+      ITER_NARROW | ITER_SRC,						\
+      NULL, bits_image_fetch_nearest_affine_ ## name, NULL		\
+    },
+
+#define AFFINE_FAST_PATHS(name, format, repeat)				\
+    SEPARABLE_CONVOLUTION_AFFINE_FAST_PATH(name, format, repeat)	\
+    BILINEAR_AFFINE_FAST_PATH(name, format, repeat)			\
+    NEAREST_AFFINE_FAST_PATH(name, format, repeat)
+    
+    AFFINE_FAST_PATHS (pad_a8r8g8b8, a8r8g8b8, PAD)
+    AFFINE_FAST_PATHS (none_a8r8g8b8, a8r8g8b8, NONE)
+    AFFINE_FAST_PATHS (reflect_a8r8g8b8, a8r8g8b8, REFLECT)
+    AFFINE_FAST_PATHS (normal_a8r8g8b8, a8r8g8b8, NORMAL)
+    AFFINE_FAST_PATHS (pad_x8r8g8b8, x8r8g8b8, PAD)
+    AFFINE_FAST_PATHS (none_x8r8g8b8, x8r8g8b8, NONE)
+    AFFINE_FAST_PATHS (reflect_x8r8g8b8, x8r8g8b8, REFLECT)
+    AFFINE_FAST_PATHS (normal_x8r8g8b8, x8r8g8b8, NORMAL)
+    AFFINE_FAST_PATHS (pad_a8, a8, PAD)
+    AFFINE_FAST_PATHS (none_a8, a8, NONE)
+    AFFINE_FAST_PATHS (reflect_a8, a8, REFLECT)
+    AFFINE_FAST_PATHS (normal_a8, a8, NORMAL)
+    AFFINE_FAST_PATHS (pad_r5g6b5, r5g6b5, PAD)
+    AFFINE_FAST_PATHS (none_r5g6b5, r5g6b5, NONE)
+    AFFINE_FAST_PATHS (reflect_r5g6b5, r5g6b5, REFLECT)
+    AFFINE_FAST_PATHS (normal_r5g6b5, r5g6b5, NORMAL)
+
+    { PIXMAN_null },
+};
+
 pixman_implementation_t *
-_pixman_implementation_create_fast_path (void)
+_pixman_implementation_create_fast_path (pixman_implementation_t *fallback)
 {
-    pixman_implementation_t *general = _pixman_implementation_create_general ();
-    pixman_implementation_t *imp = _pixman_implementation_create (general, c_fast_paths);
+    pixman_implementation_t *imp = _pixman_implementation_create (fallback, c_fast_paths);
 
     imp->fill = fast_path_fill;
+    imp->iter_info = fast_iters;
 
     return imp;
 }
diff --git a/pixman/pixman/pixman-filter.c b/pixman/pixman/pixman-filter.c
new file mode 100644
index 000000000..b2bf53fed
--- /dev/null
+++ b/pixman/pixman/pixman-filter.c
@@ -0,0 +1,350 @@
+/*
+ * Copyright 2012, Red Hat, Inc.
+ * Copyright 2012, Soren Sandmann
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Author: Soren Sandmann <soren.sandmann@gmail.com>
+ */
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <math.h>
+#include <assert.h>
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#include "pixman-private.h"
+
+typedef double (* kernel_func_t) (double x);
+
+typedef struct
+{
+    pixman_kernel_t	kernel;
+    kernel_func_t	func;
+    double		width;
+} filter_info_t;
+
+static double
+impulse_kernel (double x)
+{
+    return (x == 0.0)? 1.0 : 0.0;
+}
+
+static double
+box_kernel (double x)
+{
+    return 1;
+}
+
+static double
+linear_kernel (double x)
+{
+    return 1 - fabs (x);
+}
+
+static double
+gaussian_kernel (double x)
+{
+#define SQRT2 (1.4142135623730950488016887242096980785696718753769480)
+#define SIGMA (SQRT2 / 2.0)
+    
+    return exp (- x * x / (2 * SIGMA * SIGMA)) / (SIGMA * sqrt (2.0 * M_PI));
+}
+
+static double
+sinc (double x)
+{
+    if (x == 0.0)
+	return 1.0;
+    else
+	return sin (M_PI * x) / (M_PI * x);
+}
+
+static double
+lanczos (double x, int n)
+{
+    return sinc (x) * sinc (x * (1.0 / n));
+}
+
+static double
+lanczos2_kernel (double x)
+{
+    return lanczos (x, 2);
+}
+
+static double
+lanczos3_kernel (double x)
+{
+    return lanczos (x, 3);
+}
+
+static double
+nice_kernel (double x)
+{
+    return lanczos3_kernel (x * 0.75);
+}
+
+static double
+general_cubic (double x, double B, double C)
+{
+    double ax = fabs(x);
+
+    if (ax < 1)
+    {
+	return ((12 - 9 * B - 6 * C) * ax * ax * ax +
+		(-18 + 12 * B + 6 * C) * ax * ax + (6 - 2 * B)) / 6;
+    }
+    else if (ax >= 1 && ax < 2)
+    {
+	return ((-B - 6 * C) * ax * ax * ax +
+		(6 * B + 30 * C) * ax * ax + (-12 * B - 48 * C) *
+		ax + (8 * B + 24 * C)) / 6;
+    }
+    else
+    {
+	return 0;
+    }
+}
+
+static double
+cubic_kernel (double x)
+{
+    /* This is the Mitchell-Netravali filter.
+     *
+     * (0.0, 0.5) would give us the Catmull-Rom spline,
+     * but that one seems to be indistinguishable from Lanczos2.
+     */
+    return general_cubic (x, 1/3.0, 1/3.0);
+}
+
+static const filter_info_t filters[] =
+{
+    { PIXMAN_KERNEL_IMPULSE,	        impulse_kernel,   0.0 },
+    { PIXMAN_KERNEL_BOX,	        box_kernel,       1.0 },
+    { PIXMAN_KERNEL_LINEAR,	        linear_kernel,    2.0 },
+    { PIXMAN_KERNEL_CUBIC,		cubic_kernel,     4.0 },
+    { PIXMAN_KERNEL_GAUSSIAN,	        gaussian_kernel,  6 * SIGMA },
+    { PIXMAN_KERNEL_LANCZOS2,	        lanczos2_kernel,  4.0 },
+    { PIXMAN_KERNEL_LANCZOS3,	        lanczos3_kernel,  6.0 },
+    { PIXMAN_KERNEL_LANCZOS3_STRETCHED, nice_kernel,      8.0 },
+};
+
+/* This function scales @kernel2 by @scale, then
+ * aligns @x1 in @kernel1 with @x2 in @kernel2 and
+ * and integrates the product of the kernels across @width.
+ *
+ * This function assumes that the intervals are within
+ * the kernels in question. E.g., the caller must not
+ * try to integrate a linear kernel ouside of [-1:1]
+ */
+static double
+integral (pixman_kernel_t kernel1, double x1,
+	  pixman_kernel_t kernel2, double scale, double x2,
+	  double width)
+{
+    /* If the integration interval crosses zero, break it into
+     * two separate integrals. This ensures that filters such
+     * as LINEAR that are not differentiable at 0 will still
+     * integrate properly.
+     */
+    if (x1 < 0 && x1 + width > 0)
+    {
+	return
+	    integral (kernel1, x1, kernel2, scale, x2, - x1) +
+	    integral (kernel1, 0, kernel2, scale, x2 - x1, width + x1);
+    }
+    else if (x2 < 0 && x2 + width > 0)
+    {
+	return
+	    integral (kernel1, x1, kernel2, scale, x2, - x2) +
+	    integral (kernel1, x1 - x2, kernel2, scale, 0, width + x2);
+    }
+    else if (kernel1 == PIXMAN_KERNEL_IMPULSE)
+    {
+	assert (width == 0.0);
+	return filters[kernel2].func (x2 * scale);
+    }
+    else if (kernel2 == PIXMAN_KERNEL_IMPULSE)
+    {
+	assert (width == 0.0);
+	return filters[kernel1].func (x1);
+    }
+    else
+    {
+	/* Integration via Simpson's rule */
+#define N_SEGMENTS 128
+#define SAMPLE(a1, a2)							\
+	(filters[kernel1].func ((a1)) * filters[kernel2].func ((a2) * scale))
+	
+	double s = 0.0;
+	double h = width / (double)N_SEGMENTS;
+	int i;
+
+	s = SAMPLE (x1, x2);
+
+	for (i = 1; i < N_SEGMENTS; i += 2)
+	{
+	    double a1 = x1 + h * i;
+	    double a2 = x2 + h * i;
+
+	    s += 2 * SAMPLE (a1, a2);
+
+	    if (i >= 2 && i < N_SEGMENTS - 1)
+		s += 4 * SAMPLE (a1, a2);
+	}
+
+	s += SAMPLE (x1 + width, x2 + width);
+	
+	return h * s * (1.0 / 3.0);
+    }
+}
+
+static pixman_fixed_t *
+create_1d_filter (int             *width,
+		  pixman_kernel_t  reconstruct,
+		  pixman_kernel_t  sample,
+		  double           scale,
+		  int              n_phases)
+{
+    pixman_fixed_t *params, *p;
+    double step;
+    double size;
+    int i;
+
+    size = scale * filters[sample].width + filters[reconstruct].width;
+    *width = ceil (size);
+
+    p = params = malloc (*width * n_phases * sizeof (pixman_fixed_t));
+    if (!params)
+        return NULL;
+
+    step = 1.0 / n_phases;
+
+    for (i = 0; i < n_phases; ++i)
+    {
+        double frac = step / 2.0 + i * step;
+	pixman_fixed_t new_total;
+        int x, x1, x2;
+	double total;
+
+	/* Sample convolution of reconstruction and sampling
+	 * filter. See rounding.txt regarding the rounding
+	 * and sample positions.
+	 */
+
+	x1 = ceil (frac - *width / 2.0 - 0.5);
+        x2 = x1 + *width;
+
+	total = 0;
+        for (x = x1; x < x2; ++x)
+        {
+	    double pos = x + 0.5 - frac;
+	    double rlow = - filters[reconstruct].width / 2.0;
+	    double rhigh = rlow + filters[reconstruct].width;
+	    double slow = pos - scale * filters[sample].width / 2.0;
+	    double shigh = slow + scale * filters[sample].width;
+	    double c = 0.0;
+	    double ilow, ihigh;
+
+	    if (rhigh >= slow && rlow <= shigh)
+	    {
+		ilow = MAX (slow, rlow);
+		ihigh = MIN (shigh, rhigh);
+
+		c = integral (reconstruct, ilow,
+			      sample, 1.0 / scale, ilow - pos,
+			      ihigh - ilow);
+	    }
+
+	    total += c;
+            *p++ = (pixman_fixed_t)(c * 65536.0 + 0.5);
+        }
+
+	/* Normalize */
+	p -= *width;
+        total = 1 / total;
+        new_total = 0;
+	for (x = x1; x < x2; ++x)
+	{
+	    pixman_fixed_t t = (*p) * total + 0.5;
+
+	    new_total += t;
+	    *p++ = t;
+	}
+
+	if (new_total != pixman_fixed_1)
+	    *(p - *width / 2) += (pixman_fixed_1 - new_total);
+    }
+
+    return params;
+}
+
+/* Create the parameter list for a SEPARABLE_CONVOLUTION filter
+ * with the given kernels and scale parameters
+ */
+PIXMAN_EXPORT pixman_fixed_t *
+pixman_filter_create_separable_convolution (int             *n_values,
+					    pixman_fixed_t   scale_x,
+					    pixman_fixed_t   scale_y,
+					    pixman_kernel_t  reconstruct_x,
+					    pixman_kernel_t  reconstruct_y,
+					    pixman_kernel_t  sample_x,
+					    pixman_kernel_t  sample_y,
+					    int              subsample_bits_x,
+					    int	             subsample_bits_y)
+{
+    double sx = fabs (pixman_fixed_to_double (scale_x));
+    double sy = fabs (pixman_fixed_to_double (scale_y));
+    pixman_fixed_t *horz = NULL, *vert = NULL, *params = NULL;
+    int subsample_x, subsample_y;
+    int width, height;
+
+    subsample_x = (1 << subsample_bits_x);
+    subsample_y = (1 << subsample_bits_y);
+
+    horz = create_1d_filter (&width, reconstruct_x, sample_x, sx, subsample_x);
+    vert = create_1d_filter (&height, reconstruct_y, sample_y, sy, subsample_y);
+
+    if (!horz || !vert)
+        goto out;
+    
+    *n_values = 4 + width * subsample_x + height * subsample_y;
+    
+    params = malloc (*n_values * sizeof (pixman_fixed_t));
+    if (!params)
+        goto out;
+
+    params[0] = pixman_int_to_fixed (width);
+    params[1] = pixman_int_to_fixed (height);
+    params[2] = pixman_int_to_fixed (subsample_bits_x);
+    params[3] = pixman_int_to_fixed (subsample_bits_y);
+
+    memcpy (params + 4, horz,
+	    width * subsample_x * sizeof (pixman_fixed_t));
+    memcpy (params + 4 + width * subsample_x, vert,
+	    height * subsample_y * sizeof (pixman_fixed_t));
+
+out:
+    free (horz);
+    free (vert);
+
+    return params;
+}
diff --git a/pixman/pixman/pixman-general.c b/pixman/pixman/pixman-general.c
index bddf79aae..a653fa71a 100644
--- a/pixman/pixman/pixman-general.c
+++ b/pixman/pixman/pixman-general.c
@@ -36,231 +36,194 @@
 #include <stdlib.h>
 #include <string.h>
 #include "pixman-private.h"
-#include "pixman-combine32.h"
-#include "pixman-private.h"
+
+static void
+general_iter_init (pixman_iter_t *iter, const pixman_iter_info_t *info)
+{
+    pixman_image_t *image = iter->image;
+
+    switch (image->type)
+    {
+    case BITS:
+        if ((iter->iter_flags & ITER_SRC) == ITER_SRC)
+            _pixman_bits_image_src_iter_init (image, iter);
+        else
+            _pixman_bits_image_dest_iter_init (image, iter);
+        break;
+
+    case LINEAR:
+        _pixman_linear_gradient_iter_init (image, iter);
+        break;
+
+    case RADIAL:
+	_pixman_radial_gradient_iter_init (image, iter);
+        break;
+
+    case CONICAL:
+	_pixman_conical_gradient_iter_init (image, iter);
+        break;
+
+    case SOLID:
+        _pixman_log_error (FUNC, "Solid image not handled by noop");
+        break;
+
+    default:
+	_pixman_log_error (FUNC, "Pixman bug: unknown image type\n");
+        break;
+    }
+}
+
+static const pixman_iter_info_t general_iters[] =
+{
+    { PIXMAN_any, 0, 0, general_iter_init, NULL, NULL },
+    { PIXMAN_null },
+};
+
+typedef struct op_info_t op_info_t;
+struct op_info_t
+{
+    uint8_t src, dst;
+};
+
+#define ITER_IGNORE_BOTH						\
+    (ITER_IGNORE_ALPHA | ITER_IGNORE_RGB | ITER_LOCALIZED_ALPHA)
+
+static const op_info_t op_flags[PIXMAN_N_OPERATORS] =
+{
+    /* Src                   Dst                   */
+    { ITER_IGNORE_BOTH,      ITER_IGNORE_BOTH      }, /* CLEAR */
+    { ITER_LOCALIZED_ALPHA,  ITER_IGNORE_BOTH      }, /* SRC */
+    { ITER_IGNORE_BOTH,      ITER_LOCALIZED_ALPHA  }, /* DST */
+    { 0,                     ITER_LOCALIZED_ALPHA  }, /* OVER */
+    { ITER_LOCALIZED_ALPHA,  0                     }, /* OVER_REVERSE */
+    { ITER_LOCALIZED_ALPHA,  ITER_IGNORE_RGB       }, /* IN */
+    { ITER_IGNORE_RGB,       ITER_LOCALIZED_ALPHA  }, /* IN_REVERSE */
+    { ITER_LOCALIZED_ALPHA,  ITER_IGNORE_RGB       }, /* OUT */
+    { ITER_IGNORE_RGB,       ITER_LOCALIZED_ALPHA  }, /* OUT_REVERSE */
+    { 0,                     0                     }, /* ATOP */
+    { 0,                     0                     }, /* ATOP_REVERSE */
+    { 0,                     0                     }, /* XOR */
+    { ITER_LOCALIZED_ALPHA,  ITER_LOCALIZED_ALPHA  }, /* ADD */
+    { 0,                     0                     }, /* SATURATE */
+};
 
 #define SCANLINE_BUFFER_LENGTH 8192
 
 static void
 general_composite_rect  (pixman_implementation_t *imp,
-                         pixman_op_t              op,
-                         pixman_image_t *         src,
-                         pixman_image_t *         mask,
-                         pixman_image_t *         dest,
-                         int32_t                  src_x,
-                         int32_t                  src_y,
-                         int32_t                  mask_x,
-                         int32_t                  mask_y,
-                         int32_t                  dest_x,
-                         int32_t                  dest_y,
-                         int32_t                  width,
-                         int32_t                  height)
+                         pixman_composite_info_t *info)
 {
-    uint8_t stack_scanline_buffer[SCANLINE_BUFFER_LENGTH * 3];
-    const pixman_format_code_t src_format =
-	src->type == BITS ? src->bits.format : 0;
-    const pixman_format_code_t mask_format =
-	mask && mask->type == BITS ? mask->bits.format : 0;
-    const pixman_format_code_t dest_format =
-	dest->type == BITS ? dest->bits.format : 0;
-    const int src_wide = PIXMAN_FORMAT_IS_WIDE (src_format);
-    const int mask_wide = mask && PIXMAN_FORMAT_IS_WIDE (mask_format);
-    const int dest_wide = PIXMAN_FORMAT_IS_WIDE (dest_format);
-    const int wide = src_wide || mask_wide || dest_wide;
-    const int Bpp = wide ? 8 : 4;
-    uint8_t *scanline_buffer = stack_scanline_buffer;
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint8_t stack_scanline_buffer[3 * SCANLINE_BUFFER_LENGTH];
+    uint8_t *scanline_buffer = (uint8_t *) stack_scanline_buffer;
     uint8_t *src_buffer, *mask_buffer, *dest_buffer;
-    fetch_scanline_t fetch_src = NULL, fetch_mask = NULL, fetch_dest = NULL;
+    pixman_iter_t src_iter, mask_iter, dest_iter;
     pixman_combine_32_func_t compose;
-    store_scanline_t store;
-    source_image_class_t src_class, mask_class;
     pixman_bool_t component_alpha;
-    uint32_t *bits;
-    int32_t stride;
+    iter_flags_t width_flag, src_iter_flags;
+    int Bpp;
     int i;
 
-    if (width * Bpp > SCANLINE_BUFFER_LENGTH)
+    if ((src_image->common.flags & FAST_PATH_NARROW_FORMAT)		    &&
+	(!mask_image || mask_image->common.flags & FAST_PATH_NARROW_FORMAT) &&
+	(dest_image->common.flags & FAST_PATH_NARROW_FORMAT))
     {
-	scanline_buffer = pixman_malloc_abc (width, 3, Bpp);
-
-	if (!scanline_buffer)
-	    return;
+	width_flag = ITER_NARROW;
+	Bpp = 4;
     }
-
-    src_buffer = scanline_buffer;
-    mask_buffer = src_buffer + width * Bpp;
-    dest_buffer = mask_buffer + width * Bpp;
-
-    src_class = _pixman_image_classify (src,
-                                        src_x, src_y,
-                                        width, height);
-
-    mask_class = SOURCE_IMAGE_CLASS_UNKNOWN;
-
-    if (mask)
+    else
     {
-	mask_class = _pixman_image_classify (mask,
-	                                     src_x, src_y,
-	                                     width, height);
+	width_flag = ITER_WIDE;
+	Bpp = 16;
     }
 
-    if (op == PIXMAN_OP_CLEAR)
-	fetch_src = NULL;
-    else if (wide)
-	fetch_src = _pixman_image_get_scanline_64;
-    else
-	fetch_src = _pixman_image_get_scanline_32;
-
-    if (!mask || op == PIXMAN_OP_CLEAR)
-	fetch_mask = NULL;
-    else if (wide)
-	fetch_mask = _pixman_image_get_scanline_64;
-    else
-	fetch_mask = _pixman_image_get_scanline_32;
+#define ALIGN(addr)							\
+    ((uint8_t *)((((uintptr_t)(addr)) + 15) & (~15)))
 
-    if (op == PIXMAN_OP_CLEAR || op == PIXMAN_OP_SRC)
-	fetch_dest = NULL;
-    else if (wide)
-	fetch_dest = _pixman_image_get_scanline_64;
-    else
-	fetch_dest = _pixman_image_get_scanline_32;
+    src_buffer = ALIGN (scanline_buffer);
+    mask_buffer = ALIGN (src_buffer + width * Bpp);
+    dest_buffer = ALIGN (mask_buffer + width * Bpp);
 
-    if (wide)
-	store = _pixman_image_store_scanline_64;
-    else
-	store = _pixman_image_store_scanline_32;
-
-    /* Skip the store step and composite directly into the
-     * destination if the output format of the compose func matches
-     * the destination format.
-     *
-     * If the destination format is a8r8g8b8 then we can always do
-     * this. If it is x8r8g8b8, then we can only do it if the
-     * operator doesn't make use of destination alpha.
-     */
-    if ((dest->bits.format == PIXMAN_a8r8g8b8)	||
-	(dest->bits.format == PIXMAN_x8r8g8b8	&&
-	 (op == PIXMAN_OP_OVER		||
-	  op == PIXMAN_OP_ADD		||
-	  op == PIXMAN_OP_SRC		||
-	  op == PIXMAN_OP_CLEAR		||
-	  op == PIXMAN_OP_IN_REVERSE	||
-	  op == PIXMAN_OP_OUT_REVERSE	||
-	  op == PIXMAN_OP_DST)))
+    if (ALIGN (dest_buffer + width * Bpp) >
+	    scanline_buffer + sizeof (stack_scanline_buffer))
     {
-	if (!wide &&
-	    !dest->common.alpha_map &&
-	    !dest->bits.write_func)
-	{
-	    store = NULL;
-	}
+	scanline_buffer = pixman_malloc_ab_plus_c (width, Bpp * 3, 32 * 3);
+
+	if (!scanline_buffer)
+	    return;
+
+	src_buffer = ALIGN (scanline_buffer);
+	mask_buffer = ALIGN (src_buffer + width * Bpp);
+	dest_buffer = ALIGN (mask_buffer + width * Bpp);
     }
 
-    if (!store)
+    if (width_flag == ITER_WIDE)
     {
-	bits = dest->bits.bits;
-	stride = dest->bits.rowstride;
+	/* To make sure there aren't any NANs in the buffers */
+	memset (src_buffer, 0, width * Bpp);
+	memset (mask_buffer, 0, width * Bpp);
+	memset (dest_buffer, 0, width * Bpp);
     }
-    else
+    
+    /* src iter */
+    src_iter_flags = width_flag | op_flags[op].src | ITER_SRC;
+
+    _pixman_implementation_iter_init (imp->toplevel, &src_iter, src_image,
+                                      src_x, src_y, width, height,
+                                      src_buffer, src_iter_flags,
+                                      info->src_flags);
+
+    /* mask iter */
+    if ((src_iter_flags & (ITER_IGNORE_ALPHA | ITER_IGNORE_RGB)) ==
+	(ITER_IGNORE_ALPHA | ITER_IGNORE_RGB))
     {
-	bits = NULL;
-	stride = 0;
+	/* If it doesn't matter what the source is, then it doesn't matter
+	 * what the mask is
+	 */
+	mask_image = NULL;
     }
 
     component_alpha =
-        fetch_src                       &&
-        fetch_mask                      &&
-        mask                            &&
-        mask->common.type == BITS       &&
-        mask->common.component_alpha    &&
-        PIXMAN_FORMAT_RGB (mask->bits.format);
-
-    if (wide)
-    {
-	if (component_alpha)
-	    compose = (pixman_combine_32_func_t)_pixman_implementation_combine_64_ca;
-	else
-	    compose = (pixman_combine_32_func_t)_pixman_implementation_combine_64;
-    }
-    else
-    {
-	if (component_alpha)
-	    compose = _pixman_implementation_combine_32_ca;
-	else
-	    compose = _pixman_implementation_combine_32;
-    }
+        mask_image			      &&
+        mask_image->common.type == BITS       &&
+        mask_image->common.component_alpha    &&
+        PIXMAN_FORMAT_RGB (mask_image->bits.format);
 
-    if (!compose)
-	return;
+    _pixman_implementation_iter_init (
+	imp->toplevel, &mask_iter,
+	mask_image, mask_x, mask_y, width, height, mask_buffer,
+	ITER_SRC | width_flag | (component_alpha? 0 : ITER_IGNORE_RGB),
+	info->mask_flags);
 
-    if (!fetch_mask)
-	mask_buffer = NULL;
+    /* dest iter */
+    _pixman_implementation_iter_init (
+	imp->toplevel, &dest_iter, dest_image, dest_x, dest_y, width, height,
+	dest_buffer, ITER_DEST | width_flag | op_flags[op].dst, info->dest_flags);
+
+    compose = _pixman_implementation_lookup_combiner (
+	imp->toplevel, op, component_alpha, width_flag != ITER_WIDE);
 
     for (i = 0; i < height; ++i)
     {
-	/* fill first half of scanline with source */
-	if (fetch_src)
-	{
-	    if (fetch_mask)
-	    {
-		/* fetch mask before source so that fetching of
-		   source can be optimized */
-		fetch_mask (mask, mask_x, mask_y + i,
-		            width, (void *)mask_buffer, 0, 0);
-
-		if (mask_class == SOURCE_IMAGE_CLASS_HORIZONTAL)
-		    fetch_mask = NULL;
-	    }
-
-	    if (src_class == SOURCE_IMAGE_CLASS_HORIZONTAL)
-	    {
-		fetch_src (src, src_x, src_y + i,
-		           width, (void *)src_buffer, 0, 0);
-		fetch_src = NULL;
-	    }
-	    else
-	    {
-		fetch_src (src, src_x, src_y + i,
-		           width, (void *)src_buffer, (void *)mask_buffer,
-		           0xffffffff);
-	    }
-	}
-	else if (fetch_mask)
-	{
-	    fetch_mask (mask, mask_x, mask_y + i,
-	                width, (void *)mask_buffer, 0, 0);
-	}
-
-	if (store)
-	{
-	    /* fill dest into second half of scanline */
-	    if (fetch_dest)
-	    {
-		fetch_dest (dest, dest_x, dest_y + i,
-		            width, (void *)dest_buffer, 0, 0);
-	    }
-
-	    /* blend */
-	    compose (imp->toplevel, op,
-		     (void *)dest_buffer,
-		     (void *)src_buffer,
-		     (void *)mask_buffer,
-		     width);
-
-	    /* write back */
-	    store (&(dest->bits), dest_x, dest_y + i, width,
-	           (void *)dest_buffer);
-	}
-	else
-	{
-	    /* blend */
-	    compose (imp->toplevel, op,
-		     bits + (dest_y + i) * stride + dest_x,
-	             (void *)src_buffer, (void *)mask_buffer, width);
-	}
+	uint32_t *s, *m, *d;
+
+	m = mask_iter.get_scanline (&mask_iter, NULL);
+	s = src_iter.get_scanline (&src_iter, m);
+	d = dest_iter.get_scanline (&dest_iter, NULL);
+
+	compose (imp->toplevel, op, d, s, m, width);
+
+	dest_iter.write_back (&dest_iter);
     }
 
-    if (scanline_buffer != stack_scanline_buffer)
+    if (src_iter.fini)
+	src_iter.fini (&src_iter);
+    if (mask_iter.fini)
+	mask_iter.fini (&mask_iter);
+    if (dest_iter.fini)
+	dest_iter.fini (&dest_iter);
+    
+    if (scanline_buffer != (uint8_t *) stack_scanline_buffer)
 	free (scanline_buffer);
 }
 
@@ -270,50 +233,15 @@ static const pixman_fast_path_t general_fast_path[] =
     { PIXMAN_OP_NONE }
 };
 
-static pixman_bool_t
-general_blt (pixman_implementation_t *imp,
-             uint32_t *               src_bits,
-             uint32_t *               dst_bits,
-             int                      src_stride,
-             int                      dst_stride,
-             int                      src_bpp,
-             int                      dst_bpp,
-             int                      src_x,
-             int                      src_y,
-             int                      dst_x,
-             int                      dst_y,
-             int                      width,
-             int                      height)
-{
-    /* We can't blit unless we have sse2 or mmx */
-
-    return FALSE;
-}
-
-static pixman_bool_t
-general_fill (pixman_implementation_t *imp,
-              uint32_t *               bits,
-              int                      stride,
-              int                      bpp,
-              int                      x,
-              int                      y,
-              int                      width,
-              int                      height,
-              uint32_t xor)
-{
-    return FALSE;
-}
-
 pixman_implementation_t *
 _pixman_implementation_create_general (void)
 {
     pixman_implementation_t *imp = _pixman_implementation_create (NULL, general_fast_path);
 
     _pixman_setup_combiner_functions_32 (imp);
-    _pixman_setup_combiner_functions_64 (imp);
+    _pixman_setup_combiner_functions_float (imp);
 
-    imp->blt = general_blt;
-    imp->fill = general_fill;
+    imp->iter_info = general_iters;
 
     return imp;
 }
diff --git a/pixman/pixman/pixman-glyph.c b/pixman/pixman/pixman-glyph.c
new file mode 100644
index 000000000..96a349ab4
--- /dev/null
+++ b/pixman/pixman/pixman-glyph.c
@@ -0,0 +1,676 @@
+/*
+ * Copyright 2010, 2012, Soren Sandmann <sandmann@cs.au.dk>
+ * Copyright 2010, 2011, 2012, Red Hat, Inc
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Author: Soren Sandmann <sandmann@cs.au.dk>
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#include "pixman-private.h"
+
+#include <stdlib.h>
+
+typedef struct glyph_metrics_t glyph_metrics_t;
+typedef struct glyph_t glyph_t;
+
+#define TOMBSTONE ((glyph_t *)0x1)
+
+/* XXX: These numbers are arbitrary---we've never done any measurements.
+ */
+#define N_GLYPHS_HIGH_WATER  (16384)
+#define N_GLYPHS_LOW_WATER   (8192)
+#define HASH_SIZE (2 * N_GLYPHS_HIGH_WATER)
+#define HASH_MASK (HASH_SIZE - 1)
+
+struct glyph_t
+{
+    void *		font_key;
+    void *		glyph_key;
+    int			origin_x;
+    int			origin_y;
+    pixman_image_t *	image;
+    pixman_link_t	mru_link;
+};
+
+struct pixman_glyph_cache_t
+{
+    int			n_glyphs;
+    int			n_tombstones;
+    int			freeze_count;
+    pixman_list_t	mru;
+    glyph_t *		glyphs[HASH_SIZE];
+};
+
+static void
+free_glyph (glyph_t *glyph)
+{
+    pixman_list_unlink (&glyph->mru_link);
+    pixman_image_unref (glyph->image);
+    free (glyph);
+}
+
+static unsigned int
+hash (const void *font_key, const void *glyph_key)
+{
+    size_t key = (size_t)font_key + (size_t)glyph_key;
+
+    /* This hash function is based on one found on Thomas Wang's
+     * web page at
+     *
+     *    http://www.concentric.net/~Ttwang/tech/inthash.htm
+     *
+     */
+    key = (key << 15) - key - 1;
+    key = key ^ (key >> 12);
+    key = key + (key << 2);
+    key = key ^ (key >> 4);
+    key = key + (key << 3) + (key << 11);
+    key = key ^ (key >> 16);
+
+    return key;
+}
+
+static glyph_t *
+lookup_glyph (pixman_glyph_cache_t *cache,
+	      void                 *font_key,
+	      void                 *glyph_key)
+{
+    unsigned idx;
+    glyph_t *g;
+
+    idx = hash (font_key, glyph_key);
+    while ((g = cache->glyphs[idx++ & HASH_MASK]))
+    {
+	if (g != TOMBSTONE			&&
+	    g->font_key == font_key		&&
+	    g->glyph_key == glyph_key)
+	{
+	    return g;
+	}
+    }
+
+    return NULL;
+}
+
+static void
+insert_glyph (pixman_glyph_cache_t *cache,
+	      glyph_t              *glyph)
+{
+    unsigned idx;
+    glyph_t **loc;
+
+    idx = hash (glyph->font_key, glyph->glyph_key);
+
+    /* Note: we assume that there is room in the table. If there isn't,
+     * this will be an infinite loop.
+     */
+    do
+    {
+	loc = &cache->glyphs[idx++ & HASH_MASK];
+    } while (*loc && *loc != TOMBSTONE);
+
+    if (*loc == TOMBSTONE)
+	cache->n_tombstones--;
+    cache->n_glyphs++;
+
+    *loc = glyph;
+}
+
+static void
+remove_glyph (pixman_glyph_cache_t *cache,
+	      glyph_t              *glyph)
+{
+    unsigned idx;
+
+    idx = hash (glyph->font_key, glyph->glyph_key);
+    while (cache->glyphs[idx & HASH_MASK] != glyph)
+	idx++;
+
+    cache->glyphs[idx & HASH_MASK] = TOMBSTONE;
+    cache->n_tombstones++;
+    cache->n_glyphs--;
+
+    /* Eliminate tombstones if possible */
+    if (cache->glyphs[(idx + 1) & HASH_MASK] == NULL)
+    {
+	while (cache->glyphs[idx & HASH_MASK] == TOMBSTONE)
+	{
+	    cache->glyphs[idx & HASH_MASK] = NULL;
+	    cache->n_tombstones--;
+	    idx--;
+	}
+    }
+}
+
+static void
+clear_table (pixman_glyph_cache_t *cache)
+{
+    int i;
+
+    for (i = 0; i < HASH_SIZE; ++i)
+    {
+	glyph_t *glyph = cache->glyphs[i];
+
+	if (glyph && glyph != TOMBSTONE)
+	    free_glyph (glyph);
+
+	cache->glyphs[i] = NULL;
+    }
+
+    cache->n_glyphs = 0;
+    cache->n_tombstones = 0;
+}
+
+PIXMAN_EXPORT pixman_glyph_cache_t *
+pixman_glyph_cache_create (void)
+{
+    pixman_glyph_cache_t *cache;
+
+    if (!(cache = malloc (sizeof *cache)))
+	return NULL;
+
+    memset (cache->glyphs, 0, sizeof (cache->glyphs));
+    cache->n_glyphs = 0;
+    cache->n_tombstones = 0;
+    cache->freeze_count = 0;
+
+    pixman_list_init (&cache->mru);
+
+    return cache;
+}
+
+PIXMAN_EXPORT void
+pixman_glyph_cache_destroy (pixman_glyph_cache_t *cache)
+{
+    return_if_fail (cache->freeze_count == 0);
+
+    clear_table (cache);
+
+    free (cache);
+}
+
+PIXMAN_EXPORT void
+pixman_glyph_cache_freeze (pixman_glyph_cache_t  *cache)
+{
+    cache->freeze_count++;
+}
+
+PIXMAN_EXPORT void
+pixman_glyph_cache_thaw (pixman_glyph_cache_t  *cache)
+{
+    if (--cache->freeze_count == 0					&&
+	cache->n_glyphs + cache->n_tombstones > N_GLYPHS_HIGH_WATER)
+    {
+	if (cache->n_tombstones > N_GLYPHS_HIGH_WATER)
+	{
+	    /* More than half the entries are
+	     * tombstones. Just dump the whole table.
+	     */
+	    clear_table (cache);
+	}
+
+	while (cache->n_glyphs > N_GLYPHS_LOW_WATER)
+	{
+	    glyph_t *glyph = CONTAINER_OF (glyph_t, mru_link, cache->mru.tail);
+
+	    remove_glyph (cache, glyph);
+	    free_glyph (glyph);
+	}
+    }
+}
+
+PIXMAN_EXPORT const void *
+pixman_glyph_cache_lookup (pixman_glyph_cache_t  *cache,
+			   void                  *font_key,
+			   void                  *glyph_key)
+{
+    return lookup_glyph (cache, font_key, glyph_key);
+}
+
+PIXMAN_EXPORT const void *
+pixman_glyph_cache_insert (pixman_glyph_cache_t  *cache,
+			   void                  *font_key,
+			   void                  *glyph_key,
+			   int			  origin_x,
+			   int                    origin_y,
+			   pixman_image_t        *image)
+{
+    glyph_t *glyph;
+    int32_t width, height;
+
+    return_val_if_fail (cache->freeze_count > 0, NULL);
+    return_val_if_fail (image->type == BITS, NULL);
+
+    width = image->bits.width;
+    height = image->bits.height;
+
+    if (cache->n_glyphs >= HASH_SIZE)
+	return NULL;
+
+    if (!(glyph = malloc (sizeof *glyph)))
+	return NULL;
+
+    glyph->font_key = font_key;
+    glyph->glyph_key = glyph_key;
+    glyph->origin_x = origin_x;
+    glyph->origin_y = origin_y;
+
+    if (!(glyph->image = pixman_image_create_bits (
+	      image->bits.format, width, height, NULL, -1)))
+    {
+	free (glyph);
+	return NULL;
+    }
+
+    pixman_image_composite32 (PIXMAN_OP_SRC,
+			      image, NULL, glyph->image, 0, 0, 0, 0, 0, 0,
+			      width, height);
+
+    if (PIXMAN_FORMAT_A   (glyph->image->bits.format) != 0	&&
+	PIXMAN_FORMAT_RGB (glyph->image->bits.format) != 0)
+    {
+	pixman_image_set_component_alpha (glyph->image, TRUE);
+    }
+
+    pixman_list_prepend (&cache->mru, &glyph->mru_link);
+
+    _pixman_image_validate (glyph->image);
+    insert_glyph (cache, glyph);
+
+    return glyph;
+}
+
+PIXMAN_EXPORT void
+pixman_glyph_cache_remove (pixman_glyph_cache_t  *cache,
+			   void                  *font_key,
+			   void                  *glyph_key)
+{
+    glyph_t *glyph;
+
+    if ((glyph = lookup_glyph (cache, font_key, glyph_key)))
+    {
+	remove_glyph (cache, glyph);
+
+	free_glyph (glyph);
+    }
+}
+
+PIXMAN_EXPORT void
+pixman_glyph_get_extents (pixman_glyph_cache_t *cache,
+			  int                   n_glyphs,
+			  pixman_glyph_t       *glyphs,
+			  pixman_box32_t       *extents)
+{
+    int i;
+
+    extents->x1 = extents->y1 = INT32_MAX;
+    extents->x2 = extents->y2 = INT32_MIN;
+
+    for (i = 0; i < n_glyphs; ++i)
+    {
+	glyph_t *glyph = (glyph_t *)glyphs[i].glyph;
+	int x1, y1, x2, y2;
+
+	x1 = glyphs[i].x - glyph->origin_x;
+	y1 = glyphs[i].y - glyph->origin_y;
+	x2 = glyphs[i].x - glyph->origin_x + glyph->image->bits.width;
+	y2 = glyphs[i].y - glyph->origin_y + glyph->image->bits.height;
+
+	if (x1 < extents->x1)
+	    extents->x1 = x1;
+	if (y1 < extents->y1)
+	    extents->y1 = y1;
+	if (x2 > extents->x2)
+	    extents->x2 = x2;
+	if (y2 > extents->y2)
+	    extents->y2 = y2;
+    }
+}
+
+/* This function returns a format that is suitable for use as a mask for the
+ * set of glyphs in question.
+ */
+PIXMAN_EXPORT pixman_format_code_t
+pixman_glyph_get_mask_format (pixman_glyph_cache_t *cache,
+			      int		    n_glyphs,
+			      const pixman_glyph_t *glyphs)
+{
+    pixman_format_code_t format = PIXMAN_a1;
+    int i;
+
+    for (i = 0; i < n_glyphs; ++i)
+    {
+	const glyph_t *glyph = glyphs[i].glyph;
+	pixman_format_code_t glyph_format = glyph->image->bits.format;
+
+	if (PIXMAN_FORMAT_TYPE (glyph_format) == PIXMAN_TYPE_A)
+	{
+	    if (PIXMAN_FORMAT_A (glyph_format) > PIXMAN_FORMAT_A (format))
+		format = glyph_format;
+	}
+	else
+	{
+	    return PIXMAN_a8r8g8b8;
+	}
+    }
+
+    return format;
+}
+
+static pixman_bool_t
+box32_intersect (pixman_box32_t *dest,
+		 const pixman_box32_t *box1,
+		 const pixman_box32_t *box2)
+{
+    dest->x1 = MAX (box1->x1, box2->x1);
+    dest->y1 = MAX (box1->y1, box2->y1);
+    dest->x2 = MIN (box1->x2, box2->x2);
+    dest->y2 = MIN (box1->y2, box2->y2);
+
+    return dest->x2 > dest->x1 && dest->y2 > dest->y1;
+}
+
+#if defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
+__attribute__((__force_align_arg_pointer__))
+#endif
+PIXMAN_EXPORT void
+pixman_composite_glyphs_no_mask (pixman_op_t            op,
+				 pixman_image_t        *src,
+				 pixman_image_t        *dest,
+				 int32_t                src_x,
+				 int32_t                src_y,
+				 int32_t                dest_x,
+				 int32_t                dest_y,
+				 pixman_glyph_cache_t  *cache,
+				 int                    n_glyphs,
+				 const pixman_glyph_t  *glyphs)
+{
+    pixman_region32_t region;
+    pixman_format_code_t glyph_format = PIXMAN_null;
+    uint32_t glyph_flags = 0;
+    pixman_format_code_t dest_format;
+    uint32_t dest_flags;
+    pixman_composite_func_t func = NULL;
+    pixman_implementation_t *implementation = NULL;
+    pixman_composite_info_t info;
+    int i;
+
+    _pixman_image_validate (src);
+    _pixman_image_validate (dest);
+    
+    dest_format = dest->common.extended_format_code;
+    dest_flags = dest->common.flags;
+    
+    pixman_region32_init (&region);
+    if (!_pixman_compute_composite_region32 (
+	    &region,
+	    src, NULL, dest,
+	    src_x - dest_x, src_y - dest_y, 0, 0, 0, 0,
+	    dest->bits.width, dest->bits.height))
+    {
+	goto out;
+    }
+
+    info.op = op;
+    info.src_image = src;
+    info.dest_image = dest;
+    info.src_flags = src->common.flags;
+    info.dest_flags = dest->common.flags;
+
+    for (i = 0; i < n_glyphs; ++i)
+    {
+	glyph_t *glyph = (glyph_t *)glyphs[i].glyph;
+	pixman_image_t *glyph_img = glyph->image;
+	pixman_box32_t glyph_box;
+	pixman_box32_t *pbox;
+	uint32_t extra = FAST_PATH_SAMPLES_COVER_CLIP_NEAREST;
+	pixman_box32_t composite_box;
+	int n;
+
+	glyph_box.x1 = dest_x + glyphs[i].x - glyph->origin_x;
+	glyph_box.y1 = dest_y + glyphs[i].y - glyph->origin_y;
+	glyph_box.x2 = glyph_box.x1 + glyph->image->bits.width;
+	glyph_box.y2 = glyph_box.y1 + glyph->image->bits.height;
+	
+	pbox = pixman_region32_rectangles (&region, &n);
+	
+	info.mask_image = glyph_img;
+
+	while (n--)
+	{
+	    if (box32_intersect (&composite_box, pbox, &glyph_box))
+	    {
+		if (glyph_img->common.extended_format_code != glyph_format	||
+		    glyph_img->common.flags != glyph_flags)
+		{
+		    glyph_format = glyph_img->common.extended_format_code;
+		    glyph_flags = glyph_img->common.flags;
+
+		    _pixman_implementation_lookup_composite (
+			get_implementation(), op,
+			src->common.extended_format_code, src->common.flags,
+			glyph_format, glyph_flags | extra,
+			dest_format, dest_flags,
+			&implementation, &func);
+		}
+
+		info.src_x = src_x + composite_box.x1 - dest_x;
+		info.src_y = src_y + composite_box.y1 - dest_y;
+		info.mask_x = composite_box.x1 - (dest_x + glyphs[i].x - glyph->origin_x);
+		info.mask_y = composite_box.y1 - (dest_y + glyphs[i].y - glyph->origin_y);
+		info.dest_x = composite_box.x1;
+		info.dest_y = composite_box.y1;
+		info.width = composite_box.x2 - composite_box.x1;
+		info.height = composite_box.y2 - composite_box.y1;
+
+		info.mask_flags = glyph_flags;
+
+		func (implementation, &info);
+	    }
+
+	    pbox++;
+	}
+	pixman_list_move_to_front (&cache->mru, &glyph->mru_link);
+    }
+
+out:
+    pixman_region32_fini (&region);
+}
+
+static void
+add_glyphs (pixman_glyph_cache_t *cache,
+	    pixman_image_t *dest,
+	    int off_x, int off_y,
+	    int n_glyphs, const pixman_glyph_t *glyphs)
+{
+    pixman_format_code_t glyph_format = PIXMAN_null;
+    uint32_t glyph_flags = 0;
+    pixman_composite_func_t func = NULL;
+    pixman_implementation_t *implementation = NULL;
+    pixman_format_code_t dest_format;
+    uint32_t dest_flags;
+    pixman_box32_t dest_box;
+    pixman_composite_info_t info;
+    pixman_image_t *white_img = NULL;
+    pixman_bool_t white_src = FALSE;
+    int i;
+
+    _pixman_image_validate (dest);
+
+    dest_format = dest->common.extended_format_code;
+    dest_flags = dest->common.flags;
+
+    info.op = PIXMAN_OP_ADD;
+    info.dest_image = dest;
+    info.src_x = 0;
+    info.src_y = 0;
+    info.dest_flags = dest_flags;
+
+    dest_box.x1 = 0;
+    dest_box.y1 = 0;
+    dest_box.x2 = dest->bits.width;
+    dest_box.y2 = dest->bits.height;
+
+    for (i = 0; i < n_glyphs; ++i)
+    {
+	glyph_t *glyph = (glyph_t *)glyphs[i].glyph;
+	pixman_image_t *glyph_img = glyph->image;
+	pixman_box32_t glyph_box;
+	pixman_box32_t composite_box;
+
+	if (glyph_img->common.extended_format_code != glyph_format	||
+	    glyph_img->common.flags != glyph_flags)
+	{
+	    pixman_format_code_t src_format, mask_format;
+
+	    glyph_format = glyph_img->common.extended_format_code;
+	    glyph_flags = glyph_img->common.flags;
+
+	    if (glyph_format == dest->bits.format)
+	    {
+		src_format = glyph_format;
+		mask_format = PIXMAN_null;
+		info.src_flags = glyph_flags | FAST_PATH_SAMPLES_COVER_CLIP_NEAREST;
+		info.mask_flags = FAST_PATH_IS_OPAQUE;
+		info.mask_image = NULL;
+		white_src = FALSE;
+	    }
+	    else
+	    {
+		if (!white_img)
+		{
+		    static const pixman_color_t white = { 0xffff, 0xffff, 0xffff, 0xffff };
+
+		    if (!(white_img = pixman_image_create_solid_fill (&white)))
+			goto out;
+
+		    _pixman_image_validate (white_img);
+		}
+
+		src_format = PIXMAN_solid;
+		mask_format = glyph_format;
+		info.src_flags = white_img->common.flags;
+		info.mask_flags = glyph_flags | FAST_PATH_SAMPLES_COVER_CLIP_NEAREST;
+		info.src_image = white_img;
+		white_src = TRUE;
+	    }
+
+	    _pixman_implementation_lookup_composite (
+		get_implementation(), PIXMAN_OP_ADD,
+		src_format, info.src_flags,
+		mask_format, info.mask_flags,
+		dest_format, dest_flags,
+		&implementation, &func);
+	}
+
+	glyph_box.x1 = glyphs[i].x - glyph->origin_x + off_x;
+	glyph_box.y1 = glyphs[i].y - glyph->origin_y + off_y;
+	glyph_box.x2 = glyph_box.x1 + glyph->image->bits.width;
+	glyph_box.y2 = glyph_box.y1 + glyph->image->bits.height;
+	
+	if (box32_intersect (&composite_box, &glyph_box, &dest_box))
+	{
+	    int src_x = composite_box.x1 - glyph_box.x1;
+	    int src_y = composite_box.y1 - glyph_box.y1;
+
+	    if (white_src)
+		info.mask_image = glyph_img;
+	    else
+		info.src_image = glyph_img;
+
+	    info.mask_x = info.src_x = src_x;
+	    info.mask_y = info.src_y = src_y;
+	    info.dest_x = composite_box.x1;
+	    info.dest_y = composite_box.y1;
+	    info.width = composite_box.x2 - composite_box.x1;
+	    info.height = composite_box.y2 - composite_box.y1;
+
+	    func (implementation, &info);
+
+	    pixman_list_move_to_front (&cache->mru, &glyph->mru_link);
+	}
+    }
+
+out:
+    if (white_img)
+	pixman_image_unref (white_img);
+}
+
+/* Conceptually, for each glyph, (white IN glyph) is PIXMAN_OP_ADDed to an
+ * infinitely big mask image at the position such that the glyph origin point
+ * is positioned at the (glyphs[i].x, glyphs[i].y) point.
+ *
+ * Then (mask_x, mask_y) in the infinite mask and (src_x, src_y) in the source
+ * image are both aligned with (dest_x, dest_y) in the destination image. Then
+ * these three images are composited within the 
+ *
+ *       (dest_x, dest_y, dst_x + width, dst_y + height)
+ *
+ * rectangle.
+ *
+ * TODO:
+ *   - Trim the mask to the destination clip/image?
+ *   - Trim composite region based on sources, when the op ignores 0s.
+ */
+#if defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
+__attribute__((__force_align_arg_pointer__))
+#endif
+PIXMAN_EXPORT void
+pixman_composite_glyphs (pixman_op_t            op,
+			 pixman_image_t        *src,
+			 pixman_image_t        *dest,
+			 pixman_format_code_t   mask_format,
+			 int32_t                src_x,
+			 int32_t                src_y,
+			 int32_t		mask_x,
+			 int32_t		mask_y,
+			 int32_t                dest_x,
+			 int32_t                dest_y,
+			 int32_t                width,
+			 int32_t                height,
+			 pixman_glyph_cache_t  *cache,
+			 int			n_glyphs,
+			 const pixman_glyph_t  *glyphs)
+{
+    pixman_image_t *mask;
+
+    if (!(mask = pixman_image_create_bits (mask_format, width, height, NULL, -1)))
+	return;
+
+    if (PIXMAN_FORMAT_A   (mask_format) != 0 &&
+	PIXMAN_FORMAT_RGB (mask_format) != 0)
+    {
+	pixman_image_set_component_alpha (mask, TRUE);
+    }
+
+    add_glyphs (cache, mask, - mask_x, - mask_y, n_glyphs, glyphs);
+
+    pixman_image_composite32 (op, src, mask, dest,
+			      src_x, src_y,
+			      0, 0,
+			      dest_x, dest_y,
+			      width, height);
+
+    pixman_image_unref (mask);
+}
diff --git a/pixman/pixman/pixman-gradient-walker.c b/pixman/pixman/pixman-gradient-walker.c
index dd666b412..5944a559a 100644
--- a/pixman/pixman/pixman-gradient-walker.c
+++ b/pixman/pixman/pixman-gradient-walker.c
@@ -31,123 +31,71 @@
 void
 _pixman_gradient_walker_init (pixman_gradient_walker_t *walker,
                               gradient_t *              gradient,
-                              unsigned int              spread)
+                              pixman_repeat_t		repeat)
 {
     walker->num_stops = gradient->n_stops;
     walker->stops     = gradient->stops;
     walker->left_x    = 0;
     walker->right_x   = 0x10000;
-    walker->stepper   = 0;
-    walker->left_ag   = 0;
-    walker->left_rb   = 0;
-    walker->right_ag  = 0;
-    walker->right_rb  = 0;
-    walker->spread    = spread;
+    walker->a_s       = 0.0f;
+    walker->a_b       = 0.0f;
+    walker->r_s       = 0.0f;
+    walker->r_b       = 0.0f;
+    walker->g_s       = 0.0f;
+    walker->g_b       = 0.0f;
+    walker->b_s       = 0.0f;
+    walker->b_b       = 0.0f;
+    walker->repeat    = repeat;
 
     walker->need_reset = TRUE;
 }
 
-void
-_pixman_gradient_walker_reset (pixman_gradient_walker_t *walker,
-                               pixman_fixed_32_32_t      pos)
+static void
+gradient_walker_reset (pixman_gradient_walker_t *walker,
+		       pixman_fixed_48_16_t      pos)
 {
     int32_t x, left_x, right_x;
-    pixman_color_t          *left_c, *right_c;
+    pixman_color_t *left_c, *right_c;
     int n, count = walker->num_stops;
-    pixman_gradient_stop_t *      stops = walker->stops;
-
-    static const pixman_color_t transparent_black = { 0, 0, 0, 0 };
+    pixman_gradient_stop_t *stops = walker->stops;
+    float la, lr, lg, lb;
+    float ra, rr, rg, rb;
+    float lx, rx;
 
-    switch (walker->spread)
+    if (walker->repeat == PIXMAN_REPEAT_NORMAL)
     {
-    case PIXMAN_REPEAT_NORMAL:
-	x = (int32_t)pos & 0xFFFF;
-	for (n = 0; n < count; n++)
-	    if (x < stops[n].x)
-		break;
-	if (n == 0)
-	{
-	    left_x =  stops[count - 1].x - 0x10000;
-	    left_c = &stops[count - 1].color;
-	}
-	else
-	{
-	    left_x =  stops[n - 1].x;
-	    left_c = &stops[n - 1].color;
-	}
-
-	if (n == count)
-	{
-	    right_x =  stops[0].x + 0x10000;
-	    right_c = &stops[0].color;
-	}
-	else
-	{
-	    right_x =  stops[n].x;
-	    right_c = &stops[n].color;
-	}
-	left_x  += (pos - x);
-	right_x += (pos - x);
-	break;
-
-    case PIXMAN_REPEAT_PAD:
-	for (n = 0; n < count; n++)
-	    if (pos < stops[n].x)
-		break;
-
-	if (n == 0)
-	{
-	    left_x =  INT32_MIN;
-	    left_c = &stops[0].color;
-	}
-	else
-	{
-	    left_x =  stops[n - 1].x;
-	    left_c = &stops[n - 1].color;
-	}
-
-	if (n == count)
-	{
-	    right_x =  INT32_MAX;
-	    right_c = &stops[n - 1].color;
-	}
-	else
-	{
-	    right_x =  stops[n].x;
-	    right_c = &stops[n].color;
-	}
-	break;
-
-    case PIXMAN_REPEAT_REFLECT:
-	x = (int32_t)pos & 0xFFFF;
+	x = (int32_t)pos & 0xffff;
+    }
+    else if (walker->repeat == PIXMAN_REPEAT_REFLECT)
+    {
+	x = (int32_t)pos & 0xffff;
 	if ((int32_t)pos & 0x10000)
 	    x = 0x10000 - x;
-	for (n = 0; n < count; n++)
-	    if (x < stops[n].x)
-		break;
-
-	if (n == 0)
-	{
-	    left_x =  -stops[0].x;
-	    left_c = &stops[0].color;
-	}
-	else
-	{
-	    left_x =  stops[n - 1].x;
-	    left_c = &stops[n - 1].color;
-	}
-
-	if (n == count)
-	{
-	    right_x = 0x20000 - stops[n - 1].x;
-	    right_c = &stops[n - 1].color;
-	}
-	else
-	{
-	    right_x =  stops[n].x;
-	    right_c = &stops[n].color;
-	}
-
+    }
+    else
+    {
+	x = pos;
+    }
+    
+    for (n = 0; n < count; n++)
+    {
+	if (x < stops[n].x)
+	    break;
+    }
+    
+    left_x =  stops[n - 1].x;
+    left_c = &stops[n - 1].color;
+    
+    right_x =  stops[n].x;
+    right_c = &stops[n].color;
+
+    if (walker->repeat == PIXMAN_REPEAT_NORMAL)
+    {
+	left_x  += (pos - x);
+	right_x += (pos - x);
+    }
+    else if (walker->repeat == PIXMAN_REPEAT_REFLECT)
+    {
 	if ((int32_t)pos & 0x10000)
 	{
 	    pixman_color_t  *tmp_c;
@@ -165,90 +113,90 @@ _pixman_gradient_walker_reset (pixman_gradient_walker_t *walker,
 	}
 	left_x  += (pos - x);
 	right_x += (pos - x);
-	break;
-
-    default:  /* REPEAT_NONE */
-	for (n = 0; n < count; n++)
-	    if (pos < stops[n].x)
-		break;
-
+    }
+    else if (walker->repeat == PIXMAN_REPEAT_NONE)
+    {
 	if (n == 0)
-	{
-	    left_x  =  INT32_MIN;
-	    right_x =  stops[0].x;
-	    left_c  = right_c = (pixman_color_t*) &transparent_black;
-	}
+	    right_c = left_c;
 	else if (n == count)
-	{
-	    left_x  = stops[n - 1].x;
-	    right_x = INT32_MAX;
-	    left_c  = right_c = (pixman_color_t*) &transparent_black;
-	}
-	else
-	{
-	    left_x  =  stops[n - 1].x;
-	    right_x =  stops[n].x;
-	    left_c  = &stops[n - 1].color;
-	    right_c = &stops[n].color;
-	}
+	    left_c = right_c;
     }
 
-    walker->left_x   = left_x;
-    walker->right_x  = right_x;
-    walker->left_ag  = ((left_c->alpha >> 8) << 16)   | (left_c->green >> 8);
-    walker->left_rb  = ((left_c->red & 0xff00) << 8)  | (left_c->blue >> 8);
-    walker->right_ag = ((right_c->alpha >> 8) << 16)  | (right_c->green >> 8);
-    walker->right_rb = ((right_c->red & 0xff00) << 8) | (right_c->blue >> 8);
-
-    if (walker->left_x == walker->right_x                ||
-        ( walker->left_ag == walker->right_ag &&
-          walker->left_rb == walker->right_rb )   )
+    /* The alpha channel is scaled to be in the [0, 255] interval,
+     * and the red/green/blue channels are scaled to be in [0, 1].
+     * This ensures that after premultiplication all channels will
+     * be in the [0, 255] interval.
+     */
+    la = (left_c->alpha * (1.0f/257.0f));
+    lr = (left_c->red * (1.0f/257.0f));
+    lg = (left_c->green * (1.0f/257.0f));
+    lb = (left_c->blue * (1.0f/257.0f));
+
+    ra = (right_c->alpha * (1.0f/257.0f));
+    rr = (right_c->red * (1.0f/257.0f));
+    rg = (right_c->green * (1.0f/257.0f));
+    rb = (right_c->blue * (1.0f/257.0f));
+    
+    lx = left_x * (1.0f/65536.0f);
+    rx = right_x * (1.0f/65536.0f);
+    
+    if (FLOAT_IS_ZERO (rx - lx) || left_x == INT32_MIN || right_x == INT32_MAX)
     {
-	walker->stepper = 0;
+	walker->a_s = walker->r_s = walker->g_s = walker->b_s = 0.0f;
+	walker->a_b = (la + ra) / 2.0f;
+	walker->r_b = (lr + rr) / 510.0f;
+	walker->g_b = (lg + rg) / 510.0f;
+	walker->b_b = (lb + rb) / 510.0f;
     }
     else
     {
-	int32_t width = right_x - left_x;
-	walker->stepper = ((1 << 24) + width / 2) / width;
+	float w_rec = 1.0f / (rx - lx);
+
+	walker->a_b = (la * rx - ra * lx) * w_rec;
+	walker->r_b = (lr * rx - rr * lx) * w_rec * (1.0f/255.0f);
+	walker->g_b = (lg * rx - rg * lx) * w_rec * (1.0f/255.0f);
+	walker->b_b = (lb * rx - rb * lx) * w_rec * (1.0f/255.0f);
+
+	walker->a_s = (ra - la) * w_rec;
+	walker->r_s = (rr - lr) * w_rec * (1.0f/255.0f);
+	walker->g_s = (rg - lg) * w_rec * (1.0f/255.0f);
+	walker->b_s = (rb - lb) * w_rec * (1.0f/255.0f);
     }
+   
+    walker->left_x = left_x;
+    walker->right_x = right_x;
 
     walker->need_reset = FALSE;
 }
 
-#define  PIXMAN_GRADIENT_WALKER_NEED_RESET(w, x)                         \
-    ( (w)->need_reset || (x) < (w)->left_x || (x) >= (w)->right_x)
-
-
-/* the following assumes that PIXMAN_GRADIENT_WALKER_NEED_RESET(w,x) is FALSE */
 uint32_t
 _pixman_gradient_walker_pixel (pixman_gradient_walker_t *walker,
-                               pixman_fixed_32_32_t      x)
+                               pixman_fixed_48_16_t      x)
 {
-    int dist, idist;
-    uint32_t t1, t2, a, color;
-
-    if (PIXMAN_GRADIENT_WALKER_NEED_RESET (walker, x))
-	_pixman_gradient_walker_reset (walker, x);
+    float a, r, g, b;
+    uint8_t a8, r8, g8, b8;
+    uint32_t v;
+    float y;
 
-    dist  = ((int)(x - walker->left_x) * walker->stepper) >> 16;
-    idist = 256 - dist;
+    if (walker->need_reset || x < walker->left_x || x >= walker->right_x)
+        gradient_walker_reset (walker, x);
 
-    /* combined INTERPOLATE and premultiply */
-    t1 = walker->left_rb * idist + walker->right_rb * dist;
-    t1 = (t1 >> 8) & 0xff00ff;
+    y = x * (1.0f / 65536.0f);
 
-    t2  = walker->left_ag * idist + walker->right_ag * dist;
-    t2 &= 0xff00ff00;
+    a = walker->a_s * y + walker->a_b;
+    r = a * (walker->r_s * y + walker->r_b);
+    g = a * (walker->g_s * y + walker->g_b);
+    b = a * (walker->b_s * y + walker->b_b);
 
-    color = t2 & 0xff000000;
-    a     = t2 >> 24;
+    a8 = a + 0.5f;
+    r8 = r + 0.5f;
+    g8 = g + 0.5f;
+    b8 = b + 0.5f;
 
-    t1  = t1 * a + 0x800080;
-    t1  = (t1 + ((t1 >> 8) & 0xff00ff)) >> 8;
+    v = ((a8 << 24) & 0xff000000) |
+        ((r8 << 16) & 0x00ff0000) |
+        ((g8 <<  8) & 0x0000ff00) |
+        ((b8 >>  0) & 0x000000ff);
 
-    t2  = (t2 >> 8) * a + 0x800080;
-    t2  = (t2 + ((t2 >> 8) & 0xff00ff));
-
-    return (color | (t1 & 0xff00ff) | (t2 & 0xff00));
+    return v;
 }
-
diff --git a/pixman/pixman/pixman-image.c b/pixman/pixman/pixman-image.c
index 03a39db87..1ff1a4974 100644
--- a/pixman/pixman/pixman-image.c
+++ b/pixman/pixman/pixman-image.c
@@ -30,7 +30,50 @@
 #include <assert.h>
 
 #include "pixman-private.h"
-#include "pixman-combine32.h"
+
+static const pixman_color_t transparent_black = { 0, 0, 0, 0 };
+
+static void
+gradient_property_changed (pixman_image_t *image)
+{
+    gradient_t *gradient = &image->gradient;
+    int n = gradient->n_stops;
+    pixman_gradient_stop_t *stops = gradient->stops;
+    pixman_gradient_stop_t *begin = &(gradient->stops[-1]);
+    pixman_gradient_stop_t *end = &(gradient->stops[n]);
+
+    switch (gradient->common.repeat)
+    {
+    default:
+    case PIXMAN_REPEAT_NONE:
+	begin->x = INT32_MIN;
+	begin->color = transparent_black;
+	end->x = INT32_MAX;
+	end->color = transparent_black;
+	break;
+
+    case PIXMAN_REPEAT_NORMAL:
+	begin->x = stops[n - 1].x - pixman_fixed_1;
+	begin->color = stops[n - 1].color;
+	end->x = stops[0].x + pixman_fixed_1;
+	end->color = stops[0].color;
+	break;
+
+    case PIXMAN_REPEAT_REFLECT:
+	begin->x = - stops[0].x;
+	begin->color = stops[0].color;
+	end->x = pixman_int_to_fixed (2) - stops[n - 1].x;
+	end->color = stops[n - 1].color;
+	break;
+
+    case PIXMAN_REPEAT_PAD:
+	begin->x = INT32_MIN;
+	begin->color = stops[0].color;
+	end->x = INT32_MAX;
+	end->color = stops[n - 1].color;
+	break;
+    }
+}
 
 pixman_bool_t
 _pixman_init_gradient (gradient_t *                  gradient,
@@ -39,128 +82,111 @@ _pixman_init_gradient (gradient_t *                  gradient,
 {
     return_val_if_fail (n_stops > 0, FALSE);
 
-    gradient->stops = pixman_malloc_ab (n_stops, sizeof (pixman_gradient_stop_t));
+    /* We allocate two extra stops, one before the beginning of the stop list,
+     * and one after the end. These stops are initialized to whatever color
+     * would be used for positions outside the range of the stop list.
+     *
+     * This saves a bit of computation in the gradient walker.
+     *
+     * The pointer we store in the gradient_t struct still points to the
+     * first user-supplied struct, so when freeing, we will have to
+     * subtract one.
+     */
+    gradient->stops =
+	pixman_malloc_ab (n_stops + 2, sizeof (pixman_gradient_stop_t));
     if (!gradient->stops)
 	return FALSE;
 
+    gradient->stops += 1;
     memcpy (gradient->stops, stops, n_stops * sizeof (pixman_gradient_stop_t));
-
     gradient->n_stops = n_stops;
 
-    gradient->stop_range = 0xffff;
-    gradient->common.class = SOURCE_IMAGE_CLASS_UNKNOWN;
+    gradient->common.property_changed = gradient_property_changed;
 
     return TRUE;
 }
 
-/*
- * By default, just evaluate the image at 32bpp and expand.  Individual image
- * types can plug in a better scanline getter if they want to. For example
- * we  could produce smoother gradients by evaluating them at higher color
- * depth, but that's a project for the future.
- */
 void
-_pixman_image_get_scanline_generic_64 (pixman_image_t * image,
-                                       int              x,
-                                       int              y,
-                                       int              width,
-                                       uint32_t *       buffer,
-                                       const uint32_t * mask,
-                                       uint32_t         mask_bits)
-{
-    uint32_t *mask8 = NULL;
-
-    /* Contract the mask image, if one exists, so that the 32-bit fetch
-     * function can use it.
-     */
-    if (mask)
-    {
-	mask8 = pixman_malloc_ab (width, sizeof(uint32_t));
-	if (!mask8)
-	    return;
+_pixman_image_init (pixman_image_t *image)
+{
+    image_common_t *common = &image->common;
+
+    pixman_region32_init (&common->clip_region);
+
+    common->alpha_count = 0;
+    common->have_clip_region = FALSE;
+    common->clip_sources = FALSE;
+    common->transform = NULL;
+    common->repeat = PIXMAN_REPEAT_NONE;
+    common->filter = PIXMAN_FILTER_NEAREST;
+    common->filter_params = NULL;
+    common->n_filter_params = 0;
+    common->alpha_map = NULL;
+    common->component_alpha = FALSE;
+    common->ref_count = 1;
+    common->property_changed = NULL;
+    common->client_clip = FALSE;
+    common->destroy_func = NULL;
+    common->destroy_data = NULL;
+    common->dirty = TRUE;
+}
 
-	pixman_contract (mask8, (uint64_t *)mask, width);
-    }
+pixman_bool_t
+_pixman_image_fini (pixman_image_t *image)
+{
+    image_common_t *common = (image_common_t *)image;
 
-    /* Fetch the source image into the first half of buffer. */
-    _pixman_image_get_scanline_32 (image, x, y, width, (uint32_t*)buffer, mask8,
-                                   mask_bits);
+    common->ref_count--;
 
-    /* Expand from 32bpp to 64bpp in place. */
-    pixman_expand ((uint64_t *)buffer, buffer, PIXMAN_a8r8g8b8, width);
+    if (common->ref_count == 0)
+    {
+	if (image->common.destroy_func)
+	    image->common.destroy_func (image, image->common.destroy_data);
 
-    free (mask8);
-}
+	pixman_region32_fini (&common->clip_region);
 
-pixman_image_t *
-_pixman_image_allocate (void)
-{
-    pixman_image_t *image = malloc (sizeof (pixman_image_t));
+	free (common->transform);
+	free (common->filter_params);
 
-    if (image)
-    {
-	image_common_t *common = &image->common;
+	if (common->alpha_map)
+	    pixman_image_unref ((pixman_image_t *)common->alpha_map);
 
-	pixman_region32_init (&common->clip_region);
+	if (image->type == LINEAR ||
+	    image->type == RADIAL ||
+	    image->type == CONICAL)
+	{
+	    if (image->gradient.stops)
+	    {
+		/* See _pixman_init_gradient() for an explanation of the - 1 */
+		free (image->gradient.stops - 1);
+	    }
 
-	common->have_clip_region = FALSE;
-	common->clip_sources = FALSE;
-	common->transform = NULL;
-	common->repeat = PIXMAN_REPEAT_NONE;
-	common->filter = PIXMAN_FILTER_NEAREST;
-	common->filter_params = NULL;
-	common->n_filter_params = 0;
-	common->alpha_map = NULL;
-	common->component_alpha = FALSE;
-	common->ref_count = 1;
-	common->classify = NULL;
-	common->client_clip = FALSE;
-	common->destroy_func = NULL;
-	common->destroy_data = NULL;
-	common->dirty = TRUE;
+	    /* This will trigger if someone adds a property_changed
+	     * method to the linear/radial/conical gradient overwriting
+	     * the general one.
+	     */
+	    assert (
+		image->common.property_changed == gradient_property_changed);
+	}
+
+	if (image->type == BITS && image->bits.free_me)
+	    free (image->bits.free_me);
+
+	return TRUE;
     }
 
-    return image;
+    return FALSE;
 }
 
-source_image_class_t
-_pixman_image_classify (pixman_image_t *image,
-                        int             x,
-                        int             y,
-                        int             width,
-                        int             height)
+pixman_image_t *
+_pixman_image_allocate (void)
 {
-    if (image->common.classify)
-	return image->common.classify (image, x, y, width, height);
-    else
-	return SOURCE_IMAGE_CLASS_UNKNOWN;
-}
+    pixman_image_t *image = malloc (sizeof (pixman_image_t));
 
-void
-_pixman_image_get_scanline_32 (pixman_image_t *image,
-                               int             x,
-                               int             y,
-                               int             width,
-                               uint32_t *      buffer,
-                               const uint32_t *mask,
-                               uint32_t        mask_bits)
-{
-    image->common.get_scanline_32 (image, x, y, width, buffer, mask, mask_bits);
-}
+    if (image)
+	_pixman_image_init (image);
 
-/* Even thought the type of buffer is uint32_t *, the function actually expects
- * a uint64_t *buffer.
- */
-void
-_pixman_image_get_scanline_64 (pixman_image_t *image,
-                               int             x,
-                               int             y,
-                               int             width,
-                               uint32_t *      buffer,
-                               const uint32_t *unused,
-                               uint32_t        unused2)
-{
-    image->common.get_scanline_64 (image, x, y, width, buffer, unused, unused2);
+    return image;
 }
 
 static void
@@ -182,39 +208,9 @@ pixman_image_ref (pixman_image_t *image)
 PIXMAN_EXPORT pixman_bool_t
 pixman_image_unref (pixman_image_t *image)
 {
-    image_common_t *common = (image_common_t *)image;
-
-    common->ref_count--;
-
-    if (common->ref_count == 0)
+    if (_pixman_image_fini (image))
     {
-	if (image->common.destroy_func)
-	    image->common.destroy_func (image, image->common.destroy_data);
-
-	pixman_region32_fini (&common->clip_region);
-
-	if (common->transform)
-	    free (common->transform);
-
-	if (common->filter_params)
-	    free (common->filter_params);
-
-	if (common->alpha_map)
-	    pixman_image_unref ((pixman_image_t *)common->alpha_map);
-
-	if (image->type == LINEAR ||
-	    image->type == RADIAL ||
-	    image->type == CONICAL)
-	{
-	    if (image->gradient.stops)
-		free (image->gradient.stops);
-	}
-
-	if (image->type == BITS && image->bits.free_me)
-	    free (image->bits.free_me);
-
 	free (image);
-
 	return TRUE;
     }
 
@@ -242,54 +238,27 @@ _pixman_image_reset_clip_region (pixman_image_t *image)
     image->common.have_clip_region = FALSE;
 }
 
-static pixman_bool_t out_of_bounds_workaround = TRUE;
-
-/* Old X servers rely on out-of-bounds accesses when they are asked
- * to composite with a window as the source. They create a pixman image
- * pointing to some bogus position in memory, but then they set a clip
- * region to the position where the actual bits are.
+/* Executive Summary: This function is a no-op that only exists
+ * for historical reasons.
+ *
+ * There used to be a bug in the X server where it would rely on
+ * out-of-bounds accesses when it was asked to composite with a
+ * window as the source. It would create a pixman image pointing
+ * to some bogus position in memory, but then set a clip region
+ * to the position where the actual bits were.
  *
  * Due to a bug in old versions of pixman, where it would not clip
  * against the image bounds when a clip region was set, this would
- * actually work. So by default we allow certain out-of-bound access
- * to happen unless explicitly disabled.
+ * actually work. So when the pixman bug was fixed, a workaround was
+ * added to allow certain out-of-bound accesses. This function disabled
+ * those workarounds.
  *
- * Fixed X servers should call this function to disable the workaround.
+ * Since 0.21.2, pixman doesn't do these workarounds anymore, so now
+ * this function is a no-op.
  */
 PIXMAN_EXPORT void
 pixman_disable_out_of_bounds_workaround (void)
 {
-    out_of_bounds_workaround = FALSE;
-}
-
-static pixman_bool_t
-source_image_needs_out_of_bounds_workaround (bits_image_t *image)
-{
-    if (image->common.clip_sources                      &&
-        image->common.repeat == PIXMAN_REPEAT_NONE      &&
-	image->common.have_clip_region			&&
-        out_of_bounds_workaround)
-    {
-	if (!image->common.client_clip)
-	{
-	    /* There is no client clip, so if the clip region extends beyond the
-	     * drawable geometry, it must be because the X server generated the
-	     * bogus clip region.
-	     */
-	    const pixman_box32_t *extents =
-		pixman_region32_extents (&image->common.clip_region);
-
-	    if (extents->x1 >= 0 && extents->x2 <= image->width &&
-		extents->y1 >= 0 && extents->y2 <= image->height)
-	    {
-		return FALSE;
-	    }
-	}
-
-	return TRUE;
-    }
-
-    return FALSE;
 }
 
 static void
@@ -301,26 +270,50 @@ compute_image_info (pixman_image_t *image)
     /* Transform */
     if (!image->common.transform)
     {
-	flags |= (FAST_PATH_ID_TRANSFORM | FAST_PATH_X_UNIT_POSITIVE);
+	flags |= (FAST_PATH_ID_TRANSFORM	|
+		  FAST_PATH_X_UNIT_POSITIVE	|
+		  FAST_PATH_Y_UNIT_ZERO		|
+		  FAST_PATH_AFFINE_TRANSFORM);
     }
     else
     {
-	if (image->common.transform->matrix[0][1] == 0 &&
-	    image->common.transform->matrix[1][0] == 0 &&
-	    image->common.transform->matrix[2][0] == 0 &&
-	    image->common.transform->matrix[2][1] == 0 &&
+	flags |= FAST_PATH_HAS_TRANSFORM;
+
+	if (image->common.transform->matrix[2][0] == 0			&&
+	    image->common.transform->matrix[2][1] == 0			&&
 	    image->common.transform->matrix[2][2] == pixman_fixed_1)
 	{
-	    flags |= FAST_PATH_SCALE_TRANSFORM;
+	    flags |= FAST_PATH_AFFINE_TRANSFORM;
+
+	    if (image->common.transform->matrix[0][1] == 0 &&
+		image->common.transform->matrix[1][0] == 0)
+	    {
+		if (image->common.transform->matrix[0][0] == -pixman_fixed_1 &&
+		    image->common.transform->matrix[1][1] == -pixman_fixed_1)
+		{
+		    flags |= FAST_PATH_ROTATE_180_TRANSFORM;
+		}
+		flags |= FAST_PATH_SCALE_TRANSFORM;
+	    }
+	    else if (image->common.transform->matrix[0][0] == 0 &&
+	             image->common.transform->matrix[1][1] == 0)
+	    {
+		pixman_fixed_t m01 = image->common.transform->matrix[0][1];
+		pixman_fixed_t m10 = image->common.transform->matrix[1][0];
+
+		if (m01 == -pixman_fixed_1 && m10 == pixman_fixed_1)
+		    flags |= FAST_PATH_ROTATE_90_TRANSFORM;
+		else if (m01 == pixman_fixed_1 && m10 == -pixman_fixed_1)
+		    flags |= FAST_PATH_ROTATE_270_TRANSFORM;
+	    }
 	}
 
 	if (image->common.transform->matrix[0][0] > 0)
 	    flags |= FAST_PATH_X_UNIT_POSITIVE;
-    }
 
-    /* Alpha map */
-    if (!image->common.alpha_map)
-	flags |= FAST_PATH_NO_ALPHA_MAP;
+	if (image->common.transform->matrix[1][0] == 0)
+	    flags |= FAST_PATH_Y_UNIT_ZERO;
+    }
 
     /* Filter */
     switch (image->common.filter)
@@ -330,9 +323,60 @@ compute_image_info (pixman_image_t *image)
 	flags |= (FAST_PATH_NEAREST_FILTER | FAST_PATH_NO_CONVOLUTION_FILTER);
 	break;
 
+    case PIXMAN_FILTER_BILINEAR:
+    case PIXMAN_FILTER_GOOD:
+    case PIXMAN_FILTER_BEST:
+	flags |= (FAST_PATH_BILINEAR_FILTER | FAST_PATH_NO_CONVOLUTION_FILTER);
+
+	/* Here we have a chance to optimize BILINEAR filter to NEAREST if
+	 * they are equivalent for the currently used transformation matrix.
+	 */
+	if (flags & FAST_PATH_ID_TRANSFORM)
+	{
+	    flags |= FAST_PATH_NEAREST_FILTER;
+	}
+	else if (
+	    /* affine and integer translation components in matrix ... */
+	    ((flags & FAST_PATH_AFFINE_TRANSFORM) &&
+	     !pixman_fixed_frac (image->common.transform->matrix[0][2] |
+				 image->common.transform->matrix[1][2])) &&
+	    (
+		/* ... combined with a simple rotation */
+		(flags & (FAST_PATH_ROTATE_90_TRANSFORM |
+			  FAST_PATH_ROTATE_180_TRANSFORM |
+			  FAST_PATH_ROTATE_270_TRANSFORM)) ||
+		/* ... or combined with a simple non-rotated translation */
+		(image->common.transform->matrix[0][0] == pixman_fixed_1 &&
+		 image->common.transform->matrix[1][1] == pixman_fixed_1 &&
+		 image->common.transform->matrix[0][1] == 0 &&
+		 image->common.transform->matrix[1][0] == 0)
+		)
+	    )
+	{
+	    /* FIXME: there are some affine-test failures, showing that
+	     * handling of BILINEAR and NEAREST filter is not quite
+	     * equivalent when getting close to 32K for the translation
+	     * components of the matrix. That's likely some bug, but for
+	     * now just skip BILINEAR->NEAREST optimization in this case.
+	     */
+	    pixman_fixed_t magic_limit = pixman_int_to_fixed (30000);
+	    if (image->common.transform->matrix[0][2] <= magic_limit  &&
+	        image->common.transform->matrix[1][2] <= magic_limit  &&
+	        image->common.transform->matrix[0][2] >= -magic_limit &&
+	        image->common.transform->matrix[1][2] >= -magic_limit)
+	    {
+		flags |= FAST_PATH_NEAREST_FILTER;
+	    }
+	}
+	break;
+
     case PIXMAN_FILTER_CONVOLUTION:
 	break;
 
+    case PIXMAN_FILTER_SEPARABLE_CONVOLUTION:
+	flags |= FAST_PATH_SEPARABLE_CONVOLUTION_FILTER;
+	break;
+
     default:
 	flags |= FAST_PATH_NO_CONVOLUTION_FILTER;
 	break;
@@ -342,19 +386,31 @@ compute_image_info (pixman_image_t *image)
     switch (image->common.repeat)
     {
     case PIXMAN_REPEAT_NONE:
-	flags |= FAST_PATH_NO_REFLECT_REPEAT | FAST_PATH_NO_PAD_REPEAT;
+	flags |=
+	    FAST_PATH_NO_REFLECT_REPEAT		|
+	    FAST_PATH_NO_PAD_REPEAT		|
+	    FAST_PATH_NO_NORMAL_REPEAT;
 	break;
 
     case PIXMAN_REPEAT_REFLECT:
-	flags |= FAST_PATH_NO_PAD_REPEAT | FAST_PATH_NO_NONE_REPEAT;
+	flags |=
+	    FAST_PATH_NO_PAD_REPEAT		|
+	    FAST_PATH_NO_NONE_REPEAT		|
+	    FAST_PATH_NO_NORMAL_REPEAT;
 	break;
 
     case PIXMAN_REPEAT_PAD:
-	flags |= FAST_PATH_NO_REFLECT_REPEAT | FAST_PATH_NO_NONE_REPEAT;
+	flags |=
+	    FAST_PATH_NO_REFLECT_REPEAT		|
+	    FAST_PATH_NO_NONE_REPEAT		|
+	    FAST_PATH_NO_NORMAL_REPEAT;
 	break;
 
     default:
-	flags |= FAST_PATH_NO_REFLECT_REPEAT | FAST_PATH_NO_PAD_REPEAT | FAST_PATH_NO_NONE_REPEAT;
+	flags |=
+	    FAST_PATH_NO_REFLECT_REPEAT		|
+	    FAST_PATH_NO_PAD_REPEAT		|
+	    FAST_PATH_NO_NONE_REPEAT;
 	break;
     }
 
@@ -364,7 +420,7 @@ compute_image_info (pixman_image_t *image)
     else
 	flags |= FAST_PATH_UNIFIED_ALPHA;
 
-    flags |= (FAST_PATH_NO_ACCESSORS | FAST_PATH_NO_WIDE_FORMAT);
+    flags |= (FAST_PATH_NO_ACCESSORS | FAST_PATH_NARROW_FORMAT);
 
     /* Type specific checks */
     switch (image->type)
@@ -386,36 +442,45 @@ compute_image_info (pixman_image_t *image)
 	else
 	{
 	    code = image->bits.format;
-
-	    if (!image->common.transform &&
-		image->common.repeat == PIXMAN_REPEAT_NORMAL)
-	    {
-		flags |= FAST_PATH_SIMPLE_REPEAT;
-	    }
+	    flags |= FAST_PATH_BITS_IMAGE;
 	}
 
-	if (image->common.repeat != PIXMAN_REPEAT_NONE				&&
-	    !PIXMAN_FORMAT_A (image->bits.format)				&&
+	if (!PIXMAN_FORMAT_A (image->bits.format)				&&
 	    PIXMAN_FORMAT_TYPE (image->bits.format) != PIXMAN_TYPE_GRAY		&&
 	    PIXMAN_FORMAT_TYPE (image->bits.format) != PIXMAN_TYPE_COLOR)
 	{
-	    flags |= FAST_PATH_IS_OPAQUE;
-	}
+	    flags |= FAST_PATH_SAMPLES_OPAQUE;
 
-	if (source_image_needs_out_of_bounds_workaround (&image->bits))
-	    flags |= FAST_PATH_NEEDS_WORKAROUND;
+	    if (image->common.repeat != PIXMAN_REPEAT_NONE)
+		flags |= FAST_PATH_IS_OPAQUE;
+	}
 
 	if (image->bits.read_func || image->bits.write_func)
 	    flags &= ~FAST_PATH_NO_ACCESSORS;
 
 	if (PIXMAN_FORMAT_IS_WIDE (image->bits.format))
-	    flags &= ~FAST_PATH_NO_WIDE_FORMAT;
+	    flags &= ~FAST_PATH_NARROW_FORMAT;
 	break;
 
-    case LINEAR:
     case RADIAL:
 	code = PIXMAN_unknown;
 
+	/*
+	 * As explained in pixman-radial-gradient.c, every point of
+	 * the plane has a valid associated radius (and thus will be
+	 * colored) if and only if a is negative (i.e. one of the two
+	 * circles contains the other one).
+	 */
+
+        if (image->radial.a >= 0)
+	    break;
+
+	/* Fall through */
+
+    case CONICAL:
+    case LINEAR:
+	code = PIXMAN_unknown;
+
 	if (image->common.repeat != PIXMAN_REPEAT_NONE)
 	{
 	    int i;
@@ -437,17 +502,31 @@ compute_image_info (pixman_image_t *image)
 	break;
     }
 
+    /* Alpha maps are only supported for BITS images, so it's always
+     * safe to ignore their presense for non-BITS images
+     */
+    if (!image->common.alpha_map || image->type != BITS)
+    {
+	flags |= FAST_PATH_NO_ALPHA_MAP;
+    }
+    else
+    {
+	if (PIXMAN_FORMAT_IS_WIDE (image->common.alpha_map->format))
+	    flags &= ~FAST_PATH_NARROW_FORMAT;
+    }
+
     /* Both alpha maps and convolution filters can introduce
      * non-opaqueness in otherwise opaque images. Also
      * an image with component alpha turned on is only opaque
      * if all channels are opaque, so we simply turn it off
      * unconditionally for those images.
      */
-    if (image->common.alpha_map					||
-	image->common.filter == PIXMAN_FILTER_CONVOLUTION	||
+    if (image->common.alpha_map						||
+	image->common.filter == PIXMAN_FILTER_CONVOLUTION		||
+        image->common.filter == PIXMAN_FILTER_SEPARABLE_CONVOLUTION     ||
 	image->common.component_alpha)
     {
-	flags &= ~FAST_PATH_IS_OPAQUE;
+	flags &= ~(FAST_PATH_IS_OPAQUE | FAST_PATH_SAMPLES_OPAQUE);
     }
 
     image->common.flags = flags;
@@ -466,7 +545,8 @@ _pixman_image_validate (pixman_image_t *image)
 	 * property_changed() can make use of the flags
 	 * to set up accessors etc.
 	 */
-	image->common.property_changed (image);
+	if (image->common.property_changed)
+	    image->common.property_changed (image);
 
 	image->common.dirty = FALSE;
     }
@@ -547,7 +627,7 @@ pixman_image_set_transform (pixman_image_t *          image,
     if (common->transform == transform)
 	return TRUE;
 
-    if (memcmp (&id, transform, sizeof (pixman_transform_t)) == 0)
+    if (!transform || memcmp (&id, transform, sizeof (pixman_transform_t)) == 0)
     {
 	free (common->transform);
 	common->transform = NULL;
@@ -556,6 +636,12 @@ pixman_image_set_transform (pixman_image_t *          image,
 	goto out;
     }
 
+    if (common->transform &&
+	memcmp (common->transform, transform, sizeof (pixman_transform_t)) == 0)
+    {
+	return TRUE;
+    }
+
     if (common->transform == NULL)
 	common->transform = malloc (sizeof (pixman_transform_t));
 
@@ -580,6 +666,9 @@ PIXMAN_EXPORT void
 pixman_image_set_repeat (pixman_image_t *image,
                          pixman_repeat_t repeat)
 {
+    if (image->common.repeat == repeat)
+	return;
+
     image->common.repeat = repeat;
 
     image_property_changed (image);
@@ -597,6 +686,19 @@ pixman_image_set_filter (pixman_image_t *      image,
     if (params == common->filter_params && filter == common->filter)
 	return TRUE;
 
+    if (filter == PIXMAN_FILTER_SEPARABLE_CONVOLUTION)
+    {
+	int width = pixman_fixed_to_int (params[0]);
+	int height = pixman_fixed_to_int (params[1]);
+	int x_phase_bits = pixman_fixed_to_int (params[2]);
+	int y_phase_bits = pixman_fixed_to_int (params[3]);
+	int n_x_phases = (1 << x_phase_bits);
+	int n_y_phases = (1 << y_phase_bits);
+
+	return_val_if_fail (
+	    n_params == 4 + n_x_phases * width + n_y_phases * height, FALSE);
+    }
+    
     new_params = NULL;
     if (params)
     {
@@ -624,6 +726,9 @@ PIXMAN_EXPORT void
 pixman_image_set_source_clipping (pixman_image_t *image,
                                   pixman_bool_t   clip_sources)
 {
+    if (image->common.clip_sources == clip_sources)
+	return;
+
     image->common.clip_sources = clip_sources;
 
     image_property_changed (image);
@@ -639,6 +744,9 @@ pixman_image_set_indexed (pixman_image_t *        image,
 {
     bits_image_t *bits = (bits_image_t *)image;
 
+    if (bits->indexed == indexed)
+	return;
+
     bits->indexed = indexed;
 
     image_property_changed (image);
@@ -654,15 +762,41 @@ pixman_image_set_alpha_map (pixman_image_t *image,
 
     return_if_fail (!alpha_map || alpha_map->type == BITS);
 
+    if (alpha_map && common->alpha_count > 0)
+    {
+	/* If this image is being used as an alpha map itself,
+	 * then you can't give it an alpha map of its own.
+	 */
+	return;
+    }
+
+    if (alpha_map && alpha_map->common.alpha_map)
+    {
+	/* If the image has an alpha map of its own,
+	 * then it can't be used as an alpha map itself
+	 */
+	return;
+    }
+
     if (common->alpha_map != (bits_image_t *)alpha_map)
     {
 	if (common->alpha_map)
+	{
+	    common->alpha_map->common.alpha_count--;
+
 	    pixman_image_unref ((pixman_image_t *)common->alpha_map);
+	}
 
 	if (alpha_map)
+	{
 	    common->alpha_map = (bits_image_t *)pixman_image_ref (alpha_map);
+
+	    common->alpha_map->common.alpha_count++;
+	}
 	else
+	{
 	    common->alpha_map = NULL;
+	}
     }
 
     common->alpha_origin_x = x;
@@ -675,11 +809,20 @@ PIXMAN_EXPORT void
 pixman_image_set_component_alpha   (pixman_image_t *image,
                                     pixman_bool_t   component_alpha)
 {
+    if (image->common.component_alpha == component_alpha)
+	return;
+
     image->common.component_alpha = component_alpha;
 
     image_property_changed (image);
 }
 
+PIXMAN_EXPORT pixman_bool_t
+pixman_image_get_component_alpha   (pixman_image_t       *image)
+{
+    return image->common.component_alpha;
+}
+
 PIXMAN_EXPORT void
 pixman_image_set_accessors (pixman_image_t *           image,
                             pixman_read_memory_func_t  read_func,
@@ -741,16 +884,56 @@ pixman_image_get_depth (pixman_image_t *image)
     return 0;
 }
 
+PIXMAN_EXPORT pixman_format_code_t
+pixman_image_get_format (pixman_image_t *image)
+{
+    if (image->type == BITS)
+	return image->bits.format;
+
+    return PIXMAN_null;
+}
+
 uint32_t
-_pixman_image_get_solid (pixman_image_t *     image,
-                         pixman_format_code_t format)
+_pixman_image_get_solid (pixman_implementation_t *imp,
+			 pixman_image_t *         image,
+                         pixman_format_code_t     format)
 {
     uint32_t result;
 
-    _pixman_image_get_scanline_32 (image, 0, 0, 1, &result, NULL, 0);
+    if (image->type == SOLID)
+    {
+	result = image->solid.color_32;
+    }
+    else if (image->type == BITS)
+    {
+	if (image->bits.format == PIXMAN_a8r8g8b8)
+	    result = image->bits.bits[0];
+	else if (image->bits.format == PIXMAN_x8r8g8b8)
+	    result = image->bits.bits[0] | 0xff000000;
+	else if (image->bits.format == PIXMAN_a8)
+	    result = (*(uint8_t *)image->bits.bits) << 24;
+	else
+	    goto otherwise;
+    }
+    else
+    {
+	pixman_iter_t iter;
+
+    otherwise:
+	_pixman_implementation_iter_init (
+	    imp, &iter, image, 0, 0, 1, 1,
+	    (uint8_t *)&result,
+	    ITER_NARROW | ITER_SRC, image->common.flags);
+	
+	result = *iter.get_scanline (&iter, NULL);
+
+	if (iter.fini)
+	    iter.fini (&iter);
+    }
 
     /* If necessary, convert RGB <--> BGR. */
-    if (PIXMAN_FORMAT_TYPE (format) != PIXMAN_TYPE_ARGB)
+    if (PIXMAN_FORMAT_TYPE (format) != PIXMAN_TYPE_ARGB
+	&& PIXMAN_FORMAT_TYPE (format) != PIXMAN_TYPE_ARGB_SRGB)
     {
 	result = (((result & 0xff000000) >>  0) |
 	          ((result & 0x00ff0000) >> 16) |
diff --git a/pixman/pixman/pixman-implementation.c b/pixman/pixman/pixman-implementation.c
index bc3749ef5..588405451 100644
--- a/pixman/pixman/pixman-implementation.c
+++ b/pixman/pixman/pixman-implementation.c
@@ -27,168 +27,206 @@
 #include <stdlib.h>
 #include "pixman-private.h"
 
-static void
-delegate_combine_32 (pixman_implementation_t * imp,
-                     pixman_op_t               op,
-                     uint32_t *                dest,
-                     const uint32_t *          src,
-                     const uint32_t *          mask,
-                     int                       width)
+pixman_implementation_t *
+_pixman_implementation_create (pixman_implementation_t *fallback,
+			       const pixman_fast_path_t *fast_paths)
 {
-    _pixman_implementation_combine_32 (imp->delegate,
-                                       op, dest, src, mask, width);
-}
+    pixman_implementation_t *imp;
 
-static void
-delegate_combine_64 (pixman_implementation_t * imp,
-                     pixman_op_t               op,
-                     uint64_t *                dest,
-                     const uint64_t *          src,
-                     const uint64_t *          mask,
-                     int                       width)
-{
-    _pixman_implementation_combine_64 (imp->delegate,
-                                       op, dest, src, mask, width);
-}
+    assert (fast_paths);
 
-static void
-delegate_combine_32_ca (pixman_implementation_t * imp,
-                        pixman_op_t               op,
-                        uint32_t *                dest,
-                        const uint32_t *          src,
-                        const uint32_t *          mask,
-                        int                       width)
-{
-    _pixman_implementation_combine_32_ca (imp->delegate,
-                                          op, dest, src, mask, width);
-}
+    if ((imp = malloc (sizeof (pixman_implementation_t))))
+    {
+	pixman_implementation_t *d;
 
-static void
-delegate_combine_64_ca (pixman_implementation_t * imp,
-                        pixman_op_t               op,
-                        uint64_t *                dest,
-                        const uint64_t *          src,
-                        const uint64_t *          mask,
-                        int                       width)
-{
-    _pixman_implementation_combine_64_ca (imp->delegate,
-                                          op, dest, src, mask, width);
+	memset (imp, 0, sizeof *imp);
+
+	imp->fallback = fallback;
+	imp->fast_paths = fast_paths;
+	
+	/* Make sure the whole fallback chain has the right toplevel */
+	for (d = imp; d != NULL; d = d->fallback)
+	    d->toplevel = imp;
+    }
+
+    return imp;
 }
 
-static pixman_bool_t
-delegate_blt (pixman_implementation_t * imp,
-              uint32_t *                src_bits,
-              uint32_t *                dst_bits,
-              int                       src_stride,
-              int                       dst_stride,
-              int                       src_bpp,
-              int                       dst_bpp,
-              int                       src_x,
-              int                       src_y,
-              int                       dst_x,
-              int                       dst_y,
-              int                       width,
-              int                       height)
+#define N_CACHED_FAST_PATHS 8
+
+typedef struct
 {
-    return _pixman_implementation_blt (
-	imp->delegate, src_bits, dst_bits, src_stride, dst_stride,
-	src_bpp, dst_bpp, src_x, src_y, dst_x, dst_y,
-	width, height);
-}
+    struct
+    {
+	pixman_implementation_t *	imp;
+	pixman_fast_path_t		fast_path;
+    } cache [N_CACHED_FAST_PATHS];
+} cache_t;
+
+PIXMAN_DEFINE_THREAD_LOCAL (cache_t, fast_path_cache);
 
-static pixman_bool_t
-delegate_fill (pixman_implementation_t *imp,
-               uint32_t *               bits,
-               int                      stride,
-               int                      bpp,
-               int                      x,
-               int                      y,
-               int                      width,
-               int                      height,
-               uint32_t                 xor)
+static void
+dummy_composite_rect (pixman_implementation_t *imp,
+		      pixman_composite_info_t *info)
 {
-    return _pixman_implementation_fill (
-	imp->delegate, bits, stride, bpp, x, y, width, height, xor);
 }
 
-pixman_implementation_t *
-_pixman_implementation_create (pixman_implementation_t *delegate,
-			       const pixman_fast_path_t *fast_paths)
+void
+_pixman_implementation_lookup_composite (pixman_implementation_t  *toplevel,
+					 pixman_op_t               op,
+					 pixman_format_code_t      src_format,
+					 uint32_t                  src_flags,
+					 pixman_format_code_t      mask_format,
+					 uint32_t                  mask_flags,
+					 pixman_format_code_t      dest_format,
+					 uint32_t                  dest_flags,
+					 pixman_implementation_t **out_imp,
+					 pixman_composite_func_t  *out_func)
 {
-    pixman_implementation_t *imp = malloc (sizeof (pixman_implementation_t));
-    pixman_implementation_t *d;
+    pixman_implementation_t *imp;
+    cache_t *cache;
     int i;
 
-    if (!imp)
-	return NULL;
+    /* Check cache for fast paths */
+    cache = PIXMAN_GET_THREAD_LOCAL (fast_path_cache);
 
-    assert (fast_paths);
+    for (i = 0; i < N_CACHED_FAST_PATHS; ++i)
+    {
+	const pixman_fast_path_t *info = &(cache->cache[i].fast_path);
 
-    /* Make sure the whole delegate chain has the right toplevel */
-    imp->delegate = delegate;
-    for (d = imp; d != NULL; d = d->delegate)
-	d->toplevel = imp;
+	/* Note that we check for equality here, not whether
+	 * the cached fast path matches. This is to prevent
+	 * us from selecting an overly general fast path
+	 * when a more specific one would work.
+	 */
+	if (info->op == op			&&
+	    info->src_format == src_format	&&
+	    info->mask_format == mask_format	&&
+	    info->dest_format == dest_format	&&
+	    info->src_flags == src_flags	&&
+	    info->mask_flags == mask_flags	&&
+	    info->dest_flags == dest_flags	&&
+	    info->func)
+	{
+	    *out_imp = cache->cache[i].imp;
+	    *out_func = cache->cache[i].fast_path.func;
 
-    /* Fill out function pointers with ones that just delegate
-     */
-    imp->blt = delegate_blt;
-    imp->fill = delegate_fill;
+	    goto update_cache;
+	}
+    }
 
-    for (i = 0; i < PIXMAN_N_OPERATORS; ++i)
+    for (imp = toplevel; imp != NULL; imp = imp->fallback)
     {
-	imp->combine_32[i] = delegate_combine_32;
-	imp->combine_64[i] = delegate_combine_64;
-	imp->combine_32_ca[i] = delegate_combine_32_ca;
-	imp->combine_64_ca[i] = delegate_combine_64_ca;
+	const pixman_fast_path_t *info = imp->fast_paths;
+
+	while (info->op != PIXMAN_OP_NONE)
+	{
+	    if ((info->op == op || info->op == PIXMAN_OP_any)		&&
+		/* Formats */
+		((info->src_format == src_format) ||
+		 (info->src_format == PIXMAN_any))			&&
+		((info->mask_format == mask_format) ||
+		 (info->mask_format == PIXMAN_any))			&&
+		((info->dest_format == dest_format) ||
+		 (info->dest_format == PIXMAN_any))			&&
+		/* Flags */
+		(info->src_flags & src_flags) == info->src_flags	&&
+		(info->mask_flags & mask_flags) == info->mask_flags	&&
+		(info->dest_flags & dest_flags) == info->dest_flags)
+	    {
+		*out_imp = imp;
+		*out_func = info->func;
+
+		/* Set i to the last spot in the cache so that the
+		 * move-to-front code below will work
+		 */
+		i = N_CACHED_FAST_PATHS - 1;
+
+		goto update_cache;
+	    }
+
+	    ++info;
+	}
     }
 
-    imp->fast_paths = fast_paths;
-    
-    return imp;
-}
+    /* We should never reach this point */
+    _pixman_log_error (
+        FUNC,
+        "No composite function found\n"
+        "\n"
+        "The most likely cause of this is that this system has issues with\n"
+        "thread local storage\n");
 
-void
-_pixman_implementation_combine_32 (pixman_implementation_t * imp,
-                                   pixman_op_t               op,
-                                   uint32_t *                dest,
-                                   const uint32_t *          src,
-                                   const uint32_t *          mask,
-                                   int                       width)
-{
-    (*imp->combine_32[op]) (imp, op, dest, src, mask, width);
-}
+    *out_imp = NULL;
+    *out_func = dummy_composite_rect;
+    return;
 
-void
-_pixman_implementation_combine_64 (pixman_implementation_t * imp,
-                                   pixman_op_t               op,
-                                   uint64_t *                dest,
-                                   const uint64_t *          src,
-                                   const uint64_t *          mask,
-                                   int                       width)
-{
-    (*imp->combine_64[op]) (imp, op, dest, src, mask, width);
+update_cache:
+    if (i)
+    {
+	while (i--)
+	    cache->cache[i + 1] = cache->cache[i];
+
+	cache->cache[0].imp = *out_imp;
+	cache->cache[0].fast_path.op = op;
+	cache->cache[0].fast_path.src_format = src_format;
+	cache->cache[0].fast_path.src_flags = src_flags;
+	cache->cache[0].fast_path.mask_format = mask_format;
+	cache->cache[0].fast_path.mask_flags = mask_flags;
+	cache->cache[0].fast_path.dest_format = dest_format;
+	cache->cache[0].fast_path.dest_flags = dest_flags;
+	cache->cache[0].fast_path.func = *out_func;
+    }
 }
 
-void
-_pixman_implementation_combine_32_ca (pixman_implementation_t * imp,
-                                      pixman_op_t               op,
-                                      uint32_t *                dest,
-                                      const uint32_t *          src,
-                                      const uint32_t *          mask,
-                                      int                       width)
+static void
+dummy_combine (pixman_implementation_t *imp,
+	       pixman_op_t              op,
+	       uint32_t *               pd,
+	       const uint32_t *         ps,
+	       const uint32_t *         pm,
+	       int                      w)
 {
-    (*imp->combine_32_ca[op]) (imp, op, dest, src, mask, width);
 }
 
-void
-_pixman_implementation_combine_64_ca (pixman_implementation_t * imp,
-                                      pixman_op_t               op,
-                                      uint64_t *                dest,
-                                      const uint64_t *          src,
-                                      const uint64_t *          mask,
-                                      int                       width)
+pixman_combine_32_func_t
+_pixman_implementation_lookup_combiner (pixman_implementation_t *imp,
+					pixman_op_t		 op,
+					pixman_bool_t		 component_alpha,
+					pixman_bool_t		 narrow)
 {
-    (*imp->combine_64_ca[op]) (imp, op, dest, src, mask, width);
+    while (imp)
+    {
+	pixman_combine_32_func_t f = NULL;
+
+	switch ((narrow << 1) | component_alpha)
+	{
+	case 0: /* not narrow, not component alpha */
+	    f = (pixman_combine_32_func_t)imp->combine_float[op];
+	    break;
+	    
+	case 1: /* not narrow, component_alpha */
+	    f = (pixman_combine_32_func_t)imp->combine_float_ca[op];
+	    break;
+
+	case 2: /* narrow, not component alpha */
+	    f = imp->combine_32[op];
+	    break;
+
+	case 3: /* narrow, component_alpha */
+	    f = imp->combine_32_ca[op];
+	    break;
+	}
+
+	if (f)
+	    return f;
+
+	imp = imp->fallback;
+    }
+
+    /* We should never reach this point */
+    _pixman_log_error (FUNC, "No known combine function\n");
+    return dummy_combine;
 }
 
 pixman_bool_t
@@ -201,14 +239,25 @@ _pixman_implementation_blt (pixman_implementation_t * imp,
                             int                       dst_bpp,
                             int                       src_x,
                             int                       src_y,
-                            int                       dst_x,
-                            int                       dst_y,
+                            int                       dest_x,
+                            int                       dest_y,
                             int                       width,
                             int                       height)
 {
-    return (*imp->blt) (imp, src_bits, dst_bits, src_stride, dst_stride,
-			src_bpp, dst_bpp, src_x, src_y, dst_x, dst_y,
-			width, height);
+    while (imp)
+    {
+	if (imp->blt &&
+	    (*imp->blt) (imp, src_bits, dst_bits, src_stride, dst_stride,
+			 src_bpp, dst_bpp, src_x, src_y, dest_x, dest_y,
+			 width, height))
+	{
+	    return TRUE;
+	}
+
+	imp = imp->fallback;
+    }
+
+    return FALSE;
 }
 
 pixman_bool_t
@@ -220,8 +269,133 @@ _pixman_implementation_fill (pixman_implementation_t *imp,
                              int                      y,
                              int                      width,
                              int                      height,
-                             uint32_t                 xor)
+                             uint32_t                 filler)
+{
+    while (imp)
+    {
+	if (imp->fill &&
+	    ((*imp->fill) (imp, bits, stride, bpp, x, y, width, height, filler)))
+	{
+	    return TRUE;
+	}
+
+	imp = imp->fallback;
+    }
+
+    return FALSE;
+}
+
+static uint32_t *
+get_scanline_null (pixman_iter_t *iter, const uint32_t *mask)
+{
+    return NULL;
+}
+
+void
+_pixman_implementation_iter_init (pixman_implementation_t *imp,
+                                  pixman_iter_t           *iter,
+                                  pixman_image_t          *image,
+                                  int                      x,
+                                  int                      y,
+                                  int                      width,
+                                  int                      height,
+                                  uint8_t                 *buffer,
+                                  iter_flags_t             iter_flags,
+                                  uint32_t                 image_flags)
+{
+    pixman_format_code_t format;
+
+    iter->image = image;
+    iter->buffer = (uint32_t *)buffer;
+    iter->x = x;
+    iter->y = y;
+    iter->width = width;
+    iter->height = height;
+    iter->iter_flags = iter_flags;
+    iter->image_flags = image_flags;
+    iter->fini = NULL;
+
+    if (!iter->image)
+    {
+	iter->get_scanline = get_scanline_null;
+	return;
+    }
+
+    format = iter->image->common.extended_format_code;
+
+    while (imp)
+    {
+        if (imp->iter_info)
+        {
+            const pixman_iter_info_t *info;
+
+            for (info = imp->iter_info; info->format != PIXMAN_null; ++info)
+            {
+                if ((info->format == PIXMAN_any || info->format == format) &&
+                    (info->image_flags & image_flags) == info->image_flags &&
+                    (info->iter_flags & iter_flags) == info->iter_flags)
+                {
+                    iter->get_scanline = info->get_scanline;
+                    iter->write_back = info->write_back;
+
+                    if (info->initializer)
+                        info->initializer (iter, info);
+                    return;
+                }
+            }
+        }
+
+        imp = imp->fallback;
+    }
+}
+
+pixman_bool_t
+_pixman_disabled (const char *name)
 {
-    return (*imp->fill) (imp, bits, stride, bpp, x, y, width, height, xor);
+    const char *env;
+
+    if ((env = getenv ("PIXMAN_DISABLE")))
+    {
+	do
+	{
+	    const char *end;
+	    int len;
+
+	    if ((end = strchr (env, ' ')))
+		len = end - env;
+	    else
+		len = strlen (env);
+
+	    if (strlen (name) == len && strncmp (name, env, len) == 0)
+	    {
+		printf ("pixman: Disabled %s implementation\n", name);
+		return TRUE;
+	    }
+
+	    env += len;
+	}
+	while (*env++);
+    }
+
+    return FALSE;
 }
 
+pixman_implementation_t *
+_pixman_choose_implementation (void)
+{
+    pixman_implementation_t *imp;
+
+    imp = _pixman_implementation_create_general();
+
+    if (!_pixman_disabled ("fast"))
+	imp = _pixman_implementation_create_fast_path (imp);
+
+    imp = _pixman_x86_get_implementations (imp);
+    imp = _pixman_arm_get_implementations (imp);
+    imp = _pixman_ppc_get_implementations (imp);
+    imp = _pixman_mips_get_implementations (imp);
+
+    imp = _pixman_implementation_create_noop (imp);
+
+    return imp;
+}
diff --git a/pixman/pixman/pixman-inlines.h b/pixman/pixman/pixman-inlines.h
new file mode 100644
index 000000000..dd1c2f17f
--- /dev/null
+++ b/pixman/pixman/pixman-inlines.h
@@ -0,0 +1,1339 @@
+/* -*- Mode: c; c-basic-offset: 4; tab-width: 8; indent-tabs-mode: t; -*- */
+/*
+ * Copyright © 2000 SuSE, Inc.
+ * Copyright © 2007 Red Hat, Inc.
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of SuSE not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  SuSE makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as is"
+ * without express or implied warranty.
+ *
+ * SuSE DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL SuSE
+ * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * Author:  Keith Packard, SuSE, Inc.
+ */
+
+#ifndef PIXMAN_FAST_PATH_H__
+#define PIXMAN_FAST_PATH_H__
+
+#include "pixman-private.h"
+
+#define PIXMAN_REPEAT_COVER -1
+
+/* Flags describing input parameters to fast path macro template.
+ * Turning on some flag values may indicate that
+ * "some property X is available so template can use this" or
+ * "some property X should be handled by template".
+ *
+ * FLAG_HAVE_SOLID_MASK
+ *  Input mask is solid so template should handle this.
+ *
+ * FLAG_HAVE_NON_SOLID_MASK
+ *  Input mask is bits mask so template should handle this.
+ *
+ * FLAG_HAVE_SOLID_MASK and FLAG_HAVE_NON_SOLID_MASK are mutually
+ * exclusive. (It's not allowed to turn both flags on)
+ */
+#define FLAG_NONE				(0)
+#define FLAG_HAVE_SOLID_MASK			(1 <<   1)
+#define FLAG_HAVE_NON_SOLID_MASK		(1 <<   2)
+
+/* To avoid too short repeated scanline function calls, extend source
+ * scanlines having width less than below constant value.
+ */
+#define REPEAT_NORMAL_MIN_WIDTH			64
+
+static force_inline pixman_bool_t
+repeat (pixman_repeat_t repeat, int *c, int size)
+{
+    if (repeat == PIXMAN_REPEAT_NONE)
+    {
+	if (*c < 0 || *c >= size)
+	    return FALSE;
+    }
+    else if (repeat == PIXMAN_REPEAT_NORMAL)
+    {
+	while (*c >= size)
+	    *c -= size;
+	while (*c < 0)
+	    *c += size;
+    }
+    else if (repeat == PIXMAN_REPEAT_PAD)
+    {
+	*c = CLIP (*c, 0, size - 1);
+    }
+    else /* REFLECT */
+    {
+	*c = MOD (*c, size * 2);
+	if (*c >= size)
+	    *c = size * 2 - *c - 1;
+    }
+    return TRUE;
+}
+
+static force_inline int
+pixman_fixed_to_bilinear_weight (pixman_fixed_t x)
+{
+    return (x >> (16 - BILINEAR_INTERPOLATION_BITS)) &
+	   ((1 << BILINEAR_INTERPOLATION_BITS) - 1);
+}
+
+#if BILINEAR_INTERPOLATION_BITS <= 4
+/* Inspired by Filter_32_opaque from Skia */
+static force_inline uint32_t
+bilinear_interpolation (uint32_t tl, uint32_t tr,
+			uint32_t bl, uint32_t br,
+			int distx, int disty)
+{
+    int distxy, distxiy, distixy, distixiy;
+    uint32_t lo, hi;
+
+    distx <<= (4 - BILINEAR_INTERPOLATION_BITS);
+    disty <<= (4 - BILINEAR_INTERPOLATION_BITS);
+
+    distxy = distx * disty;
+    distxiy = (distx << 4) - distxy;	/* distx * (16 - disty) */
+    distixy = (disty << 4) - distxy;	/* disty * (16 - distx) */
+    distixiy =
+	16 * 16 - (disty << 4) -
+	(distx << 4) + distxy; /* (16 - distx) * (16 - disty) */
+
+    lo = (tl & 0xff00ff) * distixiy;
+    hi = ((tl >> 8) & 0xff00ff) * distixiy;
+
+    lo += (tr & 0xff00ff) * distxiy;
+    hi += ((tr >> 8) & 0xff00ff) * distxiy;
+
+    lo += (bl & 0xff00ff) * distixy;
+    hi += ((bl >> 8) & 0xff00ff) * distixy;
+
+    lo += (br & 0xff00ff) * distxy;
+    hi += ((br >> 8) & 0xff00ff) * distxy;
+
+    return ((lo >> 8) & 0xff00ff) | (hi & ~0xff00ff);
+}
+
+#else
+#if SIZEOF_LONG > 4
+
+static force_inline uint32_t
+bilinear_interpolation (uint32_t tl, uint32_t tr,
+			uint32_t bl, uint32_t br,
+			int distx, int disty)
+{
+    uint64_t distxy, distxiy, distixy, distixiy;
+    uint64_t tl64, tr64, bl64, br64;
+    uint64_t f, r;
+
+    distx <<= (8 - BILINEAR_INTERPOLATION_BITS);
+    disty <<= (8 - BILINEAR_INTERPOLATION_BITS);
+
+    distxy = distx * disty;
+    distxiy = distx * (256 - disty);
+    distixy = (256 - distx) * disty;
+    distixiy = (256 - distx) * (256 - disty);
+
+    /* Alpha and Blue */
+    tl64 = tl & 0xff0000ff;
+    tr64 = tr & 0xff0000ff;
+    bl64 = bl & 0xff0000ff;
+    br64 = br & 0xff0000ff;
+
+    f = tl64 * distixiy + tr64 * distxiy + bl64 * distixy + br64 * distxy;
+    r = f & 0x0000ff0000ff0000ull;
+
+    /* Red and Green */
+    tl64 = tl;
+    tl64 = ((tl64 << 16) & 0x000000ff00000000ull) | (tl64 & 0x0000ff00ull);
+
+    tr64 = tr;
+    tr64 = ((tr64 << 16) & 0x000000ff00000000ull) | (tr64 & 0x0000ff00ull);
+
+    bl64 = bl;
+    bl64 = ((bl64 << 16) & 0x000000ff00000000ull) | (bl64 & 0x0000ff00ull);
+
+    br64 = br;
+    br64 = ((br64 << 16) & 0x000000ff00000000ull) | (br64 & 0x0000ff00ull);
+
+    f = tl64 * distixiy + tr64 * distxiy + bl64 * distixy + br64 * distxy;
+    r |= ((f >> 16) & 0x000000ff00000000ull) | (f & 0xff000000ull);
+
+    return (uint32_t)(r >> 16);
+}
+
+#else
+
+static force_inline uint32_t
+bilinear_interpolation (uint32_t tl, uint32_t tr,
+			uint32_t bl, uint32_t br,
+			int distx, int disty)
+{
+    int distxy, distxiy, distixy, distixiy;
+    uint32_t f, r;
+
+    distx <<= (8 - BILINEAR_INTERPOLATION_BITS);
+    disty <<= (8 - BILINEAR_INTERPOLATION_BITS);
+
+    distxy = distx * disty;
+    distxiy = (distx << 8) - distxy;	/* distx * (256 - disty) */
+    distixy = (disty << 8) - distxy;	/* disty * (256 - distx) */
+    distixiy =
+	256 * 256 - (disty << 8) -
+	(distx << 8) + distxy;		/* (256 - distx) * (256 - disty) */
+
+    /* Blue */
+    r = (tl & 0x000000ff) * distixiy + (tr & 0x000000ff) * distxiy
+      + (bl & 0x000000ff) * distixy  + (br & 0x000000ff) * distxy;
+
+    /* Green */
+    f = (tl & 0x0000ff00) * distixiy + (tr & 0x0000ff00) * distxiy
+      + (bl & 0x0000ff00) * distixy  + (br & 0x0000ff00) * distxy;
+    r |= f & 0xff000000;
+
+    tl >>= 16;
+    tr >>= 16;
+    bl >>= 16;
+    br >>= 16;
+    r >>= 16;
+
+    /* Red */
+    f = (tl & 0x000000ff) * distixiy + (tr & 0x000000ff) * distxiy
+      + (bl & 0x000000ff) * distixy  + (br & 0x000000ff) * distxy;
+    r |= f & 0x00ff0000;
+
+    /* Alpha */
+    f = (tl & 0x0000ff00) * distixiy + (tr & 0x0000ff00) * distxiy
+      + (bl & 0x0000ff00) * distixy  + (br & 0x0000ff00) * distxy;
+    r |= f & 0xff000000;
+
+    return r;
+}
+
+#endif
+#endif // BILINEAR_INTERPOLATION_BITS <= 4
+
+/*
+ * For each scanline fetched from source image with PAD repeat:
+ * - calculate how many pixels need to be padded on the left side
+ * - calculate how many pixels need to be padded on the right side
+ * - update width to only count pixels which are fetched from the image
+ * All this information is returned via 'width', 'left_pad', 'right_pad'
+ * arguments. The code is assuming that 'unit_x' is positive.
+ *
+ * Note: 64-bit math is used in order to avoid potential overflows, which
+ *       is probably excessive in many cases. This particular function
+ *       may need its own correctness test and performance tuning.
+ */
+static force_inline void
+pad_repeat_get_scanline_bounds (int32_t         source_image_width,
+				pixman_fixed_t  vx,
+				pixman_fixed_t  unit_x,
+				int32_t *       width,
+				int32_t *       left_pad,
+				int32_t *       right_pad)
+{
+    int64_t max_vx = (int64_t) source_image_width << 16;
+    int64_t tmp;
+    if (vx < 0)
+    {
+	tmp = ((int64_t) unit_x - 1 - vx) / unit_x;
+	if (tmp > *width)
+	{
+	    *left_pad = *width;
+	    *width = 0;
+	}
+	else
+	{
+	    *left_pad = (int32_t) tmp;
+	    *width -= (int32_t) tmp;
+	}
+    }
+    else
+    {
+	*left_pad = 0;
+    }
+    tmp = ((int64_t) unit_x - 1 - vx + max_vx) / unit_x - *left_pad;
+    if (tmp < 0)
+    {
+	*right_pad = *width;
+	*width = 0;
+    }
+    else if (tmp >= *width)
+    {
+	*right_pad = 0;
+    }
+    else
+    {
+	*right_pad = *width - (int32_t) tmp;
+	*width = (int32_t) tmp;
+    }
+}
+
+/* A macroified version of specialized nearest scalers for some
+ * common 8888 and 565 formats. It supports SRC and OVER ops.
+ *
+ * There are two repeat versions, one that handles repeat normal,
+ * and one without repeat handling that only works if the src region
+ * used is completely covered by the pre-repeated source samples.
+ *
+ * The loops are unrolled to process two pixels per iteration for better
+ * performance on most CPU architectures (superscalar processors
+ * can issue several operations simultaneously, other processors can hide
+ * instructions latencies by pipelining operations). Unrolling more
+ * does not make much sense because the compiler will start running out
+ * of spare registers soon.
+ */
+
+#define GET_8888_ALPHA(s) ((s) >> 24)
+ /* This is not actually used since we don't have an OVER with
+    565 source, but it is needed to build. */
+#define GET_0565_ALPHA(s) 0xff
+#define GET_x888_ALPHA(s) 0xff
+
+#define FAST_NEAREST_SCANLINE(scanline_func_name, SRC_FORMAT, DST_FORMAT,			\
+			      src_type_t, dst_type_t, OP, repeat_mode)				\
+static force_inline void									\
+scanline_func_name (dst_type_t       *dst,							\
+		    const src_type_t *src,							\
+		    int32_t           w,							\
+		    pixman_fixed_t    vx,							\
+		    pixman_fixed_t    unit_x,							\
+		    pixman_fixed_t    src_width_fixed,						\
+		    pixman_bool_t     fully_transparent_src)					\
+{												\
+	uint32_t   d;										\
+	src_type_t s1, s2;									\
+	uint8_t    a1, a2;									\
+	int        x1, x2;									\
+												\
+	if (PIXMAN_OP_ ## OP == PIXMAN_OP_OVER && fully_transparent_src)			\
+	    return;										\
+												\
+	if (PIXMAN_OP_ ## OP != PIXMAN_OP_SRC && PIXMAN_OP_ ## OP != PIXMAN_OP_OVER)		\
+	    abort();										\
+												\
+	while ((w -= 2) >= 0)									\
+	{											\
+	    x1 = pixman_fixed_to_int (vx);							\
+	    vx += unit_x;									\
+	    if (PIXMAN_REPEAT_ ## repeat_mode == PIXMAN_REPEAT_NORMAL)				\
+	    {											\
+		/* This works because we know that unit_x is positive */			\
+		while (vx >= 0)									\
+		    vx -= src_width_fixed;							\
+	    }											\
+	    s1 = *(src + x1);									\
+												\
+	    x2 = pixman_fixed_to_int (vx);							\
+	    vx += unit_x;									\
+	    if (PIXMAN_REPEAT_ ## repeat_mode == PIXMAN_REPEAT_NORMAL)				\
+	    {											\
+		/* This works because we know that unit_x is positive */			\
+		while (vx >= 0)									\
+		    vx -= src_width_fixed;							\
+	    }											\
+	    s2 = *(src + x2);									\
+												\
+	    if (PIXMAN_OP_ ## OP == PIXMAN_OP_OVER)						\
+	    {											\
+		a1 = GET_ ## SRC_FORMAT ## _ALPHA(s1);						\
+		a2 = GET_ ## SRC_FORMAT ## _ALPHA(s2);						\
+												\
+		if (a1 == 0xff)									\
+		{										\
+		    *dst = convert_ ## SRC_FORMAT ## _to_ ## DST_FORMAT (s1);			\
+		}										\
+		else if (s1)									\
+		{										\
+		    d = convert_ ## DST_FORMAT ## _to_8888 (*dst);				\
+		    s1 = convert_ ## SRC_FORMAT ## _to_8888 (s1);				\
+		    a1 ^= 0xff;									\
+		    UN8x4_MUL_UN8_ADD_UN8x4 (d, a1, s1);					\
+		    *dst = convert_8888_to_ ## DST_FORMAT (d);					\
+		}										\
+		dst++;										\
+												\
+		if (a2 == 0xff)									\
+		{										\
+		    *dst = convert_ ## SRC_FORMAT ## _to_ ## DST_FORMAT (s2);			\
+		}										\
+		else if (s2)									\
+		{										\
+		    d = convert_## DST_FORMAT ## _to_8888 (*dst);				\
+		    s2 = convert_## SRC_FORMAT ## _to_8888 (s2);				\
+		    a2 ^= 0xff;									\
+		    UN8x4_MUL_UN8_ADD_UN8x4 (d, a2, s2);					\
+		    *dst = convert_8888_to_ ## DST_FORMAT (d);					\
+		}										\
+		dst++;										\
+	    }											\
+	    else /* PIXMAN_OP_SRC */								\
+	    {											\
+		*dst++ = convert_ ## SRC_FORMAT ## _to_ ## DST_FORMAT (s1);			\
+		*dst++ = convert_ ## SRC_FORMAT ## _to_ ## DST_FORMAT (s2);			\
+	    }											\
+	}											\
+												\
+	if (w & 1)										\
+	{											\
+	    x1 = pixman_fixed_to_int (vx);							\
+	    s1 = *(src + x1);									\
+												\
+	    if (PIXMAN_OP_ ## OP == PIXMAN_OP_OVER)						\
+	    {											\
+		a1 = GET_ ## SRC_FORMAT ## _ALPHA(s1);						\
+												\
+		if (a1 == 0xff)									\
+		{										\
+		    *dst = convert_ ## SRC_FORMAT ## _to_ ## DST_FORMAT (s1);			\
+		}										\
+		else if (s1)									\
+		{										\
+		    d = convert_## DST_FORMAT ## _to_8888 (*dst);				\
+		    s1 = convert_ ## SRC_FORMAT ## _to_8888 (s1);				\
+		    a1 ^= 0xff;									\
+		    UN8x4_MUL_UN8_ADD_UN8x4 (d, a1, s1);					\
+		    *dst = convert_8888_to_ ## DST_FORMAT (d);					\
+		}										\
+		dst++;										\
+	    }											\
+	    else /* PIXMAN_OP_SRC */								\
+	    {											\
+		*dst++ = convert_ ## SRC_FORMAT ## _to_ ## DST_FORMAT (s1);			\
+	    }											\
+	}											\
+}
+
+#define FAST_NEAREST_MAINLOOP_INT(scale_func_name, scanline_func, src_type_t, mask_type_t,	\
+				  dst_type_t, repeat_mode, have_mask, mask_is_solid)		\
+static void											\
+fast_composite_scaled_nearest  ## scale_func_name (pixman_implementation_t *imp,		\
+						   pixman_composite_info_t *info)               \
+{												\
+    PIXMAN_COMPOSITE_ARGS (info);					                        \
+    dst_type_t *dst_line;						                        \
+    mask_type_t *mask_line;									\
+    src_type_t *src_first_line;									\
+    int       y;										\
+    pixman_fixed_t src_width_fixed = pixman_int_to_fixed (src_image->bits.width);		\
+    pixman_fixed_t max_vy;									\
+    pixman_vector_t v;										\
+    pixman_fixed_t vx, vy;									\
+    pixman_fixed_t unit_x, unit_y;								\
+    int32_t left_pad, right_pad;								\
+												\
+    src_type_t *src;										\
+    dst_type_t *dst;										\
+    mask_type_t solid_mask;									\
+    const mask_type_t *mask = &solid_mask;							\
+    int src_stride, mask_stride, dst_stride;							\
+												\
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, dst_type_t, dst_stride, dst_line, 1);	\
+    if (have_mask)										\
+    {												\
+	if (mask_is_solid)									\
+	    solid_mask = _pixman_image_get_solid (imp, mask_image, dest_image->bits.format);	\
+	else											\
+	    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, mask_type_t,			\
+				   mask_stride, mask_line, 1);					\
+    }												\
+    /* pass in 0 instead of src_x and src_y because src_x and src_y need to be			\
+     * transformed from destination space to source space */					\
+    PIXMAN_IMAGE_GET_LINE (src_image, 0, 0, src_type_t, src_stride, src_first_line, 1);		\
+												\
+    /* reference point is the center of the pixel */						\
+    v.vector[0] = pixman_int_to_fixed (src_x) + pixman_fixed_1 / 2;				\
+    v.vector[1] = pixman_int_to_fixed (src_y) + pixman_fixed_1 / 2;				\
+    v.vector[2] = pixman_fixed_1;								\
+												\
+    if (!pixman_transform_point_3d (src_image->common.transform, &v))				\
+	return;											\
+												\
+    unit_x = src_image->common.transform->matrix[0][0];						\
+    unit_y = src_image->common.transform->matrix[1][1];						\
+												\
+    /* Round down to closest integer, ensuring that 0.5 rounds to 0, not 1 */			\
+    v.vector[0] -= pixman_fixed_e;								\
+    v.vector[1] -= pixman_fixed_e;								\
+												\
+    vx = v.vector[0];										\
+    vy = v.vector[1];										\
+												\
+    if (PIXMAN_REPEAT_ ## repeat_mode == PIXMAN_REPEAT_NORMAL)					\
+    {												\
+	max_vy = pixman_int_to_fixed (src_image->bits.height);					\
+												\
+	/* Clamp repeating positions inside the actual samples */				\
+	repeat (PIXMAN_REPEAT_NORMAL, &vx, src_width_fixed);					\
+	repeat (PIXMAN_REPEAT_NORMAL, &vy, max_vy);						\
+    }												\
+												\
+    if (PIXMAN_REPEAT_ ## repeat_mode == PIXMAN_REPEAT_PAD ||					\
+	PIXMAN_REPEAT_ ## repeat_mode == PIXMAN_REPEAT_NONE)					\
+    {												\
+	pad_repeat_get_scanline_bounds (src_image->bits.width, vx, unit_x,			\
+					&width, &left_pad, &right_pad);				\
+	vx += left_pad * unit_x;								\
+    }												\
+												\
+    while (--height >= 0)									\
+    {												\
+	dst = dst_line;										\
+	dst_line += dst_stride;									\
+	if (have_mask && !mask_is_solid)							\
+	{											\
+	    mask = mask_line;									\
+	    mask_line += mask_stride;								\
+	}											\
+												\
+	y = pixman_fixed_to_int (vy);								\
+	vy += unit_y;										\
+	if (PIXMAN_REPEAT_ ## repeat_mode == PIXMAN_REPEAT_NORMAL)				\
+	    repeat (PIXMAN_REPEAT_NORMAL, &vy, max_vy);						\
+	if (PIXMAN_REPEAT_ ## repeat_mode == PIXMAN_REPEAT_PAD)					\
+	{											\
+	    repeat (PIXMAN_REPEAT_PAD, &y, src_image->bits.height);				\
+	    src = src_first_line + src_stride * y;						\
+	    if (left_pad > 0)									\
+	    {											\
+		scanline_func (mask, dst,							\
+			       src + src_image->bits.width - src_image->bits.width + 1,		\
+			       left_pad, -pixman_fixed_e, 0, src_width_fixed, FALSE);		\
+	    }											\
+	    if (width > 0)									\
+	    {											\
+		scanline_func (mask + (mask_is_solid ? 0 : left_pad),				\
+			       dst + left_pad, src + src_image->bits.width, width,		\
+			       vx - src_width_fixed, unit_x, src_width_fixed, FALSE);		\
+	    }											\
+	    if (right_pad > 0)									\
+	    {											\
+		scanline_func (mask + (mask_is_solid ? 0 : left_pad + width),			\
+			       dst + left_pad + width, src + src_image->bits.width,		\
+			       right_pad, -pixman_fixed_e, 0, src_width_fixed, FALSE);		\
+	    }											\
+	}											\
+	else if (PIXMAN_REPEAT_ ## repeat_mode == PIXMAN_REPEAT_NONE)				\
+	{											\
+	    static const src_type_t zero[1] = { 0 };						\
+	    if (y < 0 || y >= src_image->bits.height)						\
+	    {											\
+		scanline_func (mask, dst, zero + 1, left_pad + width + right_pad,		\
+			       -pixman_fixed_e, 0, src_width_fixed, TRUE);			\
+		continue;									\
+	    }											\
+	    src = src_first_line + src_stride * y;						\
+	    if (left_pad > 0)									\
+	    {											\
+		scanline_func (mask, dst, zero + 1, left_pad,					\
+			       -pixman_fixed_e, 0, src_width_fixed, TRUE);			\
+	    }											\
+	    if (width > 0)									\
+	    {											\
+		scanline_func (mask + (mask_is_solid ? 0 : left_pad),				\
+			       dst + left_pad, src + src_image->bits.width, width,		\
+			       vx - src_width_fixed, unit_x, src_width_fixed, FALSE);		\
+	    }											\
+	    if (right_pad > 0)									\
+	    {											\
+		scanline_func (mask + (mask_is_solid ? 0 : left_pad + width),			\
+			       dst + left_pad + width, zero + 1, right_pad,			\
+			       -pixman_fixed_e, 0, src_width_fixed, TRUE);			\
+	    }											\
+	}											\
+	else											\
+	{											\
+	    src = src_first_line + src_stride * y;						\
+	    scanline_func (mask, dst, src + src_image->bits.width, width, vx - src_width_fixed,	\
+			   unit_x, src_width_fixed, FALSE);					\
+	}											\
+    }												\
+}
+
+/* A workaround for old sun studio, see: https://bugs.freedesktop.org/show_bug.cgi?id=32764 */
+#define FAST_NEAREST_MAINLOOP_COMMON(scale_func_name, scanline_func, src_type_t, mask_type_t,	\
+				  dst_type_t, repeat_mode, have_mask, mask_is_solid)		\
+	FAST_NEAREST_MAINLOOP_INT(_ ## scale_func_name, scanline_func, src_type_t, mask_type_t,	\
+				  dst_type_t, repeat_mode, have_mask, mask_is_solid)
+
+#define FAST_NEAREST_MAINLOOP_NOMASK(scale_func_name, scanline_func, src_type_t, dst_type_t,	\
+			      repeat_mode)							\
+    static force_inline void									\
+    scanline_func##scale_func_name##_wrapper (							\
+		    const uint8_t    *mask,							\
+		    dst_type_t       *dst,							\
+		    const src_type_t *src,							\
+		    int32_t          w,								\
+		    pixman_fixed_t   vx,							\
+		    pixman_fixed_t   unit_x,							\
+		    pixman_fixed_t   max_vx,							\
+		    pixman_bool_t    fully_transparent_src)					\
+    {												\
+	scanline_func (dst, src, w, vx, unit_x, max_vx, fully_transparent_src);			\
+    }												\
+    FAST_NEAREST_MAINLOOP_INT (scale_func_name, scanline_func##scale_func_name##_wrapper,	\
+			       src_type_t, uint8_t, dst_type_t, repeat_mode, FALSE, FALSE)
+
+#define FAST_NEAREST_MAINLOOP(scale_func_name, scanline_func, src_type_t, dst_type_t,		\
+			      repeat_mode)							\
+	FAST_NEAREST_MAINLOOP_NOMASK(_ ## scale_func_name, scanline_func, src_type_t,		\
+			      dst_type_t, repeat_mode)
+
+#define FAST_NEAREST(scale_func_name, SRC_FORMAT, DST_FORMAT,				\
+		     src_type_t, dst_type_t, OP, repeat_mode)				\
+    FAST_NEAREST_SCANLINE(scaled_nearest_scanline_ ## scale_func_name ## _ ## OP,	\
+			  SRC_FORMAT, DST_FORMAT, src_type_t, dst_type_t,		\
+			  OP, repeat_mode)						\
+    FAST_NEAREST_MAINLOOP_NOMASK(_ ## scale_func_name ## _ ## OP,			\
+			  scaled_nearest_scanline_ ## scale_func_name ## _ ## OP,	\
+			  src_type_t, dst_type_t, repeat_mode)
+
+
+#define SCALED_NEAREST_FLAGS						\
+    (FAST_PATH_SCALE_TRANSFORM	|					\
+     FAST_PATH_NO_ALPHA_MAP	|					\
+     FAST_PATH_NEAREST_FILTER	|					\
+     FAST_PATH_NO_ACCESSORS	|					\
+     FAST_PATH_NARROW_FORMAT)
+
+#define SIMPLE_NEAREST_FAST_PATH_NORMAL(op,s,d,func)			\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	(SCALED_NEAREST_FLAGS		|				\
+	 FAST_PATH_NORMAL_REPEAT	|				\
+	 FAST_PATH_X_UNIT_POSITIVE),					\
+	PIXMAN_null, 0,							\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_nearest_ ## func ## _normal ## _ ## op,	\
+    }
+
+#define SIMPLE_NEAREST_FAST_PATH_PAD(op,s,d,func)			\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	(SCALED_NEAREST_FLAGS		|				\
+	 FAST_PATH_PAD_REPEAT		|				\
+	 FAST_PATH_X_UNIT_POSITIVE),					\
+	PIXMAN_null, 0,							\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_nearest_ ## func ## _pad ## _ ## op,	\
+    }
+
+#define SIMPLE_NEAREST_FAST_PATH_NONE(op,s,d,func)			\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	(SCALED_NEAREST_FLAGS		|				\
+	 FAST_PATH_NONE_REPEAT		|				\
+	 FAST_PATH_X_UNIT_POSITIVE),					\
+	PIXMAN_null, 0,							\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_nearest_ ## func ## _none ## _ ## op,	\
+    }
+
+#define SIMPLE_NEAREST_FAST_PATH_COVER(op,s,d,func)			\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	SCALED_NEAREST_FLAGS | FAST_PATH_SAMPLES_COVER_CLIP_NEAREST,    \
+	PIXMAN_null, 0,							\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_nearest_ ## func ## _cover ## _ ## op,	\
+    }
+
+#define SIMPLE_NEAREST_A8_MASK_FAST_PATH_NORMAL(op,s,d,func)		\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	(SCALED_NEAREST_FLAGS		|				\
+	 FAST_PATH_NORMAL_REPEAT	|				\
+	 FAST_PATH_X_UNIT_POSITIVE),					\
+	PIXMAN_a8, MASK_FLAGS (a8, FAST_PATH_UNIFIED_ALPHA),		\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_nearest_ ## func ## _normal ## _ ## op,	\
+    }
+
+#define SIMPLE_NEAREST_A8_MASK_FAST_PATH_PAD(op,s,d,func)		\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	(SCALED_NEAREST_FLAGS		|				\
+	 FAST_PATH_PAD_REPEAT		|				\
+	 FAST_PATH_X_UNIT_POSITIVE),					\
+	PIXMAN_a8, MASK_FLAGS (a8, FAST_PATH_UNIFIED_ALPHA),		\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_nearest_ ## func ## _pad ## _ ## op,	\
+    }
+
+#define SIMPLE_NEAREST_A8_MASK_FAST_PATH_NONE(op,s,d,func)		\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	(SCALED_NEAREST_FLAGS		|				\
+	 FAST_PATH_NONE_REPEAT		|				\
+	 FAST_PATH_X_UNIT_POSITIVE),					\
+	PIXMAN_a8, MASK_FLAGS (a8, FAST_PATH_UNIFIED_ALPHA),		\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_nearest_ ## func ## _none ## _ ## op,	\
+    }
+
+#define SIMPLE_NEAREST_A8_MASK_FAST_PATH_COVER(op,s,d,func)		\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	SCALED_NEAREST_FLAGS | FAST_PATH_SAMPLES_COVER_CLIP_NEAREST,	\
+	PIXMAN_a8, MASK_FLAGS (a8, FAST_PATH_UNIFIED_ALPHA),		\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_nearest_ ## func ## _cover ## _ ## op,	\
+    }
+
+#define SIMPLE_NEAREST_SOLID_MASK_FAST_PATH_NORMAL(op,s,d,func)		\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	(SCALED_NEAREST_FLAGS		|				\
+	 FAST_PATH_NORMAL_REPEAT	|				\
+	 FAST_PATH_X_UNIT_POSITIVE),					\
+	PIXMAN_solid, MASK_FLAGS (solid, FAST_PATH_UNIFIED_ALPHA),	\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_nearest_ ## func ## _normal ## _ ## op,	\
+    }
+
+#define SIMPLE_NEAREST_SOLID_MASK_FAST_PATH_PAD(op,s,d,func)		\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	(SCALED_NEAREST_FLAGS		|				\
+	 FAST_PATH_PAD_REPEAT		|				\
+	 FAST_PATH_X_UNIT_POSITIVE),					\
+	PIXMAN_solid, MASK_FLAGS (solid, FAST_PATH_UNIFIED_ALPHA),	\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_nearest_ ## func ## _pad ## _ ## op,	\
+    }
+
+#define SIMPLE_NEAREST_SOLID_MASK_FAST_PATH_NONE(op,s,d,func)		\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	(SCALED_NEAREST_FLAGS		|				\
+	 FAST_PATH_NONE_REPEAT		|				\
+	 FAST_PATH_X_UNIT_POSITIVE),					\
+	PIXMAN_solid, MASK_FLAGS (solid, FAST_PATH_UNIFIED_ALPHA),	\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_nearest_ ## func ## _none ## _ ## op,	\
+    }
+
+#define SIMPLE_NEAREST_SOLID_MASK_FAST_PATH_COVER(op,s,d,func)		\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	SCALED_NEAREST_FLAGS | FAST_PATH_SAMPLES_COVER_CLIP_NEAREST,	\
+	PIXMAN_solid, MASK_FLAGS (solid, FAST_PATH_UNIFIED_ALPHA),	\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_nearest_ ## func ## _cover ## _ ## op,	\
+    }
+
+/* Prefer the use of 'cover' variant, because it is faster */
+#define SIMPLE_NEAREST_FAST_PATH(op,s,d,func)				\
+    SIMPLE_NEAREST_FAST_PATH_COVER (op,s,d,func),			\
+    SIMPLE_NEAREST_FAST_PATH_NONE (op,s,d,func),			\
+    SIMPLE_NEAREST_FAST_PATH_PAD (op,s,d,func),				\
+    SIMPLE_NEAREST_FAST_PATH_NORMAL (op,s,d,func)
+
+#define SIMPLE_NEAREST_A8_MASK_FAST_PATH(op,s,d,func)			\
+    SIMPLE_NEAREST_A8_MASK_FAST_PATH_COVER (op,s,d,func),		\
+    SIMPLE_NEAREST_A8_MASK_FAST_PATH_NONE (op,s,d,func),		\
+    SIMPLE_NEAREST_A8_MASK_FAST_PATH_PAD (op,s,d,func)
+
+#define SIMPLE_NEAREST_SOLID_MASK_FAST_PATH(op,s,d,func)		\
+    SIMPLE_NEAREST_SOLID_MASK_FAST_PATH_COVER (op,s,d,func),		\
+    SIMPLE_NEAREST_SOLID_MASK_FAST_PATH_NONE (op,s,d,func),		\
+    SIMPLE_NEAREST_SOLID_MASK_FAST_PATH_PAD (op,s,d,func)
+
+/*****************************************************************************/
+
+/*
+ * Identify 5 zones in each scanline for bilinear scaling. Depending on
+ * whether 2 pixels to be interpolated are fetched from the image itself,
+ * from the padding area around it or from both image and padding area.
+ */
+static force_inline void
+bilinear_pad_repeat_get_scanline_bounds (int32_t         source_image_width,
+					 pixman_fixed_t  vx,
+					 pixman_fixed_t  unit_x,
+					 int32_t *       left_pad,
+					 int32_t *       left_tz,
+					 int32_t *       width,
+					 int32_t *       right_tz,
+					 int32_t *       right_pad)
+{
+	int width1 = *width, left_pad1, right_pad1;
+	int width2 = *width, left_pad2, right_pad2;
+
+	pad_repeat_get_scanline_bounds (source_image_width, vx, unit_x,
+					&width1, &left_pad1, &right_pad1);
+	pad_repeat_get_scanline_bounds (source_image_width, vx + pixman_fixed_1,
+					unit_x, &width2, &left_pad2, &right_pad2);
+
+	*left_pad = left_pad2;
+	*left_tz = left_pad1 - left_pad2;
+	*right_tz = right_pad2 - right_pad1;
+	*right_pad = right_pad1;
+	*width -= *left_pad + *left_tz + *right_tz + *right_pad;
+}
+
+/*
+ * Main loop template for single pass bilinear scaling. It needs to be
+ * provided with 'scanline_func' which should do the compositing operation.
+ * The needed function has the following prototype:
+ *
+ *	scanline_func (dst_type_t *       dst,
+ *		       const mask_type_ * mask,
+ *		       const src_type_t * src_top,
+ *		       const src_type_t * src_bottom,
+ *		       int32_t            width,
+ *		       int                weight_top,
+ *		       int                weight_bottom,
+ *		       pixman_fixed_t     vx,
+ *		       pixman_fixed_t     unit_x,
+ *		       pixman_fixed_t     max_vx,
+ *		       pixman_bool_t      zero_src)
+ *
+ * Where:
+ *  dst                 - destination scanline buffer for storing results
+ *  mask                - mask buffer (or single value for solid mask)
+ *  src_top, src_bottom - two source scanlines
+ *  width               - number of pixels to process
+ *  weight_top          - weight of the top row for interpolation
+ *  weight_bottom       - weight of the bottom row for interpolation
+ *  vx                  - initial position for fetching the first pair of
+ *                        pixels from the source buffer
+ *  unit_x              - position increment needed to move to the next pair
+ *                        of pixels
+ *  max_vx              - image size as a fixed point value, can be used for
+ *                        implementing NORMAL repeat (when it is supported)
+ *  zero_src            - boolean hint variable, which is set to TRUE when
+ *                        all source pixels are fetched from zero padding
+ *                        zone for NONE repeat
+ *
+ * Note: normally the sum of 'weight_top' and 'weight_bottom' is equal to
+ *       BILINEAR_INTERPOLATION_RANGE, but sometimes it may be less than that
+ *       for NONE repeat when handling fuzzy antialiased top or bottom image
+ *       edges. Also both top and bottom weight variables are guaranteed to
+ *       have value, which is less than BILINEAR_INTERPOLATION_RANGE.
+ *       For example, the weights can fit into unsigned byte or be used
+ *       with 8-bit SIMD multiplication instructions for 8-bit interpolation
+ *       precision.
+ */
+#define FAST_BILINEAR_MAINLOOP_INT(scale_func_name, scanline_func, src_type_t, mask_type_t,	\
+				  dst_type_t, repeat_mode, flags)				\
+static void											\
+fast_composite_scaled_bilinear ## scale_func_name (pixman_implementation_t *imp,		\
+						   pixman_composite_info_t *info)		\
+{												\
+    PIXMAN_COMPOSITE_ARGS (info);								\
+    dst_type_t *dst_line;									\
+    mask_type_t *mask_line;									\
+    src_type_t *src_first_line;									\
+    int       y1, y2;										\
+    pixman_fixed_t max_vx = INT32_MAX; /* suppress uninitialized variable warning */		\
+    pixman_vector_t v;										\
+    pixman_fixed_t vx, vy;									\
+    pixman_fixed_t unit_x, unit_y;								\
+    int32_t left_pad, left_tz, right_tz, right_pad;						\
+												\
+    dst_type_t *dst;										\
+    mask_type_t solid_mask;									\
+    const mask_type_t *mask = &solid_mask;							\
+    int src_stride, mask_stride, dst_stride;							\
+												\
+    int src_width;										\
+    pixman_fixed_t src_width_fixed;								\
+    int max_x;											\
+    pixman_bool_t need_src_extension;								\
+												\
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, dst_type_t, dst_stride, dst_line, 1);	\
+    if (flags & FLAG_HAVE_SOLID_MASK)								\
+    {												\
+	solid_mask = _pixman_image_get_solid (imp, mask_image, dest_image->bits.format);	\
+	mask_stride = 0;									\
+    }												\
+    else if (flags & FLAG_HAVE_NON_SOLID_MASK)							\
+    {												\
+	PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, mask_type_t,				\
+			       mask_stride, mask_line, 1);					\
+    }												\
+												\
+    /* pass in 0 instead of src_x and src_y because src_x and src_y need to be			\
+     * transformed from destination space to source space */					\
+    PIXMAN_IMAGE_GET_LINE (src_image, 0, 0, src_type_t, src_stride, src_first_line, 1);		\
+												\
+    /* reference point is the center of the pixel */						\
+    v.vector[0] = pixman_int_to_fixed (src_x) + pixman_fixed_1 / 2;				\
+    v.vector[1] = pixman_int_to_fixed (src_y) + pixman_fixed_1 / 2;				\
+    v.vector[2] = pixman_fixed_1;								\
+												\
+    if (!pixman_transform_point_3d (src_image->common.transform, &v))				\
+	return;											\
+												\
+    unit_x = src_image->common.transform->matrix[0][0];						\
+    unit_y = src_image->common.transform->matrix[1][1];						\
+												\
+    v.vector[0] -= pixman_fixed_1 / 2;								\
+    v.vector[1] -= pixman_fixed_1 / 2;								\
+												\
+    vy = v.vector[1];										\
+												\
+    if (PIXMAN_REPEAT_ ## repeat_mode == PIXMAN_REPEAT_PAD ||					\
+	PIXMAN_REPEAT_ ## repeat_mode == PIXMAN_REPEAT_NONE)					\
+    {												\
+	bilinear_pad_repeat_get_scanline_bounds (src_image->bits.width, v.vector[0], unit_x,	\
+					&left_pad, &left_tz, &width, &right_tz, &right_pad);	\
+	if (PIXMAN_REPEAT_ ## repeat_mode == PIXMAN_REPEAT_PAD)					\
+	{											\
+	    /* PAD repeat does not need special handling for 'transition zones' and */		\
+	    /* they can be combined with 'padding zones' safely */				\
+	    left_pad += left_tz;								\
+	    right_pad += right_tz;								\
+	    left_tz = right_tz = 0;								\
+	}											\
+	v.vector[0] += left_pad * unit_x;							\
+    }												\
+												\
+    if (PIXMAN_REPEAT_ ## repeat_mode == PIXMAN_REPEAT_NORMAL)					\
+    {												\
+	vx = v.vector[0];									\
+	repeat (PIXMAN_REPEAT_NORMAL, &vx, pixman_int_to_fixed(src_image->bits.width));		\
+	max_x = pixman_fixed_to_int (vx + (width - 1) * (int64_t)unit_x) + 1;			\
+												\
+	if (src_image->bits.width < REPEAT_NORMAL_MIN_WIDTH)					\
+	{											\
+	    src_width = 0;									\
+												\
+	    while (src_width < REPEAT_NORMAL_MIN_WIDTH && src_width <= max_x)			\
+		src_width += src_image->bits.width;						\
+												\
+	    need_src_extension = TRUE;								\
+	}											\
+	else											\
+	{											\
+	    src_width = src_image->bits.width;							\
+	    need_src_extension = FALSE;								\
+	}											\
+												\
+	src_width_fixed = pixman_int_to_fixed (src_width);					\
+    }												\
+												\
+    while (--height >= 0)									\
+    {												\
+	int weight1, weight2;									\
+	dst = dst_line;										\
+	dst_line += dst_stride;									\
+	vx = v.vector[0];									\
+	if (flags & FLAG_HAVE_NON_SOLID_MASK)							\
+	{											\
+	    mask = mask_line;									\
+	    mask_line += mask_stride;								\
+	}											\
+												\
+	y1 = pixman_fixed_to_int (vy);								\
+	weight2 = pixman_fixed_to_bilinear_weight (vy);						\
+	if (weight2)										\
+	{											\
+	    /* both weight1 and weight2 are smaller than BILINEAR_INTERPOLATION_RANGE */	\
+	    y2 = y1 + 1;									\
+	    weight1 = BILINEAR_INTERPOLATION_RANGE - weight2;					\
+	}											\
+	else											\
+	{											\
+	    /* set both top and bottom row to the same scanline and tweak weights */		\
+	    y2 = y1;										\
+	    weight1 = weight2 = BILINEAR_INTERPOLATION_RANGE / 2;				\
+	}											\
+	vy += unit_y;										\
+	if (PIXMAN_REPEAT_ ## repeat_mode == PIXMAN_REPEAT_PAD)					\
+	{											\
+	    src_type_t *src1, *src2;								\
+	    src_type_t buf1[2];									\
+	    src_type_t buf2[2];									\
+	    repeat (PIXMAN_REPEAT_PAD, &y1, src_image->bits.height);				\
+	    repeat (PIXMAN_REPEAT_PAD, &y2, src_image->bits.height);				\
+	    src1 = src_first_line + src_stride * y1;						\
+	    src2 = src_first_line + src_stride * y2;						\
+												\
+	    if (left_pad > 0)									\
+	    {											\
+		buf1[0] = buf1[1] = src1[0];							\
+		buf2[0] = buf2[1] = src2[0];							\
+		scanline_func (dst, mask,							\
+			       buf1, buf2, left_pad, weight1, weight2, 0, 0, 0, FALSE);		\
+		dst += left_pad;								\
+		if (flags & FLAG_HAVE_NON_SOLID_MASK)						\
+		    mask += left_pad;								\
+	    }											\
+	    if (width > 0)									\
+	    {											\
+		scanline_func (dst, mask,							\
+			       src1, src2, width, weight1, weight2, vx, unit_x, 0, FALSE);	\
+		dst += width;									\
+		if (flags & FLAG_HAVE_NON_SOLID_MASK)						\
+		    mask += width;								\
+	    }											\
+	    if (right_pad > 0)									\
+	    {											\
+		buf1[0] = buf1[1] = src1[src_image->bits.width - 1];				\
+		buf2[0] = buf2[1] = src2[src_image->bits.width - 1];				\
+		scanline_func (dst, mask,							\
+			       buf1, buf2, right_pad, weight1, weight2, 0, 0, 0, FALSE);	\
+	    }											\
+	}											\
+	else if (PIXMAN_REPEAT_ ## repeat_mode == PIXMAN_REPEAT_NONE)				\
+	{											\
+	    src_type_t *src1, *src2;								\
+	    src_type_t buf1[2];									\
+	    src_type_t buf2[2];									\
+	    /* handle top/bottom zero padding by just setting weights to 0 if needed */		\
+	    if (y1 < 0)										\
+	    {											\
+		weight1 = 0;									\
+		y1 = 0;										\
+	    }											\
+	    if (y1 >= src_image->bits.height)							\
+	    {											\
+		weight1 = 0;									\
+		y1 = src_image->bits.height - 1;						\
+	    }											\
+	    if (y2 < 0)										\
+	    {											\
+		weight2 = 0;									\
+		y2 = 0;										\
+	    }											\
+	    if (y2 >= src_image->bits.height)							\
+	    {											\
+		weight2 = 0;									\
+		y2 = src_image->bits.height - 1;						\
+	    }											\
+	    src1 = src_first_line + src_stride * y1;						\
+	    src2 = src_first_line + src_stride * y2;						\
+												\
+	    if (left_pad > 0)									\
+	    {											\
+		buf1[0] = buf1[1] = 0;								\
+		buf2[0] = buf2[1] = 0;								\
+		scanline_func (dst, mask,							\
+			       buf1, buf2, left_pad, weight1, weight2, 0, 0, 0, TRUE);		\
+		dst += left_pad;								\
+		if (flags & FLAG_HAVE_NON_SOLID_MASK)						\
+		    mask += left_pad;								\
+	    }											\
+	    if (left_tz > 0)									\
+	    {											\
+		buf1[0] = 0;									\
+		buf1[1] = src1[0];								\
+		buf2[0] = 0;									\
+		buf2[1] = src2[0];								\
+		scanline_func (dst, mask,							\
+			       buf1, buf2, left_tz, weight1, weight2,				\
+			       pixman_fixed_frac (vx), unit_x, 0, FALSE);			\
+		dst += left_tz;									\
+		if (flags & FLAG_HAVE_NON_SOLID_MASK)						\
+		    mask += left_tz;								\
+		vx += left_tz * unit_x;								\
+	    }											\
+	    if (width > 0)									\
+	    {											\
+		scanline_func (dst, mask,							\
+			       src1, src2, width, weight1, weight2, vx, unit_x, 0, FALSE);	\
+		dst += width;									\
+		if (flags & FLAG_HAVE_NON_SOLID_MASK)						\
+		    mask += width;								\
+		vx += width * unit_x;								\
+	    }											\
+	    if (right_tz > 0)									\
+	    {											\
+		buf1[0] = src1[src_image->bits.width - 1];					\
+		buf1[1] = 0;									\
+		buf2[0] = src2[src_image->bits.width - 1];					\
+		buf2[1] = 0;									\
+		scanline_func (dst, mask,							\
+			       buf1, buf2, right_tz, weight1, weight2,				\
+			       pixman_fixed_frac (vx), unit_x, 0, FALSE);			\
+		dst += right_tz;								\
+		if (flags & FLAG_HAVE_NON_SOLID_MASK)						\
+		    mask += right_tz;								\
+	    }											\
+	    if (right_pad > 0)									\
+	    {											\
+		buf1[0] = buf1[1] = 0;								\
+		buf2[0] = buf2[1] = 0;								\
+		scanline_func (dst, mask,							\
+			       buf1, buf2, right_pad, weight1, weight2, 0, 0, 0, TRUE);		\
+	    }											\
+	}											\
+	else if (PIXMAN_REPEAT_ ## repeat_mode == PIXMAN_REPEAT_NORMAL)				\
+	{											\
+	    int32_t	    num_pixels;								\
+	    int32_t	    width_remain;							\
+	    src_type_t *    src_line_top;							\
+	    src_type_t *    src_line_bottom;							\
+	    src_type_t	    buf1[2];								\
+	    src_type_t	    buf2[2];								\
+	    src_type_t	    extended_src_line0[REPEAT_NORMAL_MIN_WIDTH*2];			\
+	    src_type_t	    extended_src_line1[REPEAT_NORMAL_MIN_WIDTH*2];			\
+	    int		    i, j;								\
+												\
+	    repeat (PIXMAN_REPEAT_NORMAL, &y1, src_image->bits.height);				\
+	    repeat (PIXMAN_REPEAT_NORMAL, &y2, src_image->bits.height);				\
+	    src_line_top = src_first_line + src_stride * y1;					\
+	    src_line_bottom = src_first_line + src_stride * y2;					\
+												\
+	    if (need_src_extension)								\
+	    {											\
+		for (i=0; i<src_width;)								\
+		{										\
+		    for (j=0; j<src_image->bits.width; j++, i++)				\
+		    {										\
+			extended_src_line0[i] = src_line_top[j];				\
+			extended_src_line1[i] = src_line_bottom[j];				\
+		    }										\
+		}										\
+												\
+		src_line_top = &extended_src_line0[0];						\
+		src_line_bottom = &extended_src_line1[0];					\
+	    }											\
+												\
+	    /* Top & Bottom wrap around buffer */						\
+	    buf1[0] = src_line_top[src_width - 1];						\
+	    buf1[1] = src_line_top[0];								\
+	    buf2[0] = src_line_bottom[src_width - 1];						\
+	    buf2[1] = src_line_bottom[0];							\
+												\
+	    width_remain = width;								\
+												\
+	    while (width_remain > 0)								\
+	    {											\
+		/* We use src_width_fixed because it can make vx in original source range */	\
+		repeat (PIXMAN_REPEAT_NORMAL, &vx, src_width_fixed);				\
+												\
+		/* Wrap around part */								\
+		if (pixman_fixed_to_int (vx) == src_width - 1)					\
+		{										\
+		    /* for positive unit_x							\
+		     * num_pixels = max(n) + 1, where vx + n*unit_x < src_width_fixed		\
+		     *										\
+		     * vx is in range [0, src_width_fixed - pixman_fixed_e]			\
+		     * So we are safe from overflow.						\
+		     */										\
+		    num_pixels = ((src_width_fixed - vx - pixman_fixed_e) / unit_x) + 1;	\
+												\
+		    if (num_pixels > width_remain)						\
+			num_pixels = width_remain;						\
+												\
+		    scanline_func (dst, mask, buf1, buf2, num_pixels,				\
+				   weight1, weight2, pixman_fixed_frac(vx),			\
+				   unit_x, src_width_fixed, FALSE);				\
+												\
+		    width_remain -= num_pixels;							\
+		    vx += num_pixels * unit_x;							\
+		    dst += num_pixels;								\
+												\
+		    if (flags & FLAG_HAVE_NON_SOLID_MASK)					\
+			mask += num_pixels;							\
+												\
+		    repeat (PIXMAN_REPEAT_NORMAL, &vx, src_width_fixed);			\
+		}										\
+												\
+		/* Normal scanline composite */							\
+		if (pixman_fixed_to_int (vx) != src_width - 1 && width_remain > 0)		\
+		{										\
+		    /* for positive unit_x							\
+		     * num_pixels = max(n) + 1, where vx + n*unit_x < (src_width_fixed - 1)	\
+		     *										\
+		     * vx is in range [0, src_width_fixed - pixman_fixed_e]			\
+		     * So we are safe from overflow here.					\
+		     */										\
+		    num_pixels = ((src_width_fixed - pixman_fixed_1 - vx - pixman_fixed_e)	\
+				  / unit_x) + 1;						\
+												\
+		    if (num_pixels > width_remain)						\
+			num_pixels = width_remain;						\
+												\
+		    scanline_func (dst, mask, src_line_top, src_line_bottom, num_pixels,	\
+				   weight1, weight2, vx, unit_x, src_width_fixed, FALSE);	\
+												\
+		    width_remain -= num_pixels;							\
+		    vx += num_pixels * unit_x;							\
+		    dst += num_pixels;								\
+												\
+		    if (flags & FLAG_HAVE_NON_SOLID_MASK)					\
+		        mask += num_pixels;							\
+		}										\
+	    }											\
+	}											\
+	else											\
+	{											\
+	    scanline_func (dst, mask, src_first_line + src_stride * y1,				\
+			   src_first_line + src_stride * y2, width,				\
+			   weight1, weight2, vx, unit_x, max_vx, FALSE);			\
+	}											\
+    }												\
+}
+
+/* A workaround for old sun studio, see: https://bugs.freedesktop.org/show_bug.cgi?id=32764 */
+#define FAST_BILINEAR_MAINLOOP_COMMON(scale_func_name, scanline_func, src_type_t, mask_type_t,	\
+				  dst_type_t, repeat_mode, flags)				\
+	FAST_BILINEAR_MAINLOOP_INT(_ ## scale_func_name, scanline_func, src_type_t, mask_type_t,\
+				  dst_type_t, repeat_mode, flags)
+
+#define SCALED_BILINEAR_FLAGS						\
+    (FAST_PATH_SCALE_TRANSFORM	|					\
+     FAST_PATH_NO_ALPHA_MAP	|					\
+     FAST_PATH_BILINEAR_FILTER	|					\
+     FAST_PATH_NO_ACCESSORS	|					\
+     FAST_PATH_NARROW_FORMAT)
+
+#define SIMPLE_BILINEAR_FAST_PATH_PAD(op,s,d,func)			\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	(SCALED_BILINEAR_FLAGS		|				\
+	 FAST_PATH_PAD_REPEAT		|				\
+	 FAST_PATH_X_UNIT_POSITIVE),					\
+	PIXMAN_null, 0,							\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_bilinear_ ## func ## _pad ## _ ## op,	\
+    }
+
+#define SIMPLE_BILINEAR_FAST_PATH_NONE(op,s,d,func)			\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	(SCALED_BILINEAR_FLAGS		|				\
+	 FAST_PATH_NONE_REPEAT		|				\
+	 FAST_PATH_X_UNIT_POSITIVE),					\
+	PIXMAN_null, 0,							\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_bilinear_ ## func ## _none ## _ ## op,	\
+    }
+
+#define SIMPLE_BILINEAR_FAST_PATH_COVER(op,s,d,func)			\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	SCALED_BILINEAR_FLAGS | FAST_PATH_SAMPLES_COVER_CLIP_BILINEAR,	\
+	PIXMAN_null, 0,							\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_bilinear_ ## func ## _cover ## _ ## op,	\
+    }
+
+#define SIMPLE_BILINEAR_FAST_PATH_NORMAL(op,s,d,func)			\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	(SCALED_BILINEAR_FLAGS		|				\
+	 FAST_PATH_NORMAL_REPEAT	|				\
+	 FAST_PATH_X_UNIT_POSITIVE),					\
+	PIXMAN_null, 0,							\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_bilinear_ ## func ## _normal ## _ ## op,	\
+    }
+
+#define SIMPLE_BILINEAR_A8_MASK_FAST_PATH_PAD(op,s,d,func)		\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	(SCALED_BILINEAR_FLAGS		|				\
+	 FAST_PATH_PAD_REPEAT		|				\
+	 FAST_PATH_X_UNIT_POSITIVE),					\
+	PIXMAN_a8, MASK_FLAGS (a8, FAST_PATH_UNIFIED_ALPHA),		\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_bilinear_ ## func ## _pad ## _ ## op,	\
+    }
+
+#define SIMPLE_BILINEAR_A8_MASK_FAST_PATH_NONE(op,s,d,func)		\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	(SCALED_BILINEAR_FLAGS		|				\
+	 FAST_PATH_NONE_REPEAT		|				\
+	 FAST_PATH_X_UNIT_POSITIVE),					\
+	PIXMAN_a8, MASK_FLAGS (a8, FAST_PATH_UNIFIED_ALPHA),		\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_bilinear_ ## func ## _none ## _ ## op,	\
+    }
+
+#define SIMPLE_BILINEAR_A8_MASK_FAST_PATH_COVER(op,s,d,func)		\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	SCALED_BILINEAR_FLAGS | FAST_PATH_SAMPLES_COVER_CLIP_BILINEAR,	\
+	PIXMAN_a8, MASK_FLAGS (a8, FAST_PATH_UNIFIED_ALPHA),		\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_bilinear_ ## func ## _cover ## _ ## op,	\
+    }
+
+#define SIMPLE_BILINEAR_A8_MASK_FAST_PATH_NORMAL(op,s,d,func)		\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	(SCALED_BILINEAR_FLAGS		|				\
+	 FAST_PATH_NORMAL_REPEAT	|				\
+	 FAST_PATH_X_UNIT_POSITIVE),					\
+	PIXMAN_a8, MASK_FLAGS (a8, FAST_PATH_UNIFIED_ALPHA),		\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_bilinear_ ## func ## _normal ## _ ## op,	\
+    }
+
+#define SIMPLE_BILINEAR_SOLID_MASK_FAST_PATH_PAD(op,s,d,func)		\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	(SCALED_BILINEAR_FLAGS		|				\
+	 FAST_PATH_PAD_REPEAT		|				\
+	 FAST_PATH_X_UNIT_POSITIVE),					\
+	PIXMAN_solid, MASK_FLAGS (solid, FAST_PATH_UNIFIED_ALPHA),	\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_bilinear_ ## func ## _pad ## _ ## op,	\
+    }
+
+#define SIMPLE_BILINEAR_SOLID_MASK_FAST_PATH_NONE(op,s,d,func)		\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	(SCALED_BILINEAR_FLAGS		|				\
+	 FAST_PATH_NONE_REPEAT		|				\
+	 FAST_PATH_X_UNIT_POSITIVE),					\
+	PIXMAN_solid, MASK_FLAGS (solid, FAST_PATH_UNIFIED_ALPHA),	\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_bilinear_ ## func ## _none ## _ ## op,	\
+    }
+
+#define SIMPLE_BILINEAR_SOLID_MASK_FAST_PATH_COVER(op,s,d,func)		\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	SCALED_BILINEAR_FLAGS | FAST_PATH_SAMPLES_COVER_CLIP_BILINEAR,	\
+	PIXMAN_solid, MASK_FLAGS (solid, FAST_PATH_UNIFIED_ALPHA),	\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_bilinear_ ## func ## _cover ## _ ## op,	\
+    }
+
+#define SIMPLE_BILINEAR_SOLID_MASK_FAST_PATH_NORMAL(op,s,d,func)	\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	(SCALED_BILINEAR_FLAGS		|				\
+	 FAST_PATH_NORMAL_REPEAT	|				\
+	 FAST_PATH_X_UNIT_POSITIVE),					\
+	PIXMAN_solid, MASK_FLAGS (solid, FAST_PATH_UNIFIED_ALPHA),	\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_bilinear_ ## func ## _normal ## _ ## op,	\
+    }
+
+/* Prefer the use of 'cover' variant, because it is faster */
+#define SIMPLE_BILINEAR_FAST_PATH(op,s,d,func)				\
+    SIMPLE_BILINEAR_FAST_PATH_COVER (op,s,d,func),			\
+    SIMPLE_BILINEAR_FAST_PATH_NONE (op,s,d,func),			\
+    SIMPLE_BILINEAR_FAST_PATH_PAD (op,s,d,func),			\
+    SIMPLE_BILINEAR_FAST_PATH_NORMAL (op,s,d,func)
+
+#define SIMPLE_BILINEAR_A8_MASK_FAST_PATH(op,s,d,func)			\
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH_COVER (op,s,d,func),		\
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH_NONE (op,s,d,func),		\
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH_PAD (op,s,d,func),		\
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH_NORMAL (op,s,d,func)
+
+#define SIMPLE_BILINEAR_SOLID_MASK_FAST_PATH(op,s,d,func)		\
+    SIMPLE_BILINEAR_SOLID_MASK_FAST_PATH_COVER (op,s,d,func),		\
+    SIMPLE_BILINEAR_SOLID_MASK_FAST_PATH_NONE (op,s,d,func),		\
+    SIMPLE_BILINEAR_SOLID_MASK_FAST_PATH_PAD (op,s,d,func),		\
+    SIMPLE_BILINEAR_SOLID_MASK_FAST_PATH_NORMAL (op,s,d,func)
+
+#endif
diff --git a/pixman/pixman/pixman-linear-gradient.c b/pixman/pixman/pixman-linear-gradient.c
index d9409fe50..40c8c9f37 100644
--- a/pixman/pixman/pixman-linear-gradient.c
+++ b/pixman/pixman/pixman-linear-gradient.c
@@ -1,3 +1,4 @@
+/* -*- Mode: c; c-basic-offset: 4; tab-width: 8; indent-tabs-mode: t; -*- */
 /*
  * Copyright © 2000 SuSE, Inc.
  * Copyright © 2007 Red Hat, Inc.
@@ -30,100 +31,96 @@
 #include <stdlib.h>
 #include "pixman-private.h"
 
-static source_image_class_t
-linear_gradient_classify (pixman_image_t *image,
-                          int             x,
-                          int             y,
-                          int             width,
-                          int             height)
+static pixman_bool_t
+linear_gradient_is_horizontal (pixman_image_t *image,
+			       int             x,
+			       int             y,
+			       int             width,
+			       int             height)
 {
     linear_gradient_t *linear = (linear_gradient_t *)image;
     pixman_vector_t v;
     pixman_fixed_32_32_t l;
-    pixman_fixed_48_16_t dx, dy, a, b, off;
-    pixman_fixed_48_16_t factors[4];
-    int i;
+    pixman_fixed_48_16_t dx, dy;
+    double inc;
 
-    image->source.class = SOURCE_IMAGE_CLASS_UNKNOWN;
-
-    dx = linear->p2.x - linear->p1.x;
-    dy = linear->p2.y - linear->p1.y;
-
-    l = dx * dx + dy * dy;
-
-    if (l)
+    if (image->common.transform)
     {
-	a = (dx << 32) / l;
-	b = (dy << 32) / l;
+	/* projective transformation */
+	if (image->common.transform->matrix[2][0] != 0 ||
+	    image->common.transform->matrix[2][1] != 0 ||
+	    image->common.transform->matrix[2][2] == 0)
+	{
+	    return FALSE;
+	}
+
+	v.vector[0] = image->common.transform->matrix[0][1];
+	v.vector[1] = image->common.transform->matrix[1][1];
+	v.vector[2] = image->common.transform->matrix[2][2];
     }
     else
     {
-	a = b = 0;
+	v.vector[0] = 0;
+	v.vector[1] = pixman_fixed_1;
+	v.vector[2] = pixman_fixed_1;
     }
 
-    off = (-a * linear->p1.x
-           -b * linear->p1.y) >> 16;
-
-    for (i = 0; i < 3; i++)
-    {
-	v.vector[0] = pixman_int_to_fixed ((i % 2) * (width  - 1) + x);
-	v.vector[1] = pixman_int_to_fixed ((i / 2) * (height - 1) + y);
-	v.vector[2] = pixman_fixed_1;
+    dx = linear->p2.x - linear->p1.x;
+    dy = linear->p2.y - linear->p1.y;
 
-	if (image->common.transform)
-	{
-	    if (!pixman_transform_point_3d (image->common.transform, &v))
-	    {
-		image->source.class = SOURCE_IMAGE_CLASS_UNKNOWN;
+    l = dx * dx + dy * dy;
 
-		return image->source.class;
-	    }
-	}
+    if (l == 0)
+	return FALSE;
 
-	factors[i] = ((a * v.vector[0] + b * v.vector[1]) >> 16) + off;
-    }
+    /*
+     * compute how much the input of the gradient walked changes
+     * when moving vertically through the whole image
+     */
+    inc = height * (double) pixman_fixed_1 * pixman_fixed_1 *
+	(dx * v.vector[0] + dy * v.vector[1]) /
+	(v.vector[2] * (double) l);
 
-    if (factors[2] == factors[0])
-	image->source.class = SOURCE_IMAGE_CLASS_HORIZONTAL;
-    else if (factors[1] == factors[0])
-	image->source.class = SOURCE_IMAGE_CLASS_VERTICAL;
+    /* check that casting to integer would result in 0 */
+    if (-1 < inc && inc < 1)
+	return TRUE;
 
-    return image->source.class;
+    return FALSE;
 }
 
-static void
-linear_gradient_get_scanline_32 (pixman_image_t *image,
-                                 int             x,
-                                 int             y,
-                                 int             width,
-                                 uint32_t *      buffer,
-                                 const uint32_t *mask,
-                                 uint32_t        mask_bits)
+static uint32_t *
+linear_get_scanline_narrow (pixman_iter_t  *iter,
+			    const uint32_t *mask)
 {
+    pixman_image_t *image  = iter->image;
+    int             x      = iter->x;
+    int             y      = iter->y;
+    int             width  = iter->width;
+    uint32_t *      buffer = iter->buffer;
+
     pixman_vector_t v, unit;
     pixman_fixed_32_32_t l;
-    pixman_fixed_48_16_t dx, dy, a, b, off;
+    pixman_fixed_48_16_t dx, dy;
     gradient_t *gradient = (gradient_t *)image;
-    source_image_t *source = (source_image_t *)image;
     linear_gradient_t *linear = (linear_gradient_t *)image;
     uint32_t *end = buffer + width;
     pixman_gradient_walker_t walker;
 
-    _pixman_gradient_walker_init (&walker, gradient, source->common.repeat);
+    _pixman_gradient_walker_init (&walker, gradient, image->common.repeat);
 
     /* reference point is the center of the pixel */
     v.vector[0] = pixman_int_to_fixed (x) + pixman_fixed_1 / 2;
     v.vector[1] = pixman_int_to_fixed (y) + pixman_fixed_1 / 2;
     v.vector[2] = pixman_fixed_1;
 
-    if (source->common.transform)
+    if (image->common.transform)
     {
-	if (!pixman_transform_point_3d (source->common.transform, &v))
-	    return;
-	
-	unit.vector[0] = source->common.transform->matrix[0][0];
-	unit.vector[1] = source->common.transform->matrix[1][0];
-	unit.vector[2] = source->common.transform->matrix[2][0];
+	if (!pixman_transform_point_3d (image->common.transform, &v))
+	    return iter->buffer;
+
+	unit.vector[0] = image->common.transform->matrix[0][0];
+	unit.vector[1] = image->common.transform->matrix[1][0];
+	unit.vector[2] = image->common.transform->matrix[2][0];
     }
     else
     {
@@ -137,31 +134,31 @@ linear_gradient_get_scanline_32 (pixman_image_t *image,
 
     l = dx * dx + dy * dy;
 
-    if (l != 0)
-    {
-	a = (dx << 32) / l;
-	b = (dy << 32) / l;
-	off = (-a * linear->p1.x
-	       -b * linear->p1.y) >> 16;
-    }
-
-    if (l == 0 || (unit.vector[2] == 0 && v.vector[2] == pixman_fixed_1))
+    if (l == 0 || unit.vector[2] == 0)
     {
-	pixman_fixed_48_16_t inc, t;
-
 	/* affine transformation only */
-	if (l == 0)
+        pixman_fixed_32_32_t t, next_inc;
+	double inc;
+
+	if (l == 0 || v.vector[2] == 0)
 	{
 	    t = 0;
 	    inc = 0;
 	}
 	else
 	{
-	    t = ((a * v.vector[0] + b * v.vector[1]) >> 16) + off;
-	    inc = (a * unit.vector[0] + b * unit.vector[1]) >> 16;
+	    double invden, v2;
+
+	    invden = pixman_fixed_1 * (double) pixman_fixed_1 /
+		(l * (double) v.vector[2]);
+	    v2 = v.vector[2] * (1. / pixman_fixed_1);
+	    t = ((dx * v.vector[0] + dy * v.vector[1]) - 
+		 (dx * linear->p1.x + dy * linear->p1.y) * v2) * invden;
+	    inc = (dx * unit.vector[0] + dy * unit.vector[1]) * invden;
 	}
+	next_inc = 0;
 
-	if (source->class == SOURCE_IMAGE_CLASS_VERTICAL)
+	if (((pixman_fixed_32_32_t )(inc * width)) == 0)
 	{
 	    register uint32_t color;
 
@@ -171,103 +168,102 @@ linear_gradient_get_scanline_32 (pixman_image_t *image,
 	}
 	else
 	{
-	    if (!mask)
-	    {
-		while (buffer < end)
-		{
-		    *buffer++ = _pixman_gradient_walker_pixel (&walker, t);
-		    
-		    t += inc;
-		}
-	    }
-	    else
+	    int i;
+
+	    i = 0;
+	    while (buffer < end)
 	    {
-		while (buffer < end)
+		if (!mask || *mask++)
 		{
-		    if (*mask++ & mask_bits)
-			*buffer = _pixman_gradient_walker_pixel (&walker, t);
-
-		    buffer++;
-		    t += inc;
+		    *buffer = _pixman_gradient_walker_pixel (&walker,
+							     t + next_inc);
 		}
+		i++;
+		next_inc = inc * i;
+		buffer++;
 	    }
 	}
     }
     else
     {
 	/* projective transformation */
-	pixman_fixed_48_16_t t;
+        double t;
 
-	if (source->class == SOURCE_IMAGE_CLASS_VERTICAL)
-	{
-	    register uint32_t color;
-
-	    if (v.vector[2] == 0)
-	    {
-		t = 0;
-	    }
-	    else
-	    {
-		pixman_fixed_48_16_t x, y;
-
-		x = ((pixman_fixed_48_16_t) v.vector[0] << 16) / v.vector[2];
-		y = ((pixman_fixed_48_16_t) v.vector[1] << 16) / v.vector[2];
-		t = ((a * x + b * y) >> 16) + off;
-	    }
+	t = 0;
 
-	    color = _pixman_gradient_walker_pixel (&walker, t);
-	    while (buffer < end)
-		*buffer++ = color;
-	}
-	else
+	while (buffer < end)
 	{
-	    while (buffer < end)
+	    if (!mask || *mask++)
 	    {
-		if (!mask || *mask++ & mask_bits)
+	        if (v.vector[2] != 0)
 		{
-		    if (v.vector[2] == 0)
-		    {
-			t = 0;
-		    }
-		    else
-		    {
-			pixman_fixed_48_16_t x, y;
-			x = ((pixman_fixed_48_16_t)v.vector[0] << 16) / v.vector[2];
-			y = ((pixman_fixed_48_16_t)v.vector[1] << 16) / v.vector[2];
-			t = ((a * x + b * y) >> 16) + off;
-		    }
-		    
-		    *buffer = _pixman_gradient_walker_pixel (&walker, t);
+		    double invden, v2;
+
+		    invden = pixman_fixed_1 * (double) pixman_fixed_1 /
+			(l * (double) v.vector[2]);
+		    v2 = v.vector[2] * (1. / pixman_fixed_1);
+		    t = ((dx * v.vector[0] + dy * v.vector[1]) - 
+			 (dx * linear->p1.x + dy * linear->p1.y) * v2) * invden;
 		}
-		
-		++buffer;
-		
-		v.vector[0] += unit.vector[0];
-		v.vector[1] += unit.vector[1];
-		v.vector[2] += unit.vector[2];
+
+		*buffer = _pixman_gradient_walker_pixel (&walker, t);
 	    }
+
+	    ++buffer;
+
+	    v.vector[0] += unit.vector[0];
+	    v.vector[1] += unit.vector[1];
+	    v.vector[2] += unit.vector[2];
 	}
     }
+
+    iter->y++;
+
+    return iter->buffer;
 }
 
-static void
-linear_gradient_property_changed (pixman_image_t *image)
+static uint32_t *
+linear_get_scanline_wide (pixman_iter_t *iter, const uint32_t *mask)
 {
-    image->common.get_scanline_32 = linear_gradient_get_scanline_32;
-    image->common.get_scanline_64 = _pixman_image_get_scanline_generic_64;
+    uint32_t *buffer = linear_get_scanline_narrow (iter, NULL);
+
+    pixman_expand_to_float (
+	(argb_t *)buffer, buffer, PIXMAN_a8r8g8b8, iter->width);
+
+    return buffer;
+}
+
+void
+_pixman_linear_gradient_iter_init (pixman_image_t *image, pixman_iter_t  *iter)
+{
+    if (linear_gradient_is_horizontal (
+	    iter->image, iter->x, iter->y, iter->width, iter->height))
+    {
+	if (iter->iter_flags & ITER_NARROW)
+	    linear_get_scanline_narrow (iter, NULL);
+	else
+	    linear_get_scanline_wide (iter, NULL);
+
+	iter->get_scanline = _pixman_iter_get_scanline_noop;
+    }
+    else
+    {
+	if (iter->iter_flags & ITER_NARROW)
+	    iter->get_scanline = linear_get_scanline_narrow;
+	else
+	    iter->get_scanline = linear_get_scanline_wide;
+    }
 }
 
 PIXMAN_EXPORT pixman_image_t *
-pixman_image_create_linear_gradient (pixman_point_fixed_t *        p1,
-                                     pixman_point_fixed_t *        p2,
+pixman_image_create_linear_gradient (const pixman_point_fixed_t *  p1,
+                                     const pixman_point_fixed_t *  p2,
                                      const pixman_gradient_stop_t *stops,
                                      int                           n_stops)
 {
     pixman_image_t *image;
     linear_gradient_t *linear;
 
-    return_val_if_fail (n_stops >= 2, NULL);
-
     image = _pixman_image_allocate ();
 
     if (!image)
@@ -285,9 +281,6 @@ pixman_image_create_linear_gradient (pixman_point_fixed_t *        p1,
     linear->p2 = *p2;
 
     image->type = LINEAR;
-    image->source.class = SOURCE_IMAGE_CLASS_UNKNOWN;
-    image->common.classify = linear_gradient_classify;
-    image->common.property_changed = linear_gradient_property_changed;
 
     return image;
 }
diff --git a/pixman/pixman/pixman-matrix.c b/pixman/pixman/pixman-matrix.c
index abdfa0525..4032c137a 100644
--- a/pixman/pixman/pixman-matrix.c
+++ b/pixman/pixman/pixman-matrix.c
@@ -25,7 +25,7 @@
  */
 
 #ifdef HAVE_CONFIG_H
-#include "config.h"
+#include <config.h>
 #endif
 
 #include <math.h>
@@ -34,6 +34,338 @@
 
 #define F(x)    pixman_int_to_fixed (x)
 
+static force_inline int
+count_leading_zeros (uint32_t x)
+{
+#ifdef HAVE_BUILTIN_CLZ
+    return __builtin_clz (x);
+#else
+    int n = 0;
+    while (x)
+    {
+        n++;
+        x >>= 1;
+    }
+    return 32 - n;
+#endif
+}
+
+/*
+ * Large signed/unsigned integer division with rounding for the platforms with
+ * only 64-bit integer data type supported (no 128-bit data type).
+ *
+ * Arguments:
+ *     hi, lo - high and low 64-bit parts of the dividend
+ *     div    - 48-bit divisor
+ *
+ * Returns: lowest 64 bits of the result as a return value and highest 64
+ *          bits of the result to "result_hi" pointer
+ */
+
+/* grade-school unsigned division (128-bit by 48-bit) with rounding to nearest */
+static force_inline uint64_t
+rounded_udiv_128_by_48 (uint64_t  hi,
+                        uint64_t  lo,
+                        uint64_t  div,
+                        uint64_t *result_hi)
+{
+    uint64_t tmp, remainder, result_lo;
+    assert(div < ((uint64_t)1 << 48));
+
+    remainder = hi % div;
+    *result_hi = hi / div;
+
+    tmp = (remainder << 16) + (lo >> 48);
+    result_lo = tmp / div;
+    remainder = tmp % div;
+
+    tmp = (remainder << 16) + ((lo >> 32) & 0xFFFF);
+    result_lo = (result_lo << 16) + (tmp / div);
+    remainder = tmp % div;
+
+    tmp = (remainder << 16) + ((lo >> 16) & 0xFFFF);
+    result_lo = (result_lo << 16) + (tmp / div);
+    remainder = tmp % div;
+
+    tmp = (remainder << 16) + (lo & 0xFFFF);
+    result_lo = (result_lo << 16) + (tmp / div);
+    remainder = tmp % div;
+
+    /* round to nearest */
+    if (remainder * 2 >= div && ++result_lo == 0)
+        *result_hi += 1;
+
+    return result_lo;
+}
+
+/* signed division (128-bit by 49-bit) with rounding to nearest */
+static inline int64_t
+rounded_sdiv_128_by_49 (int64_t   hi,
+                        uint64_t  lo,
+                        int64_t   div,
+                        int64_t  *signed_result_hi)
+{
+    uint64_t result_lo, result_hi;
+    int sign = 0;
+    if (div < 0)
+    {
+        div = -div;
+        sign ^= 1;
+    }
+    if (hi < 0)
+    {
+        if (lo != 0)
+            hi++;
+        hi = -hi;
+        lo = -lo;
+        sign ^= 1;
+    }
+    result_lo = rounded_udiv_128_by_48 (hi, lo, div, &result_hi);
+    if (sign)
+    {
+        if (result_lo != 0)
+            result_hi++;
+        result_hi = -result_hi;
+        result_lo = -result_lo;
+    }
+    if (signed_result_hi)
+    {
+        *signed_result_hi = result_hi;
+    }
+    return result_lo;
+}
+
+/*
+ * Multiply 64.16 fixed point value by (2^scalebits) and convert
+ * to 128-bit integer.
+ */
+static force_inline void
+fixed_64_16_to_int128 (int64_t  hi,
+                       int64_t  lo,
+                       int64_t *rhi,
+                       int64_t *rlo,
+                       int      scalebits)
+{
+    /* separate integer and fractional parts */
+    hi += lo >> 16;
+    lo &= 0xFFFF;
+
+    if (scalebits <= 0)
+    {
+        *rlo = hi >> (-scalebits);
+        *rhi = *rlo >> 63;
+    }
+    else
+    {
+        *rhi = hi >> (64 - scalebits);
+        *rlo = (uint64_t)hi << scalebits;
+        if (scalebits < 16)
+            *rlo += lo >> (16 - scalebits);
+        else
+            *rlo += lo << (scalebits - 16);
+    }
+}
+
+/*
+ * Convert 112.16 fixed point value to 48.16 with clamping for the out
+ * of range values.
+ */
+static force_inline pixman_fixed_48_16_t
+fixed_112_16_to_fixed_48_16 (int64_t hi, int64_t lo, pixman_bool_t *clampflag)
+{
+    if ((lo >> 63) != hi)
+    {
+        *clampflag = TRUE;
+        return hi >= 0 ? INT64_MAX : INT64_MIN;
+    }
+    else
+    {
+        return lo;
+    }
+}
+
+/*
+ * Transform a point with 31.16 fixed point coordinates from the destination
+ * space to a point with 48.16 fixed point coordinates in the source space.
+ * No overflows are possible for affine transformations and the results are
+ * accurate including the least significant bit. Projective transformations
+ * may overflow, in this case the results are just clamped to return maximum
+ * or minimum 48.16 values (so that the caller can at least handle the NONE
+ * and PAD repeats correctly) and the return value is FALSE to indicate that
+ * such clamping has happened.
+ */
+PIXMAN_EXPORT pixman_bool_t
+pixman_transform_point_31_16 (const pixman_transform_t    *t,
+                              const pixman_vector_48_16_t *v,
+                              pixman_vector_48_16_t       *result)
+{
+    pixman_bool_t clampflag = FALSE;
+    int i;
+    int64_t tmp[3][2], divint;
+    uint16_t divfrac;
+
+    /* input vector values must have no more than 31 bits (including sign)
+     * in the integer part */
+    assert (v->v[0] <   ((pixman_fixed_48_16_t)1 << (30 + 16)));
+    assert (v->v[0] >= -((pixman_fixed_48_16_t)1 << (30 + 16)));
+    assert (v->v[1] <   ((pixman_fixed_48_16_t)1 << (30 + 16)));
+    assert (v->v[1] >= -((pixman_fixed_48_16_t)1 << (30 + 16)));
+    assert (v->v[2] <   ((pixman_fixed_48_16_t)1 << (30 + 16)));
+    assert (v->v[2] >= -((pixman_fixed_48_16_t)1 << (30 + 16)));
+
+    for (i = 0; i < 3; i++)
+    {
+        tmp[i][0] = (int64_t)t->matrix[i][0] * (v->v[0] >> 16);
+        tmp[i][1] = (int64_t)t->matrix[i][0] * (v->v[0] & 0xFFFF);
+        tmp[i][0] += (int64_t)t->matrix[i][1] * (v->v[1] >> 16);
+        tmp[i][1] += (int64_t)t->matrix[i][1] * (v->v[1] & 0xFFFF);
+        tmp[i][0] += (int64_t)t->matrix[i][2] * (v->v[2] >> 16);
+        tmp[i][1] += (int64_t)t->matrix[i][2] * (v->v[2] & 0xFFFF);
+    }
+
+    /*
+     * separate 64-bit integer and 16-bit fractional parts for the divisor,
+     * which is also scaled by 65536 after fixed point multiplication.
+     */
+    divint  = tmp[2][0] + (tmp[2][1] >> 16);
+    divfrac = tmp[2][1] & 0xFFFF;
+
+    if (divint == pixman_fixed_1 && divfrac == 0)
+    {
+        /*
+         * this is a simple affine transformation
+         */
+        result->v[0] = tmp[0][0] + ((tmp[0][1] + 0x8000) >> 16);
+        result->v[1] = tmp[1][0] + ((tmp[1][1] + 0x8000) >> 16);
+        result->v[2] = pixman_fixed_1;
+    }
+    else if (divint == 0 && divfrac == 0)
+    {
+        /*
+         * handle zero divisor (if the values are non-zero, set the
+         * results to maximum positive or minimum negative)
+         */
+        clampflag = TRUE;
+
+        result->v[0] = tmp[0][0] + ((tmp[0][1] + 0x8000) >> 16);
+        result->v[1] = tmp[1][0] + ((tmp[1][1] + 0x8000) >> 16);
+
+        if (result->v[0] > 0)
+            result->v[0] = INT64_MAX;
+        else if (result->v[0] < 0)
+            result->v[0] = INT64_MIN;
+
+        if (result->v[1] > 0)
+            result->v[1] = INT64_MAX;
+        else if (result->v[1] < 0)
+            result->v[1] = INT64_MIN;
+    }
+    else
+    {
+        /*
+         * projective transformation, analyze the top 32 bits of the divisor
+         */
+        int32_t hi32divbits = divint >> 32;
+        if (hi32divbits < 0)
+            hi32divbits = ~hi32divbits;
+
+        if (hi32divbits == 0)
+        {
+            /* the divisor is small, we can actually keep all the bits */
+            int64_t hi, rhi, lo, rlo;
+            int64_t div = (divint << 16) + divfrac;
+
+            fixed_64_16_to_int128 (tmp[0][0], tmp[0][1], &hi, &lo, 32);
+            rlo = rounded_sdiv_128_by_49 (hi, lo, div, &rhi);
+            result->v[0] = fixed_112_16_to_fixed_48_16 (rhi, rlo, &clampflag);
+
+            fixed_64_16_to_int128 (tmp[1][0], tmp[1][1], &hi, &lo, 32);
+            rlo = rounded_sdiv_128_by_49 (hi, lo, div, &rhi);
+            result->v[1] = fixed_112_16_to_fixed_48_16 (rhi, rlo, &clampflag);
+        }
+        else
+        {
+            /* the divisor needs to be reduced to 48 bits */
+            int64_t hi, rhi, lo, rlo, div;
+            int shift = 32 - count_leading_zeros (hi32divbits);
+            fixed_64_16_to_int128 (divint, divfrac, &hi, &div, 16 - shift);
+
+            fixed_64_16_to_int128 (tmp[0][0], tmp[0][1], &hi, &lo, 32 - shift);
+            rlo = rounded_sdiv_128_by_49 (hi, lo, div, &rhi);
+            result->v[0] = fixed_112_16_to_fixed_48_16 (rhi, rlo, &clampflag);
+
+            fixed_64_16_to_int128 (tmp[1][0], tmp[1][1], &hi, &lo, 32 - shift);
+            rlo = rounded_sdiv_128_by_49 (hi, lo, div, &rhi);
+            result->v[1] = fixed_112_16_to_fixed_48_16 (rhi, rlo, &clampflag);
+        }
+    }
+    result->v[2] = pixman_fixed_1;
+    return !clampflag;
+}
+
+PIXMAN_EXPORT void
+pixman_transform_point_31_16_affine (const pixman_transform_t    *t,
+                                     const pixman_vector_48_16_t *v,
+                                     pixman_vector_48_16_t       *result)
+{
+    int64_t hi0, lo0, hi1, lo1;
+
+    /* input vector values must have no more than 31 bits (including sign)
+     * in the integer part */
+    assert (v->v[0] <   ((pixman_fixed_48_16_t)1 << (30 + 16)));
+    assert (v->v[0] >= -((pixman_fixed_48_16_t)1 << (30 + 16)));
+    assert (v->v[1] <   ((pixman_fixed_48_16_t)1 << (30 + 16)));
+    assert (v->v[1] >= -((pixman_fixed_48_16_t)1 << (30 + 16)));
+
+    hi0  = (int64_t)t->matrix[0][0] * (v->v[0] >> 16);
+    lo0  = (int64_t)t->matrix[0][0] * (v->v[0] & 0xFFFF);
+    hi0 += (int64_t)t->matrix[0][1] * (v->v[1] >> 16);
+    lo0 += (int64_t)t->matrix[0][1] * (v->v[1] & 0xFFFF);
+    hi0 += (int64_t)t->matrix[0][2];
+
+    hi1  = (int64_t)t->matrix[1][0] * (v->v[0] >> 16);
+    lo1  = (int64_t)t->matrix[1][0] * (v->v[0] & 0xFFFF);
+    hi1 += (int64_t)t->matrix[1][1] * (v->v[1] >> 16);
+    lo1 += (int64_t)t->matrix[1][1] * (v->v[1] & 0xFFFF);
+    hi1 += (int64_t)t->matrix[1][2];
+
+    result->v[0] = hi0 + ((lo0 + 0x8000) >> 16);
+    result->v[1] = hi1 + ((lo1 + 0x8000) >> 16);
+    result->v[2] = pixman_fixed_1;
+}
+
+PIXMAN_EXPORT void
+pixman_transform_point_31_16_3d (const pixman_transform_t    *t,
+                                 const pixman_vector_48_16_t *v,
+                                 pixman_vector_48_16_t       *result)
+{
+    int i;
+    int64_t tmp[3][2];
+
+    /* input vector values must have no more than 31 bits (including sign)
+     * in the integer part */
+    assert (v->v[0] <   ((pixman_fixed_48_16_t)1 << (30 + 16)));
+    assert (v->v[0] >= -((pixman_fixed_48_16_t)1 << (30 + 16)));
+    assert (v->v[1] <   ((pixman_fixed_48_16_t)1 << (30 + 16)));
+    assert (v->v[1] >= -((pixman_fixed_48_16_t)1 << (30 + 16)));
+    assert (v->v[2] <   ((pixman_fixed_48_16_t)1 << (30 + 16)));
+    assert (v->v[2] >= -((pixman_fixed_48_16_t)1 << (30 + 16)));
+
+    for (i = 0; i < 3; i++)
+    {
+        tmp[i][0] = (int64_t)t->matrix[i][0] * (v->v[0] >> 16);
+        tmp[i][1] = (int64_t)t->matrix[i][0] * (v->v[0] & 0xFFFF);
+        tmp[i][0] += (int64_t)t->matrix[i][1] * (v->v[1] >> 16);
+        tmp[i][1] += (int64_t)t->matrix[i][1] * (v->v[1] & 0xFFFF);
+        tmp[i][0] += (int64_t)t->matrix[i][2] * (v->v[2] >> 16);
+        tmp[i][1] += (int64_t)t->matrix[i][2] * (v->v[2] & 0xFFFF);
+    }
+
+    result->v[0] = tmp[0][0] + ((tmp[0][1] + 0x8000) >> 16);
+    result->v[1] = tmp[1][0] + ((tmp[1][1] + 0x8000) >> 16);
+    result->v[2] = tmp[2][0] + ((tmp[2][1] + 0x8000) >> 16);
+}
+
 PIXMAN_EXPORT void
 pixman_transform_init_identity (struct pixman_transform *matrix)
 {
@@ -50,69 +382,41 @@ PIXMAN_EXPORT pixman_bool_t
 pixman_transform_point_3d (const struct pixman_transform *transform,
                            struct pixman_vector *         vector)
 {
-    struct pixman_vector result;
-    pixman_fixed_32_32_t partial;
-    pixman_fixed_48_16_t v;
-    int i, j;
+    pixman_vector_48_16_t tmp;
+    tmp.v[0] = vector->vector[0];
+    tmp.v[1] = vector->vector[1];
+    tmp.v[2] = vector->vector[2];
 
-    for (j = 0; j < 3; j++)
-    {
-	v = 0;
-	for (i = 0; i < 3; i++)
-	{
-	    partial = ((pixman_fixed_48_16_t) transform->matrix[j][i] *
-	               (pixman_fixed_48_16_t) vector->vector[i]);
-	    v += partial >> 16;
-	}
-	
-	if (v > pixman_max_fixed_48_16 || v < pixman_min_fixed_48_16)
-	    return FALSE;
-	
-	result.vector[j] = (pixman_fixed_t) v;
-    }
-    
-    *vector = result;
+    pixman_transform_point_31_16_3d (transform, &tmp, &tmp);
 
-    if (!result.vector[2])
-	return FALSE;
+    vector->vector[0] = tmp.v[0];
+    vector->vector[1] = tmp.v[1];
+    vector->vector[2] = tmp.v[2];
 
-    return TRUE;
+    return vector->vector[0] == tmp.v[0] &&
+           vector->vector[1] == tmp.v[1] &&
+           vector->vector[2] == tmp.v[2];
 }
 
 PIXMAN_EXPORT pixman_bool_t
 pixman_transform_point (const struct pixman_transform *transform,
                         struct pixman_vector *         vector)
 {
-    pixman_fixed_32_32_t partial;
-    pixman_fixed_34_30_t v[3];
-    pixman_fixed_48_16_t quo;
-    int i, j;
+    pixman_vector_48_16_t tmp;
+    tmp.v[0] = vector->vector[0];
+    tmp.v[1] = vector->vector[1];
+    tmp.v[2] = vector->vector[2];
 
-    for (j = 0; j < 3; j++)
-    {
-	v[j] = 0;
-	
-	for (i = 0; i < 3; i++)
-	{
-	    partial = ((pixman_fixed_32_32_t) transform->matrix[j][i] *
-	               (pixman_fixed_32_32_t) vector->vector[i]);
-	    v[j] += partial >> 2;
-	}
-    }
-    
-    if (!(v[2] >> 16))
-	return FALSE;
+    if (!pixman_transform_point_31_16 (transform, &tmp, &tmp))
+        return FALSE;
 
-    for (j = 0; j < 2; j++)
-    {
-	quo = v[j] / (v[2] >> 16);
-	if (quo > pixman_max_fixed_48_16 || quo < pixman_min_fixed_48_16)
-	    return FALSE;
-	vector->vector[j] = (pixman_fixed_t) quo;
-    }
-    
-    vector->vector[2] = pixman_fixed_1;
-    return TRUE;
+    vector->vector[0] = tmp.v[0];
+    vector->vector[1] = tmp.v[1];
+    vector->vector[2] = tmp.v[2];
+
+    return vector->vector[0] == tmp.v[0] &&
+           vector->vector[1] == tmp.v[1] &&
+           vector->vector[2] == tmp.v[2];
 }
 
 PIXMAN_EXPORT pixman_bool_t
@@ -138,7 +442,7 @@ pixman_transform_multiply (struct pixman_transform *      dst,
 		    (pixman_fixed_32_32_t) l->matrix[dy][o] *
 		    (pixman_fixed_32_32_t) r->matrix[o][dx];
 
-		v += partial >> 16;
+		v += (partial + 0x8000) >> 16;
 	    }
 
 	    if (v > pixman_max_fixed_48_16 || v < pixman_min_fixed_48_16)
@@ -336,14 +640,14 @@ PIXMAN_EXPORT pixman_bool_t
 pixman_transform_invert (struct pixman_transform *      dst,
                          const struct pixman_transform *src)
 {
-    struct pixman_f_transform m, r;
+    struct pixman_f_transform m;
 
     pixman_f_transform_from_pixman_transform (&m, src);
 
-    if (!pixman_f_transform_invert (&r, &m))
+    if (!pixman_f_transform_invert (&m, &m))
 	return FALSE;
 
-    if (!pixman_transform_from_pixman_f_transform (dst, &r))
+    if (!pixman_transform_from_pixman_f_transform (dst, &m))
 	return FALSE;
 
     return TRUE;
@@ -425,7 +729,8 @@ pixman_transform_is_inverse (const struct pixman_transform *a,
 {
     struct pixman_transform t;
 
-    pixman_transform_multiply (&t, a, b);
+    if (!pixman_transform_multiply (&t, a, b))
+	return FALSE;
 
     return pixman_transform_is_identity (&t);
 }
@@ -464,17 +769,15 @@ pixman_transform_from_pixman_f_transform (struct pixman_transform *        t,
     return TRUE;
 }
 
-static const int a[3] = { 3, 3, 2 };
-static const int b[3] = { 2, 1, 1 };
-
 PIXMAN_EXPORT pixman_bool_t
 pixman_f_transform_invert (struct pixman_f_transform *      dst,
                            const struct pixman_f_transform *src)
 {
+    static const int a[3] = { 2, 2, 1 };
+    static const int b[3] = { 1, 0, 0 };
+    pixman_f_transform_t d;
     double det;
     int i, j;
-    static int a[3] = { 2, 2, 1 };
-    static int b[3] = { 1, 0, 0 };
 
     det = 0;
     for (i = 0; i < 3; i++)
@@ -509,10 +812,12 @@ pixman_f_transform_invert (struct pixman_f_transform *      dst,
 	    if (((i + j) & 1) != 0)
 		p = -p;
 	    
-	    dst->m[j][i] = det * p;
+	    d.m[j][i] = det * p;
 	}
     }
 
+    *dst = d;
+
     return TRUE;
 }
 
diff --git a/pixman/pixman/pixman-mips-dspr2-asm.S b/pixman/pixman/pixman-mips-dspr2-asm.S
new file mode 100644
index 000000000..866e93e58
--- /dev/null
+++ b/pixman/pixman/pixman-mips-dspr2-asm.S
@@ -0,0 +1,4283 @@
+/*
+ * Copyright (c) 2012
+ *      MIPS Technologies, Inc., California.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the MIPS Technologies, Inc., nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE MIPS TECHNOLOGIES, INC. ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE MIPS TECHNOLOGIES, INC. BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Author:  Nemanja Lukic (nlukic@mips.com)
+ */
+
+#include "pixman-private.h"
+#include "pixman-mips-dspr2-asm.h"
+
+LEAF_MIPS_DSPR2(pixman_fill_buff16_mips)
+/*
+ * a0 - *dest
+ * a1 - count (bytes)
+ * a2 - value to fill buffer with
+ */
+
+    beqz     a1, 3f
+     andi    t1, a0, 0x0002
+    beqz     t1, 0f          /* check if address is 4-byte aligned */
+     nop
+    sh       a2, 0(a0)
+    addiu    a0, a0, 2
+    addiu    a1, a1, -2
+0:
+    srl      t1, a1, 5       /* t1 how many multiples of 32 bytes */
+    replv.ph a2, a2          /* replicate fill value (16bit) in a2 */
+    beqz     t1, 2f
+     nop
+1:
+    addiu    t1, t1, -1
+    beqz     t1, 11f
+     addiu   a1, a1, -32
+    pref     30, 32(a0)
+    sw       a2, 0(a0)
+    sw       a2, 4(a0)
+    sw       a2, 8(a0)
+    sw       a2, 12(a0)
+    sw       a2, 16(a0)
+    sw       a2, 20(a0)
+    sw       a2, 24(a0)
+    sw       a2, 28(a0)
+    b        1b
+     addiu   a0, a0, 32
+11:
+    sw       a2, 0(a0)
+    sw       a2, 4(a0)
+    sw       a2, 8(a0)
+    sw       a2, 12(a0)
+    sw       a2, 16(a0)
+    sw       a2, 20(a0)
+    sw       a2, 24(a0)
+    sw       a2, 28(a0)
+    addiu    a0, a0, 32
+2:
+    blez     a1, 3f
+     addiu   a1, a1, -2
+    sh       a2, 0(a0)
+    b        2b
+     addiu   a0, a0, 2
+3:
+    jr       ra
+     nop
+
+END(pixman_fill_buff16_mips)
+
+LEAF_MIPS32R2(pixman_fill_buff32_mips)
+/*
+ * a0 - *dest
+ * a1 - count (bytes)
+ * a2 - value to fill buffer with
+ */
+
+    beqz     a1, 3f
+     nop
+    srl      t1, a1, 5 /* t1 how many multiples of 32 bytes */
+    beqz     t1, 2f
+     nop
+1:
+    addiu    t1, t1, -1
+    beqz     t1, 11f
+     addiu   a1, a1, -32
+    pref     30, 32(a0)
+    sw       a2, 0(a0)
+    sw       a2, 4(a0)
+    sw       a2, 8(a0)
+    sw       a2, 12(a0)
+    sw       a2, 16(a0)
+    sw       a2, 20(a0)
+    sw       a2, 24(a0)
+    sw       a2, 28(a0)
+    b        1b
+     addiu   a0, a0, 32
+11:
+    sw       a2, 0(a0)
+    sw       a2, 4(a0)
+    sw       a2, 8(a0)
+    sw       a2, 12(a0)
+    sw       a2, 16(a0)
+    sw       a2, 20(a0)
+    sw       a2, 24(a0)
+    sw       a2, 28(a0)
+    addiu    a0, a0, 32
+2:
+    blez     a1, 3f
+     addiu   a1, a1, -4
+    sw       a2, 0(a0)
+    b        2b
+     addiu   a0, a0, 4
+3:
+    jr       ra
+     nop
+
+END(pixman_fill_buff32_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_src_8888_0565_asm_mips)
+/*
+ * a0 - dst (r5g6b5)
+ * a1 - src (a8r8g8b8)
+ * a2 - w
+ */
+
+    beqz     a2, 3f
+     nop
+    addiu    t1, a2, -1
+    beqz     t1, 2f
+     nop
+    li       t4, 0xf800f800
+    li       t5, 0x07e007e0
+    li       t6, 0x001f001f
+1:
+    lw       t0, 0(a1)
+    lw       t1, 4(a1)
+    addiu    a1, a1, 8
+    addiu    a2, a2, -2
+
+    CONVERT_2x8888_TO_2x0565 t0, t1, t2, t3, t4, t5, t6, t7, t8
+
+    sh       t2, 0(a0)
+    sh       t3, 2(a0)
+
+    addiu    t2, a2, -1
+    bgtz     t2, 1b
+     addiu   a0, a0, 4
+2:
+    beqz     a2, 3f
+     nop
+    lw       t0, 0(a1)
+
+    CONVERT_1x8888_TO_1x0565 t0, t1, t2, t3
+
+    sh       t1, 0(a0)
+3:
+    j        ra
+     nop
+
+END(pixman_composite_src_8888_0565_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_src_0565_8888_asm_mips)
+/*
+ * a0 - dst (a8r8g8b8)
+ * a1 - src (r5g6b5)
+ * a2 - w
+ */
+
+    beqz     a2, 3f
+     nop
+    addiu    t1, a2, -1
+    beqz     t1, 2f
+     nop
+    li       t4, 0x07e007e0
+    li       t5, 0x001F001F
+1:
+    lhu      t0, 0(a1)
+    lhu      t1, 2(a1)
+    addiu    a1, a1, 4
+    addiu    a2, a2, -2
+
+    CONVERT_2x0565_TO_2x8888 t0, t1, t2, t3, t4, t5, t6, t7, t8, t9
+
+    sw       t2, 0(a0)
+    sw       t3, 4(a0)
+
+    addiu    t2, a2, -1
+    bgtz     t2, 1b
+     addiu   a0, a0, 8
+2:
+    beqz     a2, 3f
+     nop
+    lhu      t0, 0(a1)
+
+    CONVERT_1x0565_TO_1x8888 t0, t1, t2, t3
+
+    sw       t1, 0(a0)
+3:
+    j        ra
+     nop
+
+END(pixman_composite_src_0565_8888_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_src_x888_8888_asm_mips)
+/*
+ * a0 - dst (a8r8g8b8)
+ * a1 - src (x8r8g8b8)
+ * a2 - w
+ */
+
+    beqz     a2, 4f
+     nop
+    li       t9, 0xff000000
+    srl      t8, a2, 3    /* t1 = how many multiples of 8 src pixels */
+    beqz     t8, 3f       /* branch if less than 8 src pixels */
+     nop
+1:
+    addiu    t8, t8, -1
+    beqz     t8, 2f
+     addiu   a2, a2, -8
+    pref     0, 32(a1)
+    lw       t0, 0(a1)
+    lw       t1, 4(a1)
+    lw       t2, 8(a1)
+    lw       t3, 12(a1)
+    lw       t4, 16(a1)
+    lw       t5, 20(a1)
+    lw       t6, 24(a1)
+    lw       t7, 28(a1)
+    addiu    a1, a1, 32
+    or       t0, t0, t9
+    or       t1, t1, t9
+    or       t2, t2, t9
+    or       t3, t3, t9
+    or       t4, t4, t9
+    or       t5, t5, t9
+    or       t6, t6, t9
+    or       t7, t7, t9
+    pref     30, 32(a0)
+    sw       t0, 0(a0)
+    sw       t1, 4(a0)
+    sw       t2, 8(a0)
+    sw       t3, 12(a0)
+    sw       t4, 16(a0)
+    sw       t5, 20(a0)
+    sw       t6, 24(a0)
+    sw       t7, 28(a0)
+    b        1b
+     addiu   a0, a0, 32
+2:
+    lw       t0, 0(a1)
+    lw       t1, 4(a1)
+    lw       t2, 8(a1)
+    lw       t3, 12(a1)
+    lw       t4, 16(a1)
+    lw       t5, 20(a1)
+    lw       t6, 24(a1)
+    lw       t7, 28(a1)
+    addiu    a1, a1, 32
+    or       t0, t0, t9
+    or       t1, t1, t9
+    or       t2, t2, t9
+    or       t3, t3, t9
+    or       t4, t4, t9
+    or       t5, t5, t9
+    or       t6, t6, t9
+    or       t7, t7, t9
+    sw       t0, 0(a0)
+    sw       t1, 4(a0)
+    sw       t2, 8(a0)
+    sw       t3, 12(a0)
+    sw       t4, 16(a0)
+    sw       t5, 20(a0)
+    sw       t6, 24(a0)
+    sw       t7, 28(a0)
+    beqz     a2, 4f
+     addiu   a0, a0, 32
+3:
+    lw       t0, 0(a1)
+    addiu    a1, a1, 4
+    addiu    a2, a2, -1
+    or       t1, t0, t9
+    sw       t1, 0(a0)
+    bnez     a2, 3b
+     addiu   a0, a0, 4
+4:
+    jr       ra
+     nop
+
+END(pixman_composite_src_x888_8888_asm_mips)
+
+#if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL)
+LEAF_MIPS_DSPR2(pixman_composite_src_0888_8888_rev_asm_mips)
+/*
+ * a0 - dst (a8r8g8b8)
+ * a1 - src (b8g8r8)
+ * a2 - w
+ */
+
+    beqz              a2, 6f
+     nop
+
+    lui               t8, 0xff00;
+    srl               t9, a2, 2   /* t9 = how many multiples of 4 src pixels */
+    beqz              t9, 4f      /* branch if less than 4 src pixels */
+     nop
+
+    li                t0, 0x1
+    li                t1, 0x2
+    li                t2, 0x3
+    andi              t3, a1, 0x3
+    beq               t3, t0, 1f
+     nop
+    beq               t3, t1, 2f
+     nop
+    beq               t3, t2, 3f
+     nop
+
+0:
+    beqz              t9, 4f
+     addiu            t9, t9, -1
+    lw                t0, 0(a1)            /* t0 = R2 | B1 | G1 | R1 */
+    lw                t1, 4(a1)            /* t1 = G3 | R3 | B2 | G2 */
+    lw                t2, 8(a1)            /* t2 = B4 | G4 | R4 | B3 */
+
+    addiu             a1, a1, 12
+    addiu             a2, a2, -4
+
+    wsbh              t0, t0               /* t0 = B1 | R2 | R1 | G1 */
+    wsbh              t1, t1               /* t1 = R3 | G3 | G2 | B2 */
+    wsbh              t2, t2               /* t2 = G4 | B4 | B3 | R4 */
+
+    packrl.ph         t3, t1, t0           /* t3 = G2 | B2 | B1 | R2 */
+    packrl.ph         t4, t0, t0           /* t4 = R1 | G1 | B1 | R2 */
+    rotr              t3, t3, 16           /* t3 = B1 | R2 | G2 | B2 */
+    or                t3, t3, t8           /* t3 = FF | R2 | G2 | B2 */
+    srl               t4, t4, 8            /* t4 =  0 | R1 | G1 | B1 */
+    or                t4, t4, t8           /* t4 = FF | R1 | G1 | B1 */
+    packrl.ph         t5, t2, t1           /* t5 = B3 | R4 | R3 | G3 */
+    rotr              t5, t5, 24           /* t5 = R4 | R3 | G3 | B3 */
+    or                t5, t5, t8           /* t5 = FF | R3 | G3 | B3 */
+    rotr              t2, t2, 16           /* t2 = B3 | R4 | G4 | B4 */
+    or                t2, t2, t8           /* t5 = FF | R3 | G3 | B3 */
+
+    sw                t4, 0(a0)
+    sw                t3, 4(a0)
+    sw                t5, 8(a0)
+    sw                t2, 12(a0)
+    b                 0b
+     addiu            a0, a0, 16
+
+1:
+    lbu               t6, 0(a1)            /* t6 =  0 |  0 |  0 | R1 */
+    lhu               t7, 1(a1)            /* t7 =  0 |  0 | B1 | G1 */
+    sll               t6, t6, 16           /* t6 =  0 | R1 |  0 | 0  */
+    wsbh              t7, t7               /* t7 =  0 |  0 | G1 | B1 */
+    or                t7, t6, t7           /* t7 =  0 | R1 | G1 | B1 */
+11:
+    beqz              t9, 4f
+     addiu            t9, t9, -1
+    lw                t0, 3(a1)            /* t0 = R3 | B2 | G2 | R2 */
+    lw                t1, 7(a1)            /* t1 = G4 | R4 | B3 | G3 */
+    lw                t2, 11(a1)           /* t2 = B5 | G5 | R5 | B4 */
+
+    addiu             a1, a1, 12
+    addiu             a2, a2, -4
+
+    wsbh              t0, t0               /* t0 = B2 | R3 | R2 | G2 */
+    wsbh              t1, t1               /* t1 = R4 | G4 | G3 | B3 */
+    wsbh              t2, t2               /* t2 = G5 | B5 | B4 | R5 */
+
+    packrl.ph         t3, t1, t0           /* t3 = G3 | B3 | B2 | R3 */
+    packrl.ph         t4, t2, t1           /* t4 = B4 | R5 | R4 | G4 */
+    rotr              t0, t0, 24           /* t0 = R3 | R2 | G2 | B2 */
+    rotr              t3, t3, 16           /* t3 = B2 | R3 | G3 | B3 */
+    rotr              t4, t4, 24           /* t4 = R5 | R4 | G4 | B4 */
+    or                t7, t7, t8           /* t7 = FF | R1 | G1 | B1 */
+    or                t0, t0, t8           /* t0 = FF | R2 | G2 | B2 */
+    or                t3, t3, t8           /* t1 = FF | R3 | G3 | B3 */
+    or                t4, t4, t8           /* t3 = FF | R4 | G4 | B4 */
+
+    sw                t7, 0(a0)
+    sw                t0, 4(a0)
+    sw                t3, 8(a0)
+    sw                t4, 12(a0)
+    rotr              t7, t2, 16           /* t7 = xx | R5 | G5 | B5 */
+    b                 11b
+     addiu            a0, a0, 16
+
+2:
+    lhu               t7, 0(a1)            /* t7 =  0 |  0 | G1 | R1 */
+    wsbh              t7, t7               /* t7 =  0 |  0 | R1 | G1 */
+21:
+    beqz              t9, 4f
+     addiu            t9, t9, -1
+    lw                t0, 2(a1)            /* t0 = B2 | G2 | R2 | B1 */
+    lw                t1, 6(a1)            /* t1 = R4 | B3 | G3 | R3 */
+    lw                t2, 10(a1)           /* t2 = G5 | R5 | B4 | G4 */
+
+    addiu             a1, a1, 12
+    addiu             a2, a2, -4
+
+    wsbh              t0, t0               /* t0 = G2 | B2 | B1 | R2 */
+    wsbh              t1, t1               /* t1 = B3 | R4 | R3 | G3 */
+    wsbh              t2, t2               /* t2 = R5 | G5 | G4 | B4 */
+
+    precr_sra.ph.w    t7, t0, 0            /* t7 = R1 | G1 | B1 | R2 */
+    rotr              t0, t0, 16           /* t0 = B1 | R2 | G2 | B2 */
+    packrl.ph         t3, t2, t1           /* t3 = G4 | B4 | B3 | R4 */
+    rotr              t1, t1, 24           /* t1 = R4 | R3 | G3 | B3 */
+    srl               t7, t7, 8            /* t7 =  0 | R1 | G1 | B1 */
+    rotr              t3, t3, 16           /* t3 = B3 | R4 | G4 | B4 */
+    or                t7, t7, t8           /* t7 = FF | R1 | G1 | B1 */
+    or                t0, t0, t8           /* t0 = FF | R2 | G2 | B2 */
+    or                t1, t1, t8           /* t1 = FF | R3 | G3 | B3 */
+    or                t3, t3, t8           /* t3 = FF | R4 | G4 | B4 */
+
+    sw                t7, 0(a0)
+    sw                t0, 4(a0)
+    sw                t1, 8(a0)
+    sw                t3, 12(a0)
+    srl               t7, t2, 16           /* t7 =  0 |  0 | R5 | G5 */
+    b                 21b
+     addiu            a0, a0, 16
+
+3:
+    lbu               t7, 0(a1)            /* t7 =  0 |  0 |  0 | R1 */
+31:
+    beqz              t9, 4f
+     addiu            t9, t9, -1
+    lw                t0, 1(a1)            /* t0 = G2 | R2 | B1 | G1 */
+    lw                t1, 5(a1)            /* t1 = B3 | G3 | R3 | B2 */
+    lw                t2, 9(a1)            /* t2 = R5 | B4 | G4 | R4 */
+
+    addiu             a1, a1, 12
+    addiu             a2, a2, -4
+
+    wsbh              t0, t0               /* t0 = R2 | G2 | G1 | B1 */
+    wsbh              t1, t1               /* t1 = G3 | B3 | B2 | R3 */
+    wsbh              t2, t2               /* t2 = B4 | R5 | R4 | G4 */
+
+    precr_sra.ph.w    t7, t0, 0            /* t7 = xx | R1 | G1 | B1 */
+    packrl.ph         t3, t1, t0           /* t3 = B2 | R3 | R2 | G2 */
+    rotr              t1, t1, 16           /* t1 = B2 | R3 | G3 | B3 */
+    rotr              t4, t2, 24           /* t4 = R5 | R4 | G4 | B4 */
+    rotr              t3, t3, 24           /* t3 = R3 | R2 | G2 | B2 */
+    or                t7, t7, t8           /* t7 = FF | R1 | G1 | B1 */
+    or                t3, t3, t8           /* t3 = FF | R2 | G2 | B2 */
+    or                t1, t1, t8           /* t1 = FF | R3 | G3 | B3 */
+    or                t4, t4, t8           /* t4 = FF | R4 | G4 | B4 */
+
+    sw                t7, 0(a0)
+    sw                t3, 4(a0)
+    sw                t1, 8(a0)
+    sw                t4, 12(a0)
+    srl               t7, t2, 16           /* t7 =  0 |  0 | xx | R5 */
+    b                 31b
+     addiu            a0, a0, 16
+
+4:
+    beqz              a2, 6f
+     nop
+5:
+    lbu               t0, 0(a1)            /* t0 =  0 | 0 | 0 | R */
+    lbu               t1, 1(a1)            /* t1 =  0 | 0 | 0 | G */
+    lbu               t2, 2(a1)            /* t2 =  0 | 0 | 0 | B */
+    addiu             a1, a1, 3
+
+    sll               t0, t0, 16           /* t2 =  0 | R | 0 | 0 */
+    sll               t1, t1, 8            /* t1 =  0 | 0 | G | 0 */
+
+    or                t2, t2, t1           /* t2 =  0 | 0 | G | B */
+    or                t2, t2, t0           /* t2 =  0 | R | G | B */
+    or                t2, t2, t8           /* t2 = FF | R | G | B */
+
+    sw                t2, 0(a0)
+    addiu             a2, a2, -1
+    bnez              a2, 5b
+     addiu            a0, a0, 4
+6:
+    j                 ra
+     nop
+
+END(pixman_composite_src_0888_8888_rev_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_src_0888_0565_rev_asm_mips)
+/*
+ * a0 - dst (r5g6b5)
+ * a1 - src (b8g8r8)
+ * a2 - w
+ */
+
+    SAVE_REGS_ON_STACK 0, v0, v1
+    beqz              a2, 6f
+     nop
+
+    li                t6, 0xf800f800
+    li                t7, 0x07e007e0
+    li                t8, 0x001F001F
+    srl               t9, a2, 2   /* t9 = how many multiples of 4 src pixels */
+    beqz              t9, 4f      /* branch if less than 4 src pixels */
+     nop
+
+    li                t0, 0x1
+    li                t1, 0x2
+    li                t2, 0x3
+    andi              t3, a1, 0x3
+    beq               t3, t0, 1f
+     nop
+    beq               t3, t1, 2f
+     nop
+    beq               t3, t2, 3f
+     nop
+
+0:
+    beqz              t9, 4f
+     addiu            t9, t9, -1
+    lw                t0, 0(a1)            /* t0 = R2 | B1 | G1 | R1 */
+    lw                t1, 4(a1)            /* t1 = G3 | R3 | B2 | G2 */
+    lw                t2, 8(a1)            /* t2 = B4 | G4 | R4 | B3 */
+
+    addiu             a1, a1, 12
+    addiu             a2, a2, -4
+
+    wsbh              t0, t0               /* t0 = B1 | R2 | R1 | G1 */
+    wsbh              t1, t1               /* t1 = R3 | G3 | G2 | B2 */
+    wsbh              t2, t2               /* t2 = G4 | B4 | B3 | R4 */
+
+    packrl.ph         t3, t1, t0           /* t3 = G2 | B2 | B1 | R2 */
+    packrl.ph         t4, t0, t0           /* t4 = R1 | G1 | B1 | R2 */
+    rotr              t3, t3, 16           /* t3 = B1 | R2 | G2 | B2 */
+    srl               t4, t4, 8            /* t4 =  0 | R1 | G1 | B1 */
+    packrl.ph         t5, t2, t1           /* t5 = B3 | R4 | R3 | G3 */
+    rotr              t5, t5, 24           /* t5 = R4 | R3 | G3 | B3 */
+    rotr              t2, t2, 16           /* t2 = B3 | R4 | G4 | B4 */
+
+    CONVERT_2x8888_TO_2x0565 t4, t3, t4, t3, t6, t7, t8, v0, v1
+    CONVERT_2x8888_TO_2x0565 t5, t2, t5, t2, t6, t7, t8, v0, v1
+
+    sh                t4, 0(a0)
+    sh                t3, 2(a0)
+    sh                t5, 4(a0)
+    sh                t2, 6(a0)
+    b                 0b
+     addiu            a0, a0, 8
+
+1:
+    lbu               t4, 0(a1)            /* t4 =  0 |  0 |  0 | R1 */
+    lhu               t5, 1(a1)            /* t5 =  0 |  0 | B1 | G1 */
+    sll               t4, t4, 16           /* t4 =  0 | R1 |  0 | 0  */
+    wsbh              t5, t5               /* t5 =  0 |  0 | G1 | B1 */
+    or                t5, t4, t5           /* t5 =  0 | R1 | G1 | B1 */
+11:
+    beqz              t9, 4f
+     addiu            t9, t9, -1
+    lw                t0, 3(a1)            /* t0 = R3 | B2 | G2 | R2 */
+    lw                t1, 7(a1)            /* t1 = G4 | R4 | B3 | G3 */
+    lw                t2, 11(a1)           /* t2 = B5 | G5 | R5 | B4 */
+
+    addiu             a1, a1, 12
+    addiu             a2, a2, -4
+
+    wsbh              t0, t0               /* t0 = B2 | R3 | R2 | G2 */
+    wsbh              t1, t1               /* t1 = R4 | G4 | G3 | B3 */
+    wsbh              t2, t2               /* t2 = G5 | B5 | B4 | R5 */
+
+    packrl.ph         t3, t1, t0           /* t3 = G3 | B3 | B2 | R3 */
+    packrl.ph         t4, t2, t1           /* t4 = B4 | R5 | R4 | G4 */
+    rotr              t0, t0, 24           /* t0 = R3 | R2 | G2 | B2 */
+    rotr              t3, t3, 16           /* t3 = B2 | R3 | G3 | B3 */
+    rotr              t4, t4, 24           /* t4 = R5 | R4 | G4 | B4 */
+
+    CONVERT_2x8888_TO_2x0565 t5, t0, t5, t0, t6, t7, t8, v0, v1
+    CONVERT_2x8888_TO_2x0565 t3, t4, t3, t4, t6, t7, t8, v0, v1
+
+    sh                t5, 0(a0)
+    sh                t0, 2(a0)
+    sh                t3, 4(a0)
+    sh                t4, 6(a0)
+    rotr              t5, t2, 16           /* t5 = xx | R5 | G5 | B5 */
+    b                 11b
+     addiu            a0, a0, 8
+
+2:
+    lhu               t5, 0(a1)            /* t5 =  0 |  0 | G1 | R1 */
+    wsbh              t5, t5               /* t5 =  0 |  0 | R1 | G1 */
+21:
+    beqz              t9, 4f
+     addiu            t9, t9, -1
+    lw                t0, 2(a1)            /* t0 = B2 | G2 | R2 | B1 */
+    lw                t1, 6(a1)            /* t1 = R4 | B3 | G3 | R3 */
+    lw                t2, 10(a1)           /* t2 = G5 | R5 | B4 | G4 */
+
+    addiu             a1, a1, 12
+    addiu             a2, a2, -4
+
+    wsbh              t0, t0               /* t0 = G2 | B2 | B1 | R2 */
+    wsbh              t1, t1               /* t1 = B3 | R4 | R3 | G3 */
+    wsbh              t2, t2               /* t2 = R5 | G5 | G4 | B4 */
+
+    precr_sra.ph.w    t5, t0, 0            /* t5 = R1 | G1 | B1 | R2 */
+    rotr              t0, t0, 16           /* t0 = B1 | R2 | G2 | B2 */
+    packrl.ph         t3, t2, t1           /* t3 = G4 | B4 | B3 | R4 */
+    rotr              t1, t1, 24           /* t1 = R4 | R3 | G3 | B3 */
+    srl               t5, t5, 8            /* t5 =  0 | R1 | G1 | B1 */
+    rotr              t3, t3, 16           /* t3 = B3 | R4 | G4 | B4 */
+
+    CONVERT_2x8888_TO_2x0565 t5, t0, t5, t0, t6, t7, t8, v0, v1
+    CONVERT_2x8888_TO_2x0565 t1, t3, t1, t3, t6, t7, t8, v0, v1
+
+    sh                t5, 0(a0)
+    sh                t0, 2(a0)
+    sh                t1, 4(a0)
+    sh                t3, 6(a0)
+    srl               t5, t2, 16           /* t5 =  0 |  0 | R5 | G5 */
+    b                 21b
+     addiu            a0, a0, 8
+
+3:
+    lbu               t5, 0(a1)            /* t5 =  0 |  0 |  0 | R1 */
+31:
+    beqz              t9, 4f
+     addiu            t9, t9, -1
+    lw                t0, 1(a1)            /* t0 = G2 | R2 | B1 | G1 */
+    lw                t1, 5(a1)            /* t1 = B3 | G3 | R3 | B2 */
+    lw                t2, 9(a1)            /* t2 = R5 | B4 | G4 | R4 */
+
+    addiu             a1, a1, 12
+    addiu             a2, a2, -4
+
+    wsbh              t0, t0               /* t0 = R2 | G2 | G1 | B1 */
+    wsbh              t1, t1               /* t1 = G3 | B3 | B2 | R3 */
+    wsbh              t2, t2               /* t2 = B4 | R5 | R4 | G4 */
+
+    precr_sra.ph.w    t5, t0, 0            /* t5 = xx | R1 | G1 | B1 */
+    packrl.ph         t3, t1, t0           /* t3 = B2 | R3 | R2 | G2 */
+    rotr              t1, t1, 16           /* t1 = B2 | R3 | G3 | B3 */
+    rotr              t4, t2, 24           /* t4 = R5 | R4 | G4 | B4 */
+    rotr              t3, t3, 24           /* t3 = R3 | R2 | G2 | B2 */
+
+    CONVERT_2x8888_TO_2x0565 t5, t3, t5, t3, t6, t7, t8, v0, v1
+    CONVERT_2x8888_TO_2x0565 t1, t4, t1, t4, t6, t7, t8, v0, v1
+
+    sh                t5, 0(a0)
+    sh                t3, 2(a0)
+    sh                t1, 4(a0)
+    sh                t4, 6(a0)
+    srl               t5, t2, 16           /* t5 =  0 |  0 | xx | R5 */
+    b                 31b
+     addiu            a0, a0, 8
+
+4:
+    beqz              a2, 6f
+     nop
+5:
+    lbu               t0, 0(a1)            /* t0 =  0 | 0 | 0 | R */
+    lbu               t1, 1(a1)            /* t1 =  0 | 0 | 0 | G */
+    lbu               t2, 2(a1)            /* t2 =  0 | 0 | 0 | B */
+    addiu             a1, a1, 3
+
+    sll               t0, t0, 16           /* t2 =  0 | R | 0 | 0 */
+    sll               t1, t1, 8            /* t1 =  0 | 0 | G | 0 */
+
+    or                t2, t2, t1           /* t2 =  0 | 0 | G | B */
+    or                t2, t2, t0           /* t2 =  0 | R | G | B */
+
+    CONVERT_1x8888_TO_1x0565 t2, t3, t4, t5
+
+    sh                t3, 0(a0)
+    addiu             a2, a2, -1
+    bnez              a2, 5b
+     addiu            a0, a0, 2
+6:
+    RESTORE_REGS_FROM_STACK 0, v0, v1
+    j                 ra
+     nop
+
+END(pixman_composite_src_0888_0565_rev_asm_mips)
+#endif
+
+LEAF_MIPS_DSPR2(pixman_composite_src_pixbuf_8888_asm_mips)
+/*
+ * a0 - dst  (a8b8g8r8)
+ * a1 - src  (a8r8g8b8)
+ * a2 - w
+ */
+
+    SAVE_REGS_ON_STACK 0, v0
+    li       v0, 0x00ff00ff
+
+    beqz     a2, 3f
+     nop
+    addiu    t1, a2, -1
+    beqz     t1, 2f
+     nop
+1:
+    lw       t0, 0(a1)
+    lw       t1, 4(a1)
+    addiu    a1, a1, 8
+    addiu    a2, a2, -2
+    srl      t2, t0, 24
+    srl      t3, t1, 24
+
+    MIPS_2xUN8x4_MUL_2xUN8 t0, t1, t2, t3, t0, t1, v0, t4, t5, t6, t7, t8, t9
+
+    sll      t0, t0, 8
+    sll      t1, t1, 8
+    andi     t2, t2, 0xff
+    andi     t3, t3, 0xff
+    or       t0, t0, t2
+    or       t1, t1, t3
+    wsbh     t0, t0
+    wsbh     t1, t1
+    rotr     t0, t0, 16
+    rotr     t1, t1, 16
+    sw       t0, 0(a0)
+    sw       t1, 4(a0)
+
+    addiu    t2, a2, -1
+    bgtz     t2, 1b
+     addiu   a0, a0, 8
+2:
+    beqz     a2, 3f
+     nop
+    lw       t0, 0(a1)
+    srl      t1, t0, 24
+
+    MIPS_UN8x4_MUL_UN8 t0, t1, t0, v0, t3, t4, t5
+
+    sll      t0, t0, 8
+    andi     t1, t1, 0xff
+    or       t0, t0, t1
+    wsbh     t0, t0
+    rotr     t0, t0, 16
+    sw       t0, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 0, v0
+    j        ra
+     nop
+
+END(pixman_composite_src_pixbuf_8888_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_src_rpixbuf_8888_asm_mips)
+/*
+ * a0 - dst  (a8r8g8b8)
+ * a1 - src  (a8r8g8b8)
+ * a2 - w
+ */
+
+    SAVE_REGS_ON_STACK 0, v0
+    li       v0, 0x00ff00ff
+
+    beqz     a2, 3f
+     nop
+    addiu    t1, a2, -1
+    beqz     t1, 2f
+     nop
+1:
+    lw       t0, 0(a1)
+    lw       t1, 4(a1)
+    addiu    a1, a1, 8
+    addiu    a2, a2, -2
+    srl      t2, t0, 24
+    srl      t3, t1, 24
+
+    MIPS_2xUN8x4_MUL_2xUN8 t0, t1, t2, t3, t0, t1, v0, t4, t5, t6, t7, t8, t9
+
+    sll      t0, t0, 8
+    sll      t1, t1, 8
+    andi     t2, t2, 0xff
+    andi     t3, t3, 0xff
+    or       t0, t0, t2
+    or       t1, t1, t3
+    rotr     t0, t0, 8
+    rotr     t1, t1, 8
+    sw       t0, 0(a0)
+    sw       t1, 4(a0)
+
+    addiu    t2, a2, -1
+    bgtz     t2, 1b
+     addiu   a0, a0, 8
+2:
+    beqz     a2, 3f
+     nop
+    lw       t0, 0(a1)
+    srl      t1, t0, 24
+
+    MIPS_UN8x4_MUL_UN8 t0, t1, t0, v0, t3, t4, t5
+
+    sll      t0, t0, 8
+    andi     t1, t1, 0xff
+    or       t0, t0, t1
+    rotr     t0, t0, 8
+    sw       t0, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 0, v0
+    j        ra
+     nop
+
+END(pixman_composite_src_rpixbuf_8888_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_src_n_8_8888_asm_mips)
+/*
+ * a0 - dst  (a8r8g8b8)
+ * a1 - src  (32bit constant)
+ * a2 - mask (a8)
+ * a3 - w
+ */
+
+
+    SAVE_REGS_ON_STACK 0, v0
+    li       v0, 0x00ff00ff
+
+    beqz     a3, 3f
+     nop
+    addiu    t1, a3, -1
+    beqz     t1, 2f
+     nop
+
+1:
+                       /* a1 = source      (32bit constant) */
+    lbu      t0, 0(a2) /* t2 = mask        (a8) */
+    lbu      t1, 1(a2) /* t3 = mask        (a8) */
+    addiu    a2, a2, 2
+
+    MIPS_2xUN8x4_MUL_2xUN8 a1, a1, t0, t1, t2, t3, v0, t4, t5, t6, t7, t8, t9
+
+    sw       t2, 0(a0)
+    sw       t3, 4(a0)
+    addiu    a3, a3, -2
+    addiu    t2, a3, -1
+    bgtz     t2, 1b
+     addiu   a0, a0, 8
+
+    beqz     a3, 3f
+     nop
+
+2:
+    lbu      t0, 0(a2)
+    addiu    a2, a2, 1
+
+    MIPS_UN8x4_MUL_UN8 a1, t0, t1, v0, t3, t4, t5
+
+    sw       t1, 0(a0)
+    addiu    a3, a3, -1
+    addiu    a0, a0, 4
+
+3:
+    RESTORE_REGS_FROM_STACK 0, v0
+    j        ra
+     nop
+
+END(pixman_composite_src_n_8_8888_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_src_n_8_8_asm_mips)
+/*
+ * a0 - dst  (a8)
+ * a1 - src  (32bit constant)
+ * a2 - mask (a8)
+ * a3 - w
+ */
+
+    li                t9, 0x00ff00ff
+    beqz              a3, 3f
+     nop
+    srl               t7, a3, 2   /* t7 = how many multiples of 4 dst pixels */
+    beqz              t7, 1f      /* branch if less than 4 src pixels */
+     nop
+
+    srl               t8, a1, 24
+    replv.ph          t8, t8
+
+0:
+    beqz              t7, 1f
+     addiu            t7, t7, -1
+    lbu               t0, 0(a2)
+    lbu               t1, 1(a2)
+    lbu               t2, 2(a2)
+    lbu               t3, 3(a2)
+
+    addiu             a2, a2, 4
+
+    precr_sra.ph.w    t1, t0, 0
+    precr_sra.ph.w    t3, t2, 0
+    precr.qb.ph       t0, t3, t1
+
+    muleu_s.ph.qbl    t2, t0, t8
+    muleu_s.ph.qbr    t3, t0, t8
+    shra_r.ph         t4, t2, 8
+    shra_r.ph         t5, t3, 8
+    and               t4, t4, t9
+    and               t5, t5, t9
+    addq.ph           t2, t2, t4
+    addq.ph           t3, t3, t5
+    shra_r.ph         t2, t2, 8
+    shra_r.ph         t3, t3, 8
+    precr.qb.ph       t2, t2, t3
+
+    sb                t2, 0(a0)
+    srl               t2, t2, 8
+    sb                t2, 1(a0)
+    srl               t2, t2, 8
+    sb                t2, 2(a0)
+    srl               t2, t2, 8
+    sb                t2, 3(a0)
+    addiu             a3, a3, -4
+    b                 0b
+     addiu            a0, a0, 4
+
+1:
+    beqz              a3, 3f
+     nop
+    srl               t8, a1, 24
+2:
+    lbu               t0, 0(a2)
+    addiu             a2, a2, 1
+
+    mul               t2, t0, t8
+    shra_r.ph         t3, t2, 8
+    andi              t3, t3, 0x00ff
+    addq.ph           t2, t2, t3
+    shra_r.ph         t2, t2, 8
+
+    sb                t2, 0(a0)
+    addiu             a3, a3, -1
+    bnez              a3, 2b
+     addiu            a0, a0, 1
+
+3:
+    j                 ra
+     nop
+
+END(pixman_composite_src_n_8_8_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_over_n_8888_8888_ca_asm_mips)
+/*
+ * a0 - dst  (a8r8g8b8)
+ * a1 - src  (32bit constant)
+ * a2 - mask (a8r8g8b8)
+ * a3 - w
+ */
+
+    beqz         a3, 8f
+     nop
+    SAVE_REGS_ON_STACK 8, s0, s1, s2, s3, s4, s5
+
+    li           t6, 0xff
+    addiu        t7, zero, -1 /* t7 = 0xffffffff */
+    srl          t8, a1, 24   /* t8 = srca */
+    li           t9, 0x00ff00ff
+
+    addiu        t1, a3, -1
+    beqz         t1, 4f       /* last pixel */
+     nop
+
+0:
+    lw           t0, 0(a2)    /* t0 = mask */
+    lw           t1, 4(a2)    /* t1 = mask */
+    addiu        a3, a3, -2   /* w = w - 2 */
+    or           t2, t0, t1
+    beqz         t2, 3f      /* if (t0 == 0) && (t1 == 0) */
+     addiu       a2, a2, 8
+    and          t2, t0, t1
+    beq          t2, t7, 1f  /* if (t0 == 0xffffffff) && (t1 == 0xffffffff) */
+     nop
+
+//if(ma)
+    lw           t2, 0(a0)    /* t2 = dst */
+    lw           t3, 4(a0)    /* t3 = dst */
+    MIPS_2xUN8x4_MUL_2xUN8x4 a1, a1, t0, t1, t4, t5, t9, s0, s1, s2, s3, s4, s5
+    MIPS_2xUN8x4_MUL_2xUN8   t0, t1, t8, t8, t0, t1, t9, s0, s1, s2, s3, s4, s5
+    not          t0, t0
+    not          t1, t1
+    MIPS_2xUN8x4_MUL_2xUN8x4 t2, t3, t0, t1, t2, t3, t9, s0, s1, s2, s3, s4, s5
+    addu_s.qb    t2, t4, t2
+    addu_s.qb    t3, t5, t3
+    sw           t2, 0(a0)
+    sw           t3, 4(a0)
+    addiu        t1, a3, -1
+    bgtz         t1, 0b
+     addiu       a0, a0, 8
+    b            4f
+     nop
+1:
+//if (t0 == 0xffffffff) && (t1 == 0xffffffff):
+    beq          t8, t6, 2f   /* if (srca == 0xff) */
+     nop
+    lw           t2, 0(a0)    /* t2 = dst */
+    lw           t3, 4(a0)    /* t3 = dst */
+    not          t0, a1
+    not          t1, a1
+    srl          t0, t0, 24
+    srl          t1, t1, 24
+    MIPS_2xUN8x4_MUL_2xUN8 t2, t3, t0, t1, t2, t3, t9, s0, s1, s2, s3, s4, s5
+    addu_s.qb    t2, a1, t2
+    addu_s.qb    t3, a1, t3
+    sw           t2, 0(a0)
+    sw           t3, 4(a0)
+    addiu        t1, a3, -1
+    bgtz         t1, 0b
+     addiu       a0, a0, 8
+    b            4f
+     nop
+2:
+    sw           a1, 0(a0)
+    sw           a1, 4(a0)
+3:
+    addiu        t1, a3, -1
+    bgtz         t1, 0b
+     addiu       a0, a0, 8
+
+4:
+    beqz         a3, 7f
+     nop
+                              /* a1 = src */
+    lw           t0, 0(a2)    /* t0 = mask */
+    beqz         t0, 7f       /* if (t0 == 0) */
+     nop
+    beq          t0, t7, 5f  /* if (t0 == 0xffffffff) */
+     nop
+//if(ma)
+    lw           t1, 0(a0)    /* t1 = dst */
+    MIPS_UN8x4_MUL_UN8x4  a1, t0, t2, t9, t3, t4, t5, s0
+    MIPS_UN8x4_MUL_UN8    t0, t8, t0, t9, t3, t4, t5
+    not          t0, t0
+    MIPS_UN8x4_MUL_UN8x4  t1, t0, t1, t9, t3, t4, t5, s0
+    addu_s.qb    t1, t2, t1
+    sw           t1, 0(a0)
+    RESTORE_REGS_FROM_STACK 8, s0, s1, s2, s3, s4, s5
+    j            ra
+     nop
+5:
+//if (t0 == 0xffffffff)
+    beq          t8, t6, 6f   /* if (srca == 0xff) */
+     nop
+    lw           t1, 0(a0)    /* t1 = dst */
+    not          t0, a1
+    srl          t0, t0, 24
+    MIPS_UN8x4_MUL_UN8 t1, t0, t1, t9, t2, t3, t4
+    addu_s.qb    t1, a1, t1
+    sw           t1, 0(a0)
+    RESTORE_REGS_FROM_STACK 8, s0, s1, s2, s3, s4, s5
+    j            ra
+     nop
+6:
+    sw           a1, 0(a0)
+7:
+    RESTORE_REGS_FROM_STACK 8, s0, s1, s2, s3, s4, s5
+8:
+    j            ra
+     nop
+
+END(pixman_composite_over_n_8888_8888_ca_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_over_n_8888_0565_ca_asm_mips)
+/*
+ * a0 - dst  (r5g6b5)
+ * a1 - src  (32bit constant)
+ * a2 - mask (a8r8g8b8)
+ * a3 - w
+ */
+
+    beqz         a3, 8f
+     nop
+    SAVE_REGS_ON_STACK 20, s0, s1, s2, s3, s4, s5, s6, s7, s8
+
+    li           t6, 0xff
+    addiu        t7, zero, -1 /* t7 = 0xffffffff */
+    srl          t8, a1, 24   /* t8 = srca */
+    li           t9, 0x00ff00ff
+    li           s6, 0xf800f800
+    li           s7, 0x07e007e0
+    li           s8, 0x001F001F
+
+    addiu        t1, a3, -1
+    beqz         t1, 4f       /* last pixel */
+     nop
+
+0:
+    lw           t0, 0(a2)    /* t0 = mask */
+    lw           t1, 4(a2)    /* t1 = mask */
+    addiu        a3, a3, -2   /* w = w - 2 */
+    or           t2, t0, t1
+    beqz         t2, 3f      /* if (t0 == 0) && (t1 == 0) */
+     addiu       a2, a2, 8
+    and          t2, t0, t1
+    beq          t2, t7, 1f  /* if (t0 == 0xffffffff) && (t1 == 0xffffffff) */
+     nop
+
+//if(ma)
+    lhu          t2, 0(a0)    /* t2 = dst */
+    lhu          t3, 2(a0)    /* t3 = dst */
+    MIPS_2xUN8x4_MUL_2xUN8x4 a1, a1, t0, t1, t4, t5, t9, s0, s1, s2, s3, s4, s5
+    MIPS_2xUN8x4_MUL_2xUN8   t0, t1, t8, t8, t0, t1, t9, s0, s1, s2, s3, s4, s5
+    not          t0, t0
+    not          t1, t1
+    CONVERT_2x0565_TO_2x8888 t2, t3, t2, t3, s7, s8, s0, s1, s2, s3
+    MIPS_2xUN8x4_MUL_2xUN8x4 t2, t3, t0, t1, t2, t3, t9, s0, s1, s2, s3, s4, s5
+    addu_s.qb    t2, t4, t2
+    addu_s.qb    t3, t5, t3
+    CONVERT_2x8888_TO_2x0565 t2, t3, t2, t3, s6, s7, s8, s0, s1
+    sh           t2, 0(a0)
+    sh           t3, 2(a0)
+    addiu        t1, a3, -1
+    bgtz         t1, 0b
+     addiu       a0, a0, 4
+    b            4f
+     nop
+1:
+//if (t0 == 0xffffffff) && (t1 == 0xffffffff):
+    beq          t8, t6, 2f   /* if (srca == 0xff) */
+     nop
+    lhu          t2, 0(a0)    /* t2 = dst */
+    lhu          t3, 2(a0)    /* t3 = dst */
+    not          t0, a1
+    not          t1, a1
+    srl          t0, t0, 24
+    srl          t1, t1, 24
+    CONVERT_2x0565_TO_2x8888 t2, t3, t2, t3, s7, s8, s0, s1, s2, s3
+    MIPS_2xUN8x4_MUL_2xUN8   t2, t3, t0, t1, t2, t3, t9, s0, s1, s2, s3, s4, s5
+    addu_s.qb    t2, a1, t2
+    addu_s.qb    t3, a1, t3
+    CONVERT_2x8888_TO_2x0565 t2, t3, t2, t3, s6, s7, s8, s0, s1
+    sh           t2, 0(a0)
+    sh           t3, 2(a0)
+    addiu        t1, a3, -1
+    bgtz         t1, 0b
+     addiu       a0, a0, 4
+    b            4f
+     nop
+2:
+    CONVERT_1x8888_TO_1x0565 a1, t2, s0, s1
+    sh           t2, 0(a0)
+    sh           t2, 2(a0)
+3:
+    addiu        t1, a3, -1
+    bgtz         t1, 0b
+     addiu       a0, a0, 4
+
+4:
+    beqz         a3, 7f
+     nop
+                              /* a1 = src */
+    lw           t0, 0(a2)    /* t0 = mask */
+    beqz         t0, 7f       /* if (t0 == 0) */
+     nop
+    beq          t0, t7, 5f  /* if (t0 == 0xffffffff) */
+     nop
+//if(ma)
+    lhu          t1, 0(a0)    /* t1 = dst */
+    MIPS_UN8x4_MUL_UN8x4     a1, t0, t2, t9, t3, t4, t5, s0
+    MIPS_UN8x4_MUL_UN8       t0, t8, t0, t9, t3, t4, t5
+    not          t0, t0
+    CONVERT_1x0565_TO_1x8888 t1, s1, s2, s3
+    MIPS_UN8x4_MUL_UN8x4     s1, t0, s1, t9, t3, t4, t5, s0
+    addu_s.qb    s1, t2, s1
+    CONVERT_1x8888_TO_1x0565 s1, t1, s0, s2
+    sh           t1, 0(a0)
+    RESTORE_REGS_FROM_STACK 20, s0, s1, s2, s3, s4, s5, s6, s7, s8
+    j            ra
+     nop
+5:
+//if (t0 == 0xffffffff)
+    beq          t8, t6, 6f   /* if (srca == 0xff) */
+     nop
+    lhu          t1, 0(a0)    /* t1 = dst */
+    not          t0, a1
+    srl          t0, t0, 24
+    CONVERT_1x0565_TO_1x8888 t1, s1, s2, s3
+    MIPS_UN8x4_MUL_UN8       s1, t0, s1, t9, t2, t3, t4
+    addu_s.qb    s1, a1, s1
+    CONVERT_1x8888_TO_1x0565 s1, t1, s0, s2
+    sh           t1, 0(a0)
+    RESTORE_REGS_FROM_STACK 20, s0, s1, s2, s3, s4, s5, s6, s7, s8
+    j            ra
+     nop
+6:
+    CONVERT_1x8888_TO_1x0565 a1, t1, s0, s2
+    sh           t1, 0(a0)
+7:
+    RESTORE_REGS_FROM_STACK 20, s0, s1, s2, s3, s4, s5, s6, s7, s8
+8:
+    j            ra
+     nop
+
+END(pixman_composite_over_n_8888_0565_ca_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_over_n_8_8_asm_mips)
+/*
+ * a0 - dst  (a8)
+ * a1 - src  (32bit constant)
+ * a2 - mask (a8)
+ * a3 - w
+ */
+
+    SAVE_REGS_ON_STACK 0, v0
+    li                t9, 0x00ff00ff
+    beqz              a3, 3f
+     nop
+    srl               v0, a3, 2   /* v0 = how many multiples of 4 dst pixels */
+    beqz              v0, 1f      /* branch if less than 4 src pixels */
+     nop
+
+    srl               t8, a1, 24
+    replv.ph          t8, t8
+
+0:
+    beqz              v0, 1f
+     addiu            v0, v0, -1
+    lbu               t0, 0(a2)
+    lbu               t1, 1(a2)
+    lbu               t2, 2(a2)
+    lbu               t3, 3(a2)
+    lbu               t4, 0(a0)
+    lbu               t5, 1(a0)
+    lbu               t6, 2(a0)
+    lbu               t7, 3(a0)
+
+    addiu             a2, a2, 4
+
+    precr_sra.ph.w    t1, t0, 0
+    precr_sra.ph.w    t3, t2, 0
+    precr_sra.ph.w    t5, t4, 0
+    precr_sra.ph.w    t7, t6, 0
+
+    precr.qb.ph       t0, t3, t1
+    precr.qb.ph       t1, t7, t5
+
+    muleu_s.ph.qbl    t2, t0, t8
+    muleu_s.ph.qbr    t3, t0, t8
+    shra_r.ph         t4, t2, 8
+    shra_r.ph         t5, t3, 8
+    and               t4, t4, t9
+    and               t5, t5, t9
+    addq.ph           t2, t2, t4
+    addq.ph           t3, t3, t5
+    shra_r.ph         t2, t2, 8
+    shra_r.ph         t3, t3, 8
+    precr.qb.ph       t0, t2, t3
+    not               t6, t0
+
+    preceu.ph.qbl     t7, t6
+    preceu.ph.qbr     t6, t6
+
+    muleu_s.ph.qbl    t2, t1, t7
+    muleu_s.ph.qbr    t3, t1, t6
+    shra_r.ph         t4, t2, 8
+    shra_r.ph         t5, t3, 8
+    and               t4, t4, t9
+    and               t5, t5, t9
+    addq.ph           t2, t2, t4
+    addq.ph           t3, t3, t5
+    shra_r.ph         t2, t2, 8
+    shra_r.ph         t3, t3, 8
+    precr.qb.ph       t1, t2, t3
+
+    addu_s.qb         t2, t0, t1
+
+    sb                t2, 0(a0)
+    srl               t2, t2, 8
+    sb                t2, 1(a0)
+    srl               t2, t2, 8
+    sb                t2, 2(a0)
+    srl               t2, t2, 8
+    sb                t2, 3(a0)
+    addiu             a3, a3, -4
+    b                 0b
+     addiu            a0, a0, 4
+
+1:
+    beqz              a3, 3f
+     nop
+    srl               t8, a1, 24
+2:
+    lbu               t0, 0(a2)
+    lbu               t1, 0(a0)
+    addiu             a2, a2, 1
+
+    mul               t2, t0, t8
+    shra_r.ph         t3, t2, 8
+    andi              t3, t3, 0x00ff
+    addq.ph           t2, t2, t3
+    shra_r.ph         t2, t2, 8
+    not               t3, t2
+    andi              t3, t3, 0x00ff
+
+
+    mul               t4, t1, t3
+    shra_r.ph         t5, t4, 8
+    andi              t5, t5, 0x00ff
+    addq.ph           t4, t4, t5
+    shra_r.ph         t4, t4, 8
+    andi              t4, t4, 0x00ff
+
+    addu_s.qb         t2, t2, t4
+    sb                t2, 0(a0)
+    addiu             a3, a3, -1
+    bnez              a3, 2b
+     addiu            a0, a0, 1
+
+3:
+    RESTORE_REGS_FROM_STACK 0, v0
+    j                 ra
+     nop
+
+END(pixman_composite_over_n_8_8_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_over_n_8_8888_asm_mips)
+/*
+ * a0 - dst  (a8r8g8b8)
+ * a1 - src  (32bit constant)
+ * a2 - mask (a8)
+ * a3 - w
+ */
+
+    SAVE_REGS_ON_STACK 4, s0, s1, s2, s3, s4
+    beqz      a3, 4f
+     nop
+    li        t4, 0x00ff00ff
+    li        t5, 0xff
+    addiu     t0, a3, -1
+    beqz      t0, 3f         /* last pixel */
+     srl      t6, a1, 24     /* t6 = srca */
+    not       s4, a1
+    beq       t5, t6, 2f     /* if (srca == 0xff) */
+     srl      s4, s4, 24
+1:
+                             /* a1 = src */
+    lbu       t0, 0(a2)      /* t0 = mask */
+    lbu       t1, 1(a2)      /* t1 = mask */
+    or        t2, t0, t1
+    beqz      t2, 111f       /* if (t0 == 0) && (t1 == 0) */
+     addiu    a2, a2, 2
+    and       t3, t0, t1
+
+    lw        t2, 0(a0)      /* t2 = dst */
+    beq       t3, t5, 11f    /* if (t0 == 0xff) && (t1 == 0xff) */
+     lw       t3, 4(a0)      /* t3 = dst */
+
+    MIPS_2xUN8x4_MUL_2xUN8 a1, a1, t0, t1, s0, s1, t4, t6, t7, t8, t9, s2, s3
+    not       s2, s0
+    not       s3, s1
+    srl       s2, s2, 24
+    srl       s3, s3, 24
+    MIPS_2xUN8x4_MUL_2xUN8 t2, t3, s2, s3, t2, t3, t4, t0, t1, t6, t7, t8, t9
+    addu_s.qb s2, t2, s0
+    addu_s.qb s3, t3, s1
+    sw        s2, 0(a0)
+    b         111f
+     sw       s3, 4(a0)
+11:
+    MIPS_2xUN8x4_MUL_2xUN8 t2, t3, s4, s4, t2, t3, t4, t0, t1, t6, t7, t8, t9
+    addu_s.qb s2, t2, a1
+    addu_s.qb s3, t3, a1
+    sw        s2, 0(a0)
+    sw        s3, 4(a0)
+
+111:
+    addiu     a3, a3, -2
+    addiu     t0, a3, -1
+    bgtz      t0, 1b
+     addiu    a0, a0, 8
+    b         3f
+     nop
+2:
+                             /* a1 = src */
+    lbu       t0, 0(a2)      /* t0 = mask */
+    lbu       t1, 1(a2)      /* t1 = mask */
+    or        t2, t0, t1
+    beqz      t2, 222f       /* if (t0 == 0) && (t1 == 0) */
+     addiu    a2, a2, 2
+    and       t3, t0, t1
+    beq       t3, t5, 22f    /* if (t0 == 0xff) && (t1 == 0xff) */
+     nop
+    lw        t2, 0(a0)      /* t2 = dst */
+    lw        t3, 4(a0)      /* t3 = dst */
+
+    OVER_2x8888_2x8_2x8888 a1, a1, t0, t1, t2, t3, \
+                           t6, t7, t4, t8, t9, s0, s1, s2, s3
+    sw        t6, 0(a0)
+    b         222f
+     sw        t7, 4(a0)
+22:
+    sw        a1, 0(a0)
+    sw        a1, 4(a0)
+222:
+    addiu     a3, a3, -2
+    addiu     t0, a3, -1
+    bgtz      t0, 2b
+     addiu    a0, a0, 8
+3:
+    blez      a3, 4f
+     nop
+                             /* a1 = src */
+    lbu       t0, 0(a2)      /* t0 = mask */
+    beqz      t0, 4f         /* if (t0 == 0) */
+     addiu    a2, a2, 1
+    move      t3, a1
+    beq       t0, t5, 31f    /* if (t0 == 0xff) */
+     lw       t1, 0(a0)      /* t1 = dst */
+
+    MIPS_UN8x4_MUL_UN8 a1, t0, t3, t4, t6, t7, t8
+31:
+    not       t2, t3
+    srl       t2, t2, 24
+    MIPS_UN8x4_MUL_UN8 t1, t2, t1, t4, t6, t7, t8
+    addu_s.qb t2, t1, t3
+    sw        t2, 0(a0)
+4:
+    RESTORE_REGS_FROM_STACK 4, s0, s1, s2, s3, s4
+    j         ra
+     nop
+
+END(pixman_composite_over_n_8_8888_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_over_n_8_0565_asm_mips)
+/*
+ * a0 - dst  (r5g6b5)
+ * a1 - src  (32bit constant)
+ * a2 - mask (a8)
+ * a3 - w
+ */
+    SAVE_REGS_ON_STACK 24, v0, s0, s1, s2, s3, s4, s5, s6, s7, s8
+    beqz     a3, 4f
+     nop
+    li       t4, 0x00ff00ff
+    li       t5, 0xff
+    li       t6, 0xf800f800
+    li       t7, 0x07e007e0
+    li       t8, 0x001F001F
+    addiu    t1, a3, -1
+    beqz     t1, 3f         /* last pixel */
+     srl     t0, a1, 24     /* t0 = srca */
+    not      v0, a1
+    beq      t0, t5, 2f     /* if (srca == 0xff) */
+     srl     v0, v0, 24
+1:
+                            /* a1 = src */
+    lbu      t0, 0(a2)      /* t0 = mask */
+    lbu      t1, 1(a2)      /* t1 = mask */
+    or       t2, t0, t1
+    beqz     t2, 111f       /* if (t0 == 0) && (t1 == 0) */
+     addiu   a2, a2, 2
+    lhu      t2, 0(a0)      /* t2 = dst */
+    lhu      t3, 2(a0)      /* t3 = dst */
+    CONVERT_2x0565_TO_2x8888 t2, t3, s0, s1, t7, t8, t9, s2, s3, s4
+    and      t9, t0, t1
+    beq      t9, t5, 11f    /* if (t0 == 0xff) && (t1 == 0xff) */
+     nop
+
+    MIPS_2xUN8x4_MUL_2xUN8   a1, a1, t0, t1, s2, s3, t4, t9, s4, s5, s6, s7, s8
+    not      s4, s2
+    not      s5, s3
+    srl      s4, s4, 24
+    srl      s5, s5, 24
+    MIPS_2xUN8x4_MUL_2xUN8   s0, s1, s4, s5, s0, s1, t4, t9, t0, t1, s6, s7, s8
+    addu_s.qb                s4, s2, s0
+    addu_s.qb                s5, s3, s1
+    CONVERT_2x8888_TO_2x0565 s4, s5, t2, t3, t6, t7, t8, s0, s1
+    sh       t2, 0(a0)
+    b        111f
+     sh      t3, 2(a0)
+11:
+    MIPS_2xUN8x4_MUL_2xUN8   s0, s1, v0, v0, s0, s1, t4, t9, t0, t1, s6, s7, s8
+    addu_s.qb                s4, a1, s0
+    addu_s.qb                s5, a1, s1
+    CONVERT_2x8888_TO_2x0565 s4, s5, t2, t3, t6, t7, t8, s0, s1
+    sh       t2, 0(a0)
+    sh       t3, 2(a0)
+111:
+    addiu    a3, a3, -2
+    addiu    t0, a3, -1
+    bgtz     t0, 1b
+     addiu   a0, a0, 4
+    b        3f
+     nop
+2:
+    CONVERT_1x8888_TO_1x0565 a1, s0, s1, s2
+21:
+                            /* a1 = src */
+    lbu      t0, 0(a2)      /* t0 = mask */
+    lbu      t1, 1(a2)      /* t1 = mask */
+    or       t2, t0, t1
+    beqz     t2, 222f       /* if (t0 == 0) && (t1 == 0) */
+     addiu   a2, a2, 2
+    and      t9, t0, t1
+    move     s2, s0
+    beq      t9, t5, 22f    /* if (t0 == 0xff) && (t2 == 0xff) */
+     move    s3, s0
+    lhu      t2, 0(a0)      /* t2 = dst */
+    lhu      t3, 2(a0)      /* t3 = dst */
+
+    CONVERT_2x0565_TO_2x8888 t2, t3, s2, s3, t7, t8, s4, s5, s6, s7
+    OVER_2x8888_2x8_2x8888   a1, a1, t0, t1, s2, s3, \
+                             t2, t3, t4, t9, s4, s5, s6, s7, s8
+    CONVERT_2x8888_TO_2x0565 t2, t3, s2, s3, t6, t7, t8, s4, s5
+22:
+    sh       s2, 0(a0)
+    sh       s3, 2(a0)
+222:
+    addiu    a3, a3, -2
+    addiu    t0, a3, -1
+    bgtz     t0, 21b
+     addiu   a0, a0, 4
+3:
+    blez      a3, 4f
+     nop
+                            /* a1 = src */
+    lbu      t0, 0(a2)      /* t0 = mask */
+    beqz     t0, 4f         /* if (t0 == 0) */
+     nop
+    lhu      t1, 0(a0)      /* t1 = dst */
+    CONVERT_1x0565_TO_1x8888 t1, t2, t3, t7
+    beq      t0, t5, 31f    /* if (t0 == 0xff) */
+     move    t3, a1
+
+    MIPS_UN8x4_MUL_UN8       a1, t0, t3, t4, t7, t8, t9
+31:
+    not      t6, t3
+    srl      t6, t6, 24
+    MIPS_UN8x4_MUL_UN8       t2, t6, t2, t4, t7, t8, t9
+    addu_s.qb                t1, t2, t3
+    CONVERT_1x8888_TO_1x0565 t1, t2, t3, t7
+    sh       t2, 0(a0)
+4:
+    RESTORE_REGS_FROM_STACK  24, v0, s0, s1, s2, s3, s4, s5, s6, s7, s8
+    j        ra
+     nop
+
+END(pixman_composite_over_n_8_0565_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_over_8888_n_8888_asm_mips)
+/*
+ * a0 - dst  (a8r8g8b8)
+ * a1 - src  (a8r8g8b8)
+ * a2 - mask (32bit constant)
+ * a3 - w
+ */
+
+    SAVE_REGS_ON_STACK 0, s0
+    li       t4, 0x00ff00ff
+    beqz     a3, 3f
+     nop
+    addiu    t1, a3, -1
+    srl      a2, a2, 24
+    beqz     t1, 2f
+     nop
+
+1:
+    lw       t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+    lw       t1, 4(a1) /* t1 = source      (a8r8g8b8) */
+                       /* a2 = mask        (32bit constant) */
+    lw       t2, 0(a0) /* t2 = destination (a8r8g8b8) */
+    lw       t3, 4(a0) /* t3 = destination (a8r8g8b8) */
+    addiu    a1, a1, 8
+
+    OVER_2x8888_2x8_2x8888 t0, t1, a2, a2, t2, t3, \
+                           t5, t6, t4, t7, t8, t9, t0, t1, s0
+
+    sw       t5, 0(a0)
+    sw       t6, 4(a0)
+    addiu    a3, a3, -2
+    addiu    t1, a3, -1
+    bgtz     t1, 1b
+     addiu   a0, a0, 8
+2:
+    beqz     a3, 3f
+     nop
+    lw       t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+                       /* a2 = mask        (32bit constant) */
+    lw       t1, 0(a0) /* t1 = destination (a8r8g8b8) */
+
+    OVER_8888_8_8888 t0, a2, t1, t3, t4, t5, t6, t7, t8
+
+    sw       t3, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 0, s0
+    j        ra
+     nop
+
+END(pixman_composite_over_8888_n_8888_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_over_8888_n_0565_asm_mips)
+/*
+ * a0 - dst  (r5g6b5)
+ * a1 - src  (a8r8g8b8)
+ * a2 - mask (32bit constant)
+ * a3 - w
+ */
+
+    SAVE_REGS_ON_STACK 0, s0, s1, s2, s3
+    li       t6, 0x00ff00ff
+    li       t7, 0xf800f800
+    li       t8, 0x07e007e0
+    li       t9, 0x001F001F
+    beqz     a3, 3f
+     nop
+    srl      a2, a2, 24
+    addiu    t1, a3, -1
+    beqz     t1, 2f
+     nop
+1:
+    lw       t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+    lw       t1, 4(a1) /* t1 = source      (a8r8g8b8) */
+                       /* a2 = mask        (32bit constant) */
+    lhu      t2, 0(a0) /* t2 = destination (r5g6b5) */
+    lhu      t3, 2(a0) /* t2 = destination (r5g6b5) */
+    addiu    a1, a1, 8
+
+    CONVERT_2x0565_TO_2x8888 t2, t3, t4, t5, t8, t9, s0, s1, t2, t3
+    OVER_2x8888_2x8_2x8888   t0, t1, a2, a2, t4, t5, \
+                             t2, t3, t6, t0, t1, s0, s1, s2, s3
+    CONVERT_2x8888_TO_2x0565 t2, t3, t4, t5, t7, t8, t9, s0, s1
+
+    sh       t4, 0(a0)
+    sh       t5, 2(a0)
+    addiu    a3, a3, -2
+    addiu    t1, a3, -1
+    bgtz     t1, 1b
+     addiu   a0, a0, 4
+2:
+    beqz     a3, 3f
+     nop
+    lw       t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+                       /* a2 = mask        (32bit constant) */
+    lhu      t1, 0(a0) /* t1 = destination (r5g6b5) */
+
+    CONVERT_1x0565_TO_1x8888 t1, t2, t4, t5
+    OVER_8888_8_8888         t0, a2, t2, t1, t6, t3, t4, t5, t7
+    CONVERT_1x8888_TO_1x0565 t1, t3, t4, t5
+
+    sh       t3, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 0, s0, s1, s2, s3
+    j                 ra
+     nop
+
+END(pixman_composite_over_8888_n_0565_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_over_0565_n_0565_asm_mips)
+/*
+ * a0 - dst  (r5g6b5)
+ * a1 - src  (r5g6b5)
+ * a2 - mask (32bit constant)
+ * a3 - w
+ */
+
+    SAVE_REGS_ON_STACK 20, s0, s1, s2, s3, s4, s5
+    li       t6, 0x00ff00ff
+    li       t7, 0xf800f800
+    li       t8, 0x07e007e0
+    li       t9, 0x001F001F
+    beqz     a3, 3f
+     nop
+    srl      a2, a2, 24
+    addiu    t1, a3, -1
+    beqz     t1, 2f
+     nop
+1:
+    lhu      t0, 0(a1) /* t0 = source      (r5g6b5) */
+    lhu      t1, 2(a1) /* t1 = source      (r5g6b5) */
+                       /* a2 = mask        (32bit constant) */
+    lhu      t2, 0(a0) /* t2 = destination (r5g6b5) */
+    lhu      t3, 2(a0) /* t3 = destination (r5g6b5) */
+    addiu    a1, a1, 4
+
+    CONVERT_2x0565_TO_2x8888 t0, t1, t4, t5, t8, t9, s0, s1, s2, s3
+    CONVERT_2x0565_TO_2x8888 t2, t3, s0, s1, t8, t9, s2, s3, s4, s5
+    OVER_2x8888_2x8_2x8888   t4, t5, a2, a2, s0, s1, \
+                             t0, t1, t6, s2, s3, s4, s5, t4, t5
+    CONVERT_2x8888_TO_2x0565 t0, t1, s0, s1, t7, t8, t9, s2, s3
+
+    sh       s0, 0(a0)
+    sh       s1, 2(a0)
+    addiu    a3, a3, -2
+    addiu    t1, a3, -1
+    bgtz     t1, 1b
+     addiu   a0, a0, 4
+2:
+    beqz     a3, 3f
+     nop
+    lhu      t0, 0(a1) /* t0 = source      (r5g6b5) */
+                       /* a2 = mask        (32bit constant) */
+    lhu      t1, 0(a0) /* t1 = destination (r5g6b5) */
+
+    CONVERT_1x0565_TO_1x8888 t0, t2, t4, t5
+    CONVERT_1x0565_TO_1x8888 t1, t3, t4, t5
+    OVER_8888_8_8888         t2, a2, t3, t0, t6, t1, t4, t5, t7
+    CONVERT_1x8888_TO_1x0565 t0, t3, t4, t5
+
+    sh       t3, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 20, s0, s1, s2, s3, s4, s5
+    j        ra
+     nop
+
+END(pixman_composite_over_0565_n_0565_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_over_8888_8_8888_asm_mips)
+/*
+ * a0 - dst  (a8r8g8b8)
+ * a1 - src  (a8r8g8b8)
+ * a2 - mask (a8)
+ * a3 - w
+ */
+
+    SAVE_REGS_ON_STACK 0, s0, s1
+    li       t4, 0x00ff00ff
+    beqz     a3, 3f
+     nop
+    addiu    t1, a3, -1
+    beqz     t1, 2f
+     nop
+1:
+    lw       t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+    lw       t1, 4(a1) /* t1 = source      (a8r8g8b8) */
+    lbu      t2, 0(a2) /* t2 = mask        (a8) */
+    lbu      t3, 1(a2) /* t3 = mask        (a8) */
+    lw       t5, 0(a0) /* t5 = destination (a8r8g8b8) */
+    lw       t6, 4(a0) /* t6 = destination (a8r8g8b8) */
+    addiu    a1, a1, 8
+    addiu    a2, a2, 2
+
+    OVER_2x8888_2x8_2x8888 t0, t1, t2, t3, t5, t6, \
+                           t7, t8, t4, t9, s0, s1, t0, t1, t2
+
+    sw       t7, 0(a0)
+    sw       t8, 4(a0)
+    addiu    a3, a3, -2
+    addiu    t1, a3, -1
+    bgtz     t1, 1b
+     addiu   a0, a0, 8
+2:
+    beqz     a3, 3f
+     nop
+    lw       t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+    lbu      t1, 0(a2) /* t1 = mask        (a8) */
+    lw       t2, 0(a0) /* t2 = destination (a8r8g8b8) */
+
+    OVER_8888_8_8888 t0, t1, t2, t3, t4, t5, t6, t7, t8
+
+    sw       t3, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 0, s0, s1
+    j        ra
+     nop
+
+END(pixman_composite_over_8888_8_8888_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_over_8888_8_0565_asm_mips)
+/*
+ * a0 - dst  (r5g6b5)
+ * a1 - src  (a8r8g8b8)
+ * a2 - mask (a8)
+ * a3 - w
+ */
+
+    SAVE_REGS_ON_STACK 20, s0, s1, s2, s3, s4, s5
+    li       t6, 0x00ff00ff
+    li       t7, 0xf800f800
+    li       t8, 0x07e007e0
+    li       t9, 0x001F001F
+    beqz     a3, 3f
+     nop
+    addiu    t1, a3, -1
+    beqz     t1, 2f
+     nop
+1:
+    lw       t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+    lw       t1, 4(a1) /* t1 = source      (a8r8g8b8) */
+    lbu      t2, 0(a2) /* t2 = mask        (a8) */
+    lbu      t3, 1(a2) /* t3 = mask        (a8) */
+    lhu      t4, 0(a0) /* t4 = destination (r5g6b5) */
+    lhu      t5, 2(a0) /* t5 = destination (r5g6b5) */
+    addiu    a1, a1, 8
+    addiu    a2, a2, 2
+
+    CONVERT_2x0565_TO_2x8888 t4, t5, s0, s1, t8, t9, s2, s3, s4, s5
+    OVER_2x8888_2x8_2x8888   t0, t1, t2, t3, s0, s1, \
+                             t4, t5, t6, s2, s3, s4, s5, t0, t1
+    CONVERT_2x8888_TO_2x0565 t4, t5, s0, s1, t7, t8, t9, s2, s3
+
+    sh       s0, 0(a0)
+    sh       s1, 2(a0)
+    addiu    a3, a3, -2
+    addiu    t1, a3, -1
+    bgtz     t1, 1b
+     addiu   a0, a0, 4
+2:
+    beqz     a3, 3f
+     nop
+    lw       t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+    lbu      t1, 0(a2) /* t1 = mask        (a8) */
+    lhu      t2, 0(a0) /* t2 = destination (r5g6b5) */
+
+    CONVERT_1x0565_TO_1x8888 t2, t3, t4, t5
+    OVER_8888_8_8888         t0, t1, t3, t2, t6, t4, t5, t7, t8
+    CONVERT_1x8888_TO_1x0565 t2, t3, t4, t5
+
+    sh       t3, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 20, s0, s1, s2, s3, s4, s5
+    j        ra
+     nop
+
+END(pixman_composite_over_8888_8_0565_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_over_0565_8_0565_asm_mips)
+/*
+ * a0 - dst  (r5g6b5)
+ * a1 - src  (r5g6b5)
+ * a2 - mask (a8)
+ * a3 - w
+ */
+
+    SAVE_REGS_ON_STACK 20, s0, s1, s2, s3, s4, s5
+    li       t4, 0xf800f800
+    li       t5, 0x07e007e0
+    li       t6, 0x001F001F
+    li       t7, 0x00ff00ff
+    beqz     a3, 3f
+     nop
+    addiu    t1, a3, -1
+    beqz     t1, 2f
+     nop
+1:
+    lhu      t0, 0(a1) /* t0 = source      (r5g6b5) */
+    lhu      t1, 2(a1) /* t1 = source      (r5g6b5) */
+    lbu      t2, 0(a2) /* t2 = mask        (a8) */
+    lbu      t3, 1(a2) /* t3 = mask        (a8) */
+    lhu      t8, 0(a0) /* t8 = destination (r5g6b5) */
+    lhu      t9, 2(a0) /* t9 = destination (r5g6b5) */
+    addiu    a1, a1, 4
+    addiu    a2, a2, 2
+
+    CONVERT_2x0565_TO_2x8888 t0, t1, s0, s1, t5, t6, s2, s3, s4, s5
+    CONVERT_2x0565_TO_2x8888 t8, t9, s2, s3, t5, t6, s4, s5, t0, t1
+    OVER_2x8888_2x8_2x8888   s0, s1, t2, t3, s2, s3, \
+                             t0, t1, t7, s4, s5, t8, t9, s0, s1
+    CONVERT_2x8888_TO_2x0565 t0, t1, s0, s1, t4, t5, t6, s2, s3
+
+    sh       s0, 0(a0)
+    sh       s1, 2(a0)
+    addiu    a3, a3, -2
+    addiu    t1, a3, -1
+    bgtz     t1, 1b
+     addiu   a0, a0, 4
+2:
+    beqz     a3, 3f
+     nop
+    lhu      t0, 0(a1) /* t0 = source      (r5g6b5) */
+    lbu      t1, 0(a2) /* t1 = mask        (a8) */
+    lhu      t2, 0(a0) /* t2 = destination (r5g6b5) */
+
+    CONVERT_1x0565_TO_1x8888 t0, t3, t4, t5
+    CONVERT_1x0565_TO_1x8888 t2, t4, t5, t6
+    OVER_8888_8_8888         t3, t1, t4, t0, t7, t2, t5, t6, t8
+    CONVERT_1x8888_TO_1x0565 t0, t3, t4, t5
+
+    sh       t3, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 20, s0, s1, s2, s3, s4, s5
+    j        ra
+     nop
+
+END(pixman_composite_over_0565_8_0565_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_over_8888_8888_8888_asm_mips)
+/*
+ * a0 - dst  (a8r8g8b8)
+ * a1 - src  (a8r8g8b8)
+ * a2 - mask (a8r8g8b8)
+ * a3 - w
+ */
+
+    SAVE_REGS_ON_STACK 0, s0, s1, s2
+    li       t4, 0x00ff00ff
+    beqz     a3, 3f
+     nop
+    addiu    t1, a3, -1
+    beqz     t1, 2f
+     nop
+1:
+    lw       t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+    lw       t1, 4(a1) /* t1 = source      (a8r8g8b8) */
+    lw       t2, 0(a2) /* t2 = mask        (a8r8g8b8) */
+    lw       t3, 4(a2) /* t3 = mask        (a8r8g8b8) */
+    lw       t5, 0(a0) /* t5 = destination (a8r8g8b8) */
+    lw       t6, 4(a0) /* t6 = destination (a8r8g8b8) */
+    addiu    a1, a1, 8
+    addiu    a2, a2, 8
+    srl      t2, t2, 24
+    srl      t3, t3, 24
+
+    OVER_2x8888_2x8_2x8888 t0, t1, t2, t3, t5, t6, t7, t8, t4, t9, s0, s1, s2, t0, t1
+
+    sw       t7, 0(a0)
+    sw       t8, 4(a0)
+    addiu    a3, a3, -2
+    addiu    t1, a3, -1
+    bgtz     t1, 1b
+     addiu   a0, a0, 8
+2:
+    beqz     a3, 3f
+     nop
+    lw       t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+    lw       t1, 0(a2) /* t1 = mask        (a8r8g8b8) */
+    lw       t2, 0(a0) /* t2 = destination (a8r8g8b8) */
+    srl      t1, t1, 24
+
+    OVER_8888_8_8888 t0, t1, t2, t3, t4, t5, t6, t7, t8
+
+    sw       t3, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 0, s0, s1, s2
+    j        ra
+     nop
+
+END(pixman_composite_over_8888_8888_8888_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_over_8888_8888_asm_mips)
+/*
+ * a0 - dst  (a8r8g8b8)
+ * a1 - src  (a8r8g8b8)
+ * a2 - w
+ */
+
+    SAVE_REGS_ON_STACK 0, s0, s1, s2
+    li           t4, 0x00ff00ff
+    beqz         a2, 3f
+     nop
+    addiu        t1, a2, -1
+    beqz         t1, 2f
+     nop
+1:
+    lw           t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+    lw           t1, 4(a1) /* t1 = source      (a8r8g8b8) */
+    lw           t2, 0(a0) /* t2 = destination (a8r8g8b8) */
+    lw           t3, 4(a0) /* t3 = destination (a8r8g8b8) */
+    addiu        a1, a1, 8
+
+    not          t5, t0
+    srl          t5, t5, 24
+    not          t6, t1
+    srl          t6, t6, 24
+
+    or           t7, t5, t6
+    beqz         t7, 11f
+     or          t8, t0, t1
+    beqz         t8, 12f
+
+    MIPS_2xUN8x4_MUL_2xUN8 t2, t3, t5, t6, t7, t8, t4, t9, s0, s1, s2, t2, t3
+
+    addu_s.qb    t0, t7, t0
+    addu_s.qb    t1, t8, t1
+11:
+    sw           t0, 0(a0)
+    sw           t1, 4(a0)
+12:
+    addiu        a2, a2, -2
+    addiu        t1, a2, -1
+    bgtz         t1, 1b
+     addiu       a0, a0, 8
+2:
+    beqz         a2, 3f
+     nop
+
+    lw           t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+    lw           t1, 0(a0) /* t1 = destination (a8r8g8b8) */
+    addiu        a1, a1, 4
+
+    not          t2, t0
+    srl          t2, t2, 24
+
+    beqz         t2, 21f
+     nop
+    beqz         t0, 3f
+
+    MIPS_UN8x4_MUL_UN8 t1, t2, t3, t4, t5, t6, t7
+
+    addu_s.qb    t0, t3, t0
+21:
+    sw           t0, 0(a0)
+
+3:
+    RESTORE_REGS_FROM_STACK 0, s0, s1, s2
+    j            ra
+     nop
+
+END(pixman_composite_over_8888_8888_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_over_8888_0565_asm_mips)
+/*
+ * a0 - dst  (r5g6b5)
+ * a1 - src  (a8r8g8b8)
+ * a2 - w
+ */
+
+    SAVE_REGS_ON_STACK 8, s0, s1, s2, s3, s4, s5
+    li           t4, 0x00ff00ff
+    li           s3, 0xf800f800
+    li           s4, 0x07e007e0
+    li           s5, 0x001F001F
+    beqz         a2, 3f
+     nop
+    addiu        t1, a2, -1
+    beqz         t1, 2f
+     nop
+1:
+    lw           t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+    lw           t1, 4(a1) /* t1 = source      (a8r8g8b8) */
+    lhu          t2, 0(a0) /* t2 = destination (r5g6b5) */
+    lhu          t3, 2(a0) /* t3 = destination (r5g6b5) */
+    addiu        a1, a1, 8
+
+    not          t5, t0
+    srl          t5, t5, 24
+    not          t6, t1
+    srl          t6, t6, 24
+
+    or           t7, t5, t6
+    beqz         t7, 11f
+     or          t8, t0, t1
+    beqz         t8, 12f
+
+    CONVERT_2x0565_TO_2x8888 t2, t3, s0, s1, s4, s5, t7, t8, t9, s2
+    MIPS_2xUN8x4_MUL_2xUN8   s0, s1, t5, t6, t7, t8, t4, t9, t2, t3, s2, s0, s1
+
+    addu_s.qb    t0, t7, t0
+    addu_s.qb    t1, t8, t1
+11:
+    CONVERT_2x8888_TO_2x0565 t0, t1, t7, t8, s3, s4, s5, t2, t3
+    sh           t7, 0(a0)
+    sh           t8, 2(a0)
+12:
+    addiu        a2, a2, -2
+    addiu        t1, a2, -1
+    bgtz         t1, 1b
+     addiu       a0, a0, 4
+2:
+    beqz         a2, 3f
+     nop
+
+    lw           t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+    lhu          t1, 0(a0) /* t1 = destination (r5g6b5) */
+    addiu        a1, a1, 4
+
+    not          t2, t0
+    srl          t2, t2, 24
+
+    beqz         t2, 21f
+     nop
+    beqz         t0, 3f
+
+    CONVERT_1x0565_TO_1x8888 t1, s0, t8, t9
+    MIPS_UN8x4_MUL_UN8       s0, t2, t3, t4, t5, t6, t7
+
+    addu_s.qb    t0, t3, t0
+21:
+    CONVERT_1x8888_TO_1x0565 t0, s0, t8, t9
+    sh           s0, 0(a0)
+
+3:
+    RESTORE_REGS_FROM_STACK 8, s0, s1, s2, s3, s4, s5
+    j            ra
+     nop
+
+END(pixman_composite_over_8888_0565_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_over_n_0565_asm_mips)
+/*
+ * a0 - dst  (r5g6b5)
+ * a1 - src  (32bit constant)
+ * a2 - w
+ */
+
+    beqz         a2, 5f
+     nop
+
+    not          t0, a1
+    srl          t0, t0, 24
+    bgtz         t0, 1f
+     nop
+    CONVERT_1x8888_TO_1x0565 a1, t1, t2, t3
+0:
+    sh           t1, 0(a0)
+    addiu        a2, a2, -1
+    bgtz         a2, 0b
+     addiu       a0, a0, 2
+    j            ra
+     nop
+
+1:
+    SAVE_REGS_ON_STACK 0, s0, s1, s2
+    li           t4, 0x00ff00ff
+    li           t5, 0xf800f800
+    li           t6, 0x07e007e0
+    li           t7, 0x001F001F
+    addiu        t1, a2, -1
+    beqz         t1, 3f
+     nop
+2:
+    lhu          t1, 0(a0) /* t1 = destination (r5g6b5) */
+    lhu          t2, 2(a0) /* t2 = destination (r5g6b5) */
+
+    CONVERT_2x0565_TO_2x8888 t1, t2, t3, t8, t6, t7, t9, s0, s1, s2
+    MIPS_2xUN8x4_MUL_2xUN8   t3, t8, t0, t0, t1, t2, t4, t9, s0, s1, s2, t3, t8
+    addu_s.qb                t1, t1, a1
+    addu_s.qb                t2, t2, a1
+    CONVERT_2x8888_TO_2x0565 t1, t2, t3, t8, t5, t6, t7, s0, s1
+
+    sh           t3, 0(a0)
+    sh           t8, 2(a0)
+
+    addiu        a2, a2, -2
+    addiu        t1, a2, -1
+    bgtz         t1, 2b
+     addiu       a0, a0, 4
+3:
+    beqz         a2, 4f
+     nop
+
+    lhu          t1, 0(a0) /* t1 = destination (r5g6b5) */
+
+    CONVERT_1x0565_TO_1x8888 t1, t2, s0, s1
+    MIPS_UN8x4_MUL_UN8       t2, t0, t1, t4, s0, s1, s2
+    addu_s.qb                t1, t1, a1
+    CONVERT_1x8888_TO_1x0565 t1, t2, s0, s1
+
+    sh           t2, 0(a0)
+
+4:
+    RESTORE_REGS_FROM_STACK 0, s0, s1, s2
+5:
+    j            ra
+     nop
+
+END(pixman_composite_over_n_0565_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_over_n_8888_asm_mips)
+/*
+ * a0 - dst  (a8r8g8b8)
+ * a1 - src  (32bit constant)
+ * a2 - w
+ */
+
+    beqz         a2, 5f
+     nop
+
+    not          t0, a1
+    srl          t0, t0, 24
+    bgtz         t0, 1f
+     nop
+0:
+    sw           a1, 0(a0)
+    addiu        a2, a2, -1
+    bgtz         a2, 0b
+     addiu       a0, a0, 4
+    j            ra
+     nop
+
+1:
+    SAVE_REGS_ON_STACK 0, s0, s1, s2
+    li           t4, 0x00ff00ff
+    addiu        t1, a2, -1
+    beqz         t1, 3f
+     nop
+2:
+    lw           t2, 0(a0) /* t2 = destination (a8r8g8b8) */
+    lw           t3, 4(a0) /* t3 = destination (a8r8g8b8) */
+
+    MIPS_2xUN8x4_MUL_2xUN8 t2, t3, t0, t0, t7, t8, t4, t9, s0, s1, s2, t2, t3
+
+    addu_s.qb    t7, t7, a1
+    addu_s.qb    t8, t8, a1
+
+    sw           t7, 0(a0)
+    sw           t8, 4(a0)
+
+    addiu        a2, a2, -2
+    addiu        t1, a2, -1
+    bgtz         t1, 2b
+     addiu       a0, a0, 8
+3:
+    beqz         a2, 4f
+     nop
+
+    lw           t1, 0(a0) /* t1 = destination (a8r8g8b8) */
+
+    MIPS_UN8x4_MUL_UN8 t1, t0, t3, t4, t5, t6, t7
+
+    addu_s.qb    t3, t3, a1
+
+    sw           t3, 0(a0)
+
+4:
+    RESTORE_REGS_FROM_STACK 0, s0, s1, s2
+5:
+    j            ra
+     nop
+
+END(pixman_composite_over_n_8888_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_add_8_8_8_asm_mips)
+/*
+ * a0 - dst  (a8)
+ * a1 - src  (a8)
+ * a2 - mask (a8)
+ * a3 - w
+ */
+
+    SAVE_REGS_ON_STACK 0, v0, v1
+    li                t9, 0x00ff00ff
+    beqz              a3, 3f
+     nop
+
+    srl               v0, a3, 2   /* v0 = how many multiples of 4 dst pixels */
+    beqz              v0, 1f      /* branch if less than 4 src pixels */
+     nop
+
+0:
+    beqz              v0, 1f
+     addiu            v0, v0, -1
+    lbu               t0, 0(a2)
+    lbu               t1, 1(a2)
+    lbu               t2, 2(a2)
+    lbu               t3, 3(a2)
+    lbu               t4, 0(a0)
+    lbu               t5, 1(a0)
+    lbu               t6, 2(a0)
+    lbu               t7, 3(a0)
+
+    addiu             a2, a2, 4
+
+    precr_sra.ph.w    t1, t0, 0
+    precr_sra.ph.w    t3, t2, 0
+    precr_sra.ph.w    t5, t4, 0
+    precr_sra.ph.w    t7, t6, 0
+
+    precr.qb.ph       t0, t3, t1
+    precr.qb.ph       t1, t7, t5
+
+    lbu               t4, 0(a1)
+    lbu               v1, 1(a1)
+    lbu               t7, 2(a1)
+    lbu               t8, 3(a1)
+
+    addiu             a1, a1, 4
+
+    precr_sra.ph.w    v1, t4, 0
+    precr_sra.ph.w    t8, t7, 0
+
+    muleu_s.ph.qbl    t2, t0, t8
+    muleu_s.ph.qbr    t3, t0, v1
+    shra_r.ph         t4, t2, 8
+    shra_r.ph         t5, t3, 8
+    and               t4, t4, t9
+    and               t5, t5, t9
+    addq.ph           t2, t2, t4
+    addq.ph           t3, t3, t5
+    shra_r.ph         t2, t2, 8
+    shra_r.ph         t3, t3, 8
+    precr.qb.ph       t0, t2, t3
+
+    addu_s.qb         t2, t0, t1
+
+    sb                t2, 0(a0)
+    srl               t2, t2, 8
+    sb                t2, 1(a0)
+    srl               t2, t2, 8
+    sb                t2, 2(a0)
+    srl               t2, t2, 8
+    sb                t2, 3(a0)
+    addiu             a3, a3, -4
+    b                 0b
+     addiu            a0, a0, 4
+
+1:
+    beqz              a3, 3f
+     nop
+2:
+    lbu               t8, 0(a1)
+    lbu               t0, 0(a2)
+    lbu               t1, 0(a0)
+    addiu             a1, a1, 1
+    addiu             a2, a2, 1
+
+    mul               t2, t0, t8
+    shra_r.ph         t3, t2, 8
+    andi              t3, t3, 0xff
+    addq.ph           t2, t2, t3
+    shra_r.ph         t2, t2, 8
+    andi              t2, t2, 0xff
+
+    addu_s.qb         t2, t2, t1
+    sb                t2, 0(a0)
+    addiu             a3, a3, -1
+    bnez              a3, 2b
+     addiu            a0, a0, 1
+
+3:
+    RESTORE_REGS_FROM_STACK 0, v0, v1
+    j                 ra
+     nop
+
+END(pixman_composite_add_8_8_8_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_add_n_8_8_asm_mips)
+/*
+ * a0 - dst  (a8)
+ * a1 - src  (32bit constant)
+ * a2 - mask (a8)
+ * a3 - w
+ */
+
+    SAVE_REGS_ON_STACK 0, v0
+    li                t9, 0x00ff00ff
+    beqz              a3, 3f
+     nop
+
+    srl               v0, a3, 2   /* v0 = how many multiples of 4 dst pixels */
+    beqz              v0, 1f      /* branch if less than 4 src pixels */
+     nop
+
+    srl               t8, a1, 24
+    replv.ph          t8, t8
+
+0:
+    beqz              v0, 1f
+     addiu            v0, v0, -1
+    lbu               t0, 0(a2)
+    lbu               t1, 1(a2)
+    lbu               t2, 2(a2)
+    lbu               t3, 3(a2)
+    lbu               t4, 0(a0)
+    lbu               t5, 1(a0)
+    lbu               t6, 2(a0)
+    lbu               t7, 3(a0)
+
+    addiu             a2, a2, 4
+
+    precr_sra.ph.w    t1, t0, 0
+    precr_sra.ph.w    t3, t2, 0
+    precr_sra.ph.w    t5, t4, 0
+    precr_sra.ph.w    t7, t6, 0
+
+    precr.qb.ph       t0, t3, t1
+    precr.qb.ph       t1, t7, t5
+
+    muleu_s.ph.qbl    t2, t0, t8
+    muleu_s.ph.qbr    t3, t0, t8
+    shra_r.ph         t4, t2, 8
+    shra_r.ph         t5, t3, 8
+    and               t4, t4, t9
+    and               t5, t5, t9
+    addq.ph           t2, t2, t4
+    addq.ph           t3, t3, t5
+    shra_r.ph         t2, t2, 8
+    shra_r.ph         t3, t3, 8
+    precr.qb.ph       t0, t2, t3
+
+    addu_s.qb         t2, t0, t1
+
+    sb                t2, 0(a0)
+    srl               t2, t2, 8
+    sb                t2, 1(a0)
+    srl               t2, t2, 8
+    sb                t2, 2(a0)
+    srl               t2, t2, 8
+    sb                t2, 3(a0)
+    addiu             a3, a3, -4
+    b                 0b
+     addiu            a0, a0, 4
+
+1:
+    beqz              a3, 3f
+     nop
+    srl               t8, a1, 24
+2:
+    lbu               t0, 0(a2)
+    lbu               t1, 0(a0)
+    addiu             a2, a2, 1
+
+    mul               t2, t0, t8
+    shra_r.ph         t3, t2, 8
+    andi              t3, t3, 0xff
+    addq.ph           t2, t2, t3
+    shra_r.ph         t2, t2, 8
+    andi              t2, t2, 0xff
+
+    addu_s.qb         t2, t2, t1
+    sb                t2, 0(a0)
+    addiu             a3, a3, -1
+    bnez              a3, 2b
+     addiu            a0, a0, 1
+
+3:
+    RESTORE_REGS_FROM_STACK 0, v0
+    j                 ra
+     nop
+
+END(pixman_composite_add_n_8_8_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_add_n_8_8888_asm_mips)
+/*
+ * a0 - dst  (a8r8g8b8)
+ * a1 - src  (32bit constant)
+ * a2 - mask (a8)
+ * a3 - w
+ */
+
+    SAVE_REGS_ON_STACK 0, s0, s1, s2
+    li       t4, 0x00ff00ff
+    beqz     a3, 3f
+     nop
+    addiu    t1, a3, -1
+    beqz     t1, 2f
+     nop
+1:
+                       /* a1 = source      (32bit constant) */
+    lbu      t0, 0(a2) /* t0 = mask        (a8) */
+    lbu      t1, 1(a2) /* t1 = mask        (a8) */
+    lw       t2, 0(a0) /* t2 = destination (a8r8g8b8) */
+    lw       t3, 4(a0) /* t3 = destination (a8r8g8b8) */
+    addiu    a2, a2, 2
+
+    MIPS_2xUN8x4_MUL_2xUN8_ADD_2xUN8x4 a1, a1, \
+                                       t0, t1, \
+                                       t2, t3, \
+                                       t5, t6, \
+                                       t4, t7, t8, t9, s0, s1, s2
+
+    sw       t5, 0(a0)
+    sw       t6, 4(a0)
+    addiu    a3, a3, -2
+    addiu    t1, a3, -1
+    bgtz     t1, 1b
+     addiu   a0, a0, 8
+2:
+    beqz     a3, 3f
+     nop
+                       /* a1 = source      (32bit constant) */
+    lbu      t0, 0(a2) /* t0 = mask        (a8) */
+    lw       t1, 0(a0) /* t1 = destination (a8r8g8b8) */
+
+    MIPS_UN8x4_MUL_UN8_ADD_UN8x4 a1, t0, t1, t2, t4, t3, t5, t6
+
+    sw       t2, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 0, s0, s1, s2
+    j        ra
+     nop
+
+END(pixman_composite_add_n_8_8888_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_add_0565_8_0565_asm_mips)
+/*
+ * a0 - dst  (r5g6b5)
+ * a1 - src  (r5g6b5)
+ * a2 - mask (a8)
+ * a3 - w
+ */
+
+    SAVE_REGS_ON_STACK 20, s0, s1, s2, s3, s4, s5, s6, s7
+    li       t4, 0xf800f800
+    li       t5, 0x07e007e0
+    li       t6, 0x001F001F
+    li       t7, 0x00ff00ff
+    beqz     a3, 3f
+     nop
+    addiu    t1, a3, -1
+    beqz     t1, 2f
+     nop
+1:
+    lhu      t0, 0(a1) /* t0 = source      (r5g6b5) */
+    lhu      t1, 2(a1) /* t1 = source      (r5g6b5) */
+    lbu      t2, 0(a2) /* t2 = mask        (a8) */
+    lbu      t3, 1(a2) /* t3 = mask        (a8) */
+    lhu      t8, 0(a0) /* t8 = destination (r5g6b5) */
+    lhu      t9, 2(a0) /* t9 = destination (r5g6b5) */
+    addiu    a1, a1, 4
+    addiu    a2, a2, 2
+
+    CONVERT_2x0565_TO_2x8888  t0, t1, s0, s1, t5, t6, s2, s3, s4, s5
+    CONVERT_2x0565_TO_2x8888  t8, t9, s2, s3, t5, t6, s4, s5, s6, s7
+    MIPS_2xUN8x4_MUL_2xUN8_ADD_2xUN8x4  s0, s1, \
+                                        t2, t3, \
+                                        s2, s3, \
+                                        t0, t1, \
+                                        t7, s4, s5, s6, s7, t8, t9
+    CONVERT_2x8888_TO_2x0565  t0, t1, s0, s1, t4, t5, t6, s2, s3
+
+    sh       s0, 0(a0)
+    sh       s1, 2(a0)
+    addiu    a3, a3, -2
+    addiu    t1, a3, -1
+    bgtz     t1, 1b
+     addiu   a0, a0, 4
+2:
+    beqz     a3, 3f
+     nop
+    lhu      t0, 0(a1) /* t0 = source      (r5g6b5) */
+    lbu      t1, 0(a2) /* t1 = mask        (a8) */
+    lhu      t2, 0(a0) /* t2 = destination (r5g6b5) */
+
+    CONVERT_1x0565_TO_1x8888  t0, t3, t4, t5
+    CONVERT_1x0565_TO_1x8888  t2, t4, t5, t6
+    MIPS_UN8x4_MUL_UN8_ADD_UN8x4  t3, t1, t4, t0, t7, t2, t5, t6
+    CONVERT_1x8888_TO_1x0565  t0, t3, t4, t5
+
+    sh       t3, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 20, s0, s1, s2, s3, s4, s5, s6, s7
+    j        ra
+     nop
+
+END(pixman_composite_add_0565_8_0565_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_add_8888_8_8888_asm_mips)
+/*
+ * a0 - dst  (a8r8g8b8)
+ * a1 - src  (a8r8g8b8)
+ * a2 - mask (a8)
+ * a3 - w
+ */
+
+    SAVE_REGS_ON_STACK 0, s0, s1, s2
+    li       t4, 0x00ff00ff
+    beqz     a3, 3f
+     nop
+    addiu    t1, a3, -1
+    beqz     t1, 2f
+     nop
+1:
+    lw       t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+    lw       t1, 4(a1) /* t1 = source      (a8r8g8b8) */
+    lbu      t2, 0(a2) /* t2 = mask        (a8) */
+    lbu      t3, 1(a2) /* t3 = mask        (a8) */
+    lw       t5, 0(a0) /* t5 = destination (a8r8g8b8) */
+    lw       t6, 4(a0) /* t6 = destination (a8r8g8b8) */
+    addiu    a1, a1, 8
+    addiu    a2, a2, 2
+
+    MIPS_2xUN8x4_MUL_2xUN8_ADD_2xUN8x4 t0, t1, \
+                                       t2, t3, \
+                                       t5, t6, \
+                                       t7, t8, \
+                                       t4, t9, s0, s1, s2, t0, t1
+
+    sw       t7, 0(a0)
+    sw       t8, 4(a0)
+    addiu    a3, a3, -2
+    addiu    t1, a3, -1
+    bgtz     t1, 1b
+     addiu   a0, a0, 8
+2:
+    beqz     a3, 3f
+     nop
+    lw       t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+    lbu      t1, 0(a2) /* t1 = mask        (a8) */
+    lw       t2, 0(a0) /* t2 = destination (a8r8g8b8) */
+
+    MIPS_UN8x4_MUL_UN8_ADD_UN8x4 t0, t1, t2, t3, t4, t5, t6, t7
+
+    sw       t3, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 0, s0, s1, s2
+    j        ra
+     nop
+
+END(pixman_composite_add_8888_8_8888_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_add_8888_n_8888_asm_mips)
+/*
+ * a0 - dst  (a8r8g8b8)
+ * a1 - src  (a8r8g8b8)
+ * a2 - mask (32bit constant)
+ * a3 - w
+ */
+
+    SAVE_REGS_ON_STACK 0, s0, s1, s2
+    li       t4, 0x00ff00ff
+    beqz     a3, 3f
+     nop
+    srl      a2, a2, 24
+    addiu    t1, a3, -1
+    beqz     t1, 2f
+     nop
+1:
+    lw       t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+    lw       t1, 4(a1) /* t1 = source      (a8r8g8b8) */
+                       /* a2 = mask        (32bit constant) */
+    lw       t2, 0(a0) /* t2 = destination (a8r8g8b8) */
+    lw       t3, 4(a0) /* t3 = destination (a8r8g8b8) */
+    addiu    a1, a1, 8
+
+    MIPS_2xUN8x4_MUL_2xUN8_ADD_2xUN8x4 t0, t1, \
+                                       a2, a2, \
+                                       t2, t3, \
+                                       t5, t6, \
+                                       t4, t7, t8, t9, s0, s1, s2
+
+    sw       t5, 0(a0)
+    sw       t6, 4(a0)
+    addiu    a3, a3, -2
+    addiu    t1, a3, -1
+    bgtz     t1, 1b
+     addiu   a0, a0, 8
+2:
+    beqz     a3, 3f
+     nop
+    lw       t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+                       /* a2 = mask        (32bit constant) */
+    lw       t1, 0(a0) /* t1 = destination (a8r8g8b8) */
+
+    MIPS_UN8x4_MUL_UN8_ADD_UN8x4 t0, a2, t1, t3, t4, t5, t6, t7
+
+    sw       t3, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 0, s0, s1, s2
+    j        ra
+     nop
+
+END(pixman_composite_add_8888_n_8888_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_add_8888_8888_8888_asm_mips)
+/*
+ * a0 - dst  (a8r8g8b8)
+ * a1 - src  (a8r8g8b8)
+ * a2 - mask (a8r8g8b8)
+ * a3 - w
+ */
+
+    SAVE_REGS_ON_STACK 0, s0, s1, s2
+    li       t4, 0x00ff00ff
+    beqz     a3, 3f
+     nop
+    addiu    t1, a3, -1
+    beqz     t1, 2f
+     nop
+1:
+    lw       t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+    lw       t1, 4(a1) /* t1 = source      (a8r8g8b8) */
+    lw       t2, 0(a2) /* t2 = mask        (a8r8g8b8) */
+    lw       t3, 4(a2) /* t3 = mask        (a8r8g8b8) */
+    lw       t5, 0(a0) /* t5 = destination (a8r8g8b8) */
+    lw       t6, 4(a0) /* t6 = destination (a8r8g8b8) */
+    addiu    a1, a1, 8
+    addiu    a2, a2, 8
+    srl      t2, t2, 24
+    srl      t3, t3, 24
+
+    MIPS_2xUN8x4_MUL_2xUN8_ADD_2xUN8x4 t0, t1, \
+                                       t2, t3, \
+                                       t5, t6, \
+                                       t7, t8, \
+                                       t4, t9, s0, s1, s2, t0, t1
+
+    sw       t7, 0(a0)
+    sw       t8, 4(a0)
+    addiu    a3, a3, -2
+    addiu    t1, a3, -1
+    bgtz     t1, 1b
+     addiu   a0, a0, 8
+2:
+    beqz     a3, 3f
+     nop
+    lw       t0, 0(a1) /* t0 = source      (a8r8g8b8) */
+    lw       t1, 0(a2) /* t1 = mask        (a8r8g8b8) */
+    lw       t2, 0(a0) /* t2 = destination (a8r8g8b8) */
+    srl      t1, t1, 24
+
+    MIPS_UN8x4_MUL_UN8_ADD_UN8x4 t0, t1, t2, t3, t4, t5, t6, t7
+
+    sw       t3, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 0, s0, s1, s2
+    j        ra
+     nop
+
+END(pixman_composite_add_8888_8888_8888_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_add_8_8_asm_mips)
+/*
+ * a0 - dst  (a8)
+ * a1 - src  (a8)
+ * a2 - w
+ */
+
+    beqz              a2, 3f
+     nop
+    srl               t9, a2, 2   /* t9 = how many multiples of 4 dst pixels */
+    beqz              t9, 1f      /* branch if less than 4 src pixels */
+     nop
+
+0:
+    beqz              t9, 1f
+     addiu            t9, t9, -1
+    lbu               t0, 0(a1)
+    lbu               t1, 1(a1)
+    lbu               t2, 2(a1)
+    lbu               t3, 3(a1)
+    lbu               t4, 0(a0)
+    lbu               t5, 1(a0)
+    lbu               t6, 2(a0)
+    lbu               t7, 3(a0)
+
+    addiu             a1, a1, 4
+
+    precr_sra.ph.w    t1, t0, 0
+    precr_sra.ph.w    t3, t2, 0
+    precr_sra.ph.w    t5, t4, 0
+    precr_sra.ph.w    t7, t6, 0
+
+    precr.qb.ph       t0, t3, t1
+    precr.qb.ph       t1, t7, t5
+
+    addu_s.qb         t2, t0, t1
+
+    sb                t2, 0(a0)
+    srl               t2, t2, 8
+    sb                t2, 1(a0)
+    srl               t2, t2, 8
+    sb                t2, 2(a0)
+    srl               t2, t2, 8
+    sb                t2, 3(a0)
+    addiu             a2, a2, -4
+    b                 0b
+     addiu            a0, a0, 4
+
+1:
+    beqz              a2, 3f
+     nop
+2:
+    lbu               t0, 0(a1)
+    lbu               t1, 0(a0)
+    addiu             a1, a1, 1
+
+    addu_s.qb         t2, t0, t1
+    sb                t2, 0(a0)
+    addiu             a2, a2, -1
+    bnez              a2, 2b
+     addiu            a0, a0, 1
+
+3:
+    j                 ra
+     nop
+
+END(pixman_composite_add_8_8_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_add_8888_8888_asm_mips)
+/*
+ * a0 - dst (a8r8g8b8)
+ * a1 - src (a8r8g8b8)
+ * a2 - w
+ */
+
+    beqz         a2, 4f
+     nop
+
+    srl          t9, a2, 2      /* t1 = how many multiples of 4 src pixels */
+    beqz         t9, 3f         /* branch if less than 4 src pixels */
+     nop
+1:
+    addiu        t9, t9, -1
+    beqz         t9, 2f
+     addiu       a2, a2, -4
+
+    lw           t0, 0(a1)
+    lw           t1, 4(a1)
+    lw           t2, 8(a1)
+    lw           t3, 12(a1)
+    lw           t4, 0(a0)
+    lw           t5, 4(a0)
+    lw           t6, 8(a0)
+    lw           t7, 12(a0)
+    addiu        a1, a1, 16
+
+    addu_s.qb    t4, t4, t0
+    addu_s.qb    t5, t5, t1
+    addu_s.qb    t6, t6, t2
+    addu_s.qb    t7, t7, t3
+
+    sw           t4, 0(a0)
+    sw           t5, 4(a0)
+    sw           t6, 8(a0)
+    sw           t7, 12(a0)
+    b            1b
+     addiu       a0, a0, 16
+2:
+    lw           t0, 0(a1)
+    lw           t1, 4(a1)
+    lw           t2, 8(a1)
+    lw           t3, 12(a1)
+    lw           t4, 0(a0)
+    lw           t5, 4(a0)
+    lw           t6, 8(a0)
+    lw           t7, 12(a0)
+    addiu        a1, a1, 16
+
+    addu_s.qb    t4, t4, t0
+    addu_s.qb    t5, t5, t1
+    addu_s.qb    t6, t6, t2
+    addu_s.qb    t7, t7, t3
+
+    sw           t4, 0(a0)
+    sw           t5, 4(a0)
+    sw           t6, 8(a0)
+    sw           t7, 12(a0)
+
+    beqz         a2, 4f
+     addiu       a0, a0, 16
+3:
+    lw           t0, 0(a1)
+    lw           t1, 0(a0)
+    addiu        a1, a1, 4
+    addiu        a2, a2, -1
+    addu_s.qb    t1, t1, t0
+    sw           t1, 0(a0)
+    bnez         a2, 3b
+     addiu       a0, a0, 4
+4:
+    jr           ra
+     nop
+
+END(pixman_composite_add_8888_8888_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_out_reverse_8_0565_asm_mips)
+/*
+ * a0 - dst  (r5g6b5)
+ * a1 - src  (a8)
+ * a2 - w
+ */
+
+    beqz     a2, 4f
+     nop
+
+    SAVE_REGS_ON_STACK 0, s0, s1, s2, s3
+    li       t2, 0xf800f800
+    li       t3, 0x07e007e0
+    li       t4, 0x001F001F
+    li       t5, 0x00ff00ff
+
+    addiu    t1, a2, -1
+    beqz     t1, 2f
+     nop
+1:
+    lbu      t0, 0(a1) /* t0 = source      (a8) */
+    lbu      t1, 1(a1) /* t1 = source      (a8) */
+    lhu      t6, 0(a0) /* t6 = destination (r5g6b5) */
+    lhu      t7, 2(a0) /* t7 = destination (r5g6b5) */
+    addiu    a1, a1, 2
+
+    not      t0, t0
+    not      t1, t1
+    andi     t0, 0xff  /* t0 = neg source1 */
+    andi     t1, 0xff  /* t1 = neg source2 */
+    CONVERT_2x0565_TO_2x8888 t6, t7, t8, t9, t3, t4, s0, s1, s2, s3
+    MIPS_2xUN8x4_MUL_2xUN8   t8, t9, t0, t1, t6, t7, t5, s0, s1, s2, s3, t8, t9
+    CONVERT_2x8888_TO_2x0565 t6, t7, t8, t9, t2, t3, t4, s0, s1
+
+    sh       t8, 0(a0)
+    sh       t9, 2(a0)
+    addiu    a2, a2, -2
+    addiu    t1, a2, -1
+    bgtz     t1, 1b
+     addiu   a0, a0, 4
+2:
+    beqz     a2, 3f
+     nop
+    lbu      t0, 0(a1) /* t0 = source      (a8) */
+    lhu      t1, 0(a0) /* t1 = destination (r5g6b5) */
+
+    not      t0, t0
+    andi     t0, 0xff  /* t0 = neg source */
+    CONVERT_1x0565_TO_1x8888 t1, t2, t3, t4
+    MIPS_UN8x4_MUL_UN8        t2, t0, t1, t5, t3, t4, t6
+    CONVERT_1x8888_TO_1x0565 t1, t2, t3, t4
+
+    sh       t2, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 0, s0, s1, s2, s3
+4:
+    j        ra
+     nop
+
+END(pixman_composite_out_reverse_8_0565_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_out_reverse_8_8888_asm_mips)
+/*
+ * a0 - dst  (a8r8g8b8)
+ * a1 - src  (a8)
+ * a2 - w
+ */
+
+    beqz     a2, 3f
+     nop
+    li       t4, 0x00ff00ff
+    addiu    t1, a2, -1
+    beqz     t1, 2f
+     nop
+1:
+    lbu      t0, 0(a1) /* t0 = source      (a8) */
+    lbu      t1, 1(a1) /* t1 = source      (a8) */
+    lw       t2, 0(a0) /* t2 = destination (a8r8g8b8) */
+    lw       t3, 4(a0) /* t3 = destination (a8r8g8b8) */
+    addiu    a1, a1, 2
+    not      t0, t0
+    not      t1, t1
+    andi     t0, 0xff  /* t0 = neg source */
+    andi     t1, 0xff  /* t1 = neg source */
+
+    MIPS_2xUN8x4_MUL_2xUN8 t2, t3, t0, t1, t5, t6, t4, t7, t8, t9, t2, t3, t0
+
+    sw       t5, 0(a0)
+    sw       t6, 4(a0)
+    addiu    a2, a2, -2
+    addiu    t1, a2, -1
+    bgtz     t1, 1b
+     addiu   a0, a0, 8
+2:
+    beqz     a2, 3f
+     nop
+    lbu      t0, 0(a1) /* t0 = source      (a8) */
+    lw       t1, 0(a0) /* t1 = destination (a8r8g8b8) */
+    not      t0, t0
+    andi     t0, 0xff  /* t0 = neg source */
+
+    MIPS_UN8x4_MUL_UN8 t1, t0, t2, t4, t3, t5, t6
+
+    sw       t2, 0(a0)
+3:
+    j        ra
+     nop
+
+END(pixman_composite_out_reverse_8_8888_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_over_reverse_n_8888_asm_mips)
+/*
+ * a0 - dst  (a8r8g8b8)
+ * a1 - src  (32bit constant)
+ * a2 - w
+ */
+
+    beqz              a2, 5f
+     nop
+
+    SAVE_REGS_ON_STACK 20, s0, s1, s2, s3, s4, s5, s6, s7
+    li                t0, 0x00ff00ff
+    srl               t9, a2, 2   /* t9 = how many multiples of 4 src pixels */
+    beqz              t9, 2f      /* branch if less than 4 src pixels */
+     nop
+1:
+    beqz              t9, 2f
+     addiu            t9, t9, -1
+
+    lw                t1, 0(a0)
+    lw                t2, 4(a0)
+    lw                t3, 8(a0)
+    lw                t4, 12(a0)
+
+    addiu             a2, a2, -4
+
+    not               t5, t1
+    not               t6, t2
+    not               t7, t3
+    not               t8, t4
+    srl               t5, t5, 24
+    srl               t6, t6, 24
+    srl               t7, t7, 24
+    srl               t8, t8, 24
+    replv.ph          t5, t5
+    replv.ph          t6, t6
+    replv.ph          t7, t7
+    replv.ph          t8, t8
+    muleu_s.ph.qbl    s0, a1, t5
+    muleu_s.ph.qbr    s1, a1, t5
+    muleu_s.ph.qbl    s2, a1, t6
+    muleu_s.ph.qbr    s3, a1, t6
+    muleu_s.ph.qbl    s4, a1, t7
+    muleu_s.ph.qbr    s5, a1, t7
+    muleu_s.ph.qbl    s6, a1, t8
+    muleu_s.ph.qbr    s7, a1, t8
+
+    shra_r.ph         t5, s0, 8
+    shra_r.ph         t6, s1, 8
+    shra_r.ph         t7, s2, 8
+    shra_r.ph         t8, s3, 8
+    and               t5, t5, t0
+    and               t6, t6, t0
+    and               t7, t7, t0
+    and               t8, t8, t0
+    addq.ph           s0, s0, t5
+    addq.ph           s1, s1, t6
+    addq.ph           s2, s2, t7
+    addq.ph           s3, s3, t8
+    shra_r.ph         s0, s0, 8
+    shra_r.ph         s1, s1, 8
+    shra_r.ph         s2, s2, 8
+    shra_r.ph         s3, s3, 8
+    shra_r.ph         t5, s4, 8
+    shra_r.ph         t6, s5, 8
+    shra_r.ph         t7, s6, 8
+    shra_r.ph         t8, s7, 8
+    and               t5, t5, t0
+    and               t6, t6, t0
+    and               t7, t7, t0
+    and               t8, t8, t0
+    addq.ph           s4, s4, t5
+    addq.ph           s5, s5, t6
+    addq.ph           s6, s6, t7
+    addq.ph           s7, s7, t8
+    shra_r.ph         s4, s4, 8
+    shra_r.ph         s5, s5, 8
+    shra_r.ph         s6, s6, 8
+    shra_r.ph         s7, s7, 8
+
+    precr.qb.ph       t5, s0, s1
+    precr.qb.ph       t6, s2, s3
+    precr.qb.ph       t7, s4, s5
+    precr.qb.ph       t8, s6, s7
+    addu_s.qb         t5, t1, t5
+    addu_s.qb         t6, t2, t6
+    addu_s.qb         t7, t3, t7
+    addu_s.qb         t8, t4, t8
+
+    sw                t5, 0(a0)
+    sw                t6, 4(a0)
+    sw                t7, 8(a0)
+    sw                t8, 12(a0)
+    b                 1b
+     addiu            a0, a0, 16
+
+2:
+    beqz              a2, 4f
+     nop
+3:
+    lw                t1, 0(a0)
+
+    not               t2, t1
+    srl               t2, t2, 24
+    replv.ph          t2, t2
+
+    muleu_s.ph.qbl    t4, a1, t2
+    muleu_s.ph.qbr    t5, a1, t2
+    shra_r.ph         t6, t4, 8
+    shra_r.ph         t7, t5, 8
+
+    and               t6,t6,t0
+    and               t7,t7,t0
+
+    addq.ph           t8, t4, t6
+    addq.ph           t9, t5, t7
+
+    shra_r.ph         t8, t8, 8
+    shra_r.ph         t9, t9, 8
+
+    precr.qb.ph       t9, t8, t9
+
+    addu_s.qb         t9, t1, t9
+    sw                t9, 0(a0)
+
+    addiu             a2, a2, -1
+    bnez              a2, 3b
+     addiu            a0, a0, 4
+4:
+    RESTORE_REGS_FROM_STACK 20, s0, s1, s2, s3, s4, s5, s6, s7
+5:
+    j                 ra
+     nop
+
+END(pixman_composite_over_reverse_n_8888_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_composite_in_n_8_asm_mips)
+/*
+ * a0 - dst  (a8)
+ * a1 - src  (32bit constant)
+ * a2 - w
+ */
+
+    li                t9, 0x00ff00ff
+    beqz              a2, 3f
+     nop
+    srl               t7, a2, 2   /* t7 = how many multiples of 4 dst pixels */
+    beqz              t7, 1f      /* branch if less than 4 src pixels */
+     nop
+
+    srl               t8, a1, 24
+    replv.ph          t8, t8
+
+0:
+    beqz              t7, 1f
+     addiu            t7, t7, -1
+    lbu               t0, 0(a0)
+    lbu               t1, 1(a0)
+    lbu               t2, 2(a0)
+    lbu               t3, 3(a0)
+
+    precr_sra.ph.w    t1, t0, 0
+    precr_sra.ph.w    t3, t2, 0
+    precr.qb.ph       t0, t3, t1
+
+    muleu_s.ph.qbl    t2, t0, t8
+    muleu_s.ph.qbr    t3, t0, t8
+    shra_r.ph         t4, t2, 8
+    shra_r.ph         t5, t3, 8
+    and               t4, t4, t9
+    and               t5, t5, t9
+    addq.ph           t2, t2, t4
+    addq.ph           t3, t3, t5
+    shra_r.ph         t2, t2, 8
+    shra_r.ph         t3, t3, 8
+    precr.qb.ph       t2, t2, t3
+
+    sb                t2, 0(a0)
+    srl               t2, t2, 8
+    sb                t2, 1(a0)
+    srl               t2, t2, 8
+    sb                t2, 2(a0)
+    srl               t2, t2, 8
+    sb                t2, 3(a0)
+    addiu             a2, a2, -4
+    b                 0b
+     addiu            a0, a0, 4
+
+1:
+    beqz              a2, 3f
+     nop
+    srl               t8, a1, 24
+2:
+    lbu               t0, 0(a0)
+
+    mul               t2, t0, t8
+    shra_r.ph         t3, t2, 8
+    andi              t3, t3, 0x00ff
+    addq.ph           t2, t2, t3
+    shra_r.ph         t2, t2, 8
+
+    sb                t2, 0(a0)
+    addiu             a2, a2, -1
+    bnez              a2, 2b
+     addiu            a0, a0, 1
+
+3:
+    j                 ra
+     nop
+
+END(pixman_composite_in_n_8_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_scaled_nearest_scanline_8888_8888_OVER_asm_mips)
+/*
+ * a0     - dst  (a8r8g8b8)
+ * a1     - src  (a8r8g8b8)
+ * a2     - w
+ * a3     - vx
+ * 16(sp) - unit_x
+ */
+
+    SAVE_REGS_ON_STACK 0, s0, s1, s2, s3
+    lw       t8, 16(sp) /* t8 = unit_x */
+    li       t6, 0x00ff00ff
+    beqz     a2, 3f
+     nop
+    addiu    t1, a2, -1
+    beqz     t1, 2f
+     nop
+1:
+    sra      t0, a3, 16 /* t0 = vx >> 16 */
+    sll      t0, t0, 2  /* t0 = t0 * 4 (a8r8g8b8) */
+    addu     t0, a1, t0
+    lw       t0, 0(t0)  /* t0 = source      (a8r8g8b8) */
+    addu     a3, a3, t8 /* a3 = vx + unit_x */
+
+    sra      t1, a3, 16 /* t0 = vx >> 16 */
+    sll      t1, t1, 2  /* t0 = t0 * 4 (a8r8g8b8) */
+    addu     t1, a1, t1
+    lw       t1, 0(t1)  /* t1 = source      (a8r8g8b8) */
+    addu     a3, a3, t8 /* a3 = vx + unit_x */
+
+    lw       t2, 0(a0)  /* t2 = destination (a8r8g8b8) */
+    lw       t3, 4(a0)  /* t3 = destination (a8r8g8b8) */
+
+    OVER_2x8888_2x8888 t0, t1, t2, t3, t4, t5, t6, t7, t9, s0, s1, s2, s3
+
+    sw       t4, 0(a0)
+    sw       t5, 4(a0)
+    addiu    a2, a2, -2
+    addiu    t1, a2, -1
+    bgtz     t1, 1b
+     addiu   a0, a0, 8
+2:
+    beqz     a2, 3f
+     nop
+    sra      t0, a3, 16 /* t0 = vx >> 16 */
+    sll      t0, t0, 2  /* t0 = t0 * 4 (a8r8g8b8) */
+    addu     t0, a1, t0
+    lw       t0, 0(t0)  /* t0 = source      (a8r8g8b8) */
+    lw       t1, 0(a0)  /* t1 = destination (a8r8g8b8) */
+    addu     a3, a3, t8 /* a3 = vx + unit_x */
+
+    OVER_8888_8888 t0, t1, t2, t6, t4, t5, t3, t7
+
+    sw       t2, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 0, s0, s1, s2, s3
+    j        ra
+     nop
+
+END(pixman_scaled_nearest_scanline_8888_8888_OVER_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_scaled_nearest_scanline_8888_0565_OVER_asm_mips)
+/*
+ * a0     - dst  (r5g6b5)
+ * a1     - src  (a8r8g8b8)
+ * a2     - w
+ * a3     - vx
+ * 16(sp) - unit_x
+ */
+
+    SAVE_REGS_ON_STACK 24, s0, s1, s2, s3, s4, v0, v1
+    lw       t8, 40(sp) /* t8 = unit_x */
+    li       t4, 0x00ff00ff
+    li       t5, 0xf800f800
+    li       t6, 0x07e007e0
+    li       t7, 0x001F001F
+    beqz     a2, 3f
+     nop
+    addiu    t1, a2, -1
+    beqz     t1, 2f
+     nop
+1:
+    sra      t0, a3, 16 /* t0 = vx >> 16 */
+    sll      t0, t0, 2  /* t0 = t0 * 4 (a8r8g8b8) */
+    addu     t0, a1, t0
+    lw       t0, 0(t0)  /* t0 = source      (a8r8g8b8) */
+    addu     a3, a3, t8 /* a3 = vx + unit_x */
+    sra      t1, a3, 16 /* t0 = vx >> 16 */
+    sll      t1, t1, 2  /* t0 = t0 * 4 (a8r8g8b8) */
+    addu     t1, a1, t1
+    lw       t1, 0(t1)  /* t1 = source      (a8r8g8b8) */
+    addu     a3, a3, t8 /* a3 = vx + unit_x */
+    lhu      t2, 0(a0)  /* t2 = destination (r5g6b5) */
+    lhu      t3, 2(a0)  /* t3 = destination (r5g6b5) */
+
+    CONVERT_2x0565_TO_2x8888 t2, t3, v0, v1, t6, t7, s0, s1, s2, s3
+    OVER_2x8888_2x8888       t0, t1, v0, v1, t2, t3, t4, t9, s0, s1, s2, s3, s4
+    CONVERT_2x8888_TO_2x0565 t2, t3, v0, v1, t5, t6, t7, t9, s2
+
+    sh       v0, 0(a0)
+    sh       v1, 2(a0)
+    addiu    a2, a2, -2
+    addiu    t1, a2, -1
+    bgtz     t1, 1b
+     addiu   a0, a0, 4
+2:
+    beqz     a2, 3f
+     nop
+    sra      t0, a3, 16 /* t0 = vx >> 16 */
+    sll      t0, t0, 2  /* t0 = t0 * 4 (a8r8g8b8) */
+    addu     t0, a1, t0
+    lw       t0, 0(t0)  /* t0 = source      (a8r8g8b8) */
+    lhu      t1, 0(a0)  /* t1 = destination (r5g6b5) */
+    addu     a3, a3, t8 /* a3 = vx + unit_x */
+
+    CONVERT_1x0565_TO_1x8888 t1, t2, t5, t6
+    OVER_8888_8888           t0, t2, t1, t4, t3, t5, t6, t7
+    CONVERT_1x8888_TO_1x0565 t1, t2, t5, t6
+
+    sh       t2, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 24, s0, s1, s2, s3, s4, v0, v1
+    j        ra
+     nop
+
+END(pixman_scaled_nearest_scanline_8888_0565_OVER_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_scaled_nearest_scanline_0565_8888_SRC_asm_mips)
+/*
+ * a0     - dst (a8r8g8b8)
+ * a1     - src (r5g6b5)
+ * a2     - w
+ * a3     - vx
+ * 16(sp) - unit_x
+ */
+
+    SAVE_REGS_ON_STACK 0, v0
+    beqz     a2, 3f
+     nop
+
+    lw       v0, 16(sp) /* v0 = unit_x */
+    addiu    t1, a2, -1
+    beqz     t1, 2f
+     nop
+
+    li       t4, 0x07e007e0
+    li       t5, 0x001F001F
+1:
+    sra      t0, a3, 16 /* t0 = vx >> 16 */
+    sll      t0, t0, 1  /* t0 = t0 * 2 ((r5g6b5)) */
+    addu     t0, a1, t0
+    lhu      t0, 0(t0)  /* t0 = source ((r5g6b5)) */
+    addu     a3, a3, v0 /* a3 = vx + unit_x */
+    sra      t1, a3, 16 /* t1 = vx >> 16 */
+    sll      t1, t1, 1  /* t1 = t1 * 2 ((r5g6b5)) */
+    addu     t1, a1, t1
+    lhu      t1, 0(t1)  /* t1 = source ((r5g6b5)) */
+    addu     a3, a3, v0 /* a3 = vx + unit_x */
+    addiu    a2, a2, -2
+
+    CONVERT_2x0565_TO_2x8888 t0, t1, t2, t3, t4, t5, t6, t7, t8, t9
+
+    sw       t2, 0(a0)
+    sw       t3, 4(a0)
+
+    addiu    t2, a2, -1
+    bgtz     t2, 1b
+     addiu   a0, a0, 8
+2:
+    beqz     a2, 3f
+     nop
+    sra      t0, a3, 16 /* t0 = vx >> 16 */
+    sll      t0, t0, 1  /* t0 = t0 * 2 ((r5g6b5)) */
+    addu     t0, a1, t0
+    lhu      t0, 0(t0)  /* t0 = source ((r5g6b5)) */
+
+    CONVERT_1x0565_TO_1x8888 t0, t1, t2, t3
+
+    sw       t1, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 0, v0
+    j        ra
+     nop
+
+END(pixman_scaled_nearest_scanline_0565_8888_SRC_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_scaled_nearest_scanline_8888_8_0565_OVER_asm_mips)
+/*
+ * a0     - dst  (r5g6b5)
+ * a1     - src  (a8r8g8b8)
+ * a2     - mask (a8)
+ * a3     - w
+ * 16(sp) - vx
+ * 20(sp) - unit_x
+ */
+    beqz     a3, 4f
+     nop
+
+    SAVE_REGS_ON_STACK 20, v0, v1, s0, s1, s2, s3, s4, s5
+    lw       v0, 36(sp) /* v0 = vx */
+    lw       v1, 40(sp) /* v1 = unit_x */
+    li       t6, 0x00ff00ff
+    li       t7, 0xf800f800
+    li       t8, 0x07e007e0
+    li       t9, 0x001F001F
+
+    addiu    t1, a3, -1
+    beqz     t1, 2f
+     nop
+1:
+    sra      t0, v0, 16 /* t0 = vx >> 16 */
+    sll      t0, t0, 2  /* t0 = t0 * 4      (a8r8g8b8) */
+    addu     t0, a1, t0
+    lw       t0, 0(t0)  /* t0 = source      (a8r8g8b8) */
+    addu     v0, v0, v1 /* v0 = vx + unit_x */
+    sra      t1, v0, 16 /* t1 = vx >> 16 */
+    sll      t1, t1, 2  /* t1 = t1 * 4      (a8r8g8b8) */
+    addu     t1, a1, t1
+    lw       t1, 0(t1)  /* t1 = source      (a8r8g8b8) */
+    addu     v0, v0, v1 /* v0 = vx + unit_x */
+    lbu      t2, 0(a2)  /* t2 = mask        (a8) */
+    lbu      t3, 1(a2)  /* t3 = mask        (a8) */
+    lhu      t4, 0(a0)  /* t4 = destination (r5g6b5) */
+    lhu      t5, 2(a0)  /* t5 = destination (r5g6b5) */
+    addiu    a2, a2, 2
+
+    CONVERT_2x0565_TO_2x8888 t4, t5, s0, s1, t8, t9, s2, s3, s4, s5
+    OVER_2x8888_2x8_2x8888   t0, t1, \
+                             t2, t3, \
+                             s0, s1, \
+                             t4, t5, \
+                             t6, s2, s3, s4, s5, t2, t3
+    CONVERT_2x8888_TO_2x0565 t4, t5, s0, s1, t7, t8, t9, s2, s3
+
+    sh       s0, 0(a0)
+    sh       s1, 2(a0)
+    addiu    a3, a3, -2
+    addiu    t1, a3, -1
+    bgtz     t1, 1b
+     addiu   a0, a0, 4
+2:
+    beqz     a3, 3f
+     nop
+    sra      t0, v0, 16 /* t0 = vx >> 16 */
+    sll      t0, t0, 2  /* t0 = t0 * 4      (a8r8g8b8) */
+    addu     t0, a1, t0
+    lw       t0, 0(t0)  /* t0 = source      (a8r8g8b8) */
+    lbu      t1, 0(a2)  /* t1 = mask        (a8) */
+    lhu      t2, 0(a0)  /* t2 = destination (r5g6b5) */
+
+    CONVERT_1x0565_TO_1x8888 t2, t3, t4, t5
+    OVER_8888_8_8888         t0, t1, t3, t2, t6, t4, t5, t7, t8
+    CONVERT_1x8888_TO_1x0565 t2, t3, t4, t5
+
+    sh       t3, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 20, v0, v1, s0, s1, s2, s3, s4, s5
+4:
+    j        ra
+     nop
+
+END(pixman_scaled_nearest_scanline_8888_8_0565_OVER_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_scaled_nearest_scanline_0565_8_0565_OVER_asm_mips)
+/*
+ * a0     - dst  (r5g6b5)
+ * a1     - src  (r5g6b5)
+ * a2     - mask (a8)
+ * a3     - w
+ * 16(sp) - vx
+ * 20(sp) - unit_x
+ */
+
+    beqz     a3, 4f
+     nop
+    SAVE_REGS_ON_STACK 20, v0, v1, s0, s1, s2, s3, s4, s5
+    lw       v0, 36(sp) /* v0 = vx */
+    lw       v1, 40(sp) /* v1 = unit_x */
+    li       t4, 0xf800f800
+    li       t5, 0x07e007e0
+    li       t6, 0x001F001F
+    li       t7, 0x00ff00ff
+
+    addiu    t1, a3, -1
+    beqz     t1, 2f
+     nop
+1:
+    sra      t0, v0, 16 /* t0 = vx >> 16 */
+    sll      t0, t0, 1  /* t0 = t0 * 2      (r5g6b5) */
+    addu     t0, a1, t0
+    lhu      t0, 0(t0)  /* t0 = source      (r5g6b5) */
+    addu     v0, v0, v1 /* v0 = vx + unit_x */
+    sra      t1, v0, 16 /* t1 = vx >> 16 */
+    sll      t1, t1, 1  /* t1 = t1 * 2      (r5g6b5) */
+    addu     t1, a1, t1
+    lhu      t1, 0(t1)  /* t1 = source      (r5g6b5) */
+    addu     v0, v0, v1 /* v0 = vx + unit_x */
+    lbu      t2, 0(a2)  /* t2 = mask        (a8) */
+    lbu      t3, 1(a2)  /* t3 = mask        (a8) */
+    lhu      t8, 0(a0)  /* t8 = destination (r5g6b5) */
+    lhu      t9, 2(a0)  /* t9 = destination (r5g6b5) */
+    addiu    a2, a2, 2
+
+    CONVERT_2x0565_TO_2x8888 t0, t1, s0, s1, t5, t6, s2, s3, s4, s5
+    CONVERT_2x0565_TO_2x8888 t8, t9, s2, s3, t5, t6, s4, s5, t0, t1
+    OVER_2x8888_2x8_2x8888   s0, s1, \
+                             t2, t3, \
+                             s2, s3, \
+                             t0, t1, \
+                             t7, t8, t9, s4, s5, s0, s1
+    CONVERT_2x8888_TO_2x0565 t0, t1, s0, s1, t4, t5, t6, s2, s3
+
+    sh       s0, 0(a0)
+    sh       s1, 2(a0)
+    addiu    a3, a3, -2
+    addiu    t1, a3, -1
+    bgtz     t1, 1b
+     addiu   a0, a0, 4
+2:
+    beqz     a3, 3f
+     nop
+    sra      t0, v0, 16 /* t0 = vx >> 16 */
+    sll      t0, t0, 1  /* t0 = t0 * 2      (r5g6b5) */
+    addu     t0, a1, t0
+
+    lhu      t0, 0(t0)  /* t0 = source      (r5g6b5) */
+    lbu      t1, 0(a2)  /* t1 = mask        (a8) */
+    lhu      t2, 0(a0)  /* t2 = destination (r5g6b5) */
+
+    CONVERT_1x0565_TO_1x8888 t0, t3, t4, t5
+    CONVERT_1x0565_TO_1x8888 t2, t4, t5, t6
+    OVER_8888_8_8888         t3, t1, t4, t0, t7, t2, t5, t6, t8
+    CONVERT_1x8888_TO_1x0565 t0, t3, t4, t5
+
+    sh       t3, 0(a0)
+3:
+    RESTORE_REGS_FROM_STACK 20, v0, v1, s0, s1, s2, s3, s4, s5
+4:
+    j        ra
+     nop
+
+END(pixman_scaled_nearest_scanline_0565_8_0565_OVER_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_scaled_bilinear_scanline_8888_8888_SRC_asm_mips)
+/*
+ * a0     - *dst
+ * a1     - *src_top
+ * a2     - *src_bottom
+ * a3     - w
+ * 16(sp) - wt
+ * 20(sp) - wb
+ * 24(sp) - vx
+ * 28(sp) - unit_x
+ */
+
+    beqz     a3, 1f
+     nop
+
+    SAVE_REGS_ON_STACK 20, v0, s0, s1, s2, s3, s4, s5, s6, s7
+
+    lw       s0, 36(sp)     /* s0 = wt */
+    lw       s1, 40(sp)     /* s1 = wb */
+    lw       s2, 44(sp)     /* s2 = vx */
+    lw       s3, 48(sp)     /* s3 = unit_x */
+    li       v0, BILINEAR_INTERPOLATION_RANGE
+
+    sll      s0, s0, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+    sll      s1, s1, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+0:
+    andi     t4, s2, 0xffff /* t4 = (short)vx */
+    srl      t4, t4, (16 - BILINEAR_INTERPOLATION_BITS) /* t4 = vx >> 8 */
+    subu     t5, v0, t4     /* t5 = ( 256 - (vx>>8)) */
+
+    mul      s4, s0, t5     /* s4 = wt*(256-(vx>>8)) */
+    mul      s5, s0, t4     /* s5 = wt*(vx>>8) */
+    mul      s6, s1, t5     /* s6 = wb*(256-(vx>>8)) */
+    mul      s7, s1, t4     /* s7 = wb*(vx>>8) */
+
+    sra      t9, s2, 16
+    sll      t9, t9, 2
+    addiu    t8, t9, 4
+    lwx      t0, t9(a1)     /* t0 = tl */
+    lwx      t1, t8(a1)     /* t1 = tr */
+    addiu    a3, a3, -1
+    lwx      t2, t9(a2)     /* t2 = bl */
+    lwx      t3, t8(a2)     /* t3 = br */
+
+    BILINEAR_INTERPOLATE_SINGLE_PIXEL t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, s4, s5, s6, s7
+
+    addu     s2, s2, s3     /* vx += unit_x; */
+    sw       t0, 0(a0)
+    bnez     a3, 0b
+     addiu   a0, a0, 4
+
+    RESTORE_REGS_FROM_STACK 20, v0, s0, s1, s2, s3, s4, s5, s6, s7
+1:
+    j        ra
+     nop
+
+END(pixman_scaled_bilinear_scanline_8888_8888_SRC_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_scaled_bilinear_scanline_8888_0565_SRC_asm_mips)
+/*
+ * a0     - *dst
+ * a1     - *src_top
+ * a2     - *src_bottom
+ * a3     - w
+ * 16(sp) - wt
+ * 20(sp) - wb
+ * 24(sp) - vx
+ * 28(sp) - unit_x
+ */
+
+    beqz     a3, 1f
+     nop
+
+    SAVE_REGS_ON_STACK 20, v0, s0, s1, s2, s3, s4, s5, s6, s7
+
+    lw       s0, 36(sp)     /* s0 = wt */
+    lw       s1, 40(sp)     /* s1 = wb */
+    lw       s2, 44(sp)     /* s2 = vx */
+    lw       s3, 48(sp)     /* s3 = unit_x */
+    li       v0, BILINEAR_INTERPOLATION_RANGE
+
+    sll      s0, s0, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+    sll      s1, s1, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+0:
+    andi     t4, s2, 0xffff /* t4 = (short)vx */
+    srl      t4, t4, (16 - BILINEAR_INTERPOLATION_BITS) /* t4 = vx >> 8 */
+    subu     t5, v0, t4     /* t5 = ( 256 - (vx>>8)) */
+
+    mul      s4, s0, t5     /* s4 = wt*(256-(vx>>8)) */
+    mul      s5, s0, t4     /* s5 = wt*(vx>>8) */
+    mul      s6, s1, t5     /* s6 = wb*(256-(vx>>8)) */
+    mul      s7, s1, t4     /* s7 = wb*(vx>>8) */
+
+    sra      t9, s2, 16
+    sll      t9, t9, 2
+    addiu    t8, t9, 4
+    lwx      t0, t9(a1)     /* t0 = tl */
+    lwx      t1, t8(a1)     /* t1 = tr */
+    addiu    a3, a3, -1
+    lwx      t2, t9(a2)     /* t2 = bl */
+    lwx      t3, t8(a2)     /* t3 = br */
+
+    BILINEAR_INTERPOLATE_SINGLE_PIXEL t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, s4, s5, s6, s7
+    CONVERT_1x8888_TO_1x0565 t0, t1, t2, t3
+
+    addu     s2, s2, s3     /* vx += unit_x; */
+    sh       t1, 0(a0)
+    bnez     a3, 0b
+     addiu   a0, a0, 2
+
+    RESTORE_REGS_FROM_STACK 20, v0, s0, s1, s2, s3, s4, s5, s6, s7
+1:
+    j        ra
+     nop
+
+END(pixman_scaled_bilinear_scanline_8888_0565_SRC_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_scaled_bilinear_scanline_0565_8888_SRC_asm_mips)
+/*
+ * a0     - *dst
+ * a1     - *src_top
+ * a2     - *src_bottom
+ * a3     - w
+ * 16(sp) - wt
+ * 20(sp) - wb
+ * 24(sp) - vx
+ * 28(sp) - unit_x
+ */
+
+    beqz     a3, 1f
+     nop
+
+    SAVE_REGS_ON_STACK 28, v0, v1, s0, s1, s2, s3, s4, s5, s6, s7, s8
+
+    lw       s0, 44(sp)     /* s0 = wt */
+    lw       s1, 48(sp)     /* s1 = wb */
+    lw       s2, 52(sp)     /* s2 = vx */
+    lw       s3, 56(sp)     /* s3 = unit_x */
+    li       v0, BILINEAR_INTERPOLATION_RANGE
+    li       v1, 0x07e007e0
+    li       s8, 0x001f001f
+
+    sll      s0, s0, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+    sll      s1, s1, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+0:
+    andi     t4, s2, 0xffff /* t4 = (short)vx */
+    srl      t4, t4, (16 - BILINEAR_INTERPOLATION_BITS) /* t4 = vx >> 8 */
+    subu     t5, v0, t4     /* t5 = ( 256 - (vx>>8)) */
+
+    mul      s4, s0, t5     /* s4 = wt*(256-(vx>>8)) */
+    mul      s5, s0, t4     /* s5 = wt*(vx>>8) */
+    mul      s6, s1, t5     /* s6 = wb*(256-(vx>>8)) */
+    mul      s7, s1, t4     /* s7 = wb*(vx>>8) */
+
+    sra      t9, s2, 16
+    sll      t9, t9, 1
+    addiu    t8, t9, 2
+    lhx      t0, t9(a1)     /* t0 = tl */
+    lhx      t1, t8(a1)     /* t1 = tr */
+    andi     t1, t1, 0xffff
+    addiu    a3, a3, -1
+    lhx      t2, t9(a2)     /* t2 = bl */
+    lhx      t3, t8(a2)     /* t3 = br */
+    andi     t3, t3, 0xffff
+
+    CONVERT_2x0565_TO_2x8888 t0, t1, t0, t1, v1, s8, t4, t5, t6, t7
+    CONVERT_2x0565_TO_2x8888 t2, t3, t2, t3, v1, s8, t4, t5, t6, t7
+    BILINEAR_INTERPOLATE_SINGLE_PIXEL t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, s4, s5, s6, s7
+
+    addu     s2, s2, s3     /* vx += unit_x; */
+    sw       t0, 0(a0)
+    bnez     a3, 0b
+     addiu   a0, a0, 4
+
+    RESTORE_REGS_FROM_STACK 28, v0, v1, s0, s1, s2, s3, s4, s5, s6, s7, s8
+1:
+    j        ra
+     nop
+
+END(pixman_scaled_bilinear_scanline_0565_8888_SRC_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_scaled_bilinear_scanline_0565_0565_SRC_asm_mips)
+/*
+ * a0     - *dst
+ * a1     - *src_top
+ * a2     - *src_bottom
+ * a3     - w
+ * 16(sp) - wt
+ * 20(sp) - wb
+ * 24(sp) - vx
+ * 28(sp) - unit_x
+ */
+
+    beqz     a3, 1f
+     nop
+
+    SAVE_REGS_ON_STACK 28, v0, v1, s0, s1, s2, s3, s4, s5, s6, s7, s8
+
+    lw       s0, 44(sp)     /* s0 = wt */
+    lw       s1, 48(sp)     /* s1 = wb */
+    lw       s2, 52(sp)     /* s2 = vx */
+    lw       s3, 56(sp)     /* s3 = unit_x */
+    li       v0, BILINEAR_INTERPOLATION_RANGE
+    li       v1, 0x07e007e0
+    li       s8, 0x001f001f
+
+    sll      s0, s0, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+    sll      s1, s1, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+0:
+    andi     t4, s2, 0xffff /* t4 = (short)vx */
+    srl      t4, t4, (16 - BILINEAR_INTERPOLATION_BITS) /* t4 = vx >> 8 */
+    subu     t5, v0, t4     /* t5 = ( 256 - (vx>>8)) */
+
+    mul      s4, s0, t5     /* s4 = wt*(256-(vx>>8)) */
+    mul      s5, s0, t4     /* s5 = wt*(vx>>8) */
+    mul      s6, s1, t5     /* s6 = wb*(256-(vx>>8)) */
+    mul      s7, s1, t4     /* s7 = wb*(vx>>8) */
+
+    sra      t9, s2, 16
+    sll      t9, t9, 1
+    addiu    t8, t9, 2
+    lhx      t0, t9(a1)     /* t0 = tl */
+    lhx      t1, t8(a1)     /* t1 = tr */
+    andi     t1, t1, 0xffff
+    addiu    a3, a3, -1
+    lhx      t2, t9(a2)     /* t2 = bl */
+    lhx      t3, t8(a2)     /* t3 = br */
+    andi     t3, t3, 0xffff
+
+    CONVERT_2x0565_TO_2x8888 t0, t1, t0, t1, v1, s8, t4, t5, t6, t7
+    CONVERT_2x0565_TO_2x8888 t2, t3, t2, t3, v1, s8, t4, t5, t6, t7
+    BILINEAR_INTERPOLATE_SINGLE_PIXEL t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, s4, s5, s6, s7
+    CONVERT_1x8888_TO_1x0565 t0, t1, t2, t3
+
+    addu     s2, s2, s3     /* vx += unit_x; */
+    sh       t1, 0(a0)
+    bnez     a3, 0b
+     addiu   a0, a0, 2
+
+    RESTORE_REGS_FROM_STACK 28, v0, v1, s0, s1, s2, s3, s4, s5, s6, s7, s8
+1:
+    j        ra
+     nop
+
+END(pixman_scaled_bilinear_scanline_0565_0565_SRC_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_scaled_bilinear_scanline_8888_8888_OVER_asm_mips)
+/*
+ * a0     - *dst
+ * a1     - *src_top
+ * a2     - *src_bottom
+ * a3     - w
+ * 16(sp) - wt
+ * 20(sp) - wb
+ * 24(sp) - vx
+ * 28(sp) - unit_x
+ */
+
+    beqz     a3, 1f
+     nop
+
+    SAVE_REGS_ON_STACK 24, v0, s0, s1, s2, s3, s4, s5, s6, s7, s8
+
+    lw       s0, 40(sp)     /* s0 = wt */
+    lw       s1, 44(sp)     /* s1 = wb */
+    lw       s2, 48(sp)     /* s2 = vx */
+    lw       s3, 52(sp)     /* s3 = unit_x */
+    li       v0, BILINEAR_INTERPOLATION_RANGE
+    li       s8, 0x00ff00ff
+
+    sll      s0, s0, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+    sll      s1, s1, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+0:
+    andi     t4, s2, 0xffff /* t4 = (short)vx */
+    srl      t4, t4, (16 - BILINEAR_INTERPOLATION_BITS) /* t4 = vx >> 8 */
+    subu     t5, v0, t4     /* t5 = ( 256 - (vx>>8)) */
+
+    mul      s4, s0, t5     /* s4 = wt*(256-(vx>>8)) */
+    mul      s5, s0, t4     /* s5 = wt*(vx>>8) */
+    mul      s6, s1, t5     /* s6 = wb*(256-(vx>>8)) */
+    mul      s7, s1, t4     /* s7 = wb*(vx>>8) */
+
+    sra      t9, s2, 16
+    sll      t9, t9, 2
+    addiu    t8, t9, 4
+    lwx      t0, t9(a1)     /* t0 = tl */
+    lwx      t1, t8(a1)     /* t1 = tr */
+    addiu    a3, a3, -1
+    lwx      t2, t9(a2)     /* t2 = bl */
+    lwx      t3, t8(a2)     /* t3 = br */
+
+    BILINEAR_INTERPOLATE_SINGLE_PIXEL t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, s4, s5, s6, s7
+    lw       t1, 0(a0)      /* t1 = dest */
+    OVER_8888_8888 t0, t1, t2, s8, t3, t4, t5, t6
+
+    addu     s2, s2, s3     /* vx += unit_x; */
+    sw       t2, 0(a0)
+    bnez     a3, 0b
+     addiu   a0, a0, 4
+
+    RESTORE_REGS_FROM_STACK 24, v0, s0, s1, s2, s3, s4, s5, s6, s7, s8
+1:
+    j        ra
+     nop
+
+END(pixman_scaled_bilinear_scanline_8888_8888_OVER_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_scaled_bilinear_scanline_8888_8888_ADD_asm_mips)
+/*
+ * a0     - *dst
+ * a1     - *src_top
+ * a2     - *src_bottom
+ * a3     - w
+ * 16(sp) - wt
+ * 20(sp) - wb
+ * 24(sp) - vx
+ * 28(sp) - unit_x
+ */
+
+    beqz         a3, 1f
+     nop
+
+    SAVE_REGS_ON_STACK 20, v0, s0, s1, s2, s3, s4, s5, s6, s7
+
+    lw           s0, 36(sp)     /* s0 = wt */
+    lw           s1, 40(sp)     /* s1 = wb */
+    lw           s2, 44(sp)     /* s2 = vx */
+    lw           s3, 48(sp)     /* s3 = unit_x */
+    li           v0, BILINEAR_INTERPOLATION_RANGE
+
+    sll          s0, s0, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+    sll          s1, s1, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+0:
+    andi         t4, s2, 0xffff /* t4 = (short)vx */
+    srl          t4, t4, (16 - BILINEAR_INTERPOLATION_BITS) /* t4 = vx >> 8 */
+    subu         t5, v0, t4     /* t5 = ( 256 - (vx>>8)) */
+
+    mul          s4, s0, t5     /* s4 = wt*(256-(vx>>8)) */
+    mul          s5, s0, t4     /* s5 = wt*(vx>>8) */
+    mul          s6, s1, t5     /* s6 = wb*(256-(vx>>8)) */
+    mul          s7, s1, t4     /* s7 = wb*(vx>>8) */
+
+    sra          t9, s2, 16
+    sll          t9, t9, 2
+    addiu        t8, t9, 4
+    lwx          t0, t9(a1)     /* t0 = tl */
+    lwx          t1, t8(a1)     /* t1 = tr */
+    addiu        a3, a3, -1
+    lwx          t2, t9(a2)     /* t2 = bl */
+    lwx          t3, t8(a2)     /* t3 = br */
+
+    BILINEAR_INTERPOLATE_SINGLE_PIXEL t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, s4, s5, s6, s7
+    lw           t1, 0(a0)
+    addu_s.qb    t2, t0, t1
+
+    addu         s2, s2, s3     /* vx += unit_x; */
+    sw           t2, 0(a0)
+    bnez         a3, 0b
+     addiu       a0, a0, 4
+
+    RESTORE_REGS_FROM_STACK 20, v0, s0, s1, s2, s3, s4, s5, s6, s7
+1:
+    j            ra
+     nop
+
+END(pixman_scaled_bilinear_scanline_8888_8888_ADD_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_scaled_bilinear_scanline_8888_8_8888_SRC_asm_mips)
+/*
+ * a0     - *dst
+ * a1     - *mask
+ * a2     - *src_top
+ * a3     - *src_bottom
+ * 16(sp) - wt
+ * 20(sp) - wb
+ * 24(sp) - vx
+ * 28(sp) - unit_x
+ * 32(sp) - w
+ */
+
+    lw       v1, 32(sp)
+    beqz     v1, 1f
+     nop
+
+    SAVE_REGS_ON_STACK 28, v0, v1, s0, s1, s2, s3, s4, s5, s6, s7, s8
+
+    lw       s0, 44(sp)        /* s0 = wt */
+    lw       s1, 48(sp)        /* s1 = wb */
+    lw       s2, 52(sp)        /* s2 = vx */
+    lw       s3, 56(sp)        /* s3 = unit_x */
+    li       v0, BILINEAR_INTERPOLATION_RANGE
+    li       s8, 0x00ff00ff
+
+    sll      s0, s0, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+    sll      s1, s1, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+0:
+    andi     t4, s2, 0xffff    /* t4 = (short)vx */
+    srl      t4, t4, (16 - BILINEAR_INTERPOLATION_BITS) /* t4 = vx >> 8 */
+    subu     t5, v0, t4        /* t5 = ( 256 - (vx>>8)) */
+
+    mul      s4, s0, t5        /* s4 = wt*(256-(vx>>8)) */
+    mul      s5, s0, t4        /* s5 = wt*(vx>>8) */
+    mul      s6, s1, t5        /* s6 = wb*(256-(vx>>8)) */
+    mul      s7, s1, t4        /* s7 = wb*(vx>>8) */
+
+    sra      t9, s2, 16
+    sll      t9, t9, 2
+    addiu    t8, t9, 4
+    lwx      t0, t9(a2)        /* t0 = tl */
+    lwx      t1, t8(a2)        /* t1 = tr */
+    addiu    v1, v1, -1
+    lwx      t2, t9(a3)        /* t2 = bl */
+    lwx      t3, t8(a3)        /* t3 = br */
+
+    BILINEAR_INTERPOLATE_SINGLE_PIXEL t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, s4, s5, s6, s7
+    lbu      t1, 0(a1)         /* t1 = mask */
+    addiu    a1, a1, 1
+    MIPS_UN8x4_MUL_UN8 t0, t1, t0, s8, t2, t3, t4
+
+    addu     s2, s2, s3        /* vx += unit_x; */
+    sw       t0, 0(a0)
+    bnez     v1, 0b
+     addiu   a0, a0, 4
+
+    RESTORE_REGS_FROM_STACK 28, v0, v1, s0, s1, s2, s3, s4, s5, s6, s7, s8
+1:
+    j        ra
+     nop
+
+END(pixman_scaled_bilinear_scanline_8888_8_8888_SRC_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_scaled_bilinear_scanline_8888_8_0565_SRC_asm_mips)
+/*
+ * a0     - *dst
+ * a1     - *mask
+ * a2     - *src_top
+ * a3     - *src_bottom
+ * 16(sp) - wt
+ * 20(sp) - wb
+ * 24(sp) - vx
+ * 28(sp) - unit_x
+ * 32(sp) - w
+ */
+
+    lw       v1, 32(sp)
+    beqz     v1, 1f
+     nop
+
+    SAVE_REGS_ON_STACK 28, v0, v1, s0, s1, s2, s3, s4, s5, s6, s7, s8
+
+    lw       s0, 44(sp)        /* s0 = wt */
+    lw       s1, 48(sp)        /* s1 = wb */
+    lw       s2, 52(sp)        /* s2 = vx */
+    lw       s3, 56(sp)        /* s3 = unit_x */
+    li       v0, BILINEAR_INTERPOLATION_RANGE
+    li       s8, 0x00ff00ff
+
+    sll      s0, s0, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+    sll      s1, s1, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+0:
+    andi     t4, s2, 0xffff    /* t4 = (short)vx */
+    srl      t4, t4, (16 - BILINEAR_INTERPOLATION_BITS) /* t4 = vx >> 8 */
+    subu     t5, v0, t4        /* t5 = ( 256 - (vx>>8)) */
+
+    mul      s4, s0, t5        /* s4 = wt*(256-(vx>>8)) */
+    mul      s5, s0, t4        /* s5 = wt*(vx>>8) */
+    mul      s6, s1, t5        /* s6 = wb*(256-(vx>>8)) */
+    mul      s7, s1, t4        /* s7 = wb*(vx>>8) */
+
+    sra      t9, s2, 16
+    sll      t9, t9, 2
+    addiu    t8, t9, 4
+    lwx      t0, t9(a2)        /* t0 = tl */
+    lwx      t1, t8(a2)        /* t1 = tr */
+    addiu    v1, v1, -1
+    lwx      t2, t9(a3)        /* t2 = bl */
+    lwx      t3, t8(a3)        /* t3 = br */
+
+    BILINEAR_INTERPOLATE_SINGLE_PIXEL t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, s4, s5, s6, s7
+    lbu      t1, 0(a1)         /* t1 = mask */
+    addiu    a1, a1, 1
+    MIPS_UN8x4_MUL_UN8 t0, t1, t0, s8, t2, t3, t4
+    CONVERT_1x8888_TO_1x0565 t0, t1, t2, t3
+
+    addu     s2, s2, s3        /* vx += unit_x; */
+    sh       t1, 0(a0)
+    bnez     v1, 0b
+     addiu   a0, a0, 2
+
+    RESTORE_REGS_FROM_STACK 28, v0, v1, s0, s1, s2, s3, s4, s5, s6, s7, s8
+1:
+    j        ra
+     nop
+
+END(pixman_scaled_bilinear_scanline_8888_8_0565_SRC_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_scaled_bilinear_scanline_0565_8_x888_SRC_asm_mips)
+/*
+ * a0     - *dst
+ * a1     - *mask
+ * a2     - *src_top
+ * a3     - *src_bottom
+ * 16(sp) - wt
+ * 20(sp) - wb
+ * 24(sp) - vx
+ * 28(sp) - unit_x
+ * 32(sp) - w
+ */
+
+    lw       t0, 32(sp)
+    beqz     t0, 1f
+     nop
+
+    SAVE_REGS_ON_STACK 32, v0, v1, s0, s1, s2, s3, s4, s5, s6, s7, s8, ra
+
+    lw       s0, 48(sp)        /* s0 = wt */
+    lw       s1, 52(sp)        /* s1 = wb */
+    lw       s2, 56(sp)        /* s2 = vx */
+    lw       s3, 60(sp)        /* s3 = unit_x */
+    lw       ra, 64(sp)        /* ra = w */
+    li       v0, 0x00ff00ff
+    li       v1, 0x07e007e0
+    li       s8, 0x001f001f
+
+    sll      s0, s0, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+    sll      s1, s1, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+0:
+    andi     t4, s2, 0xffff    /* t4 = (short)vx */
+    srl      t4, t4, (16 - BILINEAR_INTERPOLATION_BITS) /* t4 = vx >> 8 */
+    li       t5, BILINEAR_INTERPOLATION_RANGE
+    subu     t5, t5, t4        /* t5 = ( 256 - (vx>>8)) */
+
+    mul      s4, s0, t5        /* s4 = wt*(256-(vx>>8)) */
+    mul      s5, s0, t4        /* s5 = wt*(vx>>8) */
+    mul      s6, s1, t5        /* s6 = wb*(256-(vx>>8)) */
+    mul      s7, s1, t4        /* s7 = wb*(vx>>8) */
+
+    sra      t9, s2, 16
+    sll      t9, t9, 1
+    addiu    t8, t9, 2
+    lhx      t0, t9(a2)        /* t0 = tl */
+    lhx      t1, t8(a2)        /* t1 = tr */
+    andi     t1, t1, 0xffff
+    addiu    ra, ra, -1
+    lhx      t2, t9(a3)        /* t2 = bl */
+    lhx      t3, t8(a3)        /* t3 = br */
+    andi     t3, t3, 0xffff
+
+    CONVERT_2x0565_TO_2x8888 t0, t1, t0, t1, v1, s8, t4, t5, t6, t7
+    CONVERT_2x0565_TO_2x8888 t2, t3, t2, t3, v1, s8, t4, t5, t6, t7
+    BILINEAR_INTERPOLATE_SINGLE_PIXEL t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, s4, s5, s6, s7
+    lbu      t1, 0(a1)         /* t1 = mask */
+    addiu    a1, a1, 1
+    MIPS_UN8x4_MUL_UN8 t0, t1, t0, v0, t2, t3, t4
+
+    addu     s2, s2, s3        /* vx += unit_x; */
+    sw       t0, 0(a0)
+    bnez     ra, 0b
+     addiu   a0, a0, 4
+
+    RESTORE_REGS_FROM_STACK 32, v0, v1, s0, s1, s2, s3, s4, s5, s6, s7, s8, ra
+1:
+    j        ra
+     nop
+
+END(pixman_scaled_bilinear_scanline_0565_8_x888_SRC_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_scaled_bilinear_scanline_0565_8_0565_SRC_asm_mips)
+/*
+ * a0     - *dst
+ * a1     - *mask
+ * a2     - *src_top
+ * a3     - *src_bottom
+ * 16(sp) - wt
+ * 20(sp) - wb
+ * 24(sp) - vx
+ * 28(sp) - unit_x
+ * 32(sp) - w
+ */
+
+    lw       t0, 32(sp)
+    beqz     t0, 1f
+     nop
+
+    SAVE_REGS_ON_STACK 32, v0, v1, s0, s1, s2, s3, s4, s5, s6, s7, s8, ra
+
+    lw       s0, 48(sp)        /* s0 = wt */
+    lw       s1, 52(sp)        /* s1 = wb */
+    lw       s2, 56(sp)        /* s2 = vx */
+    lw       s3, 60(sp)        /* s3 = unit_x */
+    lw       ra, 64(sp)        /* ra = w */
+    li       v0, 0x00ff00ff
+    li       v1, 0x07e007e0
+    li       s8, 0x001f001f
+
+    sll      s0, s0, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+    sll      s1, s1, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+0:
+    andi     t4, s2, 0xffff    /* t4 = (short)vx */
+    srl      t4, t4, (16 - BILINEAR_INTERPOLATION_BITS) /* t4 = vx >> 8 */
+    li       t5, BILINEAR_INTERPOLATION_RANGE
+    subu     t5, t5, t4        /* t5 = ( 256 - (vx>>8)) */
+
+    mul      s4, s0, t5        /* s4 = wt*(256-(vx>>8)) */
+    mul      s5, s0, t4        /* s5 = wt*(vx>>8) */
+    mul      s6, s1, t5        /* s6 = wb*(256-(vx>>8)) */
+    mul      s7, s1, t4        /* s7 = wb*(vx>>8) */
+
+    sra      t9, s2, 16
+    sll      t9, t9, 1
+    addiu    t8, t9, 2
+    lhx      t0, t9(a2)        /* t0 = tl */
+    lhx      t1, t8(a2)        /* t1 = tr */
+    andi     t1, t1, 0xffff
+    addiu    ra, ra, -1
+    lhx      t2, t9(a3)        /* t2 = bl */
+    lhx      t3, t8(a3)        /* t3 = br */
+    andi     t3, t3, 0xffff
+
+    CONVERT_2x0565_TO_2x8888 t0, t1, t0, t1, v1, s8, t4, t5, t6, t7
+    CONVERT_2x0565_TO_2x8888 t2, t3, t2, t3, v1, s8, t4, t5, t6, t7
+    BILINEAR_INTERPOLATE_SINGLE_PIXEL t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, s4, s5, s6, s7
+    lbu      t1, 0(a1)         /* t1 = mask */
+    addiu    a1, a1, 1
+    MIPS_UN8x4_MUL_UN8 t0, t1, t0, v0, t2, t3, t4
+    CONVERT_1x8888_TO_1x0565 t0, t1, t2, t3
+
+    addu     s2, s2, s3        /* vx += unit_x; */
+    sh       t1, 0(a0)
+    bnez     ra, 0b
+     addiu   a0, a0, 2
+
+    RESTORE_REGS_FROM_STACK 32, v0, v1, s0, s1, s2, s3, s4, s5, s6, s7, s8, ra
+1:
+    j        ra
+     nop
+
+END(pixman_scaled_bilinear_scanline_0565_8_0565_SRC_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_scaled_bilinear_scanline_8888_8_8888_OVER_asm_mips)
+/*
+ * a0     - dst        (a8r8g8b8)
+ * a1     - mask       (a8)
+ * a2     - src_top    (a8r8g8b8)
+ * a3     - src_bottom (a8r8g8b8)
+ * 16(sp) - wt
+ * 20(sp) - wb
+ * 24(sp) - vx
+ * 28(sp) - unit_x
+ * 32(sp) - w
+ */
+
+    SAVE_REGS_ON_STACK 28, v0, v1, s0, s1, s2, s3, s4, s5, s6, s7, s8
+
+    lw       v1, 60(sp)        /* v1 = w(sp + 32 + 28 save regs stack offset)*/
+    beqz     v1, 1f
+     nop
+
+    lw       s0, 44(sp)        /* s0 = wt */
+    lw       s1, 48(sp)        /* s1 = wb */
+    lw       s2, 52(sp)        /* s2 = vx */
+    lw       s3, 56(sp)        /* s3 = unit_x */
+    li       v0, BILINEAR_INTERPOLATION_RANGE
+    li       s8, 0x00ff00ff
+
+    sll      s0, s0, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+    sll      s1, s1, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+
+0:
+    andi     t4, s2, 0xffff    /* t4 = (short)vx */
+    srl      t4, t4, (16 - BILINEAR_INTERPOLATION_BITS) /* t4 = vx >> 8 */
+    subu     t5, v0, t4        /* t5 = ( 256 - (vx>>8)) */
+
+    mul      s4, s0, t5        /* s4 = wt*(256-(vx>>8)) */
+    mul      s5, s0, t4        /* s5 = wt*(vx>>8) */
+    mul      s6, s1, t5        /* s6 = wb*(256-(vx>>8)) */
+    mul      s7, s1, t4        /* s7 = wb*(vx>>8) */
+
+    sra      t9, s2, 16
+    sll      t9, t9, 2
+    addiu    t8, t9, 4
+    lwx      t0, t9(a2)        /* t0 = tl */
+    lwx      t1, t8(a2)        /* t1 = tr */
+    addiu    v1, v1, -1
+    lwx      t2, t9(a3)        /* t2 = bl */
+    lwx      t3, t8(a3)        /* t3 = br */
+
+    BILINEAR_INTERPOLATE_SINGLE_PIXEL t0, t1, t2, t3, \
+                                      t4, t5, t6, t7, t8, t9, s4, s5, s6, s7
+    lbu      t1, 0(a1)         /* t1 = mask */
+    lw       t2, 0(a0)         /* t2 = dst */
+    addiu    a1, a1, 1
+    OVER_8888_8_8888 t0, t1, t2, t0, s8, t3, t4, t5, t6
+
+    addu     s2, s2, s3        /* vx += unit_x; */
+    sw       t0, 0(a0)
+    bnez     v1, 0b
+     addiu   a0, a0, 4
+
+1:
+    RESTORE_REGS_FROM_STACK 28, v0, v1, s0, s1, s2, s3, s4, s5, s6, s7, s8
+    j        ra
+     nop
+
+END(pixman_scaled_bilinear_scanline_8888_8_8888_OVER_asm_mips)
+
+LEAF_MIPS_DSPR2(pixman_scaled_bilinear_scanline_8888_8_8888_ADD_asm_mips)
+/*
+ * a0     - *dst
+ * a1     - *mask
+ * a2     - *src_top
+ * a3     - *src_bottom
+ * 16(sp) - wt
+ * 20(sp) - wb
+ * 24(sp) - vx
+ * 28(sp) - unit_x
+ * 32(sp) - w
+ */
+
+    lw       v1, 32(sp)
+    beqz     v1, 1f
+     nop
+
+    SAVE_REGS_ON_STACK 28, v0, v1, s0, s1, s2, s3, s4, s5, s6, s7, s8
+
+    lw       s0, 44(sp)        /* s0 = wt */
+    lw       s1, 48(sp)        /* s1 = wb */
+    lw       s2, 52(sp)        /* s2 = vx */
+    lw       s3, 56(sp)        /* s3 = unit_x */
+    li       v0, BILINEAR_INTERPOLATION_RANGE
+    li       s8, 0x00ff00ff
+
+    sll      s0, s0, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+    sll      s1, s1, (2 * (8 - BILINEAR_INTERPOLATION_BITS))
+0:
+    andi     t4, s2, 0xffff    /* t4 = (short)vx */
+    srl      t4, t4, (16 - BILINEAR_INTERPOLATION_BITS) /* t4 = vx >> 8 */
+    subu     t5, v0, t4        /* t5 = ( 256 - (vx>>8)) */
+
+    mul      s4, s0, t5        /* s4 = wt*(256-(vx>>8)) */
+    mul      s5, s0, t4        /* s5 = wt*(vx>>8) */
+    mul      s6, s1, t5        /* s6 = wb*(256-(vx>>8)) */
+    mul      s7, s1, t4        /* s7 = wb*(vx>>8) */
+
+    sra      t9, s2, 16
+    sll      t9, t9, 2
+    addiu    t8, t9, 4
+    lwx      t0, t9(a2)        /* t0 = tl */
+    lwx      t1, t8(a2)        /* t1 = tr */
+    addiu    v1, v1, -1
+    lwx      t2, t9(a3)        /* t2 = bl */
+    lwx      t3, t8(a3)        /* t3 = br */
+
+    BILINEAR_INTERPOLATE_SINGLE_PIXEL t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, s4, s5, s6, s7
+    lbu      t1, 0(a1)         /* t1 = mask */
+    lw       t2, 0(a0)         /* t2 = dst */
+    addiu    a1, a1, 1
+    MIPS_UN8x4_MUL_UN8_ADD_UN8x4 t0, t1, t2, t0, s8, t3, t4, t5
+
+    addu     s2, s2, s3        /* vx += unit_x; */
+    sw       t0, 0(a0)
+    bnez     v1, 0b
+     addiu   a0, a0, 4
+
+    RESTORE_REGS_FROM_STACK 28, v0, v1, s0, s1, s2, s3, s4, s5, s6, s7, s8
+1:
+    j        ra
+     nop
+
+END(pixman_scaled_bilinear_scanline_8888_8_8888_ADD_asm_mips)
diff --git a/pixman/pixman/pixman-mips-dspr2-asm.h b/pixman/pixman/pixman-mips-dspr2-asm.h
new file mode 100644
index 000000000..11849bd66
--- /dev/null
+++ b/pixman/pixman/pixman-mips-dspr2-asm.h
@@ -0,0 +1,713 @@
+/*
+ * Copyright (c) 2012
+ *      MIPS Technologies, Inc., California.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the MIPS Technologies, Inc., nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE MIPS TECHNOLOGIES, INC. ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE MIPS TECHNOLOGIES, INC. BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Author:  Nemanja Lukic (nlukic@mips.com)
+ */
+
+#ifndef PIXMAN_MIPS_DSPR2_ASM_H
+#define PIXMAN_MIPS_DSPR2_ASM_H
+
+#define zero $0
+#define AT   $1
+#define v0   $2
+#define v1   $3
+#define a0   $4
+#define a1   $5
+#define a2   $6
+#define a3   $7
+#define t0   $8
+#define t1   $9
+#define t2   $10
+#define t3   $11
+#define t4   $12
+#define t5   $13
+#define t6   $14
+#define t7   $15
+#define s0   $16
+#define s1   $17
+#define s2   $18
+#define s3   $19
+#define s4   $20
+#define s5   $21
+#define s6   $22
+#define s7   $23
+#define t8   $24
+#define t9   $25
+#define k0   $26
+#define k1   $27
+#define gp   $28
+#define sp   $29
+#define fp   $30
+#define s8   $30
+#define ra   $31
+
+/*
+ * LEAF_MIPS32R2 - declare leaf routine for MIPS32r2
+ */
+#define LEAF_MIPS32R2(symbol)                           \
+                .globl  symbol;                         \
+                .align  2;                              \
+#ifdef __ELF__
+                .hidden symbol;                         \
+                .type   symbol, @function;              \
+#endif
+                .ent    symbol, 0;                      \
+symbol:         .frame  sp, 0, ra;                      \
+                .set    push;                           \
+                .set    arch=mips32r2;                  \
+                .set    noreorder;                      \
+                .set    noat;
+
+/*
+ * LEAF_MIPS32R2 - declare leaf routine for MIPS DSPr2
+ */
+#define LEAF_MIPS_DSPR2(symbol)                         \
+LEAF_MIPS32R2(symbol)                                   \
+                .set    dspr2;
+
+/*
+ * END - mark end of function
+ */
+#define END(function)                                   \
+                .set    pop;                            \
+                .end    function;                       \
+                .size   function,.-function
+
+/*
+ * Checks if stack offset is big enough for storing/restoring regs_num
+ * number of register to/from stack. Stack offset must be greater than
+ * or equal to the number of bytes needed for storing registers (regs_num*4).
+ * Since MIPS ABI allows usage of first 16 bytes of stack frame (this is
+ * preserved for input arguments of the functions, already stored in a0-a3),
+ * stack size can be further optimized by utilizing this space.
+ */
+.macro CHECK_STACK_OFFSET regs_num, stack_offset
+.if \stack_offset < \regs_num * 4 - 16
+.error "Stack offset too small."
+.endif
+.endm
+
+/*
+ * Saves set of registers on stack. Maximum number of registers that
+ * can be saved on stack is limitted to 14 (a0-a3, v0-v1 and s0-s7).
+ * Stack offset is number of bytes that are added to stack pointer (sp)
+ * before registers are pushed in order to provide enough space on stack
+ * (offset must be multiple of 4, and must be big enough, as described by
+ * CHECK_STACK_OFFSET macro). This macro is intended to be used in
+ * combination with RESTORE_REGS_FROM_STACK macro. Example:
+ *  SAVE_REGS_ON_STACK      4, v0, v1, s0, s1
+ *  RESTORE_REGS_FROM_STACK 4, v0, v1, s0, s1
+ */
+.macro SAVE_REGS_ON_STACK stack_offset = 0, r1, \
+                          r2  = 0, r3  = 0, r4  = 0, \
+                          r5  = 0, r6  = 0, r7  = 0, \
+                          r8  = 0, r9  = 0, r10 = 0, \
+                          r11 = 0, r12 = 0, r13 = 0, \
+                          r14 = 0
+    .if (\stack_offset < 0) || (\stack_offset - (\stack_offset / 4) * 4)
+    .error "Stack offset must be pozitive and multiple of 4."
+    .endif
+    .if \stack_offset != 0
+    addiu           sp, sp, -\stack_offset
+    .endif
+    sw              \r1, 0(sp)
+    .if \r2 != 0
+    sw              \r2, 4(sp)
+    .endif
+    .if \r3 != 0
+    sw              \r3, 8(sp)
+    .endif
+    .if \r4 != 0
+    sw              \r4, 12(sp)
+    .endif
+    .if \r5 != 0
+    CHECK_STACK_OFFSET 5, \stack_offset
+    sw              \r5, 16(sp)
+    .endif
+    .if \r6 != 0
+    CHECK_STACK_OFFSET 6, \stack_offset
+    sw              \r6, 20(sp)
+    .endif
+    .if \r7 != 0
+    CHECK_STACK_OFFSET 7, \stack_offset
+    sw              \r7, 24(sp)
+    .endif
+    .if \r8 != 0
+    CHECK_STACK_OFFSET 8, \stack_offset
+    sw              \r8, 28(sp)
+    .endif
+    .if \r9 != 0
+    CHECK_STACK_OFFSET 9, \stack_offset
+    sw              \r9, 32(sp)
+    .endif
+    .if \r10 != 0
+    CHECK_STACK_OFFSET 10, \stack_offset
+    sw              \r10, 36(sp)
+    .endif
+    .if \r11 != 0
+    CHECK_STACK_OFFSET 11, \stack_offset
+    sw              \r11, 40(sp)
+    .endif
+    .if \r12 != 0
+    CHECK_STACK_OFFSET 12, \stack_offset
+    sw              \r12, 44(sp)
+    .endif
+    .if \r13 != 0
+    CHECK_STACK_OFFSET 13, \stack_offset
+    sw              \r13, 48(sp)
+    .endif
+    .if \r14 != 0
+    CHECK_STACK_OFFSET 14, \stack_offset
+    sw              \r14, 52(sp)
+    .endif
+.endm
+
+/*
+ * Restores set of registers from stack. Maximum number of registers that
+ * can be restored from stack is limitted to 14 (a0-a3, v0-v1 and s0-s7).
+ * Stack offset is number of bytes that are added to stack pointer (sp)
+ * after registers are restored (offset must be multiple of 4, and must
+ * be big enough, as described by CHECK_STACK_OFFSET macro). This macro is
+ * intended to be used in combination with RESTORE_REGS_FROM_STACK macro.
+ * Example:
+ *  SAVE_REGS_ON_STACK      4, v0, v1, s0, s1
+ *  RESTORE_REGS_FROM_STACK 4, v0, v1, s0, s1
+ */
+.macro RESTORE_REGS_FROM_STACK stack_offset = 0, r1, \
+                               r2  = 0, r3  = 0, r4  = 0, \
+                               r5  = 0, r6  = 0, r7  = 0, \
+                               r8  = 0, r9  = 0, r10 = 0, \
+                               r11 = 0, r12 = 0, r13 = 0, \
+                               r14 = 0
+    .if (\stack_offset < 0) || (\stack_offset - (\stack_offset/4)*4)
+    .error "Stack offset must be pozitive and multiple of 4."
+    .endif
+    lw              \r1, 0(sp)
+    .if \r2 != 0
+    lw              \r2, 4(sp)
+    .endif
+    .if \r3 != 0
+    lw              \r3, 8(sp)
+    .endif
+    .if \r4 != 0
+    lw              \r4, 12(sp)
+    .endif
+    .if \r5 != 0
+    CHECK_STACK_OFFSET 5, \stack_offset
+    lw              \r5, 16(sp)
+    .endif
+    .if \r6 != 0
+    CHECK_STACK_OFFSET 6, \stack_offset
+    lw              \r6, 20(sp)
+    .endif
+    .if \r7 != 0
+    CHECK_STACK_OFFSET 7, \stack_offset
+    lw              \r7, 24(sp)
+    .endif
+    .if \r8 != 0
+    CHECK_STACK_OFFSET 8, \stack_offset
+    lw              \r8, 28(sp)
+    .endif
+    .if \r9 != 0
+    CHECK_STACK_OFFSET 9, \stack_offset
+    lw              \r9, 32(sp)
+    .endif
+    .if \r10 != 0
+    CHECK_STACK_OFFSET 10, \stack_offset
+    lw              \r10, 36(sp)
+    .endif
+    .if \r11 != 0
+    CHECK_STACK_OFFSET 11, \stack_offset
+    lw              \r11, 40(sp)
+    .endif
+    .if \r12 != 0
+    CHECK_STACK_OFFSET 12, \stack_offset
+    lw              \r12, 44(sp)
+    .endif
+    .if \r13 != 0
+    CHECK_STACK_OFFSET 13, \stack_offset
+    lw              \r13, 48(sp)
+    .endif
+    .if \r14 != 0
+    CHECK_STACK_OFFSET 14, \stack_offset
+    lw              \r14, 52(sp)
+    .endif
+    .if \stack_offset != 0
+    addiu           sp, sp, \stack_offset
+    .endif
+.endm
+
+/*
+ * Conversion of single r5g6b5 pixel (in_565) to single a8r8g8b8 pixel
+ * returned in (out_8888) register. Requires two temporary registers
+ * (scratch1 and scratch2).
+ */
+.macro CONVERT_1x0565_TO_1x8888 in_565,   \
+                                out_8888, \
+                                scratch1, scratch2
+    lui     \out_8888, 0xff00
+    sll     \scratch1, \in_565,   0x3
+    andi    \scratch2, \scratch1, 0xff
+    ext     \scratch1, \in_565,   0x2, 0x3
+    or      \scratch1, \scratch2, \scratch1
+    or      \out_8888, \out_8888, \scratch1
+
+    sll     \scratch1, \in_565,   0x5
+    andi    \scratch1, \scratch1, 0xfc00
+    srl     \scratch2, \in_565,   0x1
+    andi    \scratch2, \scratch2, 0x300
+    or      \scratch2, \scratch1, \scratch2
+    or      \out_8888, \out_8888, \scratch2
+
+    andi    \scratch1, \in_565,   0xf800
+    srl     \scratch2, \scratch1, 0x5
+    andi    \scratch2, \scratch2, 0xff00
+    or      \scratch1, \scratch1, \scratch2
+    sll     \scratch1, \scratch1, 0x8
+    or      \out_8888, \out_8888, \scratch1
+.endm
+
+/*
+ * Conversion of two r5g6b5 pixels (in1_565 and in2_565) to two a8r8g8b8 pixels
+ * returned in (out1_8888 and out2_8888) registers. Requires four scratch
+ * registers (scratch1 ... scratch4). It also requires maskG and maskB for
+ * color component extractions. These masks must have following values:
+ *   li       maskG, 0x07e007e0
+ *   li       maskB, 0x001F001F
+ */
+.macro CONVERT_2x0565_TO_2x8888 in1_565, in2_565,     \
+                                out1_8888, out2_8888, \
+                                maskG, maskB,         \
+                                scratch1, scratch2, scratch3, scratch4
+    sll               \scratch1,  \in1_565,   16
+    or                \scratch1,  \scratch1,  \in2_565
+    lui               \out2_8888, 0xff00
+    ori               \out2_8888, \out2_8888, 0xff00
+    shrl.ph           \scratch2,  \scratch1,  11
+    and               \scratch3,  \scratch1,  \maskG
+    shra.ph           \scratch4,  \scratch2,  2
+    shll.ph           \scratch2,  \scratch2,  3
+    shll.ph           \scratch3,  \scratch3,  5
+    or                \scratch2,  \scratch2,  \scratch4
+    shrl.qb           \scratch4,  \scratch3,  6
+    or                \out2_8888, \out2_8888, \scratch2
+    or                \scratch3,  \scratch3,  \scratch4
+    and               \scratch1,  \scratch1,  \maskB
+    shll.ph           \scratch2,  \scratch1,  3
+    shra.ph           \scratch4,  \scratch1,  2
+    or                \scratch2,  \scratch2,  \scratch4
+    or                \scratch3,  \scratch2,  \scratch3
+    precrq.ph.w       \out1_8888, \out2_8888, \scratch3
+    precr_sra.ph.w    \out2_8888, \scratch3,  0
+.endm
+
+/*
+ * Conversion of single a8r8g8b8 pixel (in_8888) to single r5g6b5 pixel
+ * returned in (out_565) register. Requires two temporary registers
+ * (scratch1 and scratch2).
+ */
+.macro CONVERT_1x8888_TO_1x0565 in_8888, \
+                                out_565, \
+                                scratch1, scratch2
+    ext     \out_565,  \in_8888,  0x3, 0x5
+    srl     \scratch1, \in_8888,  0x5
+    andi    \scratch1, \scratch1, 0x07e0
+    srl     \scratch2, \in_8888,  0x8
+    andi    \scratch2, \scratch2, 0xf800
+    or      \out_565,  \out_565,  \scratch1
+    or      \out_565,  \out_565,  \scratch2
+.endm
+
+/*
+ * Conversion of two a8r8g8b8 pixels (in1_8888 and in2_8888) to two r5g6b5
+ * pixels returned in (out1_565 and out2_565) registers. Requires two temporary
+ * registers (scratch1 and scratch2). It also requires maskR, maskG and maskB
+ * for color component extractions. These masks must have following values:
+ *   li       maskR, 0xf800f800
+ *   li       maskG, 0x07e007e0
+ *   li       maskB, 0x001F001F
+ * Value of input register in2_8888 is lost.
+ */
+.macro CONVERT_2x8888_TO_2x0565 in1_8888, in2_8888,  \
+                                out1_565, out2_565,  \
+                                maskR, maskG, maskB, \
+                                scratch1, scratch2
+    precr.qb.ph    \scratch1, \in2_8888, \in1_8888
+    precrq.qb.ph   \in2_8888, \in2_8888, \in1_8888
+    and            \out1_565, \scratch1, \maskR
+    shrl.ph        \scratch1, \scratch1, 3
+    shll.ph        \in2_8888, \in2_8888, 3
+    and            \scratch1, \scratch1, \maskB
+    or             \out1_565, \out1_565, \scratch1
+    and            \in2_8888, \in2_8888, \maskG
+    or             \out1_565, \out1_565, \in2_8888
+    srl            \out2_565, \out1_565, 16
+.endm
+
+/*
+ * Multiply pixel (a8) with single pixel (a8r8g8b8). It requires maskLSR needed
+ * for rounding process. maskLSR must have following value:
+ *   li       maskLSR, 0x00ff00ff
+ */
+.macro MIPS_UN8x4_MUL_UN8 s_8888,  \
+                          m_8,     \
+                          d_8888,  \
+                          maskLSR, \
+                          scratch1, scratch2, scratch3
+    replv.ph          \m_8,      \m_8                 /*   0 | M | 0 | M */
+    muleu_s.ph.qbl    \scratch1, \s_8888,   \m_8      /*    A*M  |  R*M */
+    muleu_s.ph.qbr    \scratch2, \s_8888,   \m_8      /*    G*M  |  B*M */
+    shra_r.ph         \scratch3, \scratch1, 8
+    shra_r.ph         \d_8888,   \scratch2, 8
+    and               \scratch3, \scratch3, \maskLSR  /*   0 |A*M| 0 |R*M */
+    and               \d_8888,   \d_8888,   \maskLSR  /*   0 |G*M| 0 |B*M */
+    addq.ph           \scratch1, \scratch1, \scratch3 /* A*M+A*M | R*M+R*M */
+    addq.ph           \scratch2, \scratch2, \d_8888   /* G*M+G*M | B*M+B*M */
+    shra_r.ph         \scratch1, \scratch1, 8
+    shra_r.ph         \scratch2, \scratch2, 8
+    precr.qb.ph       \d_8888,   \scratch1, \scratch2
+.endm
+
+/*
+ * Multiply two pixels (a8) with two pixels (a8r8g8b8). It requires maskLSR
+ * needed for rounding process. maskLSR must have following value:
+ *   li       maskLSR, 0x00ff00ff
+ */
+.macro MIPS_2xUN8x4_MUL_2xUN8 s1_8888, \
+                              s2_8888, \
+                              m1_8,    \
+                              m2_8,    \
+                              d1_8888, \
+                              d2_8888, \
+                              maskLSR, \
+                              scratch1, scratch2, scratch3, \
+                              scratch4, scratch5, scratch6
+    replv.ph          \m1_8,     \m1_8                /*  0 | M1 | 0 | M1 */
+    replv.ph          \m2_8,     \m2_8                /*  0 | M2 | 0 | M2 */
+    muleu_s.ph.qbl    \scratch1, \s1_8888,  \m1_8     /*  A1*M1  |  R1*M1 */
+    muleu_s.ph.qbr    \scratch2, \s1_8888,  \m1_8     /*  G1*M1  |  B1*M1 */
+    muleu_s.ph.qbl    \scratch3, \s2_8888,  \m2_8     /*  A2*M2  |  R2*M2 */
+    muleu_s.ph.qbr    \scratch4, \s2_8888,  \m2_8     /*  G2*M2  |  B2*M2 */
+    shra_r.ph         \scratch5, \scratch1, 8
+    shra_r.ph         \d1_8888,  \scratch2, 8
+    shra_r.ph         \scratch6, \scratch3, 8
+    shra_r.ph         \d2_8888,  \scratch4, 8
+    and               \scratch5, \scratch5, \maskLSR  /* 0 |A1*M1| 0 |R1*M1 */
+    and               \d1_8888,  \d1_8888,  \maskLSR  /* 0 |G1*M1| 0 |B1*M1 */
+    and               \scratch6, \scratch6, \maskLSR  /* 0 |A2*M2| 0 |R2*M2 */
+    and               \d2_8888,  \d2_8888,  \maskLSR  /* 0 |G2*M2| 0 |B2*M2 */
+    addq.ph           \scratch1, \scratch1, \scratch5
+    addq.ph           \scratch2, \scratch2, \d1_8888
+    addq.ph           \scratch3, \scratch3, \scratch6
+    addq.ph           \scratch4, \scratch4, \d2_8888
+    shra_r.ph         \scratch1, \scratch1, 8
+    shra_r.ph         \scratch2, \scratch2, 8
+    shra_r.ph         \scratch3, \scratch3, 8
+    shra_r.ph         \scratch4, \scratch4, 8
+    precr.qb.ph       \d1_8888,  \scratch1, \scratch2
+    precr.qb.ph       \d2_8888,  \scratch3, \scratch4
+.endm
+
+/*
+ * Multiply pixel (a8r8g8b8) with single pixel (a8r8g8b8). It requires maskLSR
+ * needed for rounding process. maskLSR must have following value:
+ *   li       maskLSR, 0x00ff00ff
+ */
+.macro MIPS_UN8x4_MUL_UN8x4 s_8888,  \
+                            m_8888,  \
+                            d_8888,  \
+                            maskLSR, \
+                            scratch1, scratch2, scratch3, scratch4
+    preceu.ph.qbl     \scratch1, \m_8888              /*   0 | A | 0 | R */
+    preceu.ph.qbr     \scratch2, \m_8888              /*   0 | G | 0 | B */
+    muleu_s.ph.qbl    \scratch3, \s_8888,   \scratch1 /*    A*A  |  R*R */
+    muleu_s.ph.qbr    \scratch4, \s_8888,   \scratch2 /*    G*G  |  B*B */
+    shra_r.ph         \scratch1, \scratch3, 8
+    shra_r.ph         \scratch2, \scratch4, 8
+    and               \scratch1, \scratch1, \maskLSR  /*   0 |A*A| 0 |R*R */
+    and               \scratch2, \scratch2, \maskLSR  /*   0 |G*G| 0 |B*B */
+    addq.ph           \scratch1, \scratch1, \scratch3
+    addq.ph           \scratch2, \scratch2, \scratch4
+    shra_r.ph         \scratch1, \scratch1, 8
+    shra_r.ph         \scratch2, \scratch2, 8
+    precr.qb.ph       \d_8888,   \scratch1, \scratch2
+.endm
+
+/*
+ * Multiply two pixels (a8r8g8b8) with two pixels (a8r8g8b8). It requires
+ * maskLSR needed for rounding process. maskLSR must have following value:
+ *   li       maskLSR, 0x00ff00ff
+ */
+
+.macro MIPS_2xUN8x4_MUL_2xUN8x4 s1_8888,  \
+                                s2_8888,  \
+                                m1_8888,  \
+                                m2_8888,  \
+                                d1_8888,  \
+                                d2_8888,  \
+                                maskLSR,  \
+                                scratch1, scratch2, scratch3, \
+                                scratch4, scratch5, scratch6
+    preceu.ph.qbl     \scratch1, \m1_8888             /*   0 | A | 0 | R */
+    preceu.ph.qbr     \scratch2, \m1_8888             /*   0 | G | 0 | B */
+    preceu.ph.qbl     \scratch3, \m2_8888             /*   0 | A | 0 | R */
+    preceu.ph.qbr     \scratch4, \m2_8888             /*   0 | G | 0 | B */
+    muleu_s.ph.qbl    \scratch5, \s1_8888,  \scratch1 /*    A*A  |  R*R */
+    muleu_s.ph.qbr    \scratch6, \s1_8888,  \scratch2 /*    G*G  |  B*B */
+    muleu_s.ph.qbl    \scratch1, \s2_8888,  \scratch3 /*    A*A  |  R*R */
+    muleu_s.ph.qbr    \scratch2, \s2_8888,  \scratch4 /*    G*G  |  B*B */
+    shra_r.ph         \scratch3, \scratch5, 8
+    shra_r.ph         \scratch4, \scratch6, 8
+    shra_r.ph         \d1_8888,  \scratch1, 8
+    shra_r.ph         \d2_8888,  \scratch2, 8
+    and               \scratch3, \scratch3, \maskLSR  /*   0 |A*A| 0 |R*R */
+    and               \scratch4, \scratch4, \maskLSR  /*   0 |G*G| 0 |B*B */
+    and               \d1_8888,  \d1_8888,  \maskLSR  /*   0 |A*A| 0 |R*R */
+    and               \d2_8888,  \d2_8888,  \maskLSR  /*   0 |G*G| 0 |B*B */
+    addq.ph           \scratch3, \scratch3, \scratch5
+    addq.ph           \scratch4, \scratch4, \scratch6
+    addq.ph           \d1_8888,  \d1_8888,  \scratch1
+    addq.ph           \d2_8888,  \d2_8888,  \scratch2
+    shra_r.ph         \scratch3, \scratch3, 8
+    shra_r.ph         \scratch4, \scratch4, 8
+    shra_r.ph         \scratch5, \d1_8888,  8
+    shra_r.ph         \scratch6, \d2_8888,  8
+    precr.qb.ph       \d1_8888,  \scratch3, \scratch4
+    precr.qb.ph       \d2_8888,  \scratch5, \scratch6
+.endm
+
+/*
+ * OVER operation on single a8r8g8b8 source pixel (s_8888) and single a8r8g8b8
+ * destination pixel (d_8888) using a8 mask (m_8). It also requires maskLSR
+ * needed for rounding process. maskLSR must have following value:
+ *   li       maskLSR, 0x00ff00ff
+ */
+.macro OVER_8888_8_8888 s_8888,   \
+                        m_8,      \
+                        d_8888,   \
+                        out_8888, \
+                        maskLSR,  \
+                        scratch1, scratch2, scratch3, scratch4
+    MIPS_UN8x4_MUL_UN8 \s_8888,   \m_8, \
+                       \scratch1, \maskLSR, \
+                       \scratch2, \scratch3, \scratch4
+
+    not                \scratch2, \scratch1
+    srl                \scratch2, \scratch2, 24
+
+    MIPS_UN8x4_MUL_UN8 \d_8888,   \scratch2, \
+                       \d_8888,   \maskLSR,  \
+                       \scratch3, \scratch4, \out_8888
+
+    addu_s.qb          \out_8888, \d_8888,   \scratch1
+.endm
+
+/*
+ * OVER operation on two a8r8g8b8 source pixels (s1_8888 and s2_8888) and two
+ * a8r8g8b8 destination pixels (d1_8888 and d2_8888) using a8 masks (m1_8 and
+ * m2_8). It also requires maskLSR needed for rounding process. maskLSR must
+ * have following value:
+ *   li       maskLSR, 0x00ff00ff
+ */
+.macro OVER_2x8888_2x8_2x8888 s1_8888,   \
+                              s2_8888,   \
+                              m1_8,      \
+                              m2_8,      \
+                              d1_8888,   \
+                              d2_8888,   \
+                              out1_8888, \
+                              out2_8888, \
+                              maskLSR,   \
+                              scratch1, scratch2, scratch3, \
+                              scratch4, scratch5, scratch6
+    MIPS_2xUN8x4_MUL_2xUN8 \s1_8888,   \s2_8888, \
+                           \m1_8,      \m2_8, \
+                           \scratch1,  \scratch2, \
+                           \maskLSR, \
+                           \scratch3,  \scratch4, \out1_8888, \
+                           \out2_8888, \scratch5, \scratch6
+
+    not                    \scratch3,  \scratch1
+    srl                    \scratch3,  \scratch3, 24
+    not                    \scratch4,  \scratch2
+    srl                    \scratch4,  \scratch4, 24
+
+    MIPS_2xUN8x4_MUL_2xUN8 \d1_8888,   \d2_8888, \
+                           \scratch3,  \scratch4, \
+                           \d1_8888,   \d2_8888, \
+                           \maskLSR, \
+                           \scratch5,  \scratch6, \out1_8888, \
+                           \out2_8888, \scratch3, \scratch4
+
+    addu_s.qb              \out1_8888, \d1_8888,  \scratch1
+    addu_s.qb              \out2_8888, \d2_8888,  \scratch2
+.endm
+
+/*
+ * OVER operation on single a8r8g8b8 source pixel (s_8888) and single a8r8g8b8
+ * destination pixel (d_8888). It also requires maskLSR needed for rounding
+ * process. maskLSR must have following value:
+ *   li       maskLSR, 0x00ff00ff
+ */
+.macro OVER_8888_8888 s_8888,   \
+                      d_8888,   \
+                      out_8888, \
+                      maskLSR,  \
+                      scratch1, scratch2, scratch3, scratch4
+    not                \scratch1, \s_8888
+    srl                \scratch1, \scratch1, 24
+
+    MIPS_UN8x4_MUL_UN8 \d_8888,   \scratch1, \
+                       \out_8888, \maskLSR, \
+                       \scratch2, \scratch3, \scratch4
+
+    addu_s.qb          \out_8888, \out_8888, \s_8888
+.endm
+
+/*
+ * OVER operation on two a8r8g8b8 source pixels (s1_8888 and s2_8888) and two
+ * a8r8g8b8 destination pixels (d1_8888 and d2_8888). It also requires maskLSR
+ * needed for rounding process. maskLSR must have following value:
+ *   li       maskLSR, 0x00ff00ff
+ */
+.macro OVER_2x8888_2x8888 s1_8888,   \
+                          s2_8888,   \
+                          d1_8888,   \
+                          d2_8888,   \
+                          out1_8888, \
+                          out2_8888, \
+                          maskLSR,   \
+                          scratch1, scratch2, scratch3, \
+                          scratch4, scratch5, scratch6
+    not                    \scratch1,  \s1_8888
+    srl                    \scratch1,  \scratch1,  24
+    not                    \scratch2,  \s2_8888
+    srl                    \scratch2,  \scratch2,  24
+    MIPS_2xUN8x4_MUL_2xUN8 \d1_8888,   \d2_8888, \
+                           \scratch1,  \scratch2,  \
+                           \out1_8888, \out2_8888, \
+                           \maskLSR, \
+                           \scratch3,  \scratch4, \scratch5, \
+                           \scratch6,  \d1_8888,  \d2_8888
+
+    addu_s.qb              \out1_8888, \out1_8888, \s1_8888
+    addu_s.qb              \out2_8888, \out2_8888, \s2_8888
+.endm
+
+.macro MIPS_UN8x4_MUL_UN8_ADD_UN8x4 s_8888,   \
+                                    m_8,      \
+                                    d_8888,   \
+                                    out_8888, \
+                                    maskLSR,  \
+                                    scratch1, scratch2, scratch3
+    MIPS_UN8x4_MUL_UN8 \s_8888, \m_8, \
+                       \out_8888, \maskLSR, \
+                       \scratch1, \scratch2, \scratch3
+
+    addu_s.qb          \out_8888, \out_8888, \d_8888
+.endm
+
+.macro MIPS_2xUN8x4_MUL_2xUN8_ADD_2xUN8x4 s1_8888,   \
+                             s2_8888,   \
+                             m1_8,      \
+                             m2_8,      \
+                             d1_8888,   \
+                             d2_8888,   \
+                             out1_8888, \
+                             out2_8888, \
+                             maskLSR,   \
+                             scratch1,  scratch2, scratch3, \
+                             scratch4, scratch5, scratch6
+    MIPS_2xUN8x4_MUL_2xUN8 \s1_8888,   \s2_8888, \
+                           \m1_8,      \m2_8, \
+                           \out1_8888, \out2_8888, \
+                           \maskLSR, \
+                           \scratch1,  \scratch2, \scratch3, \
+                           \scratch4,  \scratch5, \scratch6
+
+    addu_s.qb             \out1_8888, \out1_8888, \d1_8888
+    addu_s.qb             \out2_8888, \out2_8888, \d2_8888
+.endm
+
+.macro BILINEAR_INTERPOLATE_SINGLE_PIXEL tl, tr, bl, br,         \
+                                         scratch1, scratch2,     \
+                                         alpha, red, green, blue \
+                                         wt1, wt2, wb1, wb2
+    andi            \scratch1, \tl,  0xff
+    andi            \scratch2, \tr,  0xff
+    andi            \alpha,    \bl,  0xff
+    andi            \red,      \br,  0xff
+
+    multu           $ac0,      \wt1, \scratch1
+    maddu           $ac0,      \wt2, \scratch2
+    maddu           $ac0,      \wb1, \alpha
+    maddu           $ac0,      \wb2, \red
+
+    ext             \scratch1, \tl,  8, 8
+    ext             \scratch2, \tr,  8, 8
+    ext             \alpha,    \bl,  8, 8
+    ext             \red,      \br,  8, 8
+
+    multu           $ac1,      \wt1, \scratch1
+    maddu           $ac1,      \wt2, \scratch2
+    maddu           $ac1,      \wb1, \alpha
+    maddu           $ac1,      \wb2, \red
+
+    ext             \scratch1, \tl,  16, 8
+    ext             \scratch2, \tr,  16, 8
+    ext             \alpha,    \bl,  16, 8
+    ext             \red,      \br,  16, 8
+
+    mflo            \blue,     $ac0
+
+    multu           $ac2,      \wt1, \scratch1
+    maddu           $ac2,      \wt2, \scratch2
+    maddu           $ac2,      \wb1, \alpha
+    maddu           $ac2,      \wb2, \red
+
+    ext             \scratch1, \tl,  24, 8
+    ext             \scratch2, \tr,  24, 8
+    ext             \alpha,    \bl,  24, 8
+    ext             \red,      \br,  24, 8
+
+    mflo            \green,    $ac1
+
+    multu           $ac3,      \wt1, \scratch1
+    maddu           $ac3,      \wt2, \scratch2
+    maddu           $ac3,      \wb1, \alpha
+    maddu           $ac3,      \wb2, \red
+
+    mflo            \red,      $ac2
+    mflo            \alpha,    $ac3
+
+    precr.qb.ph     \alpha,    \alpha, \red
+    precr.qb.ph     \scratch1, \green, \blue
+    precrq.qb.ph    \tl,       \alpha, \scratch1
+.endm
+
+#endif //PIXMAN_MIPS_DSPR2_ASM_H
diff --git a/pixman/pixman/pixman-mips-dspr2.c b/pixman/pixman/pixman-mips-dspr2.c
new file mode 100644
index 000000000..e10c9df0a
--- /dev/null
+++ b/pixman/pixman/pixman-mips-dspr2.c
@@ -0,0 +1,459 @@
+/*
+ * Copyright (c) 2012
+ *      MIPS Technologies, Inc., California.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the MIPS Technologies, Inc., nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE MIPS TECHNOLOGIES, INC. ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE MIPS TECHNOLOGIES, INC. BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Author:  Nemanja Lukic (nlukic@mips.com)
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "pixman-private.h"
+#include "pixman-mips-dspr2.h"
+
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_DST (0, src_x888_8888,
+                                    uint32_t, 1, uint32_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_DST (0, src_8888_0565,
+                                    uint32_t, 1, uint16_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_DST (0, src_0565_8888,
+                                    uint16_t, 1, uint32_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_DST (DO_FAST_MEMCPY, src_0565_0565,
+                                    uint16_t, 1, uint16_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_DST (DO_FAST_MEMCPY, src_8888_8888,
+                                    uint32_t, 1, uint32_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_DST (DO_FAST_MEMCPY, src_0888_0888,
+                                    uint8_t, 3, uint8_t, 3)
+#if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_DST (0, src_0888_8888_rev,
+                                    uint8_t, 3, uint32_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_DST (0, src_0888_0565_rev,
+                                    uint8_t, 3, uint16_t, 1)
+#endif
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_DST (0, src_pixbuf_8888,
+                                    uint32_t, 1, uint32_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_DST (0, src_rpixbuf_8888,
+                                    uint32_t, 1, uint32_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_DST (0, over_8888_8888,
+                                    uint32_t, 1, uint32_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_DST (0, over_8888_0565,
+                                    uint32_t, 1, uint16_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_DST (0, add_8_8,
+                                    uint8_t, 1, uint8_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_DST (0, add_8888_8888,
+                                    uint32_t, 1, uint32_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_DST (0, out_reverse_8_0565,
+                                    uint8_t, 1, uint16_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_DST (0, out_reverse_8_8888,
+                                    uint8_t,  1, uint32_t, 1)
+
+PIXMAN_MIPS_BIND_FAST_PATH_N_MASK_DST (0, src_n_8_8888,
+                                       uint8_t, 1, uint32_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_N_MASK_DST (0, src_n_8_8,
+                                       uint8_t, 1, uint8_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, over_n_8888_8888_ca,
+                                       uint32_t, 1, uint32_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, over_n_8888_0565_ca,
+                                       uint32_t, 1, uint16_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, over_n_8_8,
+                                       uint8_t, 1, uint8_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, over_n_8_8888,
+                                       uint8_t, 1, uint32_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, over_n_8_0565,
+                                       uint8_t, 1, uint16_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, add_n_8_8,
+                                       uint8_t, 1, uint8_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, add_n_8_8888,
+                                       uint8_t, 1, uint32_t, 1)
+
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_N_DST (SKIP_ZERO_MASK, over_8888_n_8888,
+                                      uint32_t, 1, uint32_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_N_DST (SKIP_ZERO_MASK, over_8888_n_0565,
+                                      uint32_t, 1, uint16_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_N_DST (SKIP_ZERO_MASK, over_0565_n_0565,
+                                      uint16_t, 1, uint16_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_N_DST (SKIP_ZERO_MASK, add_8888_n_8888,
+                                      uint32_t, 1, uint32_t, 1)
+
+PIXMAN_MIPS_BIND_FAST_PATH_N_DST (SKIP_ZERO_SRC, over_n_0565,
+                                  uint16_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_N_DST (SKIP_ZERO_SRC, over_n_8888,
+                                  uint32_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_N_DST (SKIP_ZERO_SRC, over_reverse_n_8888,
+                                  uint32_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_N_DST (0, in_n_8,
+                                  uint8_t, 1)
+
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_MASK_DST (add_8_8_8, uint8_t,  1,
+                                         uint8_t,  1, uint8_t,  1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_MASK_DST (add_8888_8_8888, uint32_t, 1,
+                                         uint8_t, 1, uint32_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_MASK_DST (add_8888_8888_8888, uint32_t, 1,
+                                         uint32_t, 1, uint32_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_MASK_DST (add_0565_8_0565, uint16_t, 1,
+                                         uint8_t,  1, uint16_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_MASK_DST (over_8888_8_8888, uint32_t, 1,
+                                         uint8_t, 1, uint32_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_MASK_DST (over_8888_8_0565, uint32_t, 1,
+                                         uint8_t, 1, uint16_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_MASK_DST (over_0565_8_0565, uint16_t, 1,
+                                         uint8_t, 1, uint16_t, 1)
+PIXMAN_MIPS_BIND_FAST_PATH_SRC_MASK_DST (over_8888_8888_8888, uint32_t, 1,
+                                         uint32_t, 1, uint32_t, 1)
+
+PIXMAN_MIPS_BIND_SCALED_NEAREST_SRC_DST (8888_8888, OVER,
+                                         uint32_t, uint32_t)
+PIXMAN_MIPS_BIND_SCALED_NEAREST_SRC_DST (8888_0565, OVER,
+                                         uint32_t, uint16_t)
+PIXMAN_MIPS_BIND_SCALED_NEAREST_SRC_DST (0565_8888, SRC,
+                                         uint16_t, uint32_t)
+
+PIXMAN_MIPS_BIND_SCALED_BILINEAR_SRC_DST (0, 8888_8888, SRC,
+                                          uint32_t, uint32_t)
+PIXMAN_MIPS_BIND_SCALED_BILINEAR_SRC_DST (0, 8888_0565, SRC,
+                                          uint32_t, uint16_t)
+PIXMAN_MIPS_BIND_SCALED_BILINEAR_SRC_DST (0, 0565_8888, SRC,
+                                          uint16_t, uint32_t)
+PIXMAN_MIPS_BIND_SCALED_BILINEAR_SRC_DST (0, 0565_0565, SRC,
+                                          uint16_t, uint16_t)
+PIXMAN_MIPS_BIND_SCALED_BILINEAR_SRC_DST (SKIP_ZERO_SRC, 8888_8888, OVER,
+                                          uint32_t, uint32_t)
+PIXMAN_MIPS_BIND_SCALED_BILINEAR_SRC_DST (SKIP_ZERO_SRC, 8888_8888, ADD,
+                                          uint32_t, uint32_t)
+
+PIXMAN_MIPS_BIND_SCALED_NEAREST_SRC_A8_DST (SKIP_ZERO_SRC, 8888_8_0565,
+                                            OVER, uint32_t, uint16_t)
+PIXMAN_MIPS_BIND_SCALED_NEAREST_SRC_A8_DST (SKIP_ZERO_SRC, 0565_8_0565,
+                                            OVER, uint16_t, uint16_t)
+
+PIXMAN_MIPS_BIND_SCALED_BILINEAR_SRC_A8_DST (0, 8888_8_8888, SRC,
+                                             uint32_t, uint32_t)
+PIXMAN_MIPS_BIND_SCALED_BILINEAR_SRC_A8_DST (0, 8888_8_0565, SRC,
+                                             uint32_t, uint16_t)
+PIXMAN_MIPS_BIND_SCALED_BILINEAR_SRC_A8_DST (0, 0565_8_x888, SRC,
+                                             uint16_t, uint32_t)
+PIXMAN_MIPS_BIND_SCALED_BILINEAR_SRC_A8_DST (0, 0565_8_0565, SRC,
+                                             uint16_t, uint16_t)
+PIXMAN_MIPS_BIND_SCALED_BILINEAR_SRC_A8_DST (SKIP_ZERO_SRC, 8888_8_8888, OVER,
+                                             uint32_t, uint32_t)
+PIXMAN_MIPS_BIND_SCALED_BILINEAR_SRC_A8_DST (SKIP_ZERO_SRC, 8888_8_8888, ADD,
+                                             uint32_t, uint32_t)
+
+static pixman_bool_t
+mips_dspr2_fill (pixman_implementation_t *imp,
+                 uint32_t *               bits,
+                 int                      stride,
+                 int                      bpp,
+                 int                      x,
+                 int                      y,
+                 int                      width,
+                 int                      height,
+                 uint32_t                 _xor)
+{
+    uint8_t *byte_line;
+    uint32_t byte_width;
+    switch (bpp)
+    {
+    case 16:
+        stride = stride * (int) sizeof (uint32_t) / 2;
+        byte_line = (uint8_t *)(((uint16_t *)bits) + stride * y + x);
+        byte_width = width * 2;
+        stride *= 2;
+
+        while (height--)
+        {
+            uint8_t *dst = byte_line;
+            byte_line += stride;
+            pixman_fill_buff16_mips (dst, byte_width, _xor & 0xffff);
+        }
+        return TRUE;
+    case 32:
+        stride = stride * (int) sizeof (uint32_t) / 4;
+        byte_line = (uint8_t *)(((uint32_t *)bits) + stride * y + x);
+        byte_width = width * 4;
+        stride *= 4;
+
+        while (height--)
+        {
+            uint8_t *dst = byte_line;
+            byte_line += stride;
+            pixman_fill_buff32_mips (dst, byte_width, _xor);
+        }
+        return TRUE;
+    default:
+        return FALSE;
+    }
+}
+
+static pixman_bool_t
+mips_dspr2_blt (pixman_implementation_t *imp,
+                uint32_t *               src_bits,
+                uint32_t *               dst_bits,
+                int                      src_stride,
+                int                      dst_stride,
+                int                      src_bpp,
+                int                      dst_bpp,
+                int                      src_x,
+                int                      src_y,
+                int                      dest_x,
+                int                      dest_y,
+                int                      width,
+                int                      height)
+{
+    if (src_bpp != dst_bpp)
+        return FALSE;
+
+    uint8_t *src_bytes;
+    uint8_t *dst_bytes;
+    uint32_t byte_width;
+
+    switch (src_bpp)
+    {
+    case 16:
+        src_stride = src_stride * (int) sizeof (uint32_t) / 2;
+        dst_stride = dst_stride * (int) sizeof (uint32_t) / 2;
+        src_bytes =(uint8_t *)(((uint16_t *)src_bits)
+                                          + src_stride * (src_y) + (src_x));
+        dst_bytes = (uint8_t *)(((uint16_t *)dst_bits)
+                                           + dst_stride * (dest_y) + (dest_x));
+        byte_width = width * 2;
+        src_stride *= 2;
+        dst_stride *= 2;
+
+        while (height--)
+        {
+            uint8_t *src = src_bytes;
+            uint8_t *dst = dst_bytes;
+            src_bytes += src_stride;
+            dst_bytes += dst_stride;
+            pixman_mips_fast_memcpy (dst, src, byte_width);
+        }
+        return TRUE;
+    case 32:
+        src_stride = src_stride * (int) sizeof (uint32_t) / 4;
+        dst_stride = dst_stride * (int) sizeof (uint32_t) / 4;
+        src_bytes = (uint8_t *)(((uint32_t *)src_bits)
+                                           + src_stride * (src_y) + (src_x));
+        dst_bytes = (uint8_t *)(((uint32_t *)dst_bits)
+                                           + dst_stride * (dest_y) + (dest_x));
+        byte_width = width * 4;
+        src_stride *= 4;
+        dst_stride *= 4;
+
+        while (height--)
+        {
+            uint8_t *src = src_bytes;
+            uint8_t *dst = dst_bytes;
+            src_bytes += src_stride;
+            dst_bytes += dst_stride;
+            pixman_mips_fast_memcpy (dst, src, byte_width);
+        }
+        return TRUE;
+    default:
+        return FALSE;
+    }
+}
+
+static const pixman_fast_path_t mips_dspr2_fast_paths[] =
+{
+    PIXMAN_STD_FAST_PATH (SRC, r5g6b5,   null, r5g6b5,   mips_composite_src_0565_0565),
+    PIXMAN_STD_FAST_PATH (SRC, b5g6r5,   null, b5g6r5,   mips_composite_src_0565_0565),
+    PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, r5g6b5,   mips_composite_src_8888_0565),
+    PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, r5g6b5,   mips_composite_src_8888_0565),
+    PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, b5g6r5,   mips_composite_src_8888_0565),
+    PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, b5g6r5,   mips_composite_src_8888_0565),
+    PIXMAN_STD_FAST_PATH (SRC, r5g6b5,   null, a8r8g8b8, mips_composite_src_0565_8888),
+    PIXMAN_STD_FAST_PATH (SRC, r5g6b5,   null, x8r8g8b8, mips_composite_src_0565_8888),
+    PIXMAN_STD_FAST_PATH (SRC, b5g6r5,   null, a8b8g8r8, mips_composite_src_0565_8888),
+    PIXMAN_STD_FAST_PATH (SRC, b5g6r5,   null, x8b8g8r8, mips_composite_src_0565_8888),
+    PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, x8r8g8b8, mips_composite_src_8888_8888),
+    PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, x8r8g8b8, mips_composite_src_8888_8888),
+    PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, x8b8g8r8, mips_composite_src_8888_8888),
+    PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, x8b8g8r8, mips_composite_src_8888_8888),
+    PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, a8r8g8b8, mips_composite_src_8888_8888),
+    PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, a8b8g8r8, mips_composite_src_8888_8888),
+    PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, a8r8g8b8, mips_composite_src_x888_8888),
+    PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, a8b8g8r8, mips_composite_src_x888_8888),
+    PIXMAN_STD_FAST_PATH (SRC, r8g8b8,   null, r8g8b8,   mips_composite_src_0888_0888),
+#if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL)
+    PIXMAN_STD_FAST_PATH (SRC, b8g8r8,   null, x8r8g8b8, mips_composite_src_0888_8888_rev),
+    PIXMAN_STD_FAST_PATH (SRC, b8g8r8,   null, r5g6b5,   mips_composite_src_0888_0565_rev),
+#endif
+    PIXMAN_STD_FAST_PATH (SRC, pixbuf,   pixbuf,  a8r8g8b8, mips_composite_src_pixbuf_8888),
+    PIXMAN_STD_FAST_PATH (SRC, pixbuf,   pixbuf,  a8b8g8r8, mips_composite_src_rpixbuf_8888),
+    PIXMAN_STD_FAST_PATH (SRC, rpixbuf,  rpixbuf, a8r8g8b8, mips_composite_src_rpixbuf_8888),
+    PIXMAN_STD_FAST_PATH (SRC, rpixbuf,  rpixbuf, a8b8g8r8, mips_composite_src_pixbuf_8888),
+    PIXMAN_STD_FAST_PATH (SRC, solid,    a8,   a8r8g8b8, mips_composite_src_n_8_8888),
+    PIXMAN_STD_FAST_PATH (SRC, solid,    a8,   x8r8g8b8, mips_composite_src_n_8_8888),
+    PIXMAN_STD_FAST_PATH (SRC, solid,    a8,   a8b8g8r8, mips_composite_src_n_8_8888),
+    PIXMAN_STD_FAST_PATH (SRC, solid,    a8,   x8b8g8r8, mips_composite_src_n_8_8888),
+    PIXMAN_STD_FAST_PATH (SRC, solid,    a8,   a8,       mips_composite_src_n_8_8),
+
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, a8r8g8b8, mips_composite_over_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, x8r8g8b8, mips_composite_over_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, a8b8g8r8, mips_composite_over_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, x8b8g8r8, mips_composite_over_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, r5g6b5,   mips_composite_over_n_8888_0565_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, b5g6r5,   mips_composite_over_n_8888_0565_ca),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       a8,       mips_composite_over_n_8_8),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       a8r8g8b8, mips_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       x8r8g8b8, mips_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       a8b8g8r8, mips_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       x8b8g8r8, mips_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       r5g6b5,   mips_composite_over_n_8_0565),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       b5g6r5,   mips_composite_over_n_8_0565),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    null,     r5g6b5,   mips_composite_over_n_0565),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    null,     a8r8g8b8, mips_composite_over_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    null,     x8r8g8b8, mips_composite_over_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid,    a8r8g8b8, mips_composite_over_8888_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid,    x8r8g8b8, mips_composite_over_8888_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid,    r5g6b5,   mips_composite_over_8888_n_0565),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, solid,    b5g6r5,   mips_composite_over_8888_n_0565),
+    PIXMAN_STD_FAST_PATH (OVER, r5g6b5,   solid,    r5g6b5,   mips_composite_over_0565_n_0565),
+    PIXMAN_STD_FAST_PATH (OVER, b5g6r5,   solid,    b5g6r5,   mips_composite_over_0565_n_0565),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8,       a8r8g8b8, mips_composite_over_8888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8,       x8r8g8b8, mips_composite_over_8888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8,       a8b8g8r8, mips_composite_over_8888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8,       x8b8g8r8, mips_composite_over_8888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8,       r5g6b5,   mips_composite_over_8888_8_0565),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8,       b5g6r5,   mips_composite_over_8888_8_0565),
+    PIXMAN_STD_FAST_PATH (OVER, r5g6b5,   a8,       r5g6b5,   mips_composite_over_0565_8_0565),
+    PIXMAN_STD_FAST_PATH (OVER, b5g6r5,   a8,       b5g6r5,   mips_composite_over_0565_8_0565),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, a8r8g8b8, mips_composite_over_8888_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null,     a8r8g8b8, mips_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null,     x8r8g8b8, mips_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null,     a8b8g8r8, mips_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null,     x8b8g8r8, mips_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null,     r5g6b5,   mips_composite_over_8888_0565),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null,     b5g6r5,   mips_composite_over_8888_0565),
+    PIXMAN_STD_FAST_PATH (ADD,  solid,    a8,       a8,       mips_composite_add_n_8_8),
+    PIXMAN_STD_FAST_PATH (ADD,  solid,    a8,       a8r8g8b8, mips_composite_add_n_8_8888),
+    PIXMAN_STD_FAST_PATH (ADD,  solid,    a8,       a8b8g8r8, mips_composite_add_n_8_8888),
+    PIXMAN_STD_FAST_PATH (ADD,  a8,       a8,       a8,       mips_composite_add_8_8_8),
+    PIXMAN_STD_FAST_PATH (ADD,  r5g6b5,   a8,       r5g6b5,   mips_composite_add_0565_8_0565),
+    PIXMAN_STD_FAST_PATH (ADD,  b5g6r5,   a8,       b5g6r5,   mips_composite_add_0565_8_0565),
+    PIXMAN_STD_FAST_PATH (ADD,  a8r8g8b8, a8,       a8r8g8b8, mips_composite_add_8888_8_8888),
+    PIXMAN_STD_FAST_PATH (ADD,  a8b8g8r8, a8,       a8b8g8r8, mips_composite_add_8888_8_8888),
+    PIXMAN_STD_FAST_PATH (ADD,  a8r8g8b8, a8r8g8b8, a8r8g8b8, mips_composite_add_8888_8888_8888),
+    PIXMAN_STD_FAST_PATH (ADD,  a8r8g8b8, solid,    a8r8g8b8, mips_composite_add_8888_n_8888),
+    PIXMAN_STD_FAST_PATH (ADD,  a8b8g8r8, solid,    a8b8g8r8, mips_composite_add_8888_n_8888),
+    PIXMAN_STD_FAST_PATH (ADD,  a8,       null,     a8,       mips_composite_add_8_8),
+    PIXMAN_STD_FAST_PATH (ADD,  a8r8g8b8, null,     a8r8g8b8, mips_composite_add_8888_8888),
+    PIXMAN_STD_FAST_PATH (ADD,  a8b8g8r8, null,     a8b8g8r8, mips_composite_add_8888_8888),
+    PIXMAN_STD_FAST_PATH (OUT_REVERSE, a8,    null, r5g6b5,   mips_composite_out_reverse_8_0565),
+    PIXMAN_STD_FAST_PATH (OUT_REVERSE, a8,    null, b5g6r5,   mips_composite_out_reverse_8_0565),
+    PIXMAN_STD_FAST_PATH (OUT_REVERSE, a8,    null, a8r8g8b8, mips_composite_out_reverse_8_8888),
+    PIXMAN_STD_FAST_PATH (OUT_REVERSE, a8,    null, a8b8g8r8, mips_composite_out_reverse_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8r8g8b8, mips_composite_over_reverse_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8b8g8r8, mips_composite_over_reverse_n_8888),
+    PIXMAN_STD_FAST_PATH (IN,           solid, null, a8,       mips_composite_in_n_8),
+
+    PIXMAN_MIPS_SIMPLE_NEAREST_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, mips_8888_8888),
+    PIXMAN_MIPS_SIMPLE_NEAREST_FAST_PATH (OVER, a8b8g8r8, a8b8g8r8, mips_8888_8888),
+    PIXMAN_MIPS_SIMPLE_NEAREST_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, mips_8888_8888),
+    PIXMAN_MIPS_SIMPLE_NEAREST_FAST_PATH (OVER, a8b8g8r8, x8b8g8r8, mips_8888_8888),
+
+    PIXMAN_MIPS_SIMPLE_NEAREST_FAST_PATH (OVER, a8r8g8b8, r5g6b5, mips_8888_0565),
+    PIXMAN_MIPS_SIMPLE_NEAREST_FAST_PATH (OVER, a8b8g8r8, b5g6r5, mips_8888_0565),
+
+    PIXMAN_MIPS_SIMPLE_NEAREST_FAST_PATH (SRC, b5g6r5, x8b8g8r8, mips_0565_8888),
+    PIXMAN_MIPS_SIMPLE_NEAREST_FAST_PATH (SRC, r5g6b5, x8r8g8b8, mips_0565_8888),
+    /* Note: NONE repeat is not supported yet */
+    SIMPLE_NEAREST_FAST_PATH_COVER (SRC, r5g6b5, a8r8g8b8, mips_0565_8888),
+    SIMPLE_NEAREST_FAST_PATH_COVER (SRC, b5g6r5, a8b8g8r8, mips_0565_8888),
+    SIMPLE_NEAREST_FAST_PATH_PAD (SRC, r5g6b5, a8r8g8b8, mips_0565_8888),
+    SIMPLE_NEAREST_FAST_PATH_PAD (SRC, b5g6r5, a8b8g8r8, mips_0565_8888),
+
+    PIXMAN_MIPS_SIMPLE_NEAREST_A8_MASK_FAST_PATH (OVER, a8r8g8b8, r5g6b5, mips_8888_8_0565),
+    PIXMAN_MIPS_SIMPLE_NEAREST_A8_MASK_FAST_PATH (OVER, a8b8g8r8, b5g6r5, mips_8888_8_0565),
+
+    PIXMAN_MIPS_SIMPLE_NEAREST_A8_MASK_FAST_PATH (OVER, r5g6b5, r5g6b5, mips_0565_8_0565),
+    PIXMAN_MIPS_SIMPLE_NEAREST_A8_MASK_FAST_PATH (OVER, b5g6r5, b5g6r5, mips_0565_8_0565),
+
+    SIMPLE_BILINEAR_FAST_PATH (SRC, a8r8g8b8, a8r8g8b8, mips_8888_8888),
+    SIMPLE_BILINEAR_FAST_PATH (SRC, a8r8g8b8, x8r8g8b8, mips_8888_8888),
+    SIMPLE_BILINEAR_FAST_PATH (SRC, x8r8g8b8, x8r8g8b8, mips_8888_8888),
+
+    SIMPLE_BILINEAR_FAST_PATH (SRC, a8r8g8b8, r5g6b5, mips_8888_0565),
+    SIMPLE_BILINEAR_FAST_PATH (SRC, x8r8g8b8, r5g6b5, mips_8888_0565),
+
+    SIMPLE_BILINEAR_FAST_PATH (SRC, r5g6b5, x8r8g8b8, mips_0565_8888),
+    SIMPLE_BILINEAR_FAST_PATH (SRC, r5g6b5, r5g6b5, mips_0565_0565),
+
+    SIMPLE_BILINEAR_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, mips_8888_8888),
+    SIMPLE_BILINEAR_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, mips_8888_8888),
+
+    SIMPLE_BILINEAR_FAST_PATH (ADD, a8r8g8b8, a8r8g8b8, mips_8888_8888),
+    SIMPLE_BILINEAR_FAST_PATH (ADD, a8r8g8b8, x8r8g8b8, mips_8888_8888),
+
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, a8r8g8b8, a8r8g8b8, mips_8888_8_8888),
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, a8r8g8b8, x8r8g8b8, mips_8888_8_8888),
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, x8r8g8b8, x8r8g8b8, mips_8888_8_8888),
+
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, a8r8g8b8, r5g6b5, mips_8888_8_0565),
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, x8r8g8b8, r5g6b5, mips_8888_8_0565),
+
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, r5g6b5, x8r8g8b8, mips_0565_8_x888),
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, r5g6b5, r5g6b5, mips_0565_8_0565),
+
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, mips_8888_8_8888),
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, mips_8888_8_8888),
+
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (ADD, a8r8g8b8, a8r8g8b8, mips_8888_8_8888),
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (ADD, a8r8g8b8, x8r8g8b8, mips_8888_8_8888),
+    { PIXMAN_OP_NONE },
+};
+
+static void
+mips_dspr2_combine_over_u (pixman_implementation_t *imp,
+                           pixman_op_t              op,
+                           uint32_t *               dest,
+                           const uint32_t *         src,
+                           const uint32_t *         mask,
+                           int                      width)
+{
+    if (mask)
+        pixman_composite_over_8888_8888_8888_asm_mips (
+            dest, (uint32_t *)src, (uint32_t *)mask, width);
+    else
+        pixman_composite_over_8888_8888_asm_mips (
+		    dest, (uint32_t *)src, width);
+}
+
+pixman_implementation_t *
+_pixman_implementation_create_mips_dspr2 (pixman_implementation_t *fallback)
+{
+    pixman_implementation_t *imp =
+        _pixman_implementation_create (fallback, mips_dspr2_fast_paths);
+
+    imp->combine_32[PIXMAN_OP_OVER] = mips_dspr2_combine_over_u;
+
+    imp->blt = mips_dspr2_blt;
+    imp->fill = mips_dspr2_fill;
+
+    return imp;
+}
diff --git a/pixman/pixman/pixman-mips-dspr2.h b/pixman/pixman/pixman-mips-dspr2.h
new file mode 100644
index 000000000..955ed70b8
--- /dev/null
+++ b/pixman/pixman/pixman-mips-dspr2.h
@@ -0,0 +1,438 @@
+/*
+ * Copyright (c) 2012
+ *      MIPS Technologies, Inc., California.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the MIPS Technologies, Inc., nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE MIPS TECHNOLOGIES, INC. ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE MIPS TECHNOLOGIES, INC. BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Author:  Nemanja Lukic (nlukic@mips.com)
+ */
+
+#ifndef PIXMAN_MIPS_DSPR2_H
+#define PIXMAN_MIPS_DSPR2_H
+
+#include "pixman-private.h"
+#include "pixman-inlines.h"
+
+#define SKIP_ZERO_SRC  1
+#define SKIP_ZERO_MASK 2
+#define DO_FAST_MEMCPY 3
+
+void
+pixman_mips_fast_memcpy (void *dst, void *src, uint32_t n_bytes);
+void
+pixman_fill_buff16_mips (void *dst, uint32_t n_bytes, uint16_t value);
+void
+pixman_fill_buff32_mips (void *dst, uint32_t n_bytes, uint32_t value);
+
+/****************************************************************/
+
+#define PIXMAN_MIPS_BIND_FAST_PATH_SRC_DST(flags, name,          \
+                                           src_type, src_cnt,    \
+                                           dst_type, dst_cnt)    \
+void                                                             \
+pixman_composite_##name##_asm_mips (dst_type *dst,               \
+                                    src_type *src,               \
+                                    int32_t   w);                \
+                                                                 \
+static void                                                      \
+mips_composite_##name (pixman_implementation_t *imp,             \
+                       pixman_composite_info_t *info)            \
+{                                                                \
+    PIXMAN_COMPOSITE_ARGS (info);                                \
+    dst_type *dst_line, *dst;                                    \
+    src_type *src_line, *src;                                    \
+    int32_t dst_stride, src_stride;                              \
+    int bpp = PIXMAN_FORMAT_BPP (dest_image->bits.format) / 8;   \
+                                                                 \
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, src_type,    \
+                           src_stride, src_line, src_cnt);       \
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, dst_type, \
+                           dst_stride, dst_line, dst_cnt);       \
+                                                                 \
+    while (height--)                                             \
+    {                                                            \
+      dst = dst_line;                                            \
+      dst_line += dst_stride;                                    \
+      src = src_line;                                            \
+      src_line += src_stride;                                    \
+                                                                 \
+      if (flags == DO_FAST_MEMCPY)                               \
+        pixman_mips_fast_memcpy (dst, src, width * bpp);         \
+      else                                                       \
+        pixman_composite_##name##_asm_mips (dst, src, width);    \
+    }                                                            \
+}
+
+/****************************************************************/
+
+#define PIXMAN_MIPS_BIND_FAST_PATH_N_DST(flags, name,            \
+                                         dst_type, dst_cnt)      \
+void                                                             \
+pixman_composite_##name##_asm_mips (dst_type *dst,               \
+                                    uint32_t  src,               \
+                                    int32_t   w);                \
+                                                                 \
+static void                                                      \
+mips_composite_##name (pixman_implementation_t *imp,             \
+                       pixman_composite_info_t *info)            \
+{                                                                \
+    PIXMAN_COMPOSITE_ARGS (info);                                \
+    dst_type  *dst_line, *dst;                                   \
+    int32_t    dst_stride;                                       \
+    uint32_t   src;                                              \
+                                                                 \
+    src = _pixman_image_get_solid (                              \
+    imp, src_image, dest_image->bits.format);                    \
+                                                                 \
+    if ((flags & SKIP_ZERO_SRC) && src == 0)                     \
+        return;                                                  \
+                                                                 \
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, dst_type, \
+                           dst_stride, dst_line, dst_cnt);       \
+                                                                 \
+    while (height--)                                             \
+    {                                                            \
+        dst = dst_line;                                          \
+        dst_line += dst_stride;                                  \
+                                                                 \
+        pixman_composite_##name##_asm_mips (dst, src, width);    \
+    }                                                            \
+}
+
+/*******************************************************************/
+
+#define PIXMAN_MIPS_BIND_FAST_PATH_N_MASK_DST(flags, name,          \
+                                              mask_type, mask_cnt,  \
+                                              dst_type, dst_cnt)    \
+void                                                                \
+pixman_composite_##name##_asm_mips (dst_type  *dst,                 \
+                                    uint32_t  src,                  \
+                                    mask_type *mask,                \
+                                    int32_t   w);                   \
+                                                                    \
+static void                                                         \
+mips_composite_##name (pixman_implementation_t *imp,                \
+                       pixman_composite_info_t *info)               \
+{                                                                   \
+    PIXMAN_COMPOSITE_ARGS (info);                                   \
+    dst_type  *dst_line, *dst;                                      \
+    mask_type *mask_line, *mask;                                    \
+    int32_t    dst_stride, mask_stride;                             \
+    uint32_t   src;                                                 \
+                                                                    \
+    src = _pixman_image_get_solid (                                 \
+        imp, src_image, dest_image->bits.format);                   \
+                                                                    \
+    if ((flags & SKIP_ZERO_SRC) && src == 0)                        \
+        return;                                                     \
+                                                                    \
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, dst_type,    \
+                           dst_stride, dst_line, dst_cnt);          \
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, mask_type,   \
+                           mask_stride, mask_line, mask_cnt);       \
+                                                                    \
+    while (height--)                                                \
+    {                                                               \
+        dst = dst_line;                                             \
+        dst_line += dst_stride;                                     \
+        mask = mask_line;                                           \
+        mask_line += mask_stride;                                   \
+        pixman_composite_##name##_asm_mips (dst, src, mask, width); \
+    }                                                               \
+}
+
+/*******************************************************************/
+
+#define PIXMAN_MIPS_BIND_FAST_PATH_SRC_N_DST(flags, name,           \
+                                            src_type, src_cnt,      \
+                                            dst_type, dst_cnt)      \
+void                                                                \
+pixman_composite_##name##_asm_mips (dst_type  *dst,                 \
+                                    src_type  *src,                 \
+                                    uint32_t   mask,                \
+                                    int32_t    w);                  \
+                                                                    \
+static void                                                         \
+mips_composite_##name (pixman_implementation_t *imp,                \
+                       pixman_composite_info_t *info)               \
+{                                                                   \
+    PIXMAN_COMPOSITE_ARGS (info);                                   \
+    dst_type  *dst_line, *dst;                                      \
+    src_type  *src_line, *src;                                      \
+    int32_t    dst_stride, src_stride;                              \
+    uint32_t   mask;                                                \
+                                                                    \
+    mask = _pixman_image_get_solid (                                \
+        imp, mask_image, dest_image->bits.format);                  \
+                                                                    \
+    if ((flags & SKIP_ZERO_MASK) && mask == 0)                      \
+        return;                                                     \
+                                                                    \
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, dst_type,    \
+                           dst_stride, dst_line, dst_cnt);          \
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, src_type,       \
+                           src_stride, src_line, src_cnt);          \
+                                                                    \
+    while (height--)                                                \
+    {                                                               \
+        dst = dst_line;                                             \
+        dst_line += dst_stride;                                     \
+        src = src_line;                                             \
+        src_line += src_stride;                                     \
+                                                                    \
+        pixman_composite_##name##_asm_mips (dst, src, mask, width); \
+    }                                                               \
+}
+
+/************************************************************************/
+
+#define PIXMAN_MIPS_BIND_FAST_PATH_SRC_MASK_DST(name, src_type, src_cnt, \
+                                                mask_type, mask_cnt,     \
+                                                dst_type, dst_cnt)       \
+void                                                                     \
+pixman_composite_##name##_asm_mips (dst_type  *dst,                      \
+                                    src_type  *src,                      \
+                                    mask_type *mask,                     \
+                                    int32_t   w);                        \
+                                                                         \
+static void                                                              \
+mips_composite_##name (pixman_implementation_t *imp,                     \
+                       pixman_composite_info_t *info)                    \
+{                                                                        \
+    PIXMAN_COMPOSITE_ARGS (info);                                        \
+    dst_type  *dst_line, *dst;                                           \
+    src_type  *src_line, *src;                                           \
+    mask_type *mask_line, *mask;                                         \
+    int32_t    dst_stride, src_stride, mask_stride;                      \
+                                                                         \
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, dst_type,         \
+                           dst_stride, dst_line, dst_cnt);               \
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, src_type,            \
+                           src_stride, src_line, src_cnt);               \
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, mask_type,        \
+                           mask_stride, mask_line, mask_cnt);            \
+                                                                         \
+    while (height--)                                                     \
+    {                                                                    \
+        dst = dst_line;                                                  \
+        dst_line += dst_stride;                                          \
+        mask = mask_line;                                                \
+        mask_line += mask_stride;                                        \
+        src = src_line;                                                  \
+        src_line += src_stride;                                          \
+        pixman_composite_##name##_asm_mips (dst, src, mask, width);      \
+    }                                                                    \
+}
+
+/****************************************************************************/
+
+#define PIXMAN_MIPS_BIND_SCALED_NEAREST_SRC_DST(name, op,                    \
+                                                src_type, dst_type)          \
+void                                                                         \
+pixman_scaled_nearest_scanline_##name##_##op##_asm_mips (                    \
+                                                   dst_type *       dst,     \
+                                                   const src_type * src,     \
+                                                   int32_t          w,       \
+                                                   pixman_fixed_t   vx,      \
+                                                   pixman_fixed_t   unit_x); \
+                                                                             \
+static force_inline void                                                     \
+scaled_nearest_scanline_mips_##name##_##op (dst_type *       pd,             \
+                                            const src_type * ps,             \
+                                            int32_t          w,              \
+                                            pixman_fixed_t   vx,             \
+                                            pixman_fixed_t   unit_x,         \
+                                            pixman_fixed_t   max_vx,         \
+                                            pixman_bool_t    zero_src)       \
+{                                                                            \
+    pixman_scaled_nearest_scanline_##name##_##op##_asm_mips (pd, ps, w,      \
+                                                             vx, unit_x);    \
+}                                                                            \
+                                                                             \
+FAST_NEAREST_MAINLOOP (mips_##name##_cover_##op,                             \
+                       scaled_nearest_scanline_mips_##name##_##op,           \
+                       src_type, dst_type, COVER)                            \
+FAST_NEAREST_MAINLOOP (mips_##name##_none_##op,                              \
+                       scaled_nearest_scanline_mips_##name##_##op,           \
+                       src_type, dst_type, NONE)                             \
+FAST_NEAREST_MAINLOOP (mips_##name##_pad_##op,                               \
+                       scaled_nearest_scanline_mips_##name##_##op,           \
+                       src_type, dst_type, PAD)
+
+/* Provide entries for the fast path table */
+#define PIXMAN_MIPS_SIMPLE_NEAREST_FAST_PATH(op,s,d,func)                    \
+    SIMPLE_NEAREST_FAST_PATH_COVER (op,s,d,func),                            \
+    SIMPLE_NEAREST_FAST_PATH_NONE (op,s,d,func),                             \
+    SIMPLE_NEAREST_FAST_PATH_PAD (op,s,d,func)
+
+
+/*****************************************************************************/
+
+#define PIXMAN_MIPS_BIND_SCALED_NEAREST_SRC_A8_DST(flags, name, op,           \
+                                                  src_type, dst_type)         \
+void                                                                          \
+pixman_scaled_nearest_scanline_##name##_##op##_asm_mips (                     \
+                                                   dst_type *       dst,      \
+                                                   const src_type * src,      \
+                                                   const uint8_t *  mask,     \
+                                                   int32_t          w,        \
+                                                   pixman_fixed_t   vx,       \
+                                                   pixman_fixed_t   unit_x);  \
+                                                                              \
+static force_inline void                                                      \
+scaled_nearest_scanline_mips_##name##_##op (const uint8_t *  mask,            \
+                                            dst_type *       pd,              \
+                                            const src_type * ps,              \
+                                            int32_t          w,               \
+                                            pixman_fixed_t   vx,              \
+                                            pixman_fixed_t   unit_x,          \
+                                            pixman_fixed_t   max_vx,          \
+                                            pixman_bool_t    zero_src)        \
+{                                                                             \
+    if ((flags & SKIP_ZERO_SRC) && zero_src)                                  \
+        return;                                                               \
+    pixman_scaled_nearest_scanline_##name##_##op##_asm_mips (pd, ps,          \
+                                                             mask, w,         \
+                                                             vx, unit_x);     \
+}                                                                             \
+                                                                              \
+FAST_NEAREST_MAINLOOP_COMMON (mips_##name##_cover_##op,                       \
+                              scaled_nearest_scanline_mips_##name##_##op,     \
+                              src_type, uint8_t, dst_type, COVER, TRUE, FALSE)\
+FAST_NEAREST_MAINLOOP_COMMON (mips_##name##_none_##op,                        \
+                              scaled_nearest_scanline_mips_##name##_##op,     \
+                              src_type, uint8_t, dst_type, NONE, TRUE, FALSE) \
+FAST_NEAREST_MAINLOOP_COMMON (mips_##name##_pad_##op,                         \
+                              scaled_nearest_scanline_mips_##name##_##op,     \
+                              src_type, uint8_t, dst_type, PAD, TRUE, FALSE)
+
+/* Provide entries for the fast path table */
+#define PIXMAN_MIPS_SIMPLE_NEAREST_A8_MASK_FAST_PATH(op,s,d,func)             \
+    SIMPLE_NEAREST_A8_MASK_FAST_PATH_COVER (op,s,d,func),                     \
+    SIMPLE_NEAREST_A8_MASK_FAST_PATH_NONE (op,s,d,func),                      \
+    SIMPLE_NEAREST_A8_MASK_FAST_PATH_PAD (op,s,d,func)
+
+/****************************************************************************/
+
+#define PIXMAN_MIPS_BIND_SCALED_BILINEAR_SRC_DST(flags, name, op,            \
+                                                 src_type, dst_type)         \
+void                                                                         \
+pixman_scaled_bilinear_scanline_##name##_##op##_asm_mips(                    \
+                                             dst_type *       dst,           \
+                                             const src_type * src_top,       \
+                                             const src_type * src_bottom,    \
+                                             int32_t          w,             \
+                                             int              wt,            \
+                                             int              wb,            \
+                                             pixman_fixed_t   vx,            \
+                                             pixman_fixed_t   unit_x);       \
+static force_inline void                                                     \
+scaled_bilinear_scanline_mips_##name##_##op (dst_type *       dst,           \
+                                             const uint32_t * mask,          \
+                                             const src_type * src_top,       \
+                                             const src_type * src_bottom,    \
+                                             int32_t          w,             \
+                                             int              wt,            \
+                                             int              wb,            \
+                                             pixman_fixed_t   vx,            \
+                                             pixman_fixed_t   unit_x,        \
+                                             pixman_fixed_t   max_vx,        \
+                                             pixman_bool_t    zero_src)      \
+{                                                                            \
+    if ((flags & SKIP_ZERO_SRC) && zero_src)                                 \
+        return;                                                              \
+    pixman_scaled_bilinear_scanline_##name##_##op##_asm_mips (dst, src_top,  \
+                                                              src_bottom, w, \
+                                                              wt, wb,        \
+                                                              vx, unit_x);   \
+}                                                                            \
+                                                                             \
+FAST_BILINEAR_MAINLOOP_COMMON (mips_##name##_cover_##op,                     \
+                       scaled_bilinear_scanline_mips_##name##_##op,          \
+                       src_type, uint32_t, dst_type, COVER, FLAG_NONE)       \
+FAST_BILINEAR_MAINLOOP_COMMON (mips_##name##_none_##op,                      \
+                       scaled_bilinear_scanline_mips_##name##_##op,          \
+                       src_type, uint32_t, dst_type, NONE, FLAG_NONE)        \
+FAST_BILINEAR_MAINLOOP_COMMON (mips_##name##_pad_##op,                       \
+                       scaled_bilinear_scanline_mips_##name##_##op,          \
+                       src_type, uint32_t, dst_type, PAD, FLAG_NONE)         \
+FAST_BILINEAR_MAINLOOP_COMMON (mips_##name##_normal_##op,                    \
+                       scaled_bilinear_scanline_mips_##name##_##op,          \
+                       src_type, uint32_t, dst_type, NORMAL,                 \
+                       FLAG_NONE)
+
+/*****************************************************************************/
+
+#define PIXMAN_MIPS_BIND_SCALED_BILINEAR_SRC_A8_DST(flags, name, op,          \
+                                                src_type, dst_type)           \
+void                                                                          \
+pixman_scaled_bilinear_scanline_##name##_##op##_asm_mips (                    \
+                                             dst_type *       dst,            \
+                                             const uint8_t *  mask,           \
+                                             const src_type * top,            \
+                                             const src_type * bottom,         \
+                                             int              wt,             \
+                                             int              wb,             \
+                                             pixman_fixed_t   x,              \
+                                             pixman_fixed_t   ux,             \
+                                             int              width);         \
+                                                                              \
+static force_inline void                                                      \
+scaled_bilinear_scanline_mips_##name##_##op (dst_type *       dst,            \
+                                             const uint8_t *  mask,           \
+                                             const src_type * src_top,        \
+                                             const src_type * src_bottom,     \
+                                             int32_t          w,              \
+                                             int              wt,             \
+                                             int              wb,             \
+                                             pixman_fixed_t   vx,             \
+                                             pixman_fixed_t   unit_x,         \
+                                             pixman_fixed_t   max_vx,         \
+                                             pixman_bool_t    zero_src)       \
+{                                                                             \
+    if ((flags & SKIP_ZERO_SRC) && zero_src)                                  \
+        return;                                                               \
+    pixman_scaled_bilinear_scanline_##name##_##op##_asm_mips (                \
+                      dst, mask, src_top, src_bottom, wt, wb, vx, unit_x, w); \
+}                                                                             \
+                                                                              \
+FAST_BILINEAR_MAINLOOP_COMMON (mips_##name##_cover_##op,                      \
+                       scaled_bilinear_scanline_mips_##name##_##op,           \
+                       src_type, uint8_t, dst_type, COVER,                    \
+                       FLAG_HAVE_NON_SOLID_MASK)                              \
+FAST_BILINEAR_MAINLOOP_COMMON (mips_##name##_none_##op,                       \
+                       scaled_bilinear_scanline_mips_##name##_##op,           \
+                       src_type, uint8_t, dst_type, NONE,                     \
+                       FLAG_HAVE_NON_SOLID_MASK)                              \
+FAST_BILINEAR_MAINLOOP_COMMON (mips_##name##_pad_##op,                        \
+                       scaled_bilinear_scanline_mips_##name##_##op,           \
+                       src_type, uint8_t, dst_type, PAD,                      \
+                       FLAG_HAVE_NON_SOLID_MASK)                              \
+FAST_BILINEAR_MAINLOOP_COMMON (mips_##name##_normal_##op,                     \
+                       scaled_bilinear_scanline_mips_##name##_##op,           \
+                       src_type, uint8_t, dst_type, NORMAL,                   \
+                       FLAG_HAVE_NON_SOLID_MASK)
+
+#endif //PIXMAN_MIPS_DSPR2_H
diff --git a/pixman/pixman/pixman-mips-memcpy-asm.S b/pixman/pixman/pixman-mips-memcpy-asm.S
new file mode 100644
index 000000000..9ad6da537
--- /dev/null
+++ b/pixman/pixman/pixman-mips-memcpy-asm.S
@@ -0,0 +1,382 @@
+/*
+ * Copyright (c) 2012
+ *      MIPS Technologies, Inc., California.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the MIPS Technologies, Inc., nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE MIPS TECHNOLOGIES, INC. ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE MIPS TECHNOLOGIES, INC. BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "pixman-mips-dspr2-asm.h"
+
+/*
+ * This routine could be optimized for MIPS64. The current code only
+ * uses MIPS32 instructions.
+ */
+
+#ifdef EB
+#  define LWHI	lwl		/* high part is left in big-endian */
+#  define SWHI	swl		/* high part is left in big-endian */
+#  define LWLO	lwr		/* low part is right in big-endian */
+#  define SWLO	swr		/* low part is right in big-endian */
+#else
+#  define LWHI	lwr		/* high part is right in little-endian */
+#  define SWHI	swr		/* high part is right in little-endian */
+#  define LWLO	lwl		/* low part is left in big-endian */
+#  define SWLO	swl		/* low part is left in big-endian */
+#endif
+
+LEAF_MIPS32R2(pixman_mips_fast_memcpy)
+
+	slti	AT, a2, 8
+	bne	AT, zero, $last8
+	move	v0, a0	/* memcpy returns the dst pointer */
+
+/* Test if the src and dst are word-aligned, or can be made word-aligned */
+	xor	t8, a1, a0
+	andi	t8, t8, 0x3		/* t8 is a0/a1 word-displacement */
+
+	bne	t8, zero, $unaligned
+	negu	a3, a0
+
+	andi	a3, a3, 0x3	/* we need to copy a3 bytes to make a0/a1 aligned */
+	beq	a3, zero, $chk16w	/* when a3=0 then the dst (a0) is word-aligned */
+	subu	a2, a2, a3	/* now a2 is the remining bytes count */
+
+	LWHI	t8, 0(a1)
+	addu	a1, a1, a3
+	SWHI	t8, 0(a0)
+	addu	a0, a0, a3
+
+/* Now the dst/src are mutually word-aligned with word-aligned addresses */
+$chk16w:	andi	t8, a2, 0x3f	/* any whole 64-byte chunks? */
+				/* t8 is the byte count after 64-byte chunks */
+
+	beq	a2, t8, $chk8w	/* if a2==t8, no 64-byte chunks */
+				/* There will be at most 1 32-byte chunk after it */
+	subu	a3, a2, t8	/* subtract from a2 the reminder */
+                                /* Here a3 counts bytes in 16w chunks */
+	addu	a3, a0, a3	/* Now a3 is the final dst after 64-byte chunks */
+
+	addu	t0, a0, a2	/* t0 is the "past the end" address */
+
+/*
+ * When in the loop we exercise "pref 30, x(a0)", the a0+x should not be past
+ * the "t0-32" address
+ * This means: for x=128 the last "safe" a0 address is "t0-160"
+ * Alternatively, for x=64 the last "safe" a0 address is "t0-96"
+ * In the current version we use "pref 30, 128(a0)", so "t0-160" is the limit
+ */
+	subu	t9, t0, 160	/* t9 is the "last safe pref 30, 128(a0)" address */
+
+	pref    0, 0(a1)		/* bring the first line of src, addr 0 */
+	pref    0, 32(a1)	/* bring the second line of src, addr 32 */
+	pref    0, 64(a1)	/* bring the third line of src, addr 64 */
+	pref	30, 32(a0)	/* safe, as we have at least 64 bytes ahead */
+/* In case the a0 > t9 don't use "pref 30" at all */
+	sgtu	v1, a0, t9
+	bgtz	v1, $loop16w	/* skip "pref 30, 64(a0)" for too short arrays */
+	nop
+/* otherwise, start with using pref30 */
+	pref	30, 64(a0)
+$loop16w:
+	pref	0, 96(a1)
+	lw	t0, 0(a1)
+	bgtz	v1, $skip_pref30_96	/* skip "pref 30, 96(a0)" */
+	lw	t1, 4(a1)
+	pref    30, 96(a0)   /* continue setting up the dest, addr 96 */
+$skip_pref30_96:
+	lw	t2, 8(a1)
+	lw	t3, 12(a1)
+	lw	t4, 16(a1)
+	lw	t5, 20(a1)
+	lw	t6, 24(a1)
+	lw	t7, 28(a1)
+        pref    0, 128(a1)    /* bring the next lines of src, addr 128 */
+
+	sw	t0, 0(a0)
+	sw	t1, 4(a0)
+	sw	t2, 8(a0)
+	sw	t3, 12(a0)
+	sw	t4, 16(a0)
+	sw	t5, 20(a0)
+	sw	t6, 24(a0)
+	sw	t7, 28(a0)
+
+	lw	t0, 32(a1)
+	bgtz	v1, $skip_pref30_128	/* skip "pref 30, 128(a0)" */
+	lw	t1, 36(a1)
+	pref    30, 128(a0)   /* continue setting up the dest, addr 128 */
+$skip_pref30_128:
+	lw	t2, 40(a1)
+	lw	t3, 44(a1)
+	lw	t4, 48(a1)
+	lw	t5, 52(a1)
+	lw	t6, 56(a1)
+	lw	t7, 60(a1)
+        pref    0, 160(a1)    /* bring the next lines of src, addr 160 */
+
+	sw	t0, 32(a0)
+	sw	t1, 36(a0)
+	sw	t2, 40(a0)
+	sw	t3, 44(a0)
+	sw	t4, 48(a0)
+	sw	t5, 52(a0)
+	sw	t6, 56(a0)
+	sw	t7, 60(a0)
+
+	addiu	a0, a0, 64	/* adding 64 to dest */
+	sgtu	v1, a0, t9
+	bne	a0, a3, $loop16w
+	addiu	a1, a1, 64	/* adding 64 to src */
+	move	a2, t8
+
+/* Here we have src and dest word-aligned but less than 64-bytes to go */
+
+$chk8w:
+	pref 0, 0x0(a1)
+	andi	t8, a2, 0x1f	/* is there a 32-byte chunk? */
+				/* the t8 is the reminder count past 32-bytes */
+	beq	a2, t8, $chk1w	/* when a2=t8, no 32-byte chunk */
+	 nop
+
+	lw	t0, 0(a1)
+	lw	t1, 4(a1)
+	lw	t2, 8(a1)
+	lw	t3, 12(a1)
+	lw	t4, 16(a1)
+	lw	t5, 20(a1)
+	lw	t6, 24(a1)
+	lw	t7, 28(a1)
+	addiu	a1, a1, 32
+
+	sw	t0, 0(a0)
+	sw	t1, 4(a0)
+	sw	t2, 8(a0)
+	sw	t3, 12(a0)
+	sw	t4, 16(a0)
+	sw	t5, 20(a0)
+	sw	t6, 24(a0)
+	sw	t7, 28(a0)
+	addiu	a0, a0, 32
+
+$chk1w:
+	andi	a2, t8, 0x3	/* now a2 is the reminder past 1w chunks */
+	beq	a2, t8, $last8
+	subu	a3, t8, a2	/* a3 is count of bytes in 1w chunks */
+	addu	a3, a0, a3	/* now a3 is the dst address past the 1w chunks */
+
+/* copying in words (4-byte chunks) */
+$wordCopy_loop:
+	lw	t3, 0(a1)	/* the first t3 may be equal t0 ... optimize? */
+	addiu	a1, a1, 4
+	addiu	a0, a0, 4
+	bne	a0, a3, $wordCopy_loop
+	sw	t3, -4(a0)
+
+/* For the last (<8) bytes */
+$last8:
+	blez	a2, leave
+	addu	a3, a0, a2	/* a3 is the last dst address */
+$last8loop:
+	lb	v1, 0(a1)
+	addiu	a1, a1, 1
+	addiu	a0, a0, 1
+	bne	a0, a3, $last8loop
+	sb	v1, -1(a0)
+
+leave:	j	ra
+	nop
+
+/*
+ * UNALIGNED case
+ */
+
+$unaligned:
+	/* got here with a3="negu a0" */
+	andi	a3, a3, 0x3	/* test if the a0 is word aligned */
+	beqz	a3, $ua_chk16w
+	subu	a2, a2, a3	/* bytes left after initial a3 bytes */
+
+	LWHI	v1, 0(a1)
+	LWLO	v1, 3(a1)
+	addu	a1, a1, a3	/* a3 may be here 1, 2 or 3 */
+	SWHI	v1, 0(a0)
+	addu	a0, a0, a3	/* below the dst will be word aligned (NOTE1) */
+
+$ua_chk16w:	andi	t8, a2, 0x3f	/* any whole 64-byte chunks? */
+				/* t8 is the byte count after 64-byte chunks */
+	beq	a2, t8, $ua_chk8w	/* if a2==t8, no 64-byte chunks */
+				/* There will be at most 1 32-byte chunk after it */
+	subu	a3, a2, t8	/* subtract from a2 the reminder */
+                                /* Here a3 counts bytes in 16w chunks */
+	addu	a3, a0, a3	/* Now a3 is the final dst after 64-byte chunks */
+
+	addu	t0, a0, a2	/* t0 is the "past the end" address */
+
+	subu	t9, t0, 160	/* t9 is the "last safe pref 30, 128(a0)" address */
+
+	pref    0, 0(a1)		/* bring the first line of src, addr 0 */
+	pref    0, 32(a1)	/* bring the second line of src, addr 32 */
+	pref    0, 64(a1)	/* bring the third line of src, addr 64 */
+	pref	30, 32(a0)	/* safe, as we have at least 64 bytes ahead */
+/* In case the a0 > t9 don't use "pref 30" at all */
+	sgtu	v1, a0, t9
+	bgtz	v1, $ua_loop16w	/* skip "pref 30, 64(a0)" for too short arrays */
+	nop
+/* otherwise,  start with using pref30 */
+	pref	30, 64(a0)
+$ua_loop16w:
+	pref	0, 96(a1)
+	LWHI	t0, 0(a1)
+	LWLO	t0, 3(a1)
+	LWHI	t1, 4(a1)
+	bgtz	v1, $ua_skip_pref30_96
+	LWLO	t1, 7(a1)
+	pref    30, 96(a0)   /* continue setting up the dest, addr 96 */
+$ua_skip_pref30_96:
+	LWHI	t2, 8(a1)
+	LWLO	t2, 11(a1)
+	LWHI	t3, 12(a1)
+	LWLO	t3, 15(a1)
+	LWHI	t4, 16(a1)
+	LWLO	t4, 19(a1)
+	LWHI	t5, 20(a1)
+	LWLO	t5, 23(a1)
+	LWHI	t6, 24(a1)
+	LWLO	t6, 27(a1)
+	LWHI	t7, 28(a1)
+	LWLO	t7, 31(a1)
+        pref    0, 128(a1)    /* bring the next lines of src, addr 128 */
+
+	sw	t0, 0(a0)
+	sw	t1, 4(a0)
+	sw	t2, 8(a0)
+	sw	t3, 12(a0)
+	sw	t4, 16(a0)
+	sw	t5, 20(a0)
+	sw	t6, 24(a0)
+	sw	t7, 28(a0)
+
+	LWHI	t0, 32(a1)
+	LWLO	t0, 35(a1)
+	LWHI	t1, 36(a1)
+	bgtz	v1, $ua_skip_pref30_128
+	LWLO	t1, 39(a1)
+	pref    30, 128(a0)   /* continue setting up the dest, addr 128 */
+$ua_skip_pref30_128:
+	LWHI	t2, 40(a1)
+	LWLO	t2, 43(a1)
+	LWHI	t3, 44(a1)
+	LWLO	t3, 47(a1)
+	LWHI	t4, 48(a1)
+	LWLO	t4, 51(a1)
+	LWHI	t5, 52(a1)
+	LWLO	t5, 55(a1)
+	LWHI	t6, 56(a1)
+	LWLO	t6, 59(a1)
+	LWHI	t7, 60(a1)
+	LWLO	t7, 63(a1)
+        pref    0, 160(a1)    /* bring the next lines of src, addr 160 */
+
+	sw	t0, 32(a0)
+	sw	t1, 36(a0)
+	sw	t2, 40(a0)
+	sw	t3, 44(a0)
+	sw	t4, 48(a0)
+	sw	t5, 52(a0)
+	sw	t6, 56(a0)
+	sw	t7, 60(a0)
+
+	addiu	a0, a0, 64	/* adding 64 to dest */
+	sgtu	v1, a0, t9
+	bne	a0, a3, $ua_loop16w
+	addiu	a1, a1, 64	/* adding 64 to src */
+	move	a2, t8
+
+/* Here we have src and dest word-aligned but less than 64-bytes to go */
+
+$ua_chk8w:
+	pref 0, 0x0(a1)
+	andi	t8, a2, 0x1f	/* is there a 32-byte chunk? */
+				/* the t8 is the reminder count */
+	beq	a2, t8, $ua_chk1w	/* when a2=t8, no 32-byte chunk */
+
+	LWHI	t0, 0(a1)
+	LWLO	t0, 3(a1)
+	LWHI	t1, 4(a1)
+	LWLO	t1, 7(a1)
+	LWHI	t2, 8(a1)
+	LWLO	t2, 11(a1)
+	LWHI	t3, 12(a1)
+	LWLO	t3, 15(a1)
+	LWHI	t4, 16(a1)
+	LWLO	t4, 19(a1)
+	LWHI	t5, 20(a1)
+	LWLO	t5, 23(a1)
+	LWHI	t6, 24(a1)
+	LWLO	t6, 27(a1)
+	LWHI	t7, 28(a1)
+	LWLO	t7, 31(a1)
+	addiu	a1, a1, 32
+
+	sw	t0, 0(a0)
+	sw	t1, 4(a0)
+	sw	t2, 8(a0)
+	sw	t3, 12(a0)
+	sw	t4, 16(a0)
+	sw	t5, 20(a0)
+	sw	t6, 24(a0)
+	sw	t7, 28(a0)
+	addiu	a0, a0, 32
+
+$ua_chk1w:
+	andi	a2, t8, 0x3	/* now a2 is the reminder past 1w chunks */
+	beq	a2, t8, $ua_smallCopy
+	subu	a3, t8, a2	/* a3 is count of bytes in 1w chunks */
+	addu	a3, a0, a3	/* now a3 is the dst address past the 1w chunks */
+
+/* copying in words (4-byte chunks) */
+$ua_wordCopy_loop:
+	LWHI	v1, 0(a1)
+	LWLO	v1, 3(a1)
+	addiu	a1, a1, 4
+	addiu	a0, a0, 4		/* note: dst=a0 is word aligned here, see NOTE1 */
+	bne	a0, a3, $ua_wordCopy_loop
+	sw	v1, -4(a0)
+
+/* Now less than 4 bytes (value in a2) left to copy */
+$ua_smallCopy:
+	beqz	a2, leave
+	addu	a3, a0, a2	/* a3 is the last dst address */
+$ua_smallCopy_loop:
+	lb	v1, 0(a1)
+	addiu	a1, a1, 1
+	addiu	a0, a0, 1
+	bne	a0, a3, $ua_smallCopy_loop
+	sb	v1, -1(a0)
+
+	j	ra
+	nop
+
+END(pixman_mips_fast_memcpy)
diff --git a/pixman/pixman/pixman-mips.c b/pixman/pixman/pixman-mips.c
new file mode 100644
index 000000000..304881383
--- /dev/null
+++ b/pixman/pixman/pixman-mips.c
@@ -0,0 +1,94 @@
+/*
+ * Copyright © 2000 SuSE, Inc.
+ * Copyright © 2007 Red Hat, Inc.
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of SuSE not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  SuSE makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as is"
+ * without express or implied warranty.
+ *
+ * SuSE DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL SuSE
+ * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "pixman-private.h"
+
+#if defined(USE_MIPS_DSPR2) || defined(USE_LOONGSON_MMI)
+
+#include <string.h>
+#include <stdlib.h>
+
+static pixman_bool_t
+have_feature (const char *search_string)
+{
+#if defined (__linux__) /* linux ELF */
+    /* Simple detection of MIPS features at runtime for Linux.
+     * It is based on /proc/cpuinfo, which reveals hardware configuration
+     * to user-space applications.  According to MIPS (early 2010), no similar
+     * facility is universally available on the MIPS architectures, so it's up
+     * to individual OSes to provide such.
+     */
+    const char *file_name = "/proc/cpuinfo";
+    char cpuinfo_line[256];
+    FILE *f = NULL;
+
+    if ((f = fopen (file_name, "r")) == NULL)
+        return FALSE;
+
+    while (fgets (cpuinfo_line, sizeof (cpuinfo_line), f) != NULL)
+    {
+        if (strstr (cpuinfo_line, search_string) != NULL)
+        {
+            fclose (f);
+            return TRUE;
+        }
+    }
+
+    fclose (f);
+#endif
+
+    /* Did not find string in the proc file, or not Linux ELF. */
+    return FALSE;
+}
+
+#endif
+
+pixman_implementation_t *
+_pixman_mips_get_implementations (pixman_implementation_t *imp)
+{
+#ifdef USE_LOONGSON_MMI
+    /* I really don't know if some Loongson CPUs don't have MMI. */
+    if (!_pixman_disabled ("loongson-mmi") && have_feature ("Loongson"))
+	imp = _pixman_implementation_create_mmx (imp);
+#endif
+
+#ifdef USE_MIPS_DSPR2
+    if (!_pixman_disabled ("mips-dspr2"))
+    {
+	int already_compiling_everything_for_dspr2 = 0;
+#if defined(__mips_dsp) && (__mips_dsp_rev >= 2)
+	already_compiling_everything_for_dspr2 = 1;
+#endif
+	if (already_compiling_everything_for_dspr2 ||
+	    /* Only currently available MIPS core that supports DSPr2 is 74K. */
+	    have_feature ("MIPS 74K"))
+	{
+	    imp = _pixman_implementation_create_mips_dspr2 (imp);
+	}
+    }
+#endif
+
+    return imp;
+}
diff --git a/pixman/pixman/pixman-mmx.c b/pixman/pixman/pixman-mmx.c
index d51b40cc1..f9a92ce09 100644
--- a/pixman/pixman/pixman-mmx.c
+++ b/pixman/pixman/pixman-mmx.c
@@ -33,13 +33,16 @@
 #include <config.h>
 #endif
 
-#ifdef USE_MMX
+#if defined USE_X86_MMX || defined USE_ARM_IWMMXT || defined USE_LOONGSON_MMI
 
+#ifdef USE_LOONGSON_MMI
+#include <loongson-mmintrin.h>
+#else
 #include <mmintrin.h>
+#endif
 #include "pixman-private.h"
 #include "pixman-combine32.h"
-
-#define no_vERBOSE
+#include "pixman-inlines.h"
 
 #ifdef VERBOSE
 #define CHECKPOINT() error_f ("at %s %d\n", __FUNCTION__, __LINE__)
@@ -47,6 +50,79 @@
 #define CHECKPOINT()
 #endif
 
+#if defined USE_ARM_IWMMXT && __GNUC__ == 4 && __GNUC_MINOR__ < 8
+/* Empty the multimedia state. For some reason, ARM's mmintrin.h doesn't provide this.  */
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_empty (void)
+{
+
+}
+#endif
+
+#ifdef USE_X86_MMX
+# if (defined(__SUNPRO_C) || defined(_MSC_VER) || defined(_WIN64))
+#  include <xmmintrin.h>
+# else
+/* We have to compile with -msse to use xmmintrin.h, but that causes SSE
+ * instructions to be generated that we don't want. Just duplicate the
+ * functions we want to use.  */
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_movemask_pi8 (__m64 __A)
+{
+    int ret;
+
+    asm ("pmovmskb %1, %0\n\t"
+	: "=r" (ret)
+	: "y" (__A)
+    );
+
+    return ret;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_mulhi_pu16 (__m64 __A, __m64 __B)
+{
+    asm ("pmulhuw %1, %0\n\t"
+	: "+y" (__A)
+	: "y" (__B)
+    );
+    return __A;
+}
+
+#  ifdef __OPTIMIZE__
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_shuffle_pi16 (__m64 __A, int8_t const __N)
+{
+    __m64 ret;
+
+    asm ("pshufw %2, %1, %0\n\t"
+	: "=y" (ret)
+	: "y" (__A), "K" (__N)
+    );
+
+    return ret;
+}
+#  else
+#   define _mm_shuffle_pi16(A, N)					\
+    ({									\
+	__m64 ret;							\
+									\
+	asm ("pshufw %2, %1, %0\n\t"					\
+	     : "=y" (ret)						\
+	     : "y" (A), "K" ((const int8_t)N)				\
+	);								\
+									\
+	ret;								\
+    })
+#  endif
+# endif
+#endif
+
+#ifndef _MSC_VER
+#define _MM_SHUFFLE(fp3,fp2,fp1,fp0) \
+ (((fp3) << 6) | ((fp2) << 4) | ((fp1) << 2) | (fp0))
+#endif
+
 /* Notes about writing mmx code
  *
  * give memory operands as the second operand. If you give it as the
@@ -68,17 +144,41 @@
 
 /* --------------- MMX primitives ------------------------------------- */
 
-#ifdef __GNUC__
+/* If __m64 is defined as a struct or union, then define M64_MEMBER to be
+ * the name of the member used to access the data.
+ * If __m64 requires using mm_cvt* intrinsics functions to convert between
+ * uint64_t and __m64 values, then define USE_CVT_INTRINSICS.
+ * If __m64 and uint64_t values can just be cast to each other directly,
+ * then define USE_M64_CASTS.
+ * If __m64 is a double datatype, then define USE_M64_DOUBLE.
+ */
+#ifdef _MSC_VER
+# define M64_MEMBER m64_u64
+#elif defined(__ICC)
+# define USE_CVT_INTRINSICS
+#elif defined(USE_LOONGSON_MMI)
+# define USE_M64_DOUBLE
+#elif defined(__GNUC__)
+# define USE_M64_CASTS
+#elif defined(__SUNPRO_C)
+# if (__SUNPRO_C >= 0x5120) && !defined(__NOVECTORSIZE__)
+/* Solaris Studio 12.3 (Sun C 5.12) introduces __attribute__(__vector_size__)
+ * support, and defaults to using it to define __m64, unless __NOVECTORSIZE__
+ * is defined.   If it is used, then the mm_cvt* intrinsics must be used.
+ */
+#  define USE_CVT_INTRINSICS
+# else
+/* For Studio 12.2 or older, or when __attribute__(__vector_size__) is
+ * disabled, __m64 is defined as a struct containing "unsigned long long l_".
+ */
+#  define M64_MEMBER l_
+# endif
+#endif
+
+#if defined(USE_M64_CASTS) || defined(USE_CVT_INTRINSICS) || defined(USE_M64_DOUBLE)
 typedef uint64_t mmxdatafield;
 #else
 typedef __m64 mmxdatafield;
-/* If __m64 is defined as a struct or union, define M64_MEMBER to be the
-   name of the member used to access the data */
-# ifdef _MSC_VER
-#  define M64_MEMBER m64_u64
-# elif defined(__SUNPRO_C)
-#  define M64_MEMBER l_
-# endif
 #endif
 
 typedef struct
@@ -87,24 +187,31 @@ typedef struct
     mmxdatafield mmx_4x0080;
     mmxdatafield mmx_565_rgb;
     mmxdatafield mmx_565_unpack_multiplier;
+    mmxdatafield mmx_565_pack_multiplier;
     mmxdatafield mmx_565_r;
     mmxdatafield mmx_565_g;
     mmxdatafield mmx_565_b;
+    mmxdatafield mmx_packed_565_rb;
+    mmxdatafield mmx_packed_565_g;
+    mmxdatafield mmx_expand_565_g;
+    mmxdatafield mmx_expand_565_b;
+    mmxdatafield mmx_expand_565_r;
+#ifndef USE_LOONGSON_MMI
     mmxdatafield mmx_mask_0;
     mmxdatafield mmx_mask_1;
     mmxdatafield mmx_mask_2;
     mmxdatafield mmx_mask_3;
+#endif
     mmxdatafield mmx_full_alpha;
-    mmxdatafield mmx_ffff0000ffff0000;
-    mmxdatafield mmx_0000ffff00000000;
-    mmxdatafield mmx_000000000000ffff;
+    mmxdatafield mmx_4x0101;
+    mmxdatafield mmx_ff000000;
 } mmx_data_t;
 
 #if defined(_MSC_VER)
 # define MMXDATA_INIT(field, val) { val ## UI64 }
 #elif defined(M64_MEMBER)       /* __m64 is a struct, not an integral type */
 # define MMXDATA_INIT(field, val) field =   { val ## ULL }
-#else                           /* __m64 is an integral type */
+#else                           /* mmxdatafield is an integral type */
 # define MMXDATA_INIT(field, val) field =   val ## ULL
 #endif
 
@@ -114,53 +221,64 @@ static const mmx_data_t c =
     MMXDATA_INIT (.mmx_4x0080,                   0x0080008000800080),
     MMXDATA_INIT (.mmx_565_rgb,                  0x000001f0003f001f),
     MMXDATA_INIT (.mmx_565_unpack_multiplier,    0x0000008404100840),
+    MMXDATA_INIT (.mmx_565_pack_multiplier,      0x2000000420000004),
     MMXDATA_INIT (.mmx_565_r,                    0x000000f800000000),
     MMXDATA_INIT (.mmx_565_g,                    0x0000000000fc0000),
     MMXDATA_INIT (.mmx_565_b,                    0x00000000000000f8),
+    MMXDATA_INIT (.mmx_packed_565_rb,            0x00f800f800f800f8),
+    MMXDATA_INIT (.mmx_packed_565_g,             0x0000fc000000fc00),
+    MMXDATA_INIT (.mmx_expand_565_g,             0x07e007e007e007e0),
+    MMXDATA_INIT (.mmx_expand_565_b,             0x001f001f001f001f),
+    MMXDATA_INIT (.mmx_expand_565_r,             0xf800f800f800f800),
+#ifndef USE_LOONGSON_MMI
     MMXDATA_INIT (.mmx_mask_0,                   0xffffffffffff0000),
     MMXDATA_INIT (.mmx_mask_1,                   0xffffffff0000ffff),
     MMXDATA_INIT (.mmx_mask_2,                   0xffff0000ffffffff),
     MMXDATA_INIT (.mmx_mask_3,                   0x0000ffffffffffff),
+#endif
     MMXDATA_INIT (.mmx_full_alpha,               0x00ff000000000000),
-    MMXDATA_INIT (.mmx_ffff0000ffff0000,         0xffff0000ffff0000),
-    MMXDATA_INIT (.mmx_0000ffff00000000,         0x0000ffff00000000),
-    MMXDATA_INIT (.mmx_000000000000ffff,         0x000000000000ffff),
+    MMXDATA_INIT (.mmx_4x0101,                   0x0101010101010101),
+    MMXDATA_INIT (.mmx_ff000000,                 0xff000000ff000000),
 };
 
-#ifdef __GNUC__
-#    ifdef __ICC
-#        define MC(x)  M64 (c.mmx_ ## x)
-#    else
-#        define MC(x) ((__m64)c.mmx_ ## x)
-#    endif
+#ifdef USE_CVT_INTRINSICS
+#    define MC(x) to_m64 (c.mmx_ ## x)
+#elif defined(USE_M64_CASTS)
+#    define MC(x) ((__m64)c.mmx_ ## x)
+#elif defined(USE_M64_DOUBLE)
+#    define MC(x) (*(__m64 *)&c.mmx_ ## x)
 #else
 #    define MC(x) c.mmx_ ## x
 #endif
 
 static force_inline __m64
-M64 (uint64_t x)
+to_m64 (uint64_t x)
 {
-#ifdef __ICC
+#ifdef USE_CVT_INTRINSICS
     return _mm_cvtsi64_m64 (x);
 #elif defined M64_MEMBER        /* __m64 is a struct, not an integral type */
     __m64 res;
 
     res.M64_MEMBER = x;
     return res;
-#else                           /* __m64 is an integral type */
+#elif defined USE_M64_DOUBLE
+    return *(__m64 *)&x;
+#else /* USE_M64_CASTS */
     return (__m64)x;
 #endif
 }
 
 static force_inline uint64_t
-UINT64 (__m64 x)
+to_uint64 (__m64 x)
 {
-#ifdef __ICC
+#ifdef USE_CVT_INTRINSICS
     return _mm_cvtm64_si64 (x);
 #elif defined M64_MEMBER        /* __m64 is a struct, not an integral type */
     uint64_t res = x.M64_MEMBER;
     return res;
-#else                           /* __m64 is an integral type */
+#elif defined USE_M64_DOUBLE
+    return *(uint64_t *)&x;
+#else /* USE_M64_CASTS */
     return (uint64_t)x;
 #endif
 }
@@ -183,6 +301,29 @@ negate (__m64 mask)
     return _mm_xor_si64 (mask, MC (4x00ff));
 }
 
+/* Computes the product of two unsigned fixed-point 8-bit values from 0 to 1
+ * and maps its result to the same range.
+ *
+ * Jim Blinn gives multiple ways to compute this in "Jim Blinn's Corner:
+ * Notation, Notation, Notation", the first of which is
+ *
+ *   prod(a, b) = (a * b + 128) / 255.
+ *
+ * By approximating the division by 255 as 257/65536 it can be replaced by a
+ * multiply and a right shift. This is the implementation that we use in
+ * pix_multiply(), but we _mm_mulhi_pu16() by 257 (part of SSE1 or Extended
+ * 3DNow!, and unavailable at the time of the book's publication) to perform
+ * the multiplication and right shift in a single operation.
+ *
+ *   prod(a, b) = ((a * b + 128) * 257) >> 16.
+ *
+ * A third way (how pix_multiply() was implemented prior to 14208344) exists
+ * also that performs the multiplication by 257 with adds and shifts.
+ *
+ * Where temp = a * b + 128
+ *
+ *   prod(a, b) = (temp + (temp >> 8)) >> 8.
+ */
 static force_inline __m64
 pix_multiply (__m64 a, __m64 b)
 {
@@ -190,8 +331,7 @@ pix_multiply (__m64 a, __m64 b)
 
     res = _mm_mullo_pi16 (a, b);
     res = _mm_adds_pu16 (res, MC (4x0080));
-    res = _mm_adds_pu16 (res, _mm_srli_pi16 (res, 8));
-    res = _mm_srli_pi16 (res, 8);
+    res = _mm_mulhi_pu16 (res, MC (4x0101));
 
     return res;
 }
@@ -205,52 +345,19 @@ pix_add (__m64 a, __m64 b)
 static force_inline __m64
 expand_alpha (__m64 pixel)
 {
-    __m64 t1, t2;
-
-    t1 = shift (pixel, -48);
-    t2 = shift (t1, 16);
-    t1 = _mm_or_si64 (t1, t2);
-    t2 = shift (t1, 32);
-    t1 = _mm_or_si64 (t1, t2);
-
-    return t1;
+    return _mm_shuffle_pi16 (pixel, _MM_SHUFFLE (3, 3, 3, 3));
 }
 
 static force_inline __m64
 expand_alpha_rev (__m64 pixel)
 {
-    __m64 t1, t2;
-
-    /* move alpha to low 16 bits and zero the rest */
-    t1 = shift (pixel,  48);
-    t1 = shift (t1, -48);
-
-    t2 = shift (t1, 16);
-    t1 = _mm_or_si64 (t1, t2);
-    t2 = shift (t1, 32);
-    t1 = _mm_or_si64 (t1, t2);
-
-    return t1;
+    return _mm_shuffle_pi16 (pixel, _MM_SHUFFLE (0, 0, 0, 0));
 }
 
 static force_inline __m64
 invert_colors (__m64 pixel)
 {
-    __m64 x, y, z;
-
-    x = y = z = pixel;
-
-    x = _mm_and_si64 (x, MC (ffff0000ffff0000));
-    y = _mm_and_si64 (y, MC (000000000000ffff));
-    z = _mm_and_si64 (z, MC (0000ffff00000000));
-
-    y = shift (y, 32);
-    z = shift (z, -32);
-
-    x = _mm_or_si64 (x, y);
-    x = _mm_or_si64 (x, z);
-
-    return x;
+    return _mm_shuffle_pi16 (pixel, _MM_SHUFFLE (3, 0, 1, 2));
 }
 
 static force_inline __m64
@@ -276,14 +383,6 @@ in (__m64 src, __m64 mask)
     return pix_multiply (src, mask);
 }
 
-static force_inline __m64
-in_over_full_src_alpha (__m64 src, __m64 mask, __m64 dest)
-{
-    src = _mm_or_si64 (src, MC (full_alpha));
-
-    return over (in (src, mask), mask, dest);
-}
-
 #ifndef _MSC_VER
 static force_inline __m64
 in_over (__m64 src, __m64 srca, __m64 mask, __m64 dest)
@@ -298,10 +397,69 @@ in_over (__m64 src, __m64 srca, __m64 mask, __m64 dest)
 
 #endif
 
+/* Elemental unaligned loads */
+
+static force_inline __m64 ldq_u(__m64 *p)
+{
+#ifdef USE_X86_MMX
+    /* x86's alignment restrictions are very relaxed. */
+    return *(__m64 *)p;
+#elif defined USE_ARM_IWMMXT
+    int align = (uintptr_t)p & 7;
+    __m64 *aligned_p;
+    if (align == 0)
+	return *p;
+    aligned_p = (__m64 *)((uintptr_t)p & ~7);
+    return (__m64) _mm_align_si64 (aligned_p[0], aligned_p[1], align);
+#else
+    struct __una_u64 { __m64 x __attribute__((packed)); };
+    const struct __una_u64 *ptr = (const struct __una_u64 *) p;
+    return (__m64) ptr->x;
+#endif
+}
+
+static force_inline uint32_t ldl_u(const uint32_t *p)
+{
+#ifdef USE_X86_MMX
+    /* x86's alignment restrictions are very relaxed. */
+    return *p;
+#else
+    struct __una_u32 { uint32_t x __attribute__((packed)); };
+    const struct __una_u32 *ptr = (const struct __una_u32 *) p;
+    return ptr->x;
+#endif
+}
+
+static force_inline __m64
+load (const uint32_t *v)
+{
+#ifdef USE_LOONGSON_MMI
+    __m64 ret;
+    asm ("lwc1 %0, %1\n\t"
+	: "=f" (ret)
+	: "m" (*v)
+    );
+    return ret;
+#else
+    return _mm_cvtsi32_si64 (*v);
+#endif
+}
+
+static force_inline __m64
+load8888 (const uint32_t *v)
+{
+#ifdef USE_LOONGSON_MMI
+    return _mm_unpacklo_pi8_f (*(__m32 *)v, _mm_setzero_si64 ());
+#else
+    return _mm_unpacklo_pi8 (load (v), _mm_setzero_si64 ());
+#endif
+}
+
 static force_inline __m64
-load8888 (uint32_t v)
+load8888u (const uint32_t *v)
 {
-    return _mm_unpacklo_pi8 (_mm_cvtsi32_si64 (v), _mm_setzero_si64 ());
+    uint32_t l = ldl_u (v);
+    return load8888 (&l);
 }
 
 static force_inline __m64
@@ -310,10 +468,53 @@ pack8888 (__m64 lo, __m64 hi)
     return _mm_packs_pu16 (lo, hi);
 }
 
-static force_inline uint32_t
-store8888 (__m64 v)
+static force_inline void
+store (uint32_t *dest, __m64 v)
 {
-    return _mm_cvtsi64_si32 (pack8888 (v, _mm_setzero_si64 ()));
+#ifdef USE_LOONGSON_MMI
+    asm ("swc1 %1, %0\n\t"
+	: "=m" (*dest)
+	: "f" (v)
+	: "memory"
+    );
+#else
+    *dest = _mm_cvtsi64_si32 (v);
+#endif
+}
+
+static force_inline void
+store8888 (uint32_t *dest, __m64 v)
+{
+    v = pack8888 (v, _mm_setzero_si64 ());
+    store (dest, v);
+}
+
+static force_inline pixman_bool_t
+is_equal (__m64 a, __m64 b)
+{
+#ifdef USE_LOONGSON_MMI
+    /* __m64 is double, we can compare directly. */
+    return a == b;
+#else
+    return _mm_movemask_pi8 (_mm_cmpeq_pi8 (a, b)) == 0xff;
+#endif
+}
+
+static force_inline pixman_bool_t
+is_opaque (__m64 v)
+{
+#ifdef USE_LOONGSON_MMI
+    return is_equal (_mm_and_si64 (v, MC (full_alpha)), MC (full_alpha));
+#else
+    __m64 ffs = _mm_cmpeq_pi8 (v, v);
+    return (_mm_movemask_pi8 (_mm_cmpeq_pi8 (v, ffs)) & 0x40);
+#endif
+}
+
+static force_inline pixman_bool_t
+is_zero (__m64 v)
+{
+    return is_equal (v, _mm_setzero_si64 ());
 }
 
 /* Expand 16 bits positioned at @pos (0-3) of a mmx register into
@@ -337,7 +538,11 @@ expand565 (__m64 pixel, int pos)
     __m64 t1, t2;
 
     /* move pixel to low 16 bit and zero the rest */
+#ifdef USE_LOONGSON_MMI
+    p = loongson_extract_pi16 (p, pos);
+#else
     p = shift (shift (p, (3 - pos) * 16), -48);
+#endif
 
     t1 = shift (p, 36 - 11);
     t2 = shift (p, 16 - 5);
@@ -350,6 +555,36 @@ expand565 (__m64 pixel, int pos)
     return _mm_srli_pi16 (pixel, 8);
 }
 
+/* Expand 4 16 bit pixels in an mmx register into two mmx registers of
+ *
+ *    AARRGGBBRRGGBB
+ */
+static force_inline void
+expand_4xpacked565 (__m64 vin, __m64 *vout0, __m64 *vout1, int full_alpha)
+{
+    __m64 t0, t1, alpha = _mm_setzero_si64 ();
+    __m64 r = _mm_and_si64 (vin, MC (expand_565_r));
+    __m64 g = _mm_and_si64 (vin, MC (expand_565_g));
+    __m64 b = _mm_and_si64 (vin, MC (expand_565_b));
+    if (full_alpha)
+	alpha = _mm_cmpeq_pi32 (alpha, alpha);
+
+    /* Replicate high bits into empty low bits. */
+    r = _mm_or_si64 (_mm_srli_pi16 (r, 8), _mm_srli_pi16 (r, 13));
+    g = _mm_or_si64 (_mm_srli_pi16 (g, 3), _mm_srli_pi16 (g, 9));
+    b = _mm_or_si64 (_mm_slli_pi16 (b, 3), _mm_srli_pi16 (b, 2));
+
+    r = _mm_packs_pu16 (r, _mm_setzero_si64 ());	/* 00 00 00 00 R3 R2 R1 R0 */
+    g = _mm_packs_pu16 (g, _mm_setzero_si64 ());	/* 00 00 00 00 G3 G2 G1 G0 */
+    b = _mm_packs_pu16 (b, _mm_setzero_si64 ());	/* 00 00 00 00 B3 B2 B1 B0 */
+
+    t1 = _mm_unpacklo_pi8 (r, alpha);			/* A3 R3 A2 R2 A1 R1 A0 R0 */
+    t0 = _mm_unpacklo_pi8 (b, g);			/* G3 B3 G2 B2 G1 B1 G0 B0 */
+
+    *vout0 = _mm_unpacklo_pi16 (t0, t1);		/* A1 R1 G1 B1 A0 R0 G0 B0 */
+    *vout1 = _mm_unpackhi_pi16 (t0, t1);		/* A3 R3 G3 B3 A2 R2 G2 B2 */
+}
+
 static force_inline __m64
 expand8888 (__m64 in, int pos)
 {
@@ -365,6 +600,17 @@ expandx888 (__m64 in, int pos)
     return _mm_or_si64 (expand8888 (in, pos), MC (full_alpha));
 }
 
+static force_inline void
+expand_4x565 (__m64 vin, __m64 *vout0, __m64 *vout1, __m64 *vout2, __m64 *vout3, int full_alpha)
+{
+    __m64 v0, v1;
+    expand_4xpacked565 (vin, &v0, &v1, full_alpha);
+    *vout0 = expand8888 (v0, 0);
+    *vout1 = expand8888 (v0, 1);
+    *vout2 = expand8888 (v1, 0);
+    *vout3 = expand8888 (v1, 1);
+}
+
 static force_inline __m64
 pack_565 (__m64 pixel, __m64 target, int pos)
 {
@@ -376,6 +622,15 @@ pack_565 (__m64 pixel, __m64 target, int pos)
     g = _mm_and_si64 (p, MC (565_g));
     b = _mm_and_si64 (p, MC (565_b));
 
+#ifdef USE_LOONGSON_MMI
+    r = shift (r, -(32 - 8));
+    g = shift (g, -(16 - 3));
+    b = shift (b, -(0  + 3));
+
+    p = _mm_or_si64 (r, g);
+    p = _mm_or_si64 (p, b);
+    return loongson_insert_pi16 (t, p, pos);
+#else
     r = shift (r, -(32 - 8) + pos * 16);
     g = shift (g, -(16 - 3) + pos * 16);
     b = shift (b, -(0  + 3) + pos * 16);
@@ -393,11 +648,43 @@ pack_565 (__m64 pixel, __m64 target, int pos)
     p = _mm_or_si64 (g, p);
 
     return _mm_or_si64 (b, p);
+#endif
+}
+
+static force_inline __m64
+pack_4xpacked565 (__m64 a, __m64 b)
+{
+    __m64 rb0 = _mm_and_si64 (a, MC (packed_565_rb));
+    __m64 rb1 = _mm_and_si64 (b, MC (packed_565_rb));
+
+    __m64 t0 = _mm_madd_pi16 (rb0, MC (565_pack_multiplier));
+    __m64 t1 = _mm_madd_pi16 (rb1, MC (565_pack_multiplier));
+
+    __m64 g0 = _mm_and_si64 (a, MC (packed_565_g));
+    __m64 g1 = _mm_and_si64 (b, MC (packed_565_g));
+
+    t0 = _mm_or_si64 (t0, g0);
+    t1 = _mm_or_si64 (t1, g1);
+
+    t0 = shift(t0, -5);
+#ifdef USE_ARM_IWMMXT
+    t1 = shift(t1, -5);
+    return _mm_packs_pu32 (t0, t1);
+#else
+    t1 = shift(t1, -5 + 16);
+    return _mm_shuffle_pi16 (_mm_or_si64 (t0, t1), _MM_SHUFFLE (3, 1, 2, 0));
+#endif
 }
 
 #ifndef _MSC_VER
 
 static force_inline __m64
+pack_4x565 (__m64 v0, __m64 v1, __m64 v2, __m64 v3)
+{
+    return pack_4xpacked565 (pack8888 (v0, v1), pack8888 (v2, v3));
+}
+
+static force_inline __m64
 pix_add_mul (__m64 x, __m64 a, __m64 y, __m64 b)
 {
     x = pix_multiply (x, a);
@@ -408,32 +695,52 @@ pix_add_mul (__m64 x, __m64 a, __m64 y, __m64 b)
 
 #else
 
+/* MSVC only handles a "pass by register" of up to three SSE intrinsics */
+
+#define pack_4x565(v0, v1, v2, v3) \
+    pack_4xpacked565 (pack8888 (v0, v1), pack8888 (v2, v3))
+
 #define pix_add_mul(x, a, y, b)	 \
     ( x = pix_multiply (x, a),	 \
-      y = pix_multiply (y, a),	 \
+      y = pix_multiply (y, b),	 \
       pix_add (x, y) )
 
 #endif
 
 /* --------------- MMX code patch for fbcompose.c --------------------- */
 
-static force_inline uint32_t
+static force_inline __m64
 combine (const uint32_t *src, const uint32_t *mask)
 {
-    uint32_t ssrc = *src;
+    __m64 vsrc = load8888 (src);
 
     if (mask)
     {
-	__m64 m = load8888 (*mask);
-	__m64 s = load8888 (ssrc);
+	__m64 m = load8888 (mask);
 
 	m = expand_alpha (m);
-	s = pix_multiply (s, m);
+	vsrc = pix_multiply (vsrc, m);
+    }
+
+    return vsrc;
+}
 
-	ssrc = store8888 (s);
+static force_inline __m64
+core_combine_over_u_pixel_mmx (__m64 vsrc, __m64 vdst)
+{
+    vsrc = _mm_unpacklo_pi8 (vsrc, _mm_setzero_si64 ());
+
+    if (is_opaque (vsrc))
+    {
+	return vsrc;
+    }
+    else if (!is_zero (vsrc))
+    {
+	return over (vsrc, expand_alpha (vsrc),
+		     _mm_unpacklo_pi8 (vdst, _mm_setzero_si64 ()));
     }
 
-    return ssrc;
+    return _mm_unpacklo_pi8 (vdst, _mm_setzero_si64 ());
 }
 
 static void
@@ -448,19 +755,16 @@ mmx_combine_over_u (pixman_implementation_t *imp,
 
     while (dest < end)
     {
-	uint32_t ssrc = combine (src, mask);
-	uint32_t a = ssrc >> 24;
+	__m64 vsrc = combine (src, mask);
 
-	if (a == 0xff)
+	if (is_opaque (vsrc))
 	{
-	    *dest = ssrc;
+	    store8888 (dest, vsrc);
 	}
-	else if (ssrc)
+	else if (!is_zero (vsrc))
 	{
-	    __m64 s, sa;
-	    s = load8888 (ssrc);
-	    sa = expand_alpha (s);
-	    *dest = store8888 (over (s, sa, load8888 (*dest)));
+	    __m64 sa = expand_alpha (vsrc);
+	    store8888 (dest, over (vsrc, sa, load8888 (dest)));
 	}
 
 	++dest;
@@ -484,11 +788,11 @@ mmx_combine_over_reverse_u (pixman_implementation_t *imp,
     while (dest < end)
     {
 	__m64 d, da;
-	uint32_t s = combine (src, mask);
+	__m64 s = combine (src, mask);
 
-	d = load8888 (*dest);
+	d = load8888 (dest);
 	da = expand_alpha (d);
-	*dest = store8888 (over (d, da, load8888 (s)));
+	store8888 (dest, over (d, da, s));
 
 	++dest;
 	++src;
@@ -510,14 +814,14 @@ mmx_combine_in_u (pixman_implementation_t *imp,
 
     while (dest < end)
     {
-	__m64 x, a;
+	__m64 a;
+	__m64 x = combine (src, mask);
 
-	x = load8888 (combine (src, mask));
-	a = load8888 (*dest);
+	a = load8888 (dest);
 	a = expand_alpha (a);
 	x = pix_multiply (x, a);
 
-	*dest = store8888 (x);
+	store8888 (dest, x);
 
 	++dest;
 	++src;
@@ -539,13 +843,13 @@ mmx_combine_in_reverse_u (pixman_implementation_t *imp,
 
     while (dest < end)
     {
-	__m64 x, a;
+	__m64 a = combine (src, mask);
+	__m64 x;
 
-	x = load8888 (*dest);
-	a = load8888 (combine (src, mask));
+	x = load8888 (dest);
 	a = expand_alpha (a);
 	x = pix_multiply (x, a);
-	*dest = store8888 (x);
+	store8888 (dest, x);
 
 	++dest;
 	++src;
@@ -567,14 +871,14 @@ mmx_combine_out_u (pixman_implementation_t *imp,
 
     while (dest < end)
     {
-	__m64 x, a;
+	__m64 a;
+	__m64 x = combine (src, mask);
 
-	x = load8888 (combine (src, mask));
-	a = load8888 (*dest);
+	a = load8888 (dest);
 	a = expand_alpha (a);
 	a = negate (a);
 	x = pix_multiply (x, a);
-	*dest = store8888 (x);
+	store8888 (dest, x);
 
 	++dest;
 	++src;
@@ -596,15 +900,15 @@ mmx_combine_out_reverse_u (pixman_implementation_t *imp,
 
     while (dest < end)
     {
-	__m64 x, a;
+	__m64 a = combine (src, mask);
+	__m64 x;
 
-	x = load8888 (*dest);
-	a = load8888 (combine (src, mask));
+	x = load8888 (dest);
 	a = expand_alpha (a);
 	a = negate (a);
 	x = pix_multiply (x, a);
 
-	*dest = store8888 (x);
+	store8888 (dest, x);
 
 	++dest;
 	++src;
@@ -626,15 +930,15 @@ mmx_combine_atop_u (pixman_implementation_t *imp,
 
     while (dest < end)
     {
-	__m64 s, da, d, sia;
+	__m64 da, d, sia;
+	__m64 s = combine (src, mask);
 
-	s = load8888 (combine (src, mask));
-	d = load8888 (*dest);
+	d = load8888 (dest);
 	sia = expand_alpha (s);
 	sia = negate (sia);
 	da = expand_alpha (d);
 	s = pix_add_mul (s, da, d, sia);
-	*dest = store8888 (s);
+	store8888 (dest, s);
 
 	++dest;
 	++src;
@@ -658,15 +962,15 @@ mmx_combine_atop_reverse_u (pixman_implementation_t *imp,
 
     while (dest < end)
     {
-	__m64 s, dia, d, sa;
+	__m64 dia, d, sa;
+	__m64 s = combine (src, mask);
 
-	s = load8888 (combine (src, mask));
-	d = load8888 (*dest);
+	d = load8888 (dest);
 	sa = expand_alpha (s);
 	dia = expand_alpha (d);
 	dia = negate (dia);
 	s = pix_add_mul (s, dia, d, sa);
-	*dest = store8888 (s);
+	store8888 (dest, s);
 
 	++dest;
 	++src;
@@ -688,16 +992,16 @@ mmx_combine_xor_u (pixman_implementation_t *imp,
 
     while (dest < end)
     {
-	__m64 s, dia, d, sia;
+	__m64 dia, d, sia;
+	__m64 s = combine (src, mask);
 
-	s = load8888 (combine (src, mask));
-	d = load8888 (*dest);
+	d = load8888 (dest);
 	sia = expand_alpha (s);
 	dia = expand_alpha (d);
 	sia = negate (sia);
 	dia = negate (dia);
 	s = pix_add_mul (s, dia, d, sia);
-	*dest = store8888 (s);
+	store8888 (dest, s);
 
 	++dest;
 	++src;
@@ -719,12 +1023,12 @@ mmx_combine_add_u (pixman_implementation_t *imp,
 
     while (dest < end)
     {
-	__m64 s, d;
+	__m64 d;
+	__m64 s = combine (src, mask);
 
-	s = load8888 (combine (src, mask));
-	d = load8888 (*dest);
+	d = load8888 (dest);
 	s = pix_add (s, d);
-	*dest = store8888 (s);
+	store8888 (dest, s);
 
 	++dest;
 	++src;
@@ -746,22 +1050,25 @@ mmx_combine_saturate_u (pixman_implementation_t *imp,
 
     while (dest < end)
     {
-	uint32_t s = combine (src, mask);
+	uint32_t s, sa, da;
 	uint32_t d = *dest;
-	__m64 ms = load8888 (s);
-	__m64 md = load8888 (d);
-	uint32_t sa = s >> 24;
-	uint32_t da = ~d >> 24;
+	__m64 ms = combine (src, mask);
+	__m64 md = load8888 (dest);
+
+	store8888(&s, ms);
+	da = ~d >> 24;
+	sa = s >> 24;
 
 	if (sa > da)
 	{
-	    __m64 msa = load8888 (DIV_UN8 (da, sa) << 24);
+	    uint32_t quot = DIV_UN8 (da, sa) << 24;
+	    __m64 msa = load8888 (&quot);
 	    msa = expand_alpha (msa);
 	    ms = pix_multiply (ms, msa);
 	}
 
 	md = pix_add (md, ms);
-	*dest = store8888 (md);
+	store8888 (dest, md);
 
 	++src;
 	++dest;
@@ -783,11 +1090,11 @@ mmx_combine_src_ca (pixman_implementation_t *imp,
 
     while (src < end)
     {
-	__m64 a = load8888 (*mask);
-	__m64 s = load8888 (*src);
+	__m64 a = load8888 (mask);
+	__m64 s = load8888 (src);
 
 	s = pix_multiply (s, a);
-	*dest = store8888 (s);
+	store8888 (dest, s);
 
 	++src;
 	++mask;
@@ -808,12 +1115,12 @@ mmx_combine_over_ca (pixman_implementation_t *imp,
 
     while (src < end)
     {
-	__m64 a = load8888 (*mask);
-	__m64 s = load8888 (*src);
-	__m64 d = load8888 (*dest);
+	__m64 a = load8888 (mask);
+	__m64 s = load8888 (src);
+	__m64 d = load8888 (dest);
 	__m64 sa = expand_alpha (s);
 
-	*dest = store8888 (in_over (s, sa, a, d));
+	store8888 (dest, in_over (s, sa, a, d));
 
 	++src;
 	++dest;
@@ -834,12 +1141,12 @@ mmx_combine_over_reverse_ca (pixman_implementation_t *imp,
 
     while (src < end)
     {
-	__m64 a = load8888 (*mask);
-	__m64 s = load8888 (*src);
-	__m64 d = load8888 (*dest);
+	__m64 a = load8888 (mask);
+	__m64 s = load8888 (src);
+	__m64 d = load8888 (dest);
 	__m64 da = expand_alpha (d);
 
-	*dest = store8888 (over (d, da, in (s, a)));
+	store8888 (dest, over (d, da, in (s, a)));
 
 	++src;
 	++dest;
@@ -860,14 +1167,14 @@ mmx_combine_in_ca (pixman_implementation_t *imp,
 
     while (src < end)
     {
-	__m64 a = load8888 (*mask);
-	__m64 s = load8888 (*src);
-	__m64 d = load8888 (*dest);
+	__m64 a = load8888 (mask);
+	__m64 s = load8888 (src);
+	__m64 d = load8888 (dest);
 	__m64 da = expand_alpha (d);
 
 	s = pix_multiply (s, a);
 	s = pix_multiply (s, da);
-	*dest = store8888 (s);
+	store8888 (dest, s);
 
 	++src;
 	++dest;
@@ -888,14 +1195,14 @@ mmx_combine_in_reverse_ca (pixman_implementation_t *imp,
 
     while (src < end)
     {
-	__m64 a = load8888 (*mask);
-	__m64 s = load8888 (*src);
-	__m64 d = load8888 (*dest);
+	__m64 a = load8888 (mask);
+	__m64 s = load8888 (src);
+	__m64 d = load8888 (dest);
 	__m64 sa = expand_alpha (s);
 
 	a = pix_multiply (a, sa);
 	d = pix_multiply (d, a);
-	*dest = store8888 (d);
+	store8888 (dest, d);
 
 	++src;
 	++dest;
@@ -916,15 +1223,15 @@ mmx_combine_out_ca (pixman_implementation_t *imp,
 
     while (src < end)
     {
-	__m64 a = load8888 (*mask);
-	__m64 s = load8888 (*src);
-	__m64 d = load8888 (*dest);
+	__m64 a = load8888 (mask);
+	__m64 s = load8888 (src);
+	__m64 d = load8888 (dest);
 	__m64 da = expand_alpha (d);
 
 	da = negate (da);
 	s = pix_multiply (s, a);
 	s = pix_multiply (s, da);
-	*dest = store8888 (s);
+	store8888 (dest, s);
 
 	++src;
 	++dest;
@@ -945,15 +1252,15 @@ mmx_combine_out_reverse_ca (pixman_implementation_t *imp,
 
     while (src < end)
     {
-	__m64 a = load8888 (*mask);
-	__m64 s = load8888 (*src);
-	__m64 d = load8888 (*dest);
+	__m64 a = load8888 (mask);
+	__m64 s = load8888 (src);
+	__m64 d = load8888 (dest);
 	__m64 sa = expand_alpha (s);
 
 	a = pix_multiply (a, sa);
 	a = negate (a);
 	d = pix_multiply (d, a);
-	*dest = store8888 (d);
+	store8888 (dest, d);
 
 	++src;
 	++dest;
@@ -974,9 +1281,9 @@ mmx_combine_atop_ca (pixman_implementation_t *imp,
 
     while (src < end)
     {
-	__m64 a = load8888 (*mask);
-	__m64 s = load8888 (*src);
-	__m64 d = load8888 (*dest);
+	__m64 a = load8888 (mask);
+	__m64 s = load8888 (src);
+	__m64 d = load8888 (dest);
 	__m64 da = expand_alpha (d);
 	__m64 sa = expand_alpha (s);
 
@@ -984,7 +1291,7 @@ mmx_combine_atop_ca (pixman_implementation_t *imp,
 	a = pix_multiply (a, sa);
 	a = negate (a);
 	d = pix_add_mul (d, a, s, da);
-	*dest = store8888 (d);
+	store8888 (dest, d);
 
 	++src;
 	++dest;
@@ -1005,9 +1312,9 @@ mmx_combine_atop_reverse_ca (pixman_implementation_t *imp,
 
     while (src < end)
     {
-	__m64 a = load8888 (*mask);
-	__m64 s = load8888 (*src);
-	__m64 d = load8888 (*dest);
+	__m64 a = load8888 (mask);
+	__m64 s = load8888 (src);
+	__m64 d = load8888 (dest);
 	__m64 da = expand_alpha (d);
 	__m64 sa = expand_alpha (s);
 
@@ -1015,7 +1322,7 @@ mmx_combine_atop_reverse_ca (pixman_implementation_t *imp,
 	a = pix_multiply (a, sa);
 	da = negate (da);
 	d = pix_add_mul (d, a, s, da);
-	*dest = store8888 (d);
+	store8888 (dest, d);
 
 	++src;
 	++dest;
@@ -1036,9 +1343,9 @@ mmx_combine_xor_ca (pixman_implementation_t *imp,
 
     while (src < end)
     {
-	__m64 a = load8888 (*mask);
-	__m64 s = load8888 (*src);
-	__m64 d = load8888 (*dest);
+	__m64 a = load8888 (mask);
+	__m64 s = load8888 (src);
+	__m64 d = load8888 (dest);
 	__m64 da = expand_alpha (d);
 	__m64 sa = expand_alpha (s);
 
@@ -1047,7 +1354,7 @@ mmx_combine_xor_ca (pixman_implementation_t *imp,
 	da = negate (da);
 	a = negate (a);
 	d = pix_add_mul (d, a, s, da);
-	*dest = store8888 (d);
+	store8888 (dest, d);
 
 	++src;
 	++dest;
@@ -1068,13 +1375,13 @@ mmx_combine_add_ca (pixman_implementation_t *imp,
 
     while (src < end)
     {
-	__m64 a = load8888 (*mask);
-	__m64 s = load8888 (*src);
-	__m64 d = load8888 (*dest);
+	__m64 a = load8888 (mask);
+	__m64 s = load8888 (src);
+	__m64 d = load8888 (dest);
 
 	s = pix_multiply (s, a);
 	d = pix_add (s, d);
-	*dest = store8888 (d);
+	store8888 (dest, d);
 
 	++src;
 	++dest;
@@ -1087,19 +1394,9 @@ mmx_combine_add_ca (pixman_implementation_t *imp,
 
 static void
 mmx_composite_over_n_8888 (pixman_implementation_t *imp,
-                           pixman_op_t              op,
-                           pixman_image_t *         src_image,
-                           pixman_image_t *         mask_image,
-                           pixman_image_t *         dst_image,
-                           int32_t                  src_x,
-                           int32_t                  src_y,
-                           int32_t                  mask_x,
-                           int32_t                  mask_y,
-                           int32_t                  dest_x,
-                           int32_t                  dest_y,
-                           int32_t                  width,
-                           int32_t                  height)
+                           pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t src;
     uint32_t    *dst_line, *dst;
     int32_t w;
@@ -1108,14 +1405,14 @@ mmx_composite_over_n_8888 (pixman_implementation_t *imp,
 
     CHECKPOINT ();
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     if (src == 0)
 	return;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
 
-    vsrc = load8888 (src);
+    vsrc = load8888 (&src);
     vsrca = expand_alpha (vsrc);
 
     while (height--)
@@ -1126,9 +1423,9 @@ mmx_composite_over_n_8888 (pixman_implementation_t *imp,
 
 	CHECKPOINT ();
 
-	while (w && (unsigned long)dst & 7)
+	while (w && (uintptr_t)dst & 7)
 	{
-	    *dst = store8888 (over (vsrc, vsrca, load8888 (*dst)));
+	    store8888 (dst, over (vsrc, vsrca, load8888 (dst)));
 
 	    w--;
 	    dst++;
@@ -1152,12 +1449,9 @@ mmx_composite_over_n_8888 (pixman_implementation_t *imp,
 
 	CHECKPOINT ();
 
-	while (w)
+	if (w)
 	{
-	    *dst = store8888 (over (vsrc, vsrca, load8888 (*dst)));
-
-	    w--;
-	    dst++;
+	    store8888 (dst, over (vsrc, vsrca, load8888 (dst)));
 	}
     }
 
@@ -1166,19 +1460,9 @@ mmx_composite_over_n_8888 (pixman_implementation_t *imp,
 
 static void
 mmx_composite_over_n_0565 (pixman_implementation_t *imp,
-                           pixman_op_t              op,
-                           pixman_image_t *         src_image,
-                           pixman_image_t *         mask_image,
-                           pixman_image_t *         dst_image,
-                           int32_t                  src_x,
-                           int32_t                  src_y,
-                           int32_t                  mask_x,
-                           int32_t                  mask_y,
-                           int32_t                  dest_x,
-                           int32_t                  dest_y,
-                           int32_t                  width,
-                           int32_t                  height)
+                           pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t src;
     uint16_t    *dst_line, *dst;
     int32_t w;
@@ -1187,14 +1471,14 @@ mmx_composite_over_n_0565 (pixman_implementation_t *imp,
 
     CHECKPOINT ();
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     if (src == 0)
 	return;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
 
-    vsrc = load8888 (src);
+    vsrc = load8888 (&src);
     vsrca = expand_alpha (vsrc);
 
     while (height--)
@@ -1205,13 +1489,13 @@ mmx_composite_over_n_0565 (pixman_implementation_t *imp,
 
 	CHECKPOINT ();
 
-	while (w && (unsigned long)dst & 7)
+	while (w && (uintptr_t)dst & 7)
 	{
 	    uint64_t d = *dst;
-	    __m64 vdest = expand565 (M64 (d), 0);
+	    __m64 vdest = expand565 (to_m64 (d), 0);
 
 	    vdest = pack_565 (over (vsrc, vsrca, vdest), vdest, 0);
-	    *dst = UINT64 (vdest);
+	    *dst = to_uint64 (vdest);
 
 	    w--;
 	    dst++;
@@ -1219,16 +1503,17 @@ mmx_composite_over_n_0565 (pixman_implementation_t *imp,
 
 	while (w >= 4)
 	{
-	    __m64 vdest;
+	    __m64 vdest = *(__m64 *)dst;
+	    __m64 v0, v1, v2, v3;
 
-	    vdest = *(__m64 *)dst;
+	    expand_4x565 (vdest, &v0, &v1, &v2, &v3, 0);
 
-	    vdest = pack_565 (over (vsrc, vsrca, expand565 (vdest, 0)), vdest, 0);
-	    vdest = pack_565 (over (vsrc, vsrca, expand565 (vdest, 1)), vdest, 1);
-	    vdest = pack_565 (over (vsrc, vsrca, expand565 (vdest, 2)), vdest, 2);
-	    vdest = pack_565 (over (vsrc, vsrca, expand565 (vdest, 3)), vdest, 3);
+	    v0 = over (vsrc, vsrca, v0);
+	    v1 = over (vsrc, vsrca, v1);
+	    v2 = over (vsrc, vsrca, v2);
+	    v3 = over (vsrc, vsrca, v3);
 
-	    *(__m64 *)dst = vdest;
+	    *(__m64 *)dst = pack_4x565 (v0, v1, v2, v3);
 
 	    dst += 4;
 	    w -= 4;
@@ -1239,10 +1524,10 @@ mmx_composite_over_n_0565 (pixman_implementation_t *imp,
 	while (w)
 	{
 	    uint64_t d = *dst;
-	    __m64 vdest = expand565 (M64 (d), 0);
+	    __m64 vdest = expand565 (to_m64 (d), 0);
 
 	    vdest = pack_565 (over (vsrc, vsrca, vdest), vdest, 0);
-	    *dst = UINT64 (vdest);
+	    *dst = to_uint64 (vdest);
 
 	    w--;
 	    dst++;
@@ -1254,20 +1539,10 @@ mmx_composite_over_n_0565 (pixman_implementation_t *imp,
 
 static void
 mmx_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
-                                   pixman_op_t              op,
-                                   pixman_image_t *         src_image,
-                                   pixman_image_t *         mask_image,
-                                   pixman_image_t *         dst_image,
-                                   int32_t                  src_x,
-                                   int32_t                  src_y,
-                                   int32_t                  mask_x,
-                                   int32_t                  mask_y,
-                                   int32_t                  dest_x,
-                                   int32_t                  dest_y,
-                                   int32_t                  width,
-                                   int32_t                  height)
+                                   pixman_composite_info_t *info)
 {
-    uint32_t src, srca;
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint32_t src;
     uint32_t    *dst_line;
     uint32_t    *mask_line;
     int dst_stride, mask_stride;
@@ -1275,16 +1550,15 @@ mmx_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
 
     CHECKPOINT ();
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
-    srca = src >> 24;
     if (src == 0)
 	return;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
 
-    vsrc = load8888 (src);
+    vsrc = load8888 (&src);
     vsrca = expand_alpha (vsrc);
 
     while (height--)
@@ -1293,15 +1567,15 @@ mmx_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
 	uint32_t *p = (uint32_t *)mask_line;
 	uint32_t *q = (uint32_t *)dst_line;
 
-	while (twidth && (unsigned long)q & 7)
+	while (twidth && (uintptr_t)q & 7)
 	{
 	    uint32_t m = *(uint32_t *)p;
 
 	    if (m)
 	    {
-		__m64 vdest = load8888 (*q);
-		vdest = in_over (vsrc, vsrca, load8888 (m), vdest);
-		*q = store8888 (vdest);
+		__m64 vdest = load8888 (q);
+		vdest = in_over (vsrc, vsrca, load8888 (&m), vdest);
+		store8888 (q, vdest);
 	    }
 
 	    twidth--;
@@ -1320,9 +1594,9 @@ mmx_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
 		__m64 dest0, dest1;
 		__m64 vdest = *(__m64 *)q;
 
-		dest0 = in_over (vsrc, vsrca, load8888 (m0),
+		dest0 = in_over (vsrc, vsrca, load8888 (&m0),
 		                 expand8888 (vdest, 0));
-		dest1 = in_over (vsrc, vsrca, load8888 (m1),
+		dest1 = in_over (vsrc, vsrca, load8888 (&m1),
 		                 expand8888 (vdest, 1));
 
 		*(__m64 *)q = pack8888 (dest0, dest1);
@@ -1333,15 +1607,15 @@ mmx_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
 	    twidth -= 2;
 	}
 
-	while (twidth)
+	if (twidth)
 	{
 	    uint32_t m = *(uint32_t *)p;
 
 	    if (m)
 	    {
-		__m64 vdest = load8888 (*q);
-		vdest = in_over (vsrc, vsrca, load8888 (m), vdest);
-		*q = store8888 (vdest);
+		__m64 vdest = load8888 (q);
+		vdest = in_over (vsrc, vsrca, load8888 (&m), vdest);
+		store8888 (q, vdest);
 	    }
 
 	    twidth--;
@@ -1358,37 +1632,23 @@ mmx_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
 
 static void
 mmx_composite_over_8888_n_8888 (pixman_implementation_t *imp,
-                                pixman_op_t              op,
-                                pixman_image_t *         src_image,
-                                pixman_image_t *         mask_image,
-                                pixman_image_t *         dst_image,
-                                int32_t                  src_x,
-                                int32_t                  src_y,
-                                int32_t                  mask_x,
-                                int32_t                  mask_y,
-                                int32_t                  dest_x,
-                                int32_t                  dest_y,
-                                int32_t                  width,
-                                int32_t                  height)
+                                pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t    *dst_line, *dst;
     uint32_t    *src_line, *src;
     uint32_t mask;
     __m64 vmask;
     int dst_stride, src_stride;
     int32_t w;
-    __m64 srca;
 
     CHECKPOINT ();
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
 
-    mask = _pixman_image_get_solid (mask_image, dst_image->bits.format);
-    mask &= 0xff000000;
-    mask = mask | mask >> 8 | mask >> 16 | mask >> 24;
-    vmask = load8888 (mask);
-    srca = MC (4x00ff);
+    mask = _pixman_image_get_solid (imp, mask_image, dest_image->bits.format);
+    vmask = expand_alpha (load8888 (&mask));
 
     while (height--)
     {
@@ -1398,12 +1658,12 @@ mmx_composite_over_8888_n_8888 (pixman_implementation_t *imp,
 	src_line += src_stride;
 	w = width;
 
-	while (w && (unsigned long)dst & 7)
+	while (w && (uintptr_t)dst & 7)
 	{
-	    __m64 s = load8888 (*src);
-	    __m64 d = load8888 (*dst);
+	    __m64 s = load8888 (src);
+	    __m64 d = load8888 (dst);
 
-	    *dst = store8888 (in_over (s, expand_alpha (s), vmask, d));
+	    store8888 (dst, in_over (s, expand_alpha (s), vmask, d));
 
 	    w--;
 	    dst++;
@@ -1412,7 +1672,7 @@ mmx_composite_over_8888_n_8888 (pixman_implementation_t *imp,
 
 	while (w >= 2)
 	{
-	    __m64 vs = *(__m64 *)src;
+	    __m64 vs = ldq_u ((__m64 *)src);
 	    __m64 vd = *(__m64 *)dst;
 	    __m64 vsrc0 = expand8888 (vs, 0);
 	    __m64 vsrc1 = expand8888 (vs, 1);
@@ -1426,16 +1686,12 @@ mmx_composite_over_8888_n_8888 (pixman_implementation_t *imp,
 	    src += 2;
 	}
 
-	while (w)
+	if (w)
 	{
-	    __m64 s = load8888 (*src);
-	    __m64 d = load8888 (*dst);
+	    __m64 s = load8888 (src);
+	    __m64 d = load8888 (dst);
 
-	    *dst = store8888 (in_over (s, expand_alpha (s), vmask, d));
-
-	    w--;
-	    dst++;
-	    src++;
+	    store8888 (dst, in_over (s, expand_alpha (s), vmask, d));
 	}
     }
 
@@ -1444,19 +1700,9 @@ mmx_composite_over_8888_n_8888 (pixman_implementation_t *imp,
 
 static void
 mmx_composite_over_x888_n_8888 (pixman_implementation_t *imp,
-                                pixman_op_t              op,
-                                pixman_image_t *         src_image,
-                                pixman_image_t *         mask_image,
-                                pixman_image_t *         dst_image,
-                                int32_t                  src_x,
-                                int32_t                  src_y,
-                                int32_t                  mask_x,
-                                int32_t                  mask_y,
-                                int32_t                  dest_x,
-                                int32_t                  dest_y,
-                                int32_t                  width,
-                                int32_t                  height)
+                                pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t *dst_line, *dst;
     uint32_t *src_line, *src;
     uint32_t mask;
@@ -1467,13 +1713,11 @@ mmx_composite_over_x888_n_8888 (pixman_implementation_t *imp,
 
     CHECKPOINT ();
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
-    mask = _pixman_image_get_solid (mask_image, dst_image->bits.format);
+    mask = _pixman_image_get_solid (imp, mask_image, dest_image->bits.format);
 
-    mask &= 0xff000000;
-    mask = mask | mask >> 8 | mask >> 16 | mask >> 24;
-    vmask = load8888 (mask);
+    vmask = expand_alpha (load8888 (&mask));
     srca = MC (4x00ff);
 
     while (height--)
@@ -1484,12 +1728,13 @@ mmx_composite_over_x888_n_8888 (pixman_implementation_t *imp,
 	src_line += src_stride;
 	w = width;
 
-	while (w && (unsigned long)dst & 7)
+	while (w && (uintptr_t)dst & 7)
 	{
-	    __m64 s = load8888 (*src | 0xff000000);
-	    __m64 d = load8888 (*dst);
+	    uint32_t ssrc = *src | 0xff000000;
+	    __m64 s = load8888 (&ssrc);
+	    __m64 d = load8888 (dst);
 
-	    *dst = store8888 (in_over (s, srca, vmask, d));
+	    store8888 (dst, in_over (s, srca, vmask, d));
 
 	    w--;
 	    dst++;
@@ -1507,14 +1752,14 @@ mmx_composite_over_x888_n_8888 (pixman_implementation_t *imp,
 	    __m64 vd6 = *(__m64 *)(dst + 12);
 	    __m64 vd7 = *(__m64 *)(dst + 14);
 
-	    __m64 vs0 = *(__m64 *)(src + 0);
-	    __m64 vs1 = *(__m64 *)(src + 2);
-	    __m64 vs2 = *(__m64 *)(src + 4);
-	    __m64 vs3 = *(__m64 *)(src + 6);
-	    __m64 vs4 = *(__m64 *)(src + 8);
-	    __m64 vs5 = *(__m64 *)(src + 10);
-	    __m64 vs6 = *(__m64 *)(src + 12);
-	    __m64 vs7 = *(__m64 *)(src + 14);
+	    __m64 vs0 = ldq_u ((__m64 *)(src + 0));
+	    __m64 vs1 = ldq_u ((__m64 *)(src + 2));
+	    __m64 vs2 = ldq_u ((__m64 *)(src + 4));
+	    __m64 vs3 = ldq_u ((__m64 *)(src + 6));
+	    __m64 vs4 = ldq_u ((__m64 *)(src + 8));
+	    __m64 vs5 = ldq_u ((__m64 *)(src + 10));
+	    __m64 vs6 = ldq_u ((__m64 *)(src + 12));
+	    __m64 vs7 = ldq_u ((__m64 *)(src + 14));
 
 	    vd0 = pack8888 (
 	        in_over (expandx888 (vs0, 0), srca, vmask, expand8888 (vd0, 0)),
@@ -1564,10 +1809,11 @@ mmx_composite_over_x888_n_8888 (pixman_implementation_t *imp,
 
 	while (w)
 	{
-	    __m64 s = load8888 (*src | 0xff000000);
-	    __m64 d = load8888 (*dst);
+	    uint32_t ssrc = *src | 0xff000000;
+	    __m64 s = load8888 (&ssrc);
+	    __m64 d = load8888 (dst);
 
-	    *dst = store8888 (in_over (s, srca, vmask, d));
+	    store8888 (dst, in_over (s, srca, vmask, d));
 
 	    w--;
 	    dst++;
@@ -1580,19 +1826,9 @@ mmx_composite_over_x888_n_8888 (pixman_implementation_t *imp,
 
 static void
 mmx_composite_over_8888_8888 (pixman_implementation_t *imp,
-                              pixman_op_t              op,
-                              pixman_image_t *         src_image,
-                              pixman_image_t *         mask_image,
-                              pixman_image_t *         dst_image,
-                              int32_t                  src_x,
-                              int32_t                  src_y,
-                              int32_t                  mask_x,
-                              int32_t                  mask_y,
-                              int32_t                  dest_x,
-                              int32_t                  dest_y,
-                              int32_t                  width,
-                              int32_t                  height)
+                              pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t *dst_line, *dst;
     uint32_t *src_line, *src;
     uint32_t s;
@@ -1602,7 +1838,7 @@ mmx_composite_over_8888_8888 (pixman_implementation_t *imp,
 
     CHECKPOINT ();
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
 
     while (height--)
@@ -1625,9 +1861,9 @@ mmx_composite_over_8888_8888 (pixman_implementation_t *imp,
 	    else if (s)
 	    {
 		__m64 ms, sa;
-		ms = load8888 (s);
+		ms = load8888 (&s);
 		sa = expand_alpha (ms);
-		*dst = store8888 (over (ms, sa, load8888 (*dst)));
+		store8888 (dst, over (ms, sa, load8888 (dst)));
 	    }
 
 	    dst++;
@@ -1638,19 +1874,9 @@ mmx_composite_over_8888_8888 (pixman_implementation_t *imp,
 
 static void
 mmx_composite_over_8888_0565 (pixman_implementation_t *imp,
-                              pixman_op_t              op,
-                              pixman_image_t *         src_image,
-                              pixman_image_t *         mask_image,
-                              pixman_image_t *         dst_image,
-                              int32_t                  src_x,
-                              int32_t                  src_y,
-                              int32_t                  mask_x,
-                              int32_t                  mask_y,
-                              int32_t                  dest_x,
-                              int32_t                  dest_y,
-                              int32_t                  width,
-                              int32_t                  height)
+                              pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint16_t    *dst_line, *dst;
     uint32_t    *src_line, *src;
     int dst_stride, src_stride;
@@ -1658,7 +1884,7 @@ mmx_composite_over_8888_0565 (pixman_implementation_t *imp,
 
     CHECKPOINT ();
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
 
 #if 0
@@ -1676,16 +1902,16 @@ mmx_composite_over_8888_0565 (pixman_implementation_t *imp,
 
 	CHECKPOINT ();
 
-	while (w && (unsigned long)dst & 7)
+	while (w && (uintptr_t)dst & 7)
 	{
-	    __m64 vsrc = load8888 (*src);
+	    __m64 vsrc = load8888 (src);
 	    uint64_t d = *dst;
-	    __m64 vdest = expand565 (M64 (d), 0);
+	    __m64 vdest = expand565 (to_m64 (d), 0);
 
 	    vdest = pack_565 (
 		over (vsrc, expand_alpha (vsrc), vdest), vdest, 0);
 
-	    *dst = UINT64 (vdest);
+	    *dst = to_uint64 (vdest);
 
 	    w--;
 	    dst++;
@@ -1696,22 +1922,23 @@ mmx_composite_over_8888_0565 (pixman_implementation_t *imp,
 
 	while (w >= 4)
 	{
+	    __m64 vdest = *(__m64 *)dst;
+	    __m64 v0, v1, v2, v3;
 	    __m64 vsrc0, vsrc1, vsrc2, vsrc3;
-	    __m64 vdest;
 
-	    vsrc0 = load8888 (*(src + 0));
-	    vsrc1 = load8888 (*(src + 1));
-	    vsrc2 = load8888 (*(src + 2));
-	    vsrc3 = load8888 (*(src + 3));
+	    expand_4x565 (vdest, &v0, &v1, &v2, &v3, 0);
 
-	    vdest = *(__m64 *)dst;
+	    vsrc0 = load8888 ((src + 0));
+	    vsrc1 = load8888 ((src + 1));
+	    vsrc2 = load8888 ((src + 2));
+	    vsrc3 = load8888 ((src + 3));
 
-	    vdest = pack_565 (over (vsrc0, expand_alpha (vsrc0), expand565 (vdest, 0)), vdest, 0);
-	    vdest = pack_565 (over (vsrc1, expand_alpha (vsrc1), expand565 (vdest, 1)), vdest, 1);
-	    vdest = pack_565 (over (vsrc2, expand_alpha (vsrc2), expand565 (vdest, 2)), vdest, 2);
-	    vdest = pack_565 (over (vsrc3, expand_alpha (vsrc3), expand565 (vdest, 3)), vdest, 3);
+	    v0 = over (vsrc0, expand_alpha (vsrc0), v0);
+	    v1 = over (vsrc1, expand_alpha (vsrc1), v1);
+	    v2 = over (vsrc2, expand_alpha (vsrc2), v2);
+	    v3 = over (vsrc3, expand_alpha (vsrc3), v3);
 
-	    *(__m64 *)dst = vdest;
+	    *(__m64 *)dst = pack_4x565 (v0, v1, v2, v3);
 
 	    w -= 4;
 	    dst += 4;
@@ -1722,13 +1949,13 @@ mmx_composite_over_8888_0565 (pixman_implementation_t *imp,
 
 	while (w)
 	{
-	    __m64 vsrc = load8888 (*src);
+	    __m64 vsrc = load8888 (src);
 	    uint64_t d = *dst;
-	    __m64 vdest = expand565 (M64 (d), 0);
+	    __m64 vdest = expand565 (to_m64 (d), 0);
 
 	    vdest = pack_565 (over (vsrc, expand_alpha (vsrc), vdest), vdest, 0);
 
-	    *dst = UINT64 (vdest);
+	    *dst = to_uint64 (vdest);
 
 	    w--;
 	    dst++;
@@ -1741,19 +1968,9 @@ mmx_composite_over_8888_0565 (pixman_implementation_t *imp,
 
 static void
 mmx_composite_over_n_8_8888 (pixman_implementation_t *imp,
-                             pixman_op_t              op,
-                             pixman_image_t *         src_image,
-                             pixman_image_t *         mask_image,
-                             pixman_image_t *         dst_image,
-                             int32_t                  src_x,
-                             int32_t                  src_y,
-                             int32_t                  mask_x,
-                             int32_t                  mask_y,
-                             int32_t                  dest_x,
-                             int32_t                  dest_y,
-                             int32_t                  width,
-                             int32_t                  height)
+                             pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t src, srca;
     uint32_t *dst_line, *dst;
     uint8_t *mask_line, *mask;
@@ -1764,7 +1981,7 @@ mmx_composite_over_n_8_8888 (pixman_implementation_t *imp,
 
     CHECKPOINT ();
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     srca = src >> 24;
     if (src == 0)
@@ -1772,10 +1989,10 @@ mmx_composite_over_n_8_8888 (pixman_implementation_t *imp,
 
     srcsrc = (uint64_t)src << 32 | src;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
 
-    vsrc = load8888 (src);
+    vsrc = load8888 (&src);
     vsrca = expand_alpha (vsrc);
 
     while (height--)
@@ -1788,17 +2005,17 @@ mmx_composite_over_n_8_8888 (pixman_implementation_t *imp,
 
 	CHECKPOINT ();
 
-	while (w && (unsigned long)dst & 7)
+	while (w && (uintptr_t)dst & 7)
 	{
 	    uint64_t m = *mask;
 
 	    if (m)
 	    {
 		__m64 vdest = in_over (vsrc, vsrca,
-				       expand_alpha_rev (M64 (m)),
-				       load8888 (*dst));
+				       expand_alpha_rev (to_m64 (m)),
+				       load8888 (dst));
 
-		*dst = store8888 (vdest);
+		store8888 (dst, vdest);
 	    }
 
 	    w--;
@@ -1826,9 +2043,9 @@ mmx_composite_over_n_8_8888 (pixman_implementation_t *imp,
 
 		vdest = *(__m64 *)dst;
 
-		dest0 = in_over (vsrc, vsrca, expand_alpha_rev (M64 (m0)),
+		dest0 = in_over (vsrc, vsrca, expand_alpha_rev (to_m64 (m0)),
 				 expand8888 (vdest, 0));
-		dest1 = in_over (vsrc, vsrca, expand_alpha_rev (M64 (m1)),
+		dest1 = in_over (vsrc, vsrca, expand_alpha_rev (to_m64 (m1)),
 				 expand8888 (vdest, 1));
 
 		*(__m64 *)dst = pack8888 (dest0, dest1);
@@ -1841,44 +2058,41 @@ mmx_composite_over_n_8_8888 (pixman_implementation_t *imp,
 
 	CHECKPOINT ();
 
-	while (w)
+	if (w)
 	{
 	    uint64_t m = *mask;
 
 	    if (m)
 	    {
-		__m64 vdest = load8888 (*dst);
+		__m64 vdest = load8888 (dst);
 
 		vdest = in_over (
-		    vsrc, vsrca, expand_alpha_rev (M64 (m)), vdest);
-		*dst = store8888 (vdest);
+		    vsrc, vsrca, expand_alpha_rev (to_m64 (m)), vdest);
+		store8888 (dst, vdest);
 	    }
-
-	    w--;
-	    mask++;
-	    dst++;
 	}
     }
 
     _mm_empty ();
 }
 
-pixman_bool_t
-pixman_fill_mmx (uint32_t *bits,
-                 int       stride,
-                 int       bpp,
-                 int       x,
-                 int       y,
-                 int       width,
-                 int       height,
-                 uint32_t xor)
+static pixman_bool_t
+mmx_fill (pixman_implementation_t *imp,
+          uint32_t *               bits,
+          int                      stride,
+          int                      bpp,
+          int                      x,
+          int                      y,
+          int                      width,
+          int                      height,
+          uint32_t		   filler)
 {
     uint64_t fill;
     __m64 vfill;
     uint32_t byte_width;
     uint8_t     *byte_line;
 
-#ifdef __GNUC__
+#if defined __GNUC__ && defined USE_X86_MMX
     __m64 v1, v2, v3, v4, v5, v6, v7;
 #endif
 
@@ -1891,7 +2105,7 @@ pixman_fill_mmx (uint32_t *bits,
 	byte_line = (uint8_t *)(((uint8_t *)bits) + stride * y + x);
 	byte_width = width;
 	stride *= 1;
-        xor = (xor & 0xff) * 0x01010101;
+        filler = (filler & 0xff) * 0x01010101;
     }
     else if (bpp == 16)
     {
@@ -1899,7 +2113,7 @@ pixman_fill_mmx (uint32_t *bits,
 	byte_line = (uint8_t *)(((uint16_t *)bits) + stride * y + x);
 	byte_width = 2 * width;
 	stride *= 2;
-        xor = (xor & 0xffff) * 0x00010001;
+        filler = (filler & 0xffff) * 0x00010001;
     }
     else
     {
@@ -1909,10 +2123,10 @@ pixman_fill_mmx (uint32_t *bits,
 	stride *= 4;
     }
 
-    fill = ((uint64_t)xor << 32) | xor;
-    vfill = M64 (fill);
+    fill = ((uint64_t)filler << 32) | filler;
+    vfill = to_m64 (fill);
 
-#ifdef __GNUC__
+#if defined __GNUC__ && defined USE_X86_MMX
     __asm__ (
         "movq		%7,	%0\n"
         "movq		%7,	%1\n"
@@ -1921,8 +2135,8 @@ pixman_fill_mmx (uint32_t *bits,
         "movq		%7,	%4\n"
         "movq		%7,	%5\n"
         "movq		%7,	%6\n"
-	: "=y" (v1), "=y" (v2), "=y" (v3),
-	  "=y" (v4), "=y" (v5), "=y" (v6), "=y" (v7)
+	: "=&y" (v1), "=&y" (v2), "=&y" (v3),
+	  "=&y" (v4), "=&y" (v5), "=&y" (v6), "=y" (v7)
 	: "y" (vfill));
 #endif
 
@@ -1934,23 +2148,23 @@ pixman_fill_mmx (uint32_t *bits,
 	byte_line += stride;
 	w = byte_width;
 
-	while (w >= 1 && ((unsigned long)d & 1))
+	if (w >= 1 && ((uintptr_t)d & 1))
 	{
-	    *(uint8_t *)d = (xor & 0xff);
+	    *(uint8_t *)d = (filler & 0xff);
 	    w--;
 	    d++;
 	}
 
-	while (w >= 2 && ((unsigned long)d & 3))
+	if (w >= 2 && ((uintptr_t)d & 3))
 	{
-	    *(uint16_t *)d = xor;
+	    *(uint16_t *)d = filler;
 	    w -= 2;
 	    d += 2;
 	}
 
-	while (w >= 4 && ((unsigned long)d & 7))
+	while (w >= 4 && ((uintptr_t)d & 7))
 	{
-	    *(uint32_t *)d = xor;
+	    *(uint32_t *)d = filler;
 
 	    w -= 4;
 	    d += 4;
@@ -1958,7 +2172,7 @@ pixman_fill_mmx (uint32_t *bits,
 
 	while (w >= 64)
 	{
-#ifdef __GNUC__
+#if defined __GNUC__ && defined USE_X86_MMX
 	    __asm__ (
 	        "movq	%1,	  (%0)\n"
 	        "movq	%2,	 8(%0)\n"
@@ -1989,20 +2203,20 @@ pixman_fill_mmx (uint32_t *bits,
 
 	while (w >= 4)
 	{
-	    *(uint32_t *)d = xor;
+	    *(uint32_t *)d = filler;
 
 	    w -= 4;
 	    d += 4;
 	}
-	while (w >= 2)
+	if (w >= 2)
 	{
-	    *(uint16_t *)d = xor;
+	    *(uint16_t *)d = filler;
 	    w -= 2;
 	    d += 2;
 	}
-	while (w >= 1)
+	if (w >= 1)
 	{
-	    *(uint8_t *)d = (xor & 0xff);
+	    *(uint8_t *)d = (filler & 0xff);
 	    w--;
 	    d++;
 	}
@@ -2014,48 +2228,93 @@ pixman_fill_mmx (uint32_t *bits,
 }
 
 static void
+mmx_composite_src_x888_0565 (pixman_implementation_t *imp,
+                             pixman_composite_info_t *info)
+{
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint16_t    *dst_line, *dst;
+    uint32_t    *src_line, *src, s;
+    int dst_stride, src_stride;
+    int32_t w;
+
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w && (uintptr_t)dst & 7)
+	{
+	    s = *src++;
+	    *dst = convert_8888_to_0565 (s);
+	    dst++;
+	    w--;
+	}
+
+	while (w >= 4)
+	{
+	    __m64 vdest;
+	    __m64 vsrc0 = ldq_u ((__m64 *)(src + 0));
+	    __m64 vsrc1 = ldq_u ((__m64 *)(src + 2));
+
+	    vdest = pack_4xpacked565 (vsrc0, vsrc1);
+
+	    *(__m64 *)dst = vdest;
+
+	    w -= 4;
+	    src += 4;
+	    dst += 4;
+	}
+
+	while (w)
+	{
+	    s = *src++;
+	    *dst = convert_8888_to_0565 (s);
+	    dst++;
+	    w--;
+	}
+    }
+
+    _mm_empty ();
+}
+
+static void
 mmx_composite_src_n_8_8888 (pixman_implementation_t *imp,
-                            pixman_op_t              op,
-                            pixman_image_t *         src_image,
-                            pixman_image_t *         mask_image,
-                            pixman_image_t *         dst_image,
-                            int32_t                  src_x,
-                            int32_t                  src_y,
-                            int32_t                  mask_x,
-                            int32_t                  mask_y,
-                            int32_t                  dest_x,
-                            int32_t                  dest_y,
-                            int32_t                  width,
-                            int32_t                  height)
+                            pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t src, srca;
     uint32_t    *dst_line, *dst;
     uint8_t     *mask_line, *mask;
     int dst_stride, mask_stride;
     int32_t w;
-    __m64 vsrc, vsrca;
+    __m64 vsrc;
     uint64_t srcsrc;
 
     CHECKPOINT ();
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     srca = src >> 24;
     if (src == 0)
     {
-	pixman_fill_mmx (dst_image->bits.bits, dst_image->bits.rowstride,
-			 PIXMAN_FORMAT_BPP (dst_image->bits.format),
-	                 dest_x, dest_y, width, height, 0);
+	mmx_fill (imp, dest_image->bits.bits, dest_image->bits.rowstride,
+		  PIXMAN_FORMAT_BPP (dest_image->bits.format),
+		  dest_x, dest_y, width, height, 0);
 	return;
     }
 
     srcsrc = (uint64_t)src << 32 | src;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
 
-    vsrc = load8888 (src);
-    vsrca = expand_alpha (vsrc);
+    vsrc = load8888 (&src);
 
     while (height--)
     {
@@ -2067,15 +2326,15 @@ mmx_composite_src_n_8_8888 (pixman_implementation_t *imp,
 
 	CHECKPOINT ();
 
-	while (w && (unsigned long)dst & 7)
+	while (w && (uintptr_t)dst & 7)
 	{
 	    uint64_t m = *mask;
 
 	    if (m)
 	    {
-		__m64 vdest = in (vsrc, expand_alpha_rev (M64 (m)));
+		__m64 vdest = in (vsrc, expand_alpha_rev (to_m64 (m)));
 
-		*dst = store8888 (vdest);
+		store8888 (dst, vdest);
 	    }
 	    else
 	    {
@@ -2101,13 +2360,10 @@ mmx_composite_src_n_8_8888 (pixman_implementation_t *imp,
 	    }
 	    else if (m0 | m1)
 	    {
-		__m64 vdest;
 		__m64 dest0, dest1;
 
-		vdest = *(__m64 *)dst;
-
-		dest0 = in (vsrc, expand_alpha_rev (M64 (m0)));
-		dest1 = in (vsrc, expand_alpha_rev (M64 (m1)));
+		dest0 = in (vsrc, expand_alpha_rev (to_m64 (m0)));
+		dest1 = in (vsrc, expand_alpha_rev (to_m64 (m1)));
 
 		*(__m64 *)dst = pack8888 (dest0, dest1);
 	    }
@@ -2123,25 +2379,21 @@ mmx_composite_src_n_8_8888 (pixman_implementation_t *imp,
 
 	CHECKPOINT ();
 
-	while (w)
+	if (w)
 	{
 	    uint64_t m = *mask;
 
 	    if (m)
 	    {
-		__m64 vdest = load8888 (*dst);
+		__m64 vdest = load8888 (dst);
 
-		vdest = in (vsrc, expand_alpha_rev (M64 (m)));
-		*dst = store8888 (vdest);
+		vdest = in (vsrc, expand_alpha_rev (to_m64 (m)));
+		store8888 (dst, vdest);
 	    }
 	    else
 	    {
 		*dst = 0;
 	    }
-
-	    w--;
-	    mask++;
-	    dst++;
 	}
     }
 
@@ -2150,47 +2402,33 @@ mmx_composite_src_n_8_8888 (pixman_implementation_t *imp,
 
 static void
 mmx_composite_over_n_8_0565 (pixman_implementation_t *imp,
-                             pixman_op_t              op,
-                             pixman_image_t *         src_image,
-                             pixman_image_t *         mask_image,
-                             pixman_image_t *         dst_image,
-                             int32_t                  src_x,
-                             int32_t                  src_y,
-                             int32_t                  mask_x,
-                             int32_t                  mask_y,
-                             int32_t                  dest_x,
-                             int32_t                  dest_y,
-                             int32_t                  width,
-                             int32_t                  height)
+                             pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t src, srca;
     uint16_t *dst_line, *dst;
     uint8_t *mask_line, *mask;
     int dst_stride, mask_stride;
     int32_t w;
     __m64 vsrc, vsrca, tmp;
-    uint64_t srcsrcsrcsrc, src16;
+    __m64 srcsrcsrcsrc;
 
     CHECKPOINT ();
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     srca = src >> 24;
     if (src == 0)
 	return;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
 
-    vsrc = load8888 (src);
+    vsrc = load8888 (&src);
     vsrca = expand_alpha (vsrc);
 
     tmp = pack_565 (vsrc, _mm_setzero_si64 (), 0);
-    src16 = UINT64 (tmp);
-
-    srcsrcsrcsrc =
-	(uint64_t)src16 << 48 | (uint64_t)src16 << 32 |
-	(uint64_t)src16 << 16 | (uint64_t)src16;
+    srcsrcsrcsrc = expand_alpha_rev (tmp);
 
     while (height--)
     {
@@ -2202,19 +2440,19 @@ mmx_composite_over_n_8_0565 (pixman_implementation_t *imp,
 
 	CHECKPOINT ();
 
-	while (w && (unsigned long)dst & 7)
+	while (w && (uintptr_t)dst & 7)
 	{
 	    uint64_t m = *mask;
 
 	    if (m)
 	    {
 		uint64_t d = *dst;
-		__m64 vd = M64 (d);
+		__m64 vd = to_m64 (d);
 		__m64 vdest = in_over (
-		    vsrc, vsrca, expand_alpha_rev (M64 (m)), expand565 (vd, 0));
+		    vsrc, vsrca, expand_alpha_rev (to_m64 (m)), expand565 (vd, 0));
 
 		vd = pack_565 (vdest, _mm_setzero_si64 (), 0);
-		*dst = UINT64 (vd);
+		*dst = to_uint64 (vd);
 	    }
 
 	    w--;
@@ -2234,29 +2472,29 @@ mmx_composite_over_n_8_0565 (pixman_implementation_t *imp,
 
 	    if (srca == 0xff && (m0 & m1 & m2 & m3) == 0xff)
 	    {
-		*(uint64_t *)dst = srcsrcsrcsrc;
+		*(__m64 *)dst = srcsrcsrcsrc;
 	    }
 	    else if (m0 | m1 | m2 | m3)
 	    {
-		__m64 vdest;
+		__m64 vdest = *(__m64 *)dst;
+		__m64 v0, v1, v2, v3;
 		__m64 vm0, vm1, vm2, vm3;
 
-		vdest = *(__m64 *)dst;
+		expand_4x565 (vdest, &v0, &v1, &v2, &v3, 0);
+
+		vm0 = to_m64 (m0);
+		v0 = in_over (vsrc, vsrca, expand_alpha_rev (vm0), v0);
 
-		vm0 = M64 (m0);
-		vdest = pack_565 (in_over (vsrc, vsrca, expand_alpha_rev (vm0),
-					   expand565 (vdest, 0)), vdest, 0);
-		vm1 = M64 (m1);
-		vdest = pack_565 (in_over (vsrc, vsrca, expand_alpha_rev (vm1),
-					   expand565 (vdest, 1)), vdest, 1);
-		vm2 = M64 (m2);
-		vdest = pack_565 (in_over (vsrc, vsrca, expand_alpha_rev (vm2),
-					   expand565 (vdest, 2)), vdest, 2);
-		vm3 = M64 (m3);
-		vdest = pack_565 (in_over (vsrc, vsrca, expand_alpha_rev (vm3),
-					   expand565 (vdest, 3)), vdest, 3);
-
-		*(__m64 *)dst = vdest;
+		vm1 = to_m64 (m1);
+		v1 = in_over (vsrc, vsrca, expand_alpha_rev (vm1), v1);
+
+		vm2 = to_m64 (m2);
+		v2 = in_over (vsrc, vsrca, expand_alpha_rev (vm2), v2);
+
+		vm3 = to_m64 (m3);
+		v3 = in_over (vsrc, vsrca, expand_alpha_rev (vm3), v3);
+
+		*(__m64 *)dst = pack_4x565 (v0, v1, v2, v3);;
 	    }
 
 	    w -= 4;
@@ -2273,11 +2511,11 @@ mmx_composite_over_n_8_0565 (pixman_implementation_t *imp,
 	    if (m)
 	    {
 		uint64_t d = *dst;
-		__m64 vd = M64 (d);
-		__m64 vdest = in_over (vsrc, vsrca, expand_alpha_rev (M64 (m)),
+		__m64 vd = to_m64 (d);
+		__m64 vdest = in_over (vsrc, vsrca, expand_alpha_rev (to_m64 (m)),
 				       expand565 (vd, 0));
 		vd = pack_565 (vdest, _mm_setzero_si64 (), 0);
-		*dst = UINT64 (vd);
+		*dst = to_uint64 (vd);
 	    }
 
 	    w--;
@@ -2291,19 +2529,9 @@ mmx_composite_over_n_8_0565 (pixman_implementation_t *imp,
 
 static void
 mmx_composite_over_pixbuf_0565 (pixman_implementation_t *imp,
-                                pixman_op_t              op,
-                                pixman_image_t *         src_image,
-                                pixman_image_t *         mask_image,
-                                pixman_image_t *         dst_image,
-                                int32_t                  src_x,
-                                int32_t                  src_y,
-                                int32_t                  mask_x,
-                                int32_t                  mask_y,
-                                int32_t                  dest_x,
-                                int32_t                  dest_y,
-                                int32_t                  width,
-                                int32_t                  height)
+                                pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint16_t    *dst_line, *dst;
     uint32_t    *src_line, *src;
     int dst_stride, src_stride;
@@ -2311,7 +2539,7 @@ mmx_composite_over_pixbuf_0565 (pixman_implementation_t *imp,
 
     CHECKPOINT ();
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
 
 #if 0
@@ -2329,15 +2557,15 @@ mmx_composite_over_pixbuf_0565 (pixman_implementation_t *imp,
 
 	CHECKPOINT ();
 
-	while (w && (unsigned long)dst & 7)
+	while (w && (uintptr_t)dst & 7)
 	{
-	    __m64 vsrc = load8888 (*src);
+	    __m64 vsrc = load8888 (src);
 	    uint64_t d = *dst;
-	    __m64 vdest = expand565 (M64 (d), 0);
+	    __m64 vdest = expand565 (to_m64 (d), 0);
 
 	    vdest = pack_565 (over_rev_non_pre (vsrc, vdest), vdest, 0);
 
-	    *dst = UINT64 (vdest);
+	    *dst = to_uint64 (vdest);
 
 	    w--;
 	    dst++;
@@ -2363,24 +2591,31 @@ mmx_composite_over_pixbuf_0565 (pixman_implementation_t *imp,
 
 	    if ((a0 & a1 & a2 & a3) == 0xFF)
 	    {
-		__m64 vdest;
-		vdest = pack_565 (invert_colors (load8888 (s0)), _mm_setzero_si64 (), 0);
-		vdest = pack_565 (invert_colors (load8888 (s1)), vdest, 1);
-		vdest = pack_565 (invert_colors (load8888 (s2)), vdest, 2);
-		vdest = pack_565 (invert_colors (load8888 (s3)), vdest, 3);
+		__m64 v0 = invert_colors (load8888 (&s0));
+		__m64 v1 = invert_colors (load8888 (&s1));
+		__m64 v2 = invert_colors (load8888 (&s2));
+		__m64 v3 = invert_colors (load8888 (&s3));
 
-		*(__m64 *)dst = vdest;
+		*(__m64 *)dst = pack_4x565 (v0, v1, v2, v3);
 	    }
 	    else if (s0 | s1 | s2 | s3)
 	    {
 		__m64 vdest = *(__m64 *)dst;
+		__m64 v0, v1, v2, v3;
 
-		vdest = pack_565 (over_rev_non_pre (load8888 (s0), expand565 (vdest, 0)), vdest, 0);
-		vdest = pack_565 (over_rev_non_pre (load8888 (s1), expand565 (vdest, 1)), vdest, 1);
-		vdest = pack_565 (over_rev_non_pre (load8888 (s2), expand565 (vdest, 2)), vdest, 2);
-		vdest = pack_565 (over_rev_non_pre (load8888 (s3), expand565 (vdest, 3)), vdest, 3);
+		__m64 vsrc0 = load8888 (&s0);
+		__m64 vsrc1 = load8888 (&s1);
+		__m64 vsrc2 = load8888 (&s2);
+		__m64 vsrc3 = load8888 (&s3);
 
-		*(__m64 *)dst = vdest;
+		expand_4x565 (vdest, &v0, &v1, &v2, &v3, 0);
+
+		v0 = over_rev_non_pre (vsrc0, v0);
+		v1 = over_rev_non_pre (vsrc1, v1);
+		v2 = over_rev_non_pre (vsrc2, v2);
+		v3 = over_rev_non_pre (vsrc3, v3);
+
+		*(__m64 *)dst = pack_4x565 (v0, v1, v2, v3);
 	    }
 
 	    w -= 4;
@@ -2392,13 +2627,13 @@ mmx_composite_over_pixbuf_0565 (pixman_implementation_t *imp,
 
 	while (w)
 	{
-	    __m64 vsrc = load8888 (*src);
+	    __m64 vsrc = load8888 (src);
 	    uint64_t d = *dst;
-	    __m64 vdest = expand565 (M64 (d), 0);
+	    __m64 vdest = expand565 (to_m64 (d), 0);
 
 	    vdest = pack_565 (over_rev_non_pre (vsrc, vdest), vdest, 0);
 
-	    *dst = UINT64 (vdest);
+	    *dst = to_uint64 (vdest);
 
 	    w--;
 	    dst++;
@@ -2411,19 +2646,9 @@ mmx_composite_over_pixbuf_0565 (pixman_implementation_t *imp,
 
 static void
 mmx_composite_over_pixbuf_8888 (pixman_implementation_t *imp,
-                                pixman_op_t              op,
-                                pixman_image_t *         src_image,
-                                pixman_image_t *         mask_image,
-                                pixman_image_t *         dst_image,
-                                int32_t                  src_x,
-                                int32_t                  src_y,
-                                int32_t                  mask_x,
-                                int32_t                  mask_y,
-                                int32_t                  dest_x,
-                                int32_t                  dest_y,
-                                int32_t                  width,
-                                int32_t                  height)
+                                pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t    *dst_line, *dst;
     uint32_t    *src_line, *src;
     int dst_stride, src_stride;
@@ -2431,7 +2656,7 @@ mmx_composite_over_pixbuf_8888 (pixman_implementation_t *imp,
 
     CHECKPOINT ();
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
 
 #if 0
@@ -2447,12 +2672,12 @@ mmx_composite_over_pixbuf_8888 (pixman_implementation_t *imp,
 	src_line += src_stride;
 	w = width;
 
-	while (w && (unsigned long)dst & 7)
+	while (w && (uintptr_t)dst & 7)
 	{
-	    __m64 s = load8888 (*src);
-	    __m64 d = load8888 (*dst);
+	    __m64 s = load8888 (src);
+	    __m64 d = load8888 (dst);
 
-	    *dst = store8888 (over_rev_non_pre (s, d));
+	    store8888 (dst, over_rev_non_pre (s, d));
 
 	    w--;
 	    dst++;
@@ -2461,7 +2686,7 @@ mmx_composite_over_pixbuf_8888 (pixman_implementation_t *imp,
 
 	while (w >= 2)
 	{
-	    uint64_t s0, s1;
+	    uint32_t s0, s1;
 	    unsigned char a0, a1;
 	    __m64 d0, d1;
 
@@ -2473,8 +2698,8 @@ mmx_composite_over_pixbuf_8888 (pixman_implementation_t *imp,
 
 	    if ((a0 & a1) == 0xFF)
 	    {
-		d0 = invert_colors (load8888 (s0));
-		d1 = invert_colors (load8888 (s1));
+		d0 = invert_colors (load8888 (&s0));
+		d1 = invert_colors (load8888 (&s1));
 
 		*(__m64 *)dst = pack8888 (d0, d1);
 	    }
@@ -2482,8 +2707,8 @@ mmx_composite_over_pixbuf_8888 (pixman_implementation_t *imp,
 	    {
 		__m64 vdest = *(__m64 *)dst;
 
-		d0 = over_rev_non_pre (load8888 (s0), expand8888 (vdest, 0));
-		d1 = over_rev_non_pre (load8888 (s1), expand8888 (vdest, 1));
+		d0 = over_rev_non_pre (load8888 (&s0), expand8888 (vdest, 0));
+		d1 = over_rev_non_pre (load8888 (&s1), expand8888 (vdest, 1));
 
 		*(__m64 *)dst = pack8888 (d0, d1);
 	    }
@@ -2493,16 +2718,12 @@ mmx_composite_over_pixbuf_8888 (pixman_implementation_t *imp,
 	    src += 2;
 	}
 
-	while (w)
+	if (w)
 	{
-	    __m64 s = load8888 (*src);
-	    __m64 d = load8888 (*dst);
-
-	    *dst = store8888 (over_rev_non_pre (s, d));
+	    __m64 s = load8888 (src);
+	    __m64 d = load8888 (dst);
 
-	    w--;
-	    dst++;
-	    src++;
+	    store8888 (dst, over_rev_non_pre (s, d));
 	}
     }
 
@@ -2511,20 +2732,10 @@ mmx_composite_over_pixbuf_8888 (pixman_implementation_t *imp,
 
 static void
 mmx_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
-                                   pixman_op_t              op,
-                                   pixman_image_t *         src_image,
-                                   pixman_image_t *         mask_image,
-                                   pixman_image_t *         dst_image,
-                                   int32_t                  src_x,
-                                   int32_t                  src_y,
-                                   int32_t                  mask_x,
-                                   int32_t                  mask_y,
-                                   int32_t                  dest_x,
-                                   int32_t                  dest_y,
-                                   int32_t                  width,
-                                   int32_t                  height)
+                                   pixman_composite_info_t *info)
 {
-    uint32_t src, srca;
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint32_t src;
     uint16_t    *dst_line;
     uint32_t    *mask_line;
     int dst_stride, mask_stride;
@@ -2532,16 +2743,15 @@ mmx_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
 
     CHECKPOINT ();
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
-    srca = src >> 24;
     if (src == 0)
 	return;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
 
-    vsrc = load8888 (src);
+    vsrc = load8888 (&src);
     vsrca = expand_alpha (vsrc);
 
     while (height--)
@@ -2550,16 +2760,16 @@ mmx_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
 	uint32_t *p = (uint32_t *)mask_line;
 	uint16_t *q = (uint16_t *)dst_line;
 
-	while (twidth && ((unsigned long)q & 7))
+	while (twidth && ((uintptr_t)q & 7))
 	{
 	    uint32_t m = *(uint32_t *)p;
 
 	    if (m)
 	    {
 		uint64_t d = *q;
-		__m64 vdest = expand565 (M64 (d), 0);
-		vdest = pack_565 (in_over (vsrc, vsrca, load8888 (m), vdest), vdest, 0);
-		*q = UINT64 (vdest);
+		__m64 vdest = expand565 (to_m64 (d), 0);
+		vdest = pack_565 (in_over (vsrc, vsrca, load8888 (&m), vdest), vdest, 0);
+		*q = to_uint64 (vdest);
 	    }
 
 	    twidth--;
@@ -2579,13 +2789,16 @@ mmx_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
 	    if ((m0 | m1 | m2 | m3))
 	    {
 		__m64 vdest = *(__m64 *)q;
+		__m64 v0, v1, v2, v3;
+
+		expand_4x565 (vdest, &v0, &v1, &v2, &v3, 0);
 
-		vdest = pack_565 (in_over (vsrc, vsrca, load8888 (m0), expand565 (vdest, 0)), vdest, 0);
-		vdest = pack_565 (in_over (vsrc, vsrca, load8888 (m1), expand565 (vdest, 1)), vdest, 1);
-		vdest = pack_565 (in_over (vsrc, vsrca, load8888 (m2), expand565 (vdest, 2)), vdest, 2);
-		vdest = pack_565 (in_over (vsrc, vsrca, load8888 (m3), expand565 (vdest, 3)), vdest, 3);
+		v0 = in_over (vsrc, vsrca, load8888 (&m0), v0);
+		v1 = in_over (vsrc, vsrca, load8888 (&m1), v1);
+		v2 = in_over (vsrc, vsrca, load8888 (&m2), v2);
+		v3 = in_over (vsrc, vsrca, load8888 (&m3), v3);
 
-		*(__m64 *)q = vdest;
+		*(__m64 *)q = pack_4x565 (v0, v1, v2, v3);
 	    }
 	    twidth -= 4;
 	    p += 4;
@@ -2600,9 +2813,9 @@ mmx_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
 	    if (m)
 	    {
 		uint64_t d = *q;
-		__m64 vdest = expand565 (M64 (d), 0);
-		vdest = pack_565 (in_over (vsrc, vsrca, load8888 (m), vdest), vdest, 0);
-		*q = UINT64 (vdest);
+		__m64 vdest = expand565 (to_m64 (d), 0);
+		vdest = pack_565 (in_over (vsrc, vsrca, load8888 (&m), vdest), vdest, 0);
+		*q = to_uint64 (vdest);
 	    }
 
 	    twidth--;
@@ -2619,19 +2832,9 @@ mmx_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
 
 static void
 mmx_composite_in_n_8_8 (pixman_implementation_t *imp,
-                        pixman_op_t              op,
-                        pixman_image_t *         src_image,
-                        pixman_image_t *         mask_image,
-                        pixman_image_t *         dst_image,
-                        int32_t                  src_x,
-                        int32_t                  src_y,
-                        int32_t                  mask_x,
-                        int32_t                  mask_y,
-                        int32_t                  dest_x,
-                        int32_t                  dest_y,
-                        int32_t                  width,
-                        int32_t                  height)
+                        pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint8_t *dst_line, *dst;
     uint8_t *mask_line, *mask;
     int dst_stride, mask_stride;
@@ -2640,14 +2843,14 @@ mmx_composite_in_n_8_8 (pixman_implementation_t *imp,
     uint8_t sa;
     __m64 vsrc, vsrca;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     sa = src >> 24;
 
-    vsrc = load8888 (src);
+    vsrc = load8888 (&src);
     vsrca = expand_alpha (vsrc);
 
     while (height--)
@@ -2658,26 +2861,35 @@ mmx_composite_in_n_8_8 (pixman_implementation_t *imp,
 	mask_line += mask_stride;
 	w = width;
 
-	if ((((unsigned long)dst_image & 3) == 0) &&
-	    (((unsigned long)src_image & 3) == 0))
+	while (w && (uintptr_t)dst & 7)
 	{
-	    while (w >= 4)
-	    {
-		uint32_t m;
-		__m64 vmask;
-		__m64 vdest;
+	    uint16_t tmp;
+	    uint8_t a;
+	    uint32_t m, d;
 
-		m = 0;
+	    a = *mask++;
+	    d = *dst;
 
-		vmask = load8888 (*(uint32_t *)mask);
-		vdest = load8888 (*(uint32_t *)dst);
+	    m = MUL_UN8 (sa, a, tmp);
+	    d = MUL_UN8 (m, d, tmp);
 
-		*(uint32_t *)dst = store8888 (in (in (vsrca, vmask), vdest));
+	    *dst++ = d;
+	    w--;
+	}
 
-		dst += 4;
-		mask += 4;
-		w -= 4;
-	    }
+	while (w >= 4)
+	{
+	    __m64 vmask;
+	    __m64 vdest;
+
+	    vmask = load8888u ((uint32_t *)mask);
+	    vdest = load8888 ((uint32_t *)dst);
+
+	    store8888 ((uint32_t *)dst, in (in (vsrca, vmask), vdest));
+
+	    dst += 4;
+	    mask += 4;
+	    w -= 4;
 	}
 
 	while (w--)
@@ -2701,25 +2913,15 @@ mmx_composite_in_n_8_8 (pixman_implementation_t *imp,
 
 static void
 mmx_composite_in_8_8 (pixman_implementation_t *imp,
-                      pixman_op_t              op,
-                      pixman_image_t *         src_image,
-                      pixman_image_t *         mask_image,
-                      pixman_image_t *         dst_image,
-                      int32_t                  src_x,
-                      int32_t                  src_y,
-                      int32_t                  mask_x,
-                      int32_t                  mask_y,
-                      int32_t                  dest_x,
-                      int32_t                  dest_y,
-                      int32_t                  width,
-                      int32_t                  height)
+                      pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint8_t     *dst_line, *dst;
     uint8_t     *src_line, *src;
     int src_stride, dst_stride;
     int32_t w;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
 
     while (height--)
@@ -2730,20 +2932,31 @@ mmx_composite_in_8_8 (pixman_implementation_t *imp,
 	src_line += src_stride;
 	w = width;
 
-	if ((((unsigned long)dst_image & 3) == 0) &&
-	    (((unsigned long)src_image & 3) == 0))
+	while (w && (uintptr_t)dst & 3)
 	{
-	    while (w >= 4)
-	    {
-		uint32_t *s = (uint32_t *)src;
-		uint32_t *d = (uint32_t *)dst;
+	    uint8_t s, d;
+	    uint16_t tmp;
 
-		*d = store8888 (in (load8888 (*s), load8888 (*d)));
+	    s = *src;
+	    d = *dst;
 
-		w -= 4;
-		dst += 4;
-		src += 4;
-	    }
+	    *dst = MUL_UN8 (s, d, tmp);
+
+	    src++;
+	    dst++;
+	    w--;
+	}
+
+	while (w >= 4)
+	{
+	    uint32_t *s = (uint32_t *)src;
+	    uint32_t *d = (uint32_t *)dst;
+
+	    store8888 (d, in (load8888u (s), load8888 (d)));
+
+	    w -= 4;
+	    dst += 4;
+	    src += 4;
 	}
 
 	while (w--)
@@ -2766,19 +2979,9 @@ mmx_composite_in_8_8 (pixman_implementation_t *imp,
 
 static void
 mmx_composite_add_n_8_8 (pixman_implementation_t *imp,
-			 pixman_op_t              op,
-			 pixman_image_t *         src_image,
-			 pixman_image_t *         mask_image,
-			 pixman_image_t *         dst_image,
-			 int32_t                  src_x,
-			 int32_t                  src_y,
-			 int32_t                  mask_x,
-			 int32_t                  mask_y,
-			 int32_t                  dest_x,
-			 int32_t                  dest_y,
-			 int32_t                  width,
-			 int32_t                  height)
+			 pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint8_t     *dst_line, *dst;
     uint8_t     *mask_line, *mask;
     int dst_stride, mask_stride;
@@ -2787,17 +2990,17 @@ mmx_composite_add_n_8_8 (pixman_implementation_t *imp,
     uint8_t sa;
     __m64 vsrc, vsrca;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     sa = src >> 24;
 
     if (src == 0)
 	return;
 
-    vsrc = load8888 (src);
+    vsrc = load8888 (&src);
     vsrca = expand_alpha (vsrc);
 
     while (height--)
@@ -2808,20 +3011,36 @@ mmx_composite_add_n_8_8 (pixman_implementation_t *imp,
 	mask_line += mask_stride;
 	w = width;
 
-	if ((((unsigned long)mask_image & 3) == 0) &&
-	    (((unsigned long)dst_image  & 3) == 0))
+	while (w && (uintptr_t)dst & 3)
 	{
-	    while (w >= 4)
-	    {
-		__m64 vmask = load8888 (*(uint32_t *)mask);
-		__m64 vdest = load8888 (*(uint32_t *)dst);
+	    uint16_t tmp;
+	    uint16_t a;
+	    uint32_t m, d;
+	    uint32_t r;
 
-		*(uint32_t *)dst = store8888 (_mm_adds_pu8 (in (vsrca, vmask), vdest));
+	    a = *mask++;
+	    d = *dst;
 
-		w -= 4;
-		dst += 4;
-		mask += 4;
-	    }
+	    m = MUL_UN8 (sa, a, tmp);
+	    r = ADD_UN8 (m, d, tmp);
+
+	    *dst++ = r;
+	    w--;
+	}
+
+	while (w >= 4)
+	{
+	    __m64 vmask;
+	    __m64 vdest;
+
+	    vmask = load8888u ((uint32_t *)mask);
+	    vdest = load8888 ((uint32_t *)dst);
+
+	    store8888 ((uint32_t *)dst, _mm_adds_pu8 (in (vsrca, vmask), vdest));
+
+	    dst += 4;
+	    mask += 4;
+	    w -= 4;
 	}
 
 	while (w--)
@@ -2845,20 +3064,10 @@ mmx_composite_add_n_8_8 (pixman_implementation_t *imp,
 }
 
 static void
-mmx_composite_add_8000_8000 (pixman_implementation_t *imp,
-                             pixman_op_t              op,
-                             pixman_image_t *         src_image,
-                             pixman_image_t *         mask_image,
-                             pixman_image_t *         dst_image,
-                             int32_t                  src_x,
-                             int32_t                  src_y,
-                             int32_t                  mask_x,
-                             int32_t                  mask_y,
-                             int32_t                  dest_x,
-                             int32_t                  dest_y,
-                             int32_t                  width,
-                             int32_t                  height)
+mmx_composite_add_8_8 (pixman_implementation_t *imp,
+		       pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint8_t *dst_line, *dst;
     uint8_t *src_line, *src;
     int dst_stride, src_stride;
@@ -2869,7 +3078,7 @@ mmx_composite_add_8000_8000 (pixman_implementation_t *imp,
     CHECKPOINT ();
 
     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
 
     while (height--)
     {
@@ -2879,7 +3088,7 @@ mmx_composite_add_8000_8000 (pixman_implementation_t *imp,
 	src_line += src_stride;
 	w = width;
 
-	while (w && (unsigned long)dst & 7)
+	while (w && (uintptr_t)dst & 7)
 	{
 	    s = *src;
 	    d = *dst;
@@ -2894,7 +3103,7 @@ mmx_composite_add_8000_8000 (pixman_implementation_t *imp,
 
 	while (w >= 8)
 	{
-	    *(__m64*)dst = _mm_adds_pu8 (*(__m64*)src, *(__m64*)dst);
+	    *(__m64*)dst = _mm_adds_pu8 (ldq_u ((__m64 *)src), *(__m64*)dst);
 	    dst += 8;
 	    src += 8;
 	    w -= 8;
@@ -2918,21 +3127,94 @@ mmx_composite_add_8000_8000 (pixman_implementation_t *imp,
 }
 
 static void
+mmx_composite_add_0565_0565 (pixman_implementation_t *imp,
+                             pixman_composite_info_t *info)
+{
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint16_t    *dst_line, *dst;
+    uint32_t	d;
+    uint16_t    *src_line, *src;
+    uint32_t	s;
+    int dst_stride, src_stride;
+    int32_t w;
+
+    CHECKPOINT ();
+
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint16_t, src_stride, src_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w && (uintptr_t)dst & 7)
+	{
+	    s = *src++;
+	    if (s)
+	    {
+		d = *dst;
+		s = convert_0565_to_8888 (s);
+		if (d)
+		{
+		    d = convert_0565_to_8888 (d);
+		    UN8x4_ADD_UN8x4 (s, d);
+		}
+		*dst = convert_8888_to_0565 (s);
+	    }
+	    dst++;
+	    w--;
+	}
+
+	while (w >= 4)
+	{
+	    __m64 vdest = *(__m64 *)dst;
+	    __m64 vsrc = ldq_u ((__m64 *)src);
+	    __m64 vd0, vd1;
+	    __m64 vs0, vs1;
+
+	    expand_4xpacked565 (vdest, &vd0, &vd1, 0);
+	    expand_4xpacked565 (vsrc, &vs0, &vs1, 0);
+
+	    vd0 = _mm_adds_pu8 (vd0, vs0);
+	    vd1 = _mm_adds_pu8 (vd1, vs1);
+
+	    *(__m64 *)dst = pack_4xpacked565 (vd0, vd1);
+
+	    dst += 4;
+	    src += 4;
+	    w -= 4;
+	}
+
+	while (w--)
+	{
+	    s = *src++;
+	    if (s)
+	    {
+		d = *dst;
+		s = convert_0565_to_8888 (s);
+		if (d)
+		{
+		    d = convert_0565_to_8888 (d);
+		    UN8x4_ADD_UN8x4 (s, d);
+		}
+		*dst = convert_8888_to_0565 (s);
+	    }
+	    dst++;
+	}
+    }
+
+    _mm_empty ();
+}
+
+static void
 mmx_composite_add_8888_8888 (pixman_implementation_t *imp,
-                             pixman_op_t              op,
-                             pixman_image_t *         src_image,
-                             pixman_image_t *         mask_image,
-                             pixman_image_t *         dst_image,
-                             int32_t                  src_x,
-                             int32_t                  src_y,
-                             int32_t                  mask_x,
-                             int32_t                  mask_y,
-                             int32_t                  dest_x,
-                             int32_t                  dest_y,
-                             int32_t                  width,
-                             int32_t                  height)
-{
-    __m64 dst64;
+                             pixman_composite_info_t *info)
+{
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t    *dst_line, *dst;
     uint32_t    *src_line, *src;
     int dst_stride, src_stride;
@@ -2941,7 +3223,7 @@ mmx_composite_add_8888_8888 (pixman_implementation_t *imp,
     CHECKPOINT ();
 
     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
 
     while (height--)
     {
@@ -2951,10 +3233,10 @@ mmx_composite_add_8888_8888 (pixman_implementation_t *imp,
 	src_line += src_stride;
 	w = width;
 
-	while (w && (unsigned long)dst & 7)
+	while (w && (uintptr_t)dst & 7)
 	{
-	    *dst = _mm_cvtsi64_si32 (_mm_adds_pu8 (_mm_cvtsi32_si64 (*src),
-	                                           _mm_cvtsi32_si64 (*dst)));
+	    store (dst, _mm_adds_pu8 (load ((const uint32_t *)src),
+	                              load ((const uint32_t *)dst)));
 	    dst++;
 	    src++;
 	    w--;
@@ -2962,8 +3244,7 @@ mmx_composite_add_8888_8888 (pixman_implementation_t *imp,
 
 	while (w >= 2)
 	{
-	    dst64 = _mm_adds_pu8 (*(__m64*)src, *(__m64*)dst);
-	    *(uint64_t*)dst = UINT64 (dst64);
+	    *(__m64 *)dst = _mm_adds_pu8 (ldq_u ((__m64 *)src), *(__m64*)dst);
 	    dst += 2;
 	    src += 2;
 	    w -= 2;
@@ -2971,8 +3252,8 @@ mmx_composite_add_8888_8888 (pixman_implementation_t *imp,
 
 	if (w)
 	{
-	    *dst = _mm_cvtsi64_si32 (_mm_adds_pu8 (_mm_cvtsi32_si64 (*src),
-	                                           _mm_cvtsi32_si64 (*dst)));
+	    store (dst, _mm_adds_pu8 (load ((const uint32_t *)src),
+	                              load ((const uint32_t *)dst)));
 
 	}
     }
@@ -2981,18 +3262,19 @@ mmx_composite_add_8888_8888 (pixman_implementation_t *imp,
 }
 
 static pixman_bool_t
-pixman_blt_mmx (uint32_t *src_bits,
-                uint32_t *dst_bits,
-                int       src_stride,
-                int       dst_stride,
-                int       src_bpp,
-                int       dst_bpp,
-                int       src_x,
-                int       src_y,
-                int       dst_x,
-                int       dst_y,
-                int       width,
-                int       height)
+mmx_blt (pixman_implementation_t *imp,
+         uint32_t *               src_bits,
+         uint32_t *               dst_bits,
+         int                      src_stride,
+         int                      dst_stride,
+         int                      src_bpp,
+         int                      dst_bpp,
+         int                      src_x,
+         int                      src_y,
+         int                      dest_x,
+         int                      dest_y,
+         int                      width,
+         int                      height)
 {
     uint8_t *   src_bytes;
     uint8_t *   dst_bytes;
@@ -3006,7 +3288,7 @@ pixman_blt_mmx (uint32_t *src_bits,
 	src_stride = src_stride * (int) sizeof (uint32_t) / 2;
 	dst_stride = dst_stride * (int) sizeof (uint32_t) / 2;
 	src_bytes = (uint8_t *)(((uint16_t *)src_bits) + src_stride * (src_y) + (src_x));
-	dst_bytes = (uint8_t *)(((uint16_t *)dst_bits) + dst_stride * (dst_y) + (dst_x));
+	dst_bytes = (uint8_t *)(((uint16_t *)dst_bits) + dst_stride * (dest_y) + (dest_x));
 	byte_width = 2 * width;
 	src_stride *= 2;
 	dst_stride *= 2;
@@ -3016,7 +3298,7 @@ pixman_blt_mmx (uint32_t *src_bits,
 	src_stride = src_stride * (int) sizeof (uint32_t) / 4;
 	dst_stride = dst_stride * (int) sizeof (uint32_t) / 4;
 	src_bytes = (uint8_t *)(((uint32_t *)src_bits) + src_stride * (src_y) + (src_x));
-	dst_bytes = (uint8_t *)(((uint32_t *)dst_bits) + dst_stride * (dst_y) + (dst_x));
+	dst_bytes = (uint8_t *)(((uint32_t *)dst_bits) + dst_stride * (dest_y) + (dest_x));
 	byte_width = 4 * width;
 	src_stride *= 4;
 	dst_stride *= 4;
@@ -3035,7 +3317,15 @@ pixman_blt_mmx (uint32_t *src_bits,
 	dst_bytes += dst_stride;
 	w = byte_width;
 
-	while (w >= 2 && ((unsigned long)d & 3))
+	if (w >= 1 && ((uintptr_t)d & 1))
+	{
+	    *(uint8_t *)d = *(uint8_t *)s;
+	    w -= 1;
+	    s += 1;
+	    d += 1;
+	}
+
+	if (w >= 2 && ((uintptr_t)d & 3))
 	{
 	    *(uint16_t *)d = *(uint16_t *)s;
 	    w -= 2;
@@ -3043,9 +3333,9 @@ pixman_blt_mmx (uint32_t *src_bits,
 	    d += 2;
 	}
 
-	while (w >= 4 && ((unsigned long)d & 7))
+	while (w >= 4 && ((uintptr_t)d & 7))
 	{
-	    *(uint32_t *)d = *(uint32_t *)s;
+	    *(uint32_t *)d = ldl_u ((uint32_t *)s);
 
 	    w -= 4;
 	    s += 4;
@@ -3054,7 +3344,7 @@ pixman_blt_mmx (uint32_t *src_bits,
 
 	while (w >= 64)
 	{
-#if defined (__GNUC__) || (defined(__SUNPRO_C) && (__SUNPRO_C >= 0x590))
+#if (defined (__GNUC__) || (defined(__SUNPRO_C) && (__SUNPRO_C >= 0x590))) && defined USE_X86_MMX
 	    __asm__ (
 	        "movq	  (%1),	  %%mm0\n"
 	        "movq	 8(%1),	  %%mm1\n"
@@ -3079,14 +3369,14 @@ pixman_blt_mmx (uint32_t *src_bits,
 		  "%mm0", "%mm1", "%mm2", "%mm3",
 		  "%mm4", "%mm5", "%mm6", "%mm7");
 #else
-	    __m64 v0 = *(__m64 *)(s + 0);
-	    __m64 v1 = *(__m64 *)(s + 8);
-	    __m64 v2 = *(__m64 *)(s + 16);
-	    __m64 v3 = *(__m64 *)(s + 24);
-	    __m64 v4 = *(__m64 *)(s + 32);
-	    __m64 v5 = *(__m64 *)(s + 40);
-	    __m64 v6 = *(__m64 *)(s + 48);
-	    __m64 v7 = *(__m64 *)(s + 56);
+	    __m64 v0 = ldq_u ((__m64 *)(s + 0));
+	    __m64 v1 = ldq_u ((__m64 *)(s + 8));
+	    __m64 v2 = ldq_u ((__m64 *)(s + 16));
+	    __m64 v3 = ldq_u ((__m64 *)(s + 24));
+	    __m64 v4 = ldq_u ((__m64 *)(s + 32));
+	    __m64 v5 = ldq_u ((__m64 *)(s + 40));
+	    __m64 v6 = ldq_u ((__m64 *)(s + 48));
+	    __m64 v7 = ldq_u ((__m64 *)(s + 56));
 	    *(__m64 *)(d + 0)  = v0;
 	    *(__m64 *)(d + 8)  = v1;
 	    *(__m64 *)(d + 16) = v2;
@@ -3103,7 +3393,7 @@ pixman_blt_mmx (uint32_t *src_bits,
 	}
 	while (w >= 4)
 	{
-	    *(uint32_t *)d = *(uint32_t *)s;
+	    *(uint32_t *)d = ldl_u ((uint32_t *)s);
 
 	    w -= 4;
 	    s += 4;
@@ -3125,51 +3415,31 @@ pixman_blt_mmx (uint32_t *src_bits,
 
 static void
 mmx_composite_copy_area (pixman_implementation_t *imp,
-                         pixman_op_t              op,
-                         pixman_image_t *         src_image,
-                         pixman_image_t *         mask_image,
-                         pixman_image_t *         dst_image,
-                         int32_t                  src_x,
-                         int32_t                  src_y,
-                         int32_t                  mask_x,
-                         int32_t                  mask_y,
-                         int32_t                  dest_x,
-                         int32_t                  dest_y,
-                         int32_t                  width,
-                         int32_t                  height)
-{
-    pixman_blt_mmx (src_image->bits.bits,
-                    dst_image->bits.bits,
-                    src_image->bits.rowstride,
-                    dst_image->bits.rowstride,
-                    PIXMAN_FORMAT_BPP (src_image->bits.format),
-                    PIXMAN_FORMAT_BPP (dst_image->bits.format),
-                    src_x, src_y, dest_x, dest_y, width, height);
+                         pixman_composite_info_t *info)
+{
+    PIXMAN_COMPOSITE_ARGS (info);
+
+    mmx_blt (imp, src_image->bits.bits,
+	     dest_image->bits.bits,
+	     src_image->bits.rowstride,
+	     dest_image->bits.rowstride,
+	     PIXMAN_FORMAT_BPP (src_image->bits.format),
+	     PIXMAN_FORMAT_BPP (dest_image->bits.format),
+	     src_x, src_y, dest_x, dest_y, width, height);
 }
 
-#if 0
 static void
 mmx_composite_over_x888_8_8888 (pixman_implementation_t *imp,
-                                pixman_op_t              op,
-                                pixman_image_t *         src_image,
-                                pixman_image_t *         mask_image,
-                                pixman_image_t *         dst_image,
-                                int32_t                  src_x,
-                                int32_t                  src_y,
-                                int32_t                  mask_x,
-                                int32_t                  mask_y,
-                                int32_t                  dest_x,
-                                int32_t                  dest_y,
-                                int32_t                  width,
-                                int32_t                  height)
+                                pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t  *src, *src_line;
     uint32_t  *dst, *dst_line;
     uint8_t  *mask, *mask_line;
     int src_stride, mask_stride, dst_stride;
     int32_t w;
 
-    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
 
@@ -3190,19 +3460,20 @@ mmx_composite_over_x888_8_8888 (pixman_implementation_t *imp,
 
 	    if (m)
 	    {
-		__m64 s = load8888 (*src | 0xff000000);
+		uint32_t ssrc = *src | 0xff000000;
+		__m64 s = load8888 (&ssrc);
 
 		if (m == 0xff)
 		{
-		    *dst = store8888 (s);
+		    store8888 (dst, s);
 		}
 		else
 		{
 		    __m64 sa = expand_alpha (s);
-		    __m64 vm = expand_alpha_rev (M64 (m));
-		    __m64 vdest = in_over (s, sa, vm, load8888 (*dst));
+		    __m64 vm = expand_alpha_rev (to_m64 (m));
+		    __m64 vdest = in_over (s, sa, vm, load8888 (dst));
 
-		    *dst = store8888 (vdest);
+		    store8888 (dst, vdest);
 		}
 	    }
 
@@ -3214,7 +3485,439 @@ mmx_composite_over_x888_8_8888 (pixman_implementation_t *imp,
 
     _mm_empty ();
 }
-#endif
+
+static void
+mmx_composite_over_reverse_n_8888 (pixman_implementation_t *imp,
+                                   pixman_composite_info_t *info)
+{
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint32_t src;
+    uint32_t    *dst_line, *dst;
+    int32_t w;
+    int dst_stride;
+    __m64 vsrc;
+
+    CHECKPOINT ();
+
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
+
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+
+    vsrc = load8888 (&src);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	w = width;
+
+	CHECKPOINT ();
+
+	while (w && (uintptr_t)dst & 7)
+	{
+	    __m64 vdest = load8888 (dst);
+
+	    store8888 (dst, over (vdest, expand_alpha (vdest), vsrc));
+
+	    w--;
+	    dst++;
+	}
+
+	while (w >= 2)
+	{
+	    __m64 vdest = *(__m64 *)dst;
+	    __m64 dest0 = expand8888 (vdest, 0);
+	    __m64 dest1 = expand8888 (vdest, 1);
+
+
+	    dest0 = over (dest0, expand_alpha (dest0), vsrc);
+	    dest1 = over (dest1, expand_alpha (dest1), vsrc);
+
+	    *(__m64 *)dst = pack8888 (dest0, dest1);
+
+	    dst += 2;
+	    w -= 2;
+	}
+
+	CHECKPOINT ();
+
+	if (w)
+	{
+	    __m64 vdest = load8888 (dst);
+
+	    store8888 (dst, over (vdest, expand_alpha (vdest), vsrc));
+	}
+    }
+
+    _mm_empty ();
+}
+
+#define BSHIFT ((1 << BILINEAR_INTERPOLATION_BITS))
+#define BMSK (BSHIFT - 1)
+
+#define BILINEAR_DECLARE_VARIABLES						\
+    const __m64 mm_wt = _mm_set_pi16 (wt, wt, wt, wt);				\
+    const __m64 mm_wb = _mm_set_pi16 (wb, wb, wb, wb);				\
+    const __m64 mm_addc7 = _mm_set_pi16 (0, 1, 0, 1);				\
+    const __m64 mm_xorc7 = _mm_set_pi16 (0, BMSK, 0, BMSK);			\
+    const __m64 mm_ux = _mm_set_pi16 (unit_x, unit_x, unit_x, unit_x);		\
+    const __m64 mm_zero = _mm_setzero_si64 ();					\
+    __m64 mm_x = _mm_set_pi16 (vx, vx, vx, vx)
+
+#define BILINEAR_INTERPOLATE_ONE_PIXEL(pix)					\
+do {										\
+    /* fetch 2x2 pixel block into 2 mmx registers */				\
+    __m64 t = ldq_u ((__m64 *)&src_top [pixman_fixed_to_int (vx)]);		\
+    __m64 b = ldq_u ((__m64 *)&src_bottom [pixman_fixed_to_int (vx)]);		\
+    /* vertical interpolation */						\
+    __m64 t_hi = _mm_mullo_pi16 (_mm_unpackhi_pi8 (t, mm_zero), mm_wt);		\
+    __m64 t_lo = _mm_mullo_pi16 (_mm_unpacklo_pi8 (t, mm_zero), mm_wt);		\
+    __m64 b_hi = _mm_mullo_pi16 (_mm_unpackhi_pi8 (b, mm_zero), mm_wb);		\
+    __m64 b_lo = _mm_mullo_pi16 (_mm_unpacklo_pi8 (b, mm_zero), mm_wb);		\
+    __m64 hi = _mm_add_pi16 (t_hi, b_hi);					\
+    __m64 lo = _mm_add_pi16 (t_lo, b_lo);					\
+    /* calculate horizontal weights */						\
+    __m64 mm_wh = _mm_add_pi16 (mm_addc7, _mm_xor_si64 (mm_xorc7,		\
+			  _mm_srli_pi16 (mm_x,					\
+					 16 - BILINEAR_INTERPOLATION_BITS)));	\
+    /* horizontal interpolation */						\
+    __m64 p = _mm_unpacklo_pi16 (lo, hi);					\
+    __m64 q = _mm_unpackhi_pi16 (lo, hi);					\
+    vx += unit_x;								\
+    lo = _mm_madd_pi16 (p, mm_wh);						\
+    hi = _mm_madd_pi16 (q, mm_wh);						\
+    mm_x = _mm_add_pi16 (mm_x, mm_ux);						\
+    /* shift and pack the result */						\
+    hi = _mm_srli_pi32 (hi, BILINEAR_INTERPOLATION_BITS * 2);			\
+    lo = _mm_srli_pi32 (lo, BILINEAR_INTERPOLATION_BITS * 2);			\
+    lo = _mm_packs_pi32 (lo, hi);						\
+    lo = _mm_packs_pu16 (lo, lo);						\
+    pix = lo;									\
+} while (0)
+
+#define BILINEAR_SKIP_ONE_PIXEL()						\
+do {										\
+    vx += unit_x;								\
+    mm_x = _mm_add_pi16 (mm_x, mm_ux);						\
+} while(0)
+
+static force_inline void
+scaled_bilinear_scanline_mmx_8888_8888_SRC (uint32_t *       dst,
+					    const uint32_t * mask,
+					    const uint32_t * src_top,
+					    const uint32_t * src_bottom,
+					    int32_t          w,
+					    int              wt,
+					    int              wb,
+					    pixman_fixed_t   vx,
+					    pixman_fixed_t   unit_x,
+					    pixman_fixed_t   max_vx,
+					    pixman_bool_t    zero_src)
+{
+    BILINEAR_DECLARE_VARIABLES;
+    __m64 pix;
+
+    while (w--)
+    {
+	BILINEAR_INTERPOLATE_ONE_PIXEL (pix);
+	store (dst, pix);
+	dst++;
+    }
+
+    _mm_empty ();
+}
+
+FAST_BILINEAR_MAINLOOP_COMMON (mmx_8888_8888_cover_SRC,
+			       scaled_bilinear_scanline_mmx_8888_8888_SRC,
+			       uint32_t, uint32_t, uint32_t,
+			       COVER, FLAG_NONE)
+FAST_BILINEAR_MAINLOOP_COMMON (mmx_8888_8888_pad_SRC,
+			       scaled_bilinear_scanline_mmx_8888_8888_SRC,
+			       uint32_t, uint32_t, uint32_t,
+			       PAD, FLAG_NONE)
+FAST_BILINEAR_MAINLOOP_COMMON (mmx_8888_8888_none_SRC,
+			       scaled_bilinear_scanline_mmx_8888_8888_SRC,
+			       uint32_t, uint32_t, uint32_t,
+			       NONE, FLAG_NONE)
+FAST_BILINEAR_MAINLOOP_COMMON (mmx_8888_8888_normal_SRC,
+			       scaled_bilinear_scanline_mmx_8888_8888_SRC,
+			       uint32_t, uint32_t, uint32_t,
+			       NORMAL, FLAG_NONE)
+
+static force_inline void
+scaled_bilinear_scanline_mmx_8888_8888_OVER (uint32_t *       dst,
+					     const uint32_t * mask,
+					     const uint32_t * src_top,
+					     const uint32_t * src_bottom,
+					     int32_t          w,
+					     int              wt,
+					     int              wb,
+					     pixman_fixed_t   vx,
+					     pixman_fixed_t   unit_x,
+					     pixman_fixed_t   max_vx,
+					     pixman_bool_t    zero_src)
+{
+    BILINEAR_DECLARE_VARIABLES;
+    __m64 pix1, pix2;
+
+    while (w)
+    {
+	BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
+
+	if (!is_zero (pix1))
+	{
+	    pix2 = load (dst);
+	    store8888 (dst, core_combine_over_u_pixel_mmx (pix1, pix2));
+	}
+
+	w--;
+	dst++;
+    }
+
+    _mm_empty ();
+}
+
+FAST_BILINEAR_MAINLOOP_COMMON (mmx_8888_8888_cover_OVER,
+			       scaled_bilinear_scanline_mmx_8888_8888_OVER,
+			       uint32_t, uint32_t, uint32_t,
+			       COVER, FLAG_NONE)
+FAST_BILINEAR_MAINLOOP_COMMON (mmx_8888_8888_pad_OVER,
+			       scaled_bilinear_scanline_mmx_8888_8888_OVER,
+			       uint32_t, uint32_t, uint32_t,
+			       PAD, FLAG_NONE)
+FAST_BILINEAR_MAINLOOP_COMMON (mmx_8888_8888_none_OVER,
+			       scaled_bilinear_scanline_mmx_8888_8888_OVER,
+			       uint32_t, uint32_t, uint32_t,
+			       NONE, FLAG_NONE)
+FAST_BILINEAR_MAINLOOP_COMMON (mmx_8888_8888_normal_OVER,
+			       scaled_bilinear_scanline_mmx_8888_8888_OVER,
+			       uint32_t, uint32_t, uint32_t,
+			       NORMAL, FLAG_NONE)
+
+static force_inline void
+scaled_bilinear_scanline_mmx_8888_8_8888_OVER (uint32_t *       dst,
+					       const uint8_t  * mask,
+					       const uint32_t * src_top,
+					       const uint32_t * src_bottom,
+					       int32_t          w,
+					       int              wt,
+					       int              wb,
+					       pixman_fixed_t   vx,
+					       pixman_fixed_t   unit_x,
+					       pixman_fixed_t   max_vx,
+					       pixman_bool_t    zero_src)
+{
+    BILINEAR_DECLARE_VARIABLES;
+    __m64 pix1, pix2;
+    uint32_t m;
+
+    while (w)
+    {
+	m = (uint32_t) *mask++;
+
+	if (m)
+	{
+	    BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
+
+	    if (m == 0xff && is_opaque (pix1))
+	    {
+		store (dst, pix1);
+	    }
+	    else
+	    {
+		__m64 ms, md, ma, msa;
+
+		pix2 = load (dst);
+		ma = expand_alpha_rev (to_m64 (m));
+		ms = _mm_unpacklo_pi8 (pix1, _mm_setzero_si64 ());
+		md = _mm_unpacklo_pi8 (pix2, _mm_setzero_si64 ());
+
+		msa = expand_alpha (ms);
+
+		store8888 (dst, (in_over (ms, msa, ma, md)));
+	    }
+	}
+	else
+	{
+	    BILINEAR_SKIP_ONE_PIXEL ();
+	}
+
+	w--;
+	dst++;
+    }
+
+    _mm_empty ();
+}
+
+FAST_BILINEAR_MAINLOOP_COMMON (mmx_8888_8_8888_cover_OVER,
+			       scaled_bilinear_scanline_mmx_8888_8_8888_OVER,
+			       uint32_t, uint8_t, uint32_t,
+			       COVER, FLAG_HAVE_NON_SOLID_MASK)
+FAST_BILINEAR_MAINLOOP_COMMON (mmx_8888_8_8888_pad_OVER,
+			       scaled_bilinear_scanline_mmx_8888_8_8888_OVER,
+			       uint32_t, uint8_t, uint32_t,
+			       PAD, FLAG_HAVE_NON_SOLID_MASK)
+FAST_BILINEAR_MAINLOOP_COMMON (mmx_8888_8_8888_none_OVER,
+			       scaled_bilinear_scanline_mmx_8888_8_8888_OVER,
+			       uint32_t, uint8_t, uint32_t,
+			       NONE, FLAG_HAVE_NON_SOLID_MASK)
+FAST_BILINEAR_MAINLOOP_COMMON (mmx_8888_8_8888_normal_OVER,
+			       scaled_bilinear_scanline_mmx_8888_8_8888_OVER,
+			       uint32_t, uint8_t, uint32_t,
+			       NORMAL, FLAG_HAVE_NON_SOLID_MASK)
+
+static uint32_t *
+mmx_fetch_x8r8g8b8 (pixman_iter_t *iter, const uint32_t *mask)
+{
+    int w = iter->width;
+    uint32_t *dst = iter->buffer;
+    uint32_t *src = (uint32_t *)iter->bits;
+
+    iter->bits += iter->stride;
+
+    while (w && ((uintptr_t)dst) & 7)
+    {
+	*dst++ = (*src++) | 0xff000000;
+	w--;
+    }
+
+    while (w >= 8)
+    {
+	__m64 vsrc1 = ldq_u ((__m64 *)(src + 0));
+	__m64 vsrc2 = ldq_u ((__m64 *)(src + 2));
+	__m64 vsrc3 = ldq_u ((__m64 *)(src + 4));
+	__m64 vsrc4 = ldq_u ((__m64 *)(src + 6));
+
+	*(__m64 *)(dst + 0) = _mm_or_si64 (vsrc1, MC (ff000000));
+	*(__m64 *)(dst + 2) = _mm_or_si64 (vsrc2, MC (ff000000));
+	*(__m64 *)(dst + 4) = _mm_or_si64 (vsrc3, MC (ff000000));
+	*(__m64 *)(dst + 6) = _mm_or_si64 (vsrc4, MC (ff000000));
+
+	dst += 8;
+	src += 8;
+	w -= 8;
+    }
+
+    while (w)
+    {
+	*dst++ = (*src++) | 0xff000000;
+	w--;
+    }
+
+    _mm_empty ();
+    return iter->buffer;
+}
+
+static uint32_t *
+mmx_fetch_r5g6b5 (pixman_iter_t *iter, const uint32_t *mask)
+{
+    int w = iter->width;
+    uint32_t *dst = iter->buffer;
+    uint16_t *src = (uint16_t *)iter->bits;
+
+    iter->bits += iter->stride;
+
+    while (w && ((uintptr_t)dst) & 0x0f)
+    {
+	uint16_t s = *src++;
+
+	*dst++ = convert_0565_to_8888 (s);
+	w--;
+    }
+
+    while (w >= 4)
+    {
+	__m64 vsrc = ldq_u ((__m64 *)src);
+	__m64 mm0, mm1;
+
+	expand_4xpacked565 (vsrc, &mm0, &mm1, 1);
+
+	*(__m64 *)(dst + 0) = mm0;
+	*(__m64 *)(dst + 2) = mm1;
+
+	dst += 4;
+	src += 4;
+	w -= 4;
+    }
+
+    while (w)
+    {
+	uint16_t s = *src++;
+
+	*dst++ = convert_0565_to_8888 (s);
+	w--;
+    }
+
+    _mm_empty ();
+    return iter->buffer;
+}
+
+static uint32_t *
+mmx_fetch_a8 (pixman_iter_t *iter, const uint32_t *mask)
+{
+    int w = iter->width;
+    uint32_t *dst = iter->buffer;
+    uint8_t *src = iter->bits;
+
+    iter->bits += iter->stride;
+
+    while (w && (((uintptr_t)dst) & 15))
+    {
+        *dst++ = *(src++) << 24;
+        w--;
+    }
+
+    while (w >= 8)
+    {
+	__m64 mm0 = ldq_u ((__m64 *)src);
+
+	__m64 mm1 = _mm_unpacklo_pi8  (_mm_setzero_si64(), mm0);
+	__m64 mm2 = _mm_unpackhi_pi8  (_mm_setzero_si64(), mm0);
+	__m64 mm3 = _mm_unpacklo_pi16 (_mm_setzero_si64(), mm1);
+	__m64 mm4 = _mm_unpackhi_pi16 (_mm_setzero_si64(), mm1);
+	__m64 mm5 = _mm_unpacklo_pi16 (_mm_setzero_si64(), mm2);
+	__m64 mm6 = _mm_unpackhi_pi16 (_mm_setzero_si64(), mm2);
+
+	*(__m64 *)(dst + 0) = mm3;
+	*(__m64 *)(dst + 2) = mm4;
+	*(__m64 *)(dst + 4) = mm5;
+	*(__m64 *)(dst + 6) = mm6;
+
+	dst += 8;
+	src += 8;
+	w -= 8;
+    }
+
+    while (w)
+    {
+	*dst++ = *(src++) << 24;
+	w--;
+    }
+
+    _mm_empty ();
+    return iter->buffer;
+}
+
+#define IMAGE_FLAGS							\
+    (FAST_PATH_STANDARD_FLAGS | FAST_PATH_ID_TRANSFORM |		\
+     FAST_PATH_BITS_IMAGE | FAST_PATH_SAMPLES_COVER_CLIP_NEAREST)
+
+static const pixman_iter_info_t mmx_iters[] = 
+{
+    { PIXMAN_x8r8g8b8, IMAGE_FLAGS, ITER_NARROW,
+      _pixman_iter_init_bits_stride, mmx_fetch_x8r8g8b8, NULL
+    },
+    { PIXMAN_r5g6b5, IMAGE_FLAGS, ITER_NARROW,
+      _pixman_iter_init_bits_stride, mmx_fetch_r5g6b5, NULL
+    },
+    { PIXMAN_a8, IMAGE_FLAGS, ITER_NARROW,
+      _pixman_iter_init_bits_stride, mmx_fetch_a8, NULL
+    },
+    { PIXMAN_null },
+};
 
 static const pixman_fast_path_t mmx_fast_paths[] =
 {
@@ -3244,18 +3947,14 @@ static const pixman_fast_path_t mmx_fast_paths[] =
     PIXMAN_STD_FAST_PATH    (OVER, a8r8g8b8, solid,    x8r8g8b8, mmx_composite_over_8888_n_8888    ),
     PIXMAN_STD_FAST_PATH    (OVER, a8b8g8r8, solid,    a8b8g8r8, mmx_composite_over_8888_n_8888    ),
     PIXMAN_STD_FAST_PATH    (OVER, a8b8g8r8, solid,    x8b8g8r8, mmx_composite_over_8888_n_8888    ),
-#if 0
-    /* FIXME: This code is commented out since it's apparently
-     * not actually faster than the generic code.
-     */
     PIXMAN_STD_FAST_PATH    (OVER, x8r8g8b8, a8,       x8r8g8b8, mmx_composite_over_x888_8_8888    ),
     PIXMAN_STD_FAST_PATH    (OVER, x8r8g8b8, a8,       a8r8g8b8, mmx_composite_over_x888_8_8888    ),
-    PIXMAN_STD_FAST_PATH    (OVER, x8b8r8g8, a8,       x8b8g8r8, mmx_composite_over_x888_8_8888    ),
-    PIXMAN_STD_FAST_PATH    (OVER, x8b8r8g8, a8,       a8r8g8b8, mmx_composite_over_x888_8_8888    ),
-#endif
+    PIXMAN_STD_FAST_PATH    (OVER, x8b8g8r8, a8,       x8b8g8r8, mmx_composite_over_x888_8_8888    ),
+    PIXMAN_STD_FAST_PATH    (OVER, x8b8g8r8, a8,       a8b8g8r8, mmx_composite_over_x888_8_8888    ),
     PIXMAN_STD_FAST_PATH    (OVER, solid,    null,     a8r8g8b8, mmx_composite_over_n_8888         ),
     PIXMAN_STD_FAST_PATH    (OVER, solid,    null,     x8r8g8b8, mmx_composite_over_n_8888         ),
     PIXMAN_STD_FAST_PATH    (OVER, solid,    null,     r5g6b5,   mmx_composite_over_n_0565         ),
+    PIXMAN_STD_FAST_PATH    (OVER, solid,    null,     b5g6r5,   mmx_composite_over_n_0565         ),
     PIXMAN_STD_FAST_PATH    (OVER, x8r8g8b8, null,     x8r8g8b8, mmx_composite_copy_area           ),
     PIXMAN_STD_FAST_PATH    (OVER, x8b8g8r8, null,     x8b8g8r8, mmx_composite_copy_area           ),
 
@@ -3266,11 +3965,20 @@ static const pixman_fast_path_t mmx_fast_paths[] =
     PIXMAN_STD_FAST_PATH    (OVER, a8b8g8r8, null,     x8b8g8r8, mmx_composite_over_8888_8888      ),
     PIXMAN_STD_FAST_PATH    (OVER, a8b8g8r8, null,     b5g6r5,   mmx_composite_over_8888_0565      ),
 
+    PIXMAN_STD_FAST_PATH    (OVER_REVERSE, solid, null, a8r8g8b8, mmx_composite_over_reverse_n_8888),
+    PIXMAN_STD_FAST_PATH    (OVER_REVERSE, solid, null, a8b8g8r8, mmx_composite_over_reverse_n_8888),
+
+    PIXMAN_STD_FAST_PATH    (ADD,  r5g6b5,   null,     r5g6b5,   mmx_composite_add_0565_0565       ),
+    PIXMAN_STD_FAST_PATH    (ADD,  b5g6r5,   null,     b5g6r5,   mmx_composite_add_0565_0565       ),
     PIXMAN_STD_FAST_PATH    (ADD,  a8r8g8b8, null,     a8r8g8b8, mmx_composite_add_8888_8888       ),
     PIXMAN_STD_FAST_PATH    (ADD,  a8b8g8r8, null,     a8b8g8r8, mmx_composite_add_8888_8888       ),
-    PIXMAN_STD_FAST_PATH    (ADD,  a8,       null,     a8,       mmx_composite_add_8000_8000       ),
+    PIXMAN_STD_FAST_PATH    (ADD,  a8,       null,     a8,       mmx_composite_add_8_8		   ),
     PIXMAN_STD_FAST_PATH    (ADD,  solid,    a8,       a8,       mmx_composite_add_n_8_8           ),
 
+    PIXMAN_STD_FAST_PATH    (SRC,  a8r8g8b8, null,     r5g6b5,   mmx_composite_src_x888_0565       ),
+    PIXMAN_STD_FAST_PATH    (SRC,  a8b8g8r8, null,     b5g6r5,   mmx_composite_src_x888_0565       ),
+    PIXMAN_STD_FAST_PATH    (SRC,  x8r8g8b8, null,     r5g6b5,   mmx_composite_src_x888_0565       ),
+    PIXMAN_STD_FAST_PATH    (SRC,  x8b8g8r8, null,     b5g6r5,   mmx_composite_src_x888_0565       ),
     PIXMAN_STD_FAST_PATH    (SRC,  solid,    a8,       a8r8g8b8, mmx_composite_src_n_8_8888        ),
     PIXMAN_STD_FAST_PATH    (SRC,  solid,    a8,       x8r8g8b8, mmx_composite_src_n_8_8888        ),
     PIXMAN_STD_FAST_PATH    (SRC,  solid,    a8,       a8b8g8r8, mmx_composite_src_n_8_8888        ),
@@ -3287,63 +3995,30 @@ static const pixman_fast_path_t mmx_fast_paths[] =
     PIXMAN_STD_FAST_PATH    (IN,   a8,       null,     a8,       mmx_composite_in_8_8              ),
     PIXMAN_STD_FAST_PATH    (IN,   solid,    a8,       a8,       mmx_composite_in_n_8_8            ),
 
-    { PIXMAN_OP_NONE },
-};
-
-static pixman_bool_t
-mmx_blt (pixman_implementation_t *imp,
-         uint32_t *               src_bits,
-         uint32_t *               dst_bits,
-         int                      src_stride,
-         int                      dst_stride,
-         int                      src_bpp,
-         int                      dst_bpp,
-         int                      src_x,
-         int                      src_y,
-         int                      dst_x,
-         int                      dst_y,
-         int                      width,
-         int                      height)
-{
-    if (!pixman_blt_mmx (
-            src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
-            src_x, src_y, dst_x, dst_y, width, height))
-
-    {
-	return _pixman_implementation_blt (
-	    imp->delegate,
-	    src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
-	    src_x, src_y, dst_x, dst_y, width, height);
-    }
+    SIMPLE_BILINEAR_FAST_PATH (SRC, a8r8g8b8,          a8r8g8b8, mmx_8888_8888                     ),
+    SIMPLE_BILINEAR_FAST_PATH (SRC, a8r8g8b8,          x8r8g8b8, mmx_8888_8888                     ),
+    SIMPLE_BILINEAR_FAST_PATH (SRC, x8r8g8b8,          x8r8g8b8, mmx_8888_8888                     ),
+    SIMPLE_BILINEAR_FAST_PATH (SRC, a8b8g8r8,          a8b8g8r8, mmx_8888_8888                     ),
+    SIMPLE_BILINEAR_FAST_PATH (SRC, a8b8g8r8,          x8b8g8r8, mmx_8888_8888                     ),
+    SIMPLE_BILINEAR_FAST_PATH (SRC, x8b8g8r8,          x8b8g8r8, mmx_8888_8888                     ),
 
-    return TRUE;
-}
+    SIMPLE_BILINEAR_FAST_PATH (OVER, a8r8g8b8,         x8r8g8b8, mmx_8888_8888                     ),
+    SIMPLE_BILINEAR_FAST_PATH (OVER, a8b8g8r8,         x8b8g8r8, mmx_8888_8888                     ),
+    SIMPLE_BILINEAR_FAST_PATH (OVER, a8r8g8b8,         a8r8g8b8, mmx_8888_8888                     ),
+    SIMPLE_BILINEAR_FAST_PATH (OVER, a8b8g8r8,         a8b8g8r8, mmx_8888_8888                     ),
 
-static pixman_bool_t
-mmx_fill (pixman_implementation_t *imp,
-          uint32_t *               bits,
-          int                      stride,
-          int                      bpp,
-          int                      x,
-          int                      y,
-          int                      width,
-          int                      height,
-          uint32_t xor)
-{
-    if (!pixman_fill_mmx (bits, stride, bpp, x, y, width, height, xor))
-    {
-	return _pixman_implementation_fill (
-	    imp->delegate, bits, stride, bpp, x, y, width, height, xor);
-    }
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, mmx_8888_8_8888                   ),
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (OVER, a8b8g8r8, x8b8g8r8, mmx_8888_8_8888                   ),
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, mmx_8888_8_8888                   ),
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (OVER, a8b8g8r8, a8b8g8r8, mmx_8888_8_8888                   ),
 
-    return TRUE;
-}
+    { PIXMAN_OP_NONE },
+};
 
 pixman_implementation_t *
-_pixman_implementation_create_mmx (void)
+_pixman_implementation_create_mmx (pixman_implementation_t *fallback)
 {
-    pixman_implementation_t *general = _pixman_implementation_create_fast_path ();
-    pixman_implementation_t *imp = _pixman_implementation_create (general, mmx_fast_paths);
+    pixman_implementation_t *imp = _pixman_implementation_create (fallback, mmx_fast_paths);
 
     imp->combine_32[PIXMAN_OP_OVER] = mmx_combine_over_u;
     imp->combine_32[PIXMAN_OP_OVER_REVERSE] = mmx_combine_over_reverse_u;
@@ -3372,7 +4047,9 @@ _pixman_implementation_create_mmx (void)
     imp->blt = mmx_blt;
     imp->fill = mmx_fill;
 
+    imp->iter_info = mmx_iters;
+
     return imp;
 }
 
-#endif /* USE_MMX */
+#endif /* USE_X86_MMX || USE_ARM_IWMMXT || USE_LOONGSON_MMI */
diff --git a/pixman/pixman/pixman-noop.c b/pixman/pixman/pixman-noop.c
new file mode 100644
index 000000000..e59890492
--- /dev/null
+++ b/pixman/pixman/pixman-noop.c
@@ -0,0 +1,161 @@
+/* -*- Mode: c; c-basic-offset: 4; tab-width: 8; indent-tabs-mode: t; -*- */
+/*
+ * Copyright © 2011 Red Hat, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#include <string.h>
+#include <stdlib.h>
+#include "pixman-private.h"
+#include "pixman-combine32.h"
+#include "pixman-inlines.h"
+
+static void
+noop_composite (pixman_implementation_t *imp,
+		pixman_composite_info_t *info)
+{
+    return;
+}
+
+static uint32_t *
+noop_get_scanline (pixman_iter_t *iter, const uint32_t *mask)
+{
+    uint32_t *result = iter->buffer;
+
+    iter->buffer += iter->image->bits.rowstride;
+
+    return result;
+}
+
+static void
+noop_init_solid_narrow (pixman_iter_t *iter,
+			const pixman_iter_info_t *info)
+{ 
+    pixman_image_t *image = iter->image;
+    uint32_t *buffer = iter->buffer;
+    uint32_t *end = buffer + iter->width;
+    uint32_t color;
+
+    if (iter->image->type == SOLID)
+	color = image->solid.color_32;
+    else
+	color = image->bits.fetch_pixel_32 (&image->bits, 0, 0);
+
+    while (buffer < end)
+	*(buffer++) = color;
+}
+
+static void
+noop_init_solid_wide (pixman_iter_t *iter,
+		      const pixman_iter_info_t *info)
+{
+    pixman_image_t *image = iter->image;
+    argb_t *buffer = (argb_t *)iter->buffer;
+    argb_t *end = buffer + iter->width;
+    argb_t color;
+
+    if (iter->image->type == SOLID)
+	color = image->solid.color_float;
+    else
+	color = image->bits.fetch_pixel_float (&image->bits, 0, 0);
+
+    while (buffer < end)
+	*(buffer++) = color;
+}
+
+static void
+noop_init_direct_buffer (pixman_iter_t *iter, const pixman_iter_info_t *info)
+{
+    pixman_image_t *image = iter->image;
+
+    iter->buffer =
+	image->bits.bits + iter->y * image->bits.rowstride + iter->x;
+}
+
+static void
+dest_write_back_direct (pixman_iter_t *iter)
+{
+    iter->buffer += iter->image->bits.rowstride;
+}
+
+static const pixman_iter_info_t noop_iters[] =
+{
+    /* Source iters */
+    { PIXMAN_any,
+      0, ITER_IGNORE_ALPHA | ITER_IGNORE_RGB | ITER_SRC,
+      NULL,
+      _pixman_iter_get_scanline_noop,
+      NULL
+    },
+    { PIXMAN_solid,
+      FAST_PATH_NO_ALPHA_MAP, ITER_NARROW | ITER_SRC,
+      noop_init_solid_narrow,
+      _pixman_iter_get_scanline_noop,
+      NULL,
+    },
+    { PIXMAN_solid,
+      FAST_PATH_NO_ALPHA_MAP, ITER_WIDE | ITER_SRC,
+      noop_init_solid_wide,
+      _pixman_iter_get_scanline_noop,
+      NULL
+    },
+    { PIXMAN_a8r8g8b8,
+      FAST_PATH_STANDARD_FLAGS | FAST_PATH_ID_TRANSFORM |
+          FAST_PATH_BITS_IMAGE | FAST_PATH_SAMPLES_COVER_CLIP_NEAREST,
+      ITER_NARROW | ITER_SRC,
+      noop_init_direct_buffer,
+      noop_get_scanline,
+      NULL
+    },
+    /* Dest iters */
+    { PIXMAN_a8r8g8b8,
+      FAST_PATH_STD_DEST_FLAGS, ITER_NARROW | ITER_DEST,
+      noop_init_direct_buffer,
+      _pixman_iter_get_scanline_noop,
+      dest_write_back_direct
+    },
+    { PIXMAN_x8r8g8b8,
+      FAST_PATH_STD_DEST_FLAGS, ITER_NARROW | ITER_DEST | ITER_LOCALIZED_ALPHA,
+      noop_init_direct_buffer,
+      _pixman_iter_get_scanline_noop,
+      dest_write_back_direct
+    },
+    { PIXMAN_null },
+};
+
+static const pixman_fast_path_t noop_fast_paths[] =
+{
+    { PIXMAN_OP_DST, PIXMAN_any, 0, PIXMAN_any, 0, PIXMAN_any, 0, noop_composite },
+    { PIXMAN_OP_NONE },
+};
+
+pixman_implementation_t *
+_pixman_implementation_create_noop (pixman_implementation_t *fallback)
+{
+    pixman_implementation_t *imp =
+	_pixman_implementation_create (fallback, noop_fast_paths);
+ 
+    imp->iter_info = noop_iters;
+
+    return imp;
+}
diff --git a/pixman/pixman/pixman-ppc.c b/pixman/pixman/pixman-ppc.c
new file mode 100644
index 000000000..a6e7bb0cf
--- /dev/null
+++ b/pixman/pixman/pixman-ppc.c
@@ -0,0 +1,155 @@
+/*
+ * Copyright © 2000 SuSE, Inc.
+ * Copyright © 2007 Red Hat, Inc.
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of SuSE not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  SuSE makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as is"
+ * without express or implied warranty.
+ *
+ * SuSE DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL SuSE
+ * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "pixman-private.h"
+
+#ifdef USE_VMX
+
+/* The CPU detection code needs to be in a file not compiled with
+ * "-maltivec -mabi=altivec", as gcc would try to save vector register
+ * across function calls causing SIGILL on cpus without Altivec/vmx.
+ */
+#ifdef __APPLE__
+#include <sys/sysctl.h>
+
+static pixman_bool_t
+pixman_have_vmx (void)
+{
+    int error, have_vmx;
+    size_t length = sizeof(have_vmx);
+
+    error = sysctlbyname ("hw.optional.altivec", &have_vmx, &length, NULL, 0);
+
+    if (error)
+	return FALSE;
+
+    return have_vmx;
+}
+
+#elif defined (__OpenBSD__)
+#include <sys/param.h>
+#include <sys/sysctl.h>
+#include <machine/cpu.h>
+
+static pixman_bool_t
+pixman_have_vmx (void)
+{
+    int error, have_vmx;
+    int mib[2] = { CTL_MACHDEP, CPU_ALTIVEC };
+    size_t length = sizeof(have_vmx);
+
+    error = sysctl (mib, 2, &have_vmx, &length, NULL, 0);
+
+    if (error != 0)
+	return FALSE;
+
+    return have_vmx;
+}
+
+#elif defined (__linux__)
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <linux/auxvec.h>
+#include <asm/cputable.h>
+
+static pixman_bool_t
+pixman_have_vmx (void)
+{
+    int have_vmx = FALSE;
+    int fd;
+    struct
+    {
+	unsigned long type;
+	unsigned long value;
+    } aux;
+
+    fd = open ("/proc/self/auxv", O_RDONLY);
+    if (fd >= 0)
+    {
+	while (read (fd, &aux, sizeof (aux)) == sizeof (aux))
+	{
+	    if (aux.type == AT_HWCAP && (aux.value & PPC_FEATURE_HAS_ALTIVEC))
+	    {
+		have_vmx = TRUE;
+		break;
+	    }
+	}
+
+	close (fd);
+    }
+
+    return have_vmx;
+}
+
+#else /* !__APPLE__ && !__OpenBSD__ && !__linux__ */
+#include <signal.h>
+#include <setjmp.h>
+
+static jmp_buf jump_env;
+
+static void
+vmx_test (int        sig,
+	  siginfo_t *si,
+	  void *     unused)
+{
+    longjmp (jump_env, 1);
+}
+
+static pixman_bool_t
+pixman_have_vmx (void)
+{
+    struct sigaction sa, osa;
+    int jmp_result;
+
+    sa.sa_flags = SA_SIGINFO;
+    sigemptyset (&sa.sa_mask);
+    sa.sa_sigaction = vmx_test;
+    sigaction (SIGILL, &sa, &osa);
+    jmp_result = setjmp (jump_env);
+    if (jmp_result == 0)
+    {
+	asm volatile ( "vor 0, 0, 0" );
+    }
+    sigaction (SIGILL, &osa, NULL);
+    return (jmp_result == 0);
+}
+
+#endif /* __APPLE__ */
+#endif /* USE_VMX */
+
+pixman_implementation_t *
+_pixman_ppc_get_implementations (pixman_implementation_t *imp)
+{
+#ifdef USE_VMX
+    if (!_pixman_disabled ("vmx") && pixman_have_vmx ())
+	imp = _pixman_implementation_create_vmx (imp);
+#endif
+
+    return imp;
+}
diff --git a/pixman/pixman/pixman-private.h b/pixman/pixman/pixman-private.h
index d5767af48..6ca13b216 100644
--- a/pixman/pixman/pixman-private.h
+++ b/pixman/pixman/pixman-private.h
@@ -1,10 +1,26 @@
-#ifndef PACKAGE
-#  error config.h must be included before pixman-private.h
-#endif
+#include <float.h>
 
 #ifndef PIXMAN_PRIVATE_H
 #define PIXMAN_PRIVATE_H
 
+/*
+ * The defines which are shared between C and assembly code
+ */
+
+/* bilinear interpolation precision (must be <= 8) */
+#define BILINEAR_INTERPOLATION_BITS 7
+#define BILINEAR_INTERPOLATION_RANGE (1 << BILINEAR_INTERPOLATION_BITS)
+
+/*
+ * C specific part
+ */
+
+#ifndef __ASSEMBLER__
+
+#ifndef PACKAGE
+#  error config.h must be included before pixman-private.h
+#endif
+
 #define PIXMAN_DISABLE_DEPRECATED
 #define PIXMAN_USE_INTERNAL_API
 
@@ -13,6 +29,7 @@
 #include <assert.h>
 #include <stdio.h>
 #include <string.h>
+#include <stddef.h>
 
 #include "pixman-compiler.h"
 
@@ -20,7 +37,6 @@
  * Images
  */
 typedef struct image_common image_common_t;
-typedef struct source_image source_image_t;
 typedef struct solid_fill solid_fill_t;
 typedef struct gradient gradient_t;
 typedef struct linear_gradient linear_gradient_t;
@@ -31,21 +47,30 @@ typedef struct radial_gradient radial_gradient_t;
 typedef struct bits_image bits_image_t;
 typedef struct circle circle_t;
 
-typedef void (*fetch_scanline_t) (pixman_image_t *image,
+typedef struct argb_t argb_t;
+
+struct argb_t
+{
+    float a;
+    float r;
+    float g;
+    float b;
+};
+
+typedef void (*fetch_scanline_t) (bits_image_t   *image,
 				  int             x,
 				  int             y,
 				  int             width,
 				  uint32_t       *buffer,
-				  const uint32_t *mask,
-				  uint32_t        mask_bits);
+				  const uint32_t *mask);
 
 typedef uint32_t (*fetch_pixel_32_t) (bits_image_t *image,
 				      int           x,
 				      int           y);
 
-typedef uint64_t (*fetch_pixel_64_t) (bits_image_t *image,
-				      int           x,
-				      int           y);
+typedef argb_t (*fetch_pixel_float_t) (bits_image_t *image,
+				       int           x,
+				       int           y);
 
 typedef void (*store_scanline_t) (bits_image_t *  image,
 				  int             x,
@@ -62,18 +87,6 @@ typedef enum
     SOLID
 } image_type_t;
 
-typedef enum
-{
-    SOURCE_IMAGE_CLASS_UNKNOWN,
-    SOURCE_IMAGE_CLASS_HORIZONTAL,
-    SOURCE_IMAGE_CLASS_VERTICAL,
-} source_image_class_t;
-
-typedef source_image_class_t (*classify_func_t) (pixman_image_t *image,
-						int             x,
-						int             y,
-						int             width,
-						int             height);
 typedef void (*property_changed_func_t) (pixman_image_t *image);
 
 struct image_common
@@ -81,6 +94,7 @@ struct image_common
     image_type_t                type;
     int32_t                     ref_count;
     pixman_region32_t           clip_region;
+    int32_t			alpha_count;	    /* How many times this image is being used as an alpha map */
     pixman_bool_t               have_clip_region;   /* FALSE if there is no clip */
     pixman_bool_t               client_clip;        /* Whether the source clip was
 						       set by a client */
@@ -97,10 +111,7 @@ struct image_common
     int                         alpha_origin_x;
     int                         alpha_origin_y;
     pixman_bool_t               component_alpha;
-    classify_func_t             classify;
     property_changed_func_t     property_changed;
-    fetch_scanline_t            get_scanline_32;
-    fetch_scanline_t            get_scanline_64;
 
     pixman_image_destroy_func_t destroy_func;
     void *                      destroy_data;
@@ -109,27 +120,20 @@ struct image_common
     pixman_format_code_t	extended_format_code;
 };
 
-struct source_image
-{
-    image_common_t common;
-    source_image_class_t class;
-};
-
 struct solid_fill
 {
-    source_image_t common;
+    image_common_t common;
     pixman_color_t color;
-    
+
     uint32_t	   color_32;
-    uint64_t	   color_64;
+    argb_t	   color_float;
 };
 
 struct gradient
 {
-    source_image_t          common;
+    image_common_t	    common;
     int                     n_stops;
     pixman_gradient_stop_t *stops;
-    int                     stop_range;
 };
 
 struct linear_gradient
@@ -152,17 +156,18 @@ struct radial_gradient
 
     circle_t   c1;
     circle_t   c2;
-    double     cdx;
-    double     cdy;
-    double     dr;
-    double     A;
+
+    circle_t   delta;
+    double     a;
+    double     inva;
+    double     mindr;
 };
 
 struct conical_gradient
 {
     gradient_t           common;
     pixman_point_fixed_t center;
-    pixman_fixed_t       angle;
+    double		 angle;
 };
 
 struct bits_image
@@ -176,25 +181,13 @@ struct bits_image
     uint32_t *                 free_me;
     int                        rowstride;  /* in number of uint32_t's */
 
-    /* Fetch a pixel, disregarding alpha maps, transformations etc. */
-    fetch_pixel_32_t	       fetch_pixel_raw_32;
-    fetch_pixel_64_t	       fetch_pixel_raw_64;
-
-    /* Fetch a pixel, taking alpha maps into account */
+    fetch_scanline_t           fetch_scanline_32;
     fetch_pixel_32_t	       fetch_pixel_32;
-    fetch_pixel_64_t	       fetch_pixel_64;
-
-    /* Fetch raw scanlines, with no regard for transformations, alpha maps etc. */
-    fetch_scanline_t           fetch_scanline_raw_32;
-    fetch_scanline_t           fetch_scanline_raw_64;
-
-    /* Store scanlines with no regard for alpha maps */
-    store_scanline_t           store_scanline_raw_32;
-    store_scanline_t           store_scanline_raw_64;
-
-    /* Store a scanline, taking alpha maps into account */
     store_scanline_t           store_scanline_32;
-    store_scanline_t           store_scanline_64;
+
+    fetch_scanline_t	       fetch_scanline_float;
+    fetch_pixel_float_t	       fetch_pixel_float;
+    store_scanline_t           store_scanline_float;
 
     /* Used for indirect access to the bits */
     pixman_read_memory_func_t  read_func;
@@ -206,7 +199,6 @@ union pixman_image
     image_type_t       type;
     image_common_t     common;
     bits_image_t       bits;
-    source_image_t     source;
     gradient_t         gradient;
     linear_gradient_t  linear;
     conical_gradient_t conical;
@@ -214,63 +206,108 @@ union pixman_image
     solid_fill_t       solid;
 };
 
+typedef struct pixman_iter_t pixman_iter_t;
+typedef uint32_t *(* pixman_iter_get_scanline_t) (pixman_iter_t *iter, const uint32_t *mask);
+typedef void      (* pixman_iter_write_back_t)   (pixman_iter_t *iter);
+typedef void	  (* pixman_iter_fini_t)	 (pixman_iter_t *iter);
+
+typedef enum
+{
+    ITER_NARROW =               (1 << 0),
+    ITER_WIDE =                 (1 << 1),
+
+    /* "Localized alpha" is when the alpha channel is used only to compute
+     * the alpha value of the destination. This means that the computation
+     * of the RGB values of the result is independent of the alpha value.
+     *
+     * For example, the OVER operator has localized alpha for the
+     * destination, because the RGB values of the result can be computed
+     * without knowing the destination alpha. Similarly, ADD has localized
+     * alpha for both source and destination because the RGB values of the
+     * result can be computed without knowing the alpha value of source or
+     * destination.
+     *
+     * When he destination is xRGB, this is useful knowledge, because then
+     * we can treat it as if it were ARGB, which means in some cases we can
+     * avoid copying it to a temporary buffer.
+     */
+    ITER_LOCALIZED_ALPHA =	(1 << 2),
+    ITER_IGNORE_ALPHA =		(1 << 3),
+    ITER_IGNORE_RGB =		(1 << 4),
+
+    /* These indicate whether the iterator is for a source
+     * or a destination image
+     */
+    ITER_SRC =			(1 << 5),
+    ITER_DEST =			(1 << 6)
+} iter_flags_t;
+
+struct pixman_iter_t
+{
+    /* These are initialized by _pixman_implementation_{src,dest}_init */
+    pixman_image_t *		image;
+    uint32_t *			buffer;
+    int				x, y;
+    int				width;
+    int				height;
+    iter_flags_t		iter_flags;
+    uint32_t			image_flags;
+
+    /* These function pointers are initialized by the implementation */
+    pixman_iter_get_scanline_t	get_scanline;
+    pixman_iter_write_back_t	write_back;
+    pixman_iter_fini_t          fini;
+
+    /* These fields are scratch data that implementations can use */
+    void *			data;
+    uint8_t *			bits;
+    int				stride;
+};
+
+typedef struct pixman_iter_info_t pixman_iter_info_t;
+typedef void (* pixman_iter_initializer_t) (pixman_iter_t *iter,
+                                            const pixman_iter_info_t *info);
+struct pixman_iter_info_t
+{
+    pixman_format_code_t	format;
+    uint32_t			image_flags;
+    iter_flags_t		iter_flags;
+    pixman_iter_initializer_t	initializer;
+    pixman_iter_get_scanline_t	get_scanline;
+    pixman_iter_write_back_t	write_back;
+};
 
 void
-_pixman_bits_image_setup_raw_accessors (bits_image_t *image);
+_pixman_bits_image_setup_accessors (bits_image_t *image);
 
 void
-_pixman_image_get_scanline_generic_64  (pixman_image_t *image,
-                                        int             x,
-                                        int             y,
-                                        int             width,
-                                        uint32_t *      buffer,
-                                        const uint32_t *mask,
-                                        uint32_t        mask_bits);
-
-source_image_class_t
-_pixman_image_classify (pixman_image_t *image,
-                        int             x,
-                        int             y,
-                        int             width,
-                        int             height);
+_pixman_bits_image_src_iter_init (pixman_image_t *image, pixman_iter_t *iter);
 
 void
-_pixman_image_get_scanline_32 (pixman_image_t *image,
-                               int             x,
-                               int             y,
-                               int             width,
-                               uint32_t *      buffer,
-                               const uint32_t *mask,
-                               uint32_t        mask_bits);
-
-/* Even thought the type of buffer is uint32_t *, the function actually expects
- * a uint64_t *buffer.
- */
+_pixman_bits_image_dest_iter_init (pixman_image_t *image, pixman_iter_t *iter);
+
 void
-_pixman_image_get_scanline_64 (pixman_image_t *image,
-                               int             x,
-                               int             y,
-                               int             width,
-                               uint32_t *      buffer,
-                               const uint32_t *unused,
-                               uint32_t        unused2);
+_pixman_linear_gradient_iter_init (pixman_image_t *image, pixman_iter_t  *iter);
 
 void
-_pixman_image_store_scanline_32 (bits_image_t *  image,
-                                 int             x,
-                                 int             y,
-                                 int             width,
-                                 const uint32_t *buffer);
-
-/* Even though the type of buffer is uint32_t *, the function
- * actually expects a uint64_t *buffer.
- */
+_pixman_radial_gradient_iter_init (pixman_image_t *image, pixman_iter_t *iter);
+
 void
-_pixman_image_store_scanline_64 (bits_image_t *  image,
-                                 int             x,
-                                 int             y,
-                                 int             width,
-                                 const uint32_t *buffer);
+_pixman_conical_gradient_iter_init (pixman_image_t *image, pixman_iter_t *iter);
+
+void
+_pixman_image_init (pixman_image_t *image);
+
+pixman_bool_t
+_pixman_bits_image_init (pixman_image_t *     image,
+                         pixman_format_code_t format,
+                         int                  width,
+                         int                  height,
+                         uint32_t *           bits,
+                         int                  rowstride,
+			 pixman_bool_t	      clear);
+pixman_bool_t
+_pixman_image_fini (pixman_image_t *image);
 
 pixman_image_t *
 _pixman_image_allocate (void);
@@ -285,10 +322,6 @@ _pixman_image_reset_clip_region (pixman_image_t *image);
 void
 _pixman_image_validate (pixman_image_t *image);
 
-uint32_t
-_pixman_image_get_solid (pixman_image_t *     image,
-                         pixman_format_code_t format);
-
 #define PIXMAN_IMAGE_GET_LINE(image, x, y, type, out_stride, line, mul)	\
     do									\
     {									\
@@ -308,33 +341,32 @@ _pixman_image_get_solid (pixman_image_t *     image,
  */
 typedef struct
 {
-    uint32_t                left_ag;
-    uint32_t                left_rb;
-    uint32_t                right_ag;
-    uint32_t                right_rb;
-    int32_t                 left_x;
-    int32_t                 right_x;
-    int32_t                 stepper;
+    float		    a_s, a_b;
+    float		    r_s, r_b;
+    float		    g_s, g_b;
+    float		    b_s, b_b;
+    pixman_fixed_t	    left_x;
+    pixman_fixed_t          right_x;
 
     pixman_gradient_stop_t *stops;
     int                     num_stops;
-    unsigned int            spread;
+    pixman_repeat_t	    repeat;
 
-    int                     need_reset;
+    pixman_bool_t           need_reset;
 } pixman_gradient_walker_t;
 
 void
 _pixman_gradient_walker_init (pixman_gradient_walker_t *walker,
                               gradient_t *              gradient,
-                              unsigned int              spread);
+			      pixman_repeat_t           repeat);
 
 void
 _pixman_gradient_walker_reset (pixman_gradient_walker_t *walker,
-                               pixman_fixed_32_32_t      pos);
+                               pixman_fixed_48_16_t      pos);
 
 uint32_t
 _pixman_gradient_walker_pixel (pixman_gradient_walker_t *walker,
-                               pixman_fixed_32_32_t      x);
+                               pixman_fixed_48_16_t      x);
 
 /*
  * Edges
@@ -372,6 +404,40 @@ pixman_rasterize_edges_accessors (pixman_image_t *image,
  */
 typedef struct pixman_implementation_t pixman_implementation_t;
 
+typedef struct
+{
+    pixman_op_t              op;
+    pixman_image_t *         src_image;
+    pixman_image_t *         mask_image;
+    pixman_image_t *         dest_image;
+    int32_t                  src_x;
+    int32_t                  src_y;
+    int32_t                  mask_x;
+    int32_t                  mask_y;
+    int32_t                  dest_x;
+    int32_t                  dest_y;
+    int32_t                  width;
+    int32_t                  height;
+
+    uint32_t                 src_flags;
+    uint32_t                 mask_flags;
+    uint32_t                 dest_flags;
+} pixman_composite_info_t;
+
+#define PIXMAN_COMPOSITE_ARGS(info)					\
+    MAYBE_UNUSED pixman_op_t        op = info->op;			\
+    MAYBE_UNUSED pixman_image_t *   src_image = info->src_image;	\
+    MAYBE_UNUSED pixman_image_t *   mask_image = info->mask_image;	\
+    MAYBE_UNUSED pixman_image_t *   dest_image = info->dest_image;	\
+    MAYBE_UNUSED int32_t            src_x = info->src_x;		\
+    MAYBE_UNUSED int32_t            src_y = info->src_y;		\
+    MAYBE_UNUSED int32_t            mask_x = info->mask_x;		\
+    MAYBE_UNUSED int32_t            mask_y = info->mask_y;		\
+    MAYBE_UNUSED int32_t            dest_x = info->dest_x;		\
+    MAYBE_UNUSED int32_t            dest_y = info->dest_y;		\
+    MAYBE_UNUSED int32_t            width = info->width;		\
+    MAYBE_UNUSED int32_t            height = info->height
+
 typedef void (*pixman_combine_32_func_t) (pixman_implementation_t *imp,
 					  pixman_op_t              op,
 					  uint32_t *               dest,
@@ -379,26 +445,15 @@ typedef void (*pixman_combine_32_func_t) (pixman_implementation_t *imp,
 					  const uint32_t *         mask,
 					  int                      width);
 
-typedef void (*pixman_combine_64_func_t) (pixman_implementation_t *imp,
-					  pixman_op_t              op,
-					  uint64_t *               dest,
-					  const uint64_t *         src,
-					  const uint64_t *         mask,
-					  int                      width);
+typedef void (*pixman_combine_float_func_t) (pixman_implementation_t *imp,
+					     pixman_op_t	      op,
+					     float *		      dest,
+					     const float *	      src,
+					     const float *	      mask,
+					     int		      n_pixels);
 
 typedef void (*pixman_composite_func_t) (pixman_implementation_t *imp,
-					 pixman_op_t              op,
-					 pixman_image_t *         src,
-					 pixman_image_t *         mask,
-					 pixman_image_t *         dest,
-					 int32_t                  src_x,
-					 int32_t                  src_y,
-					 int32_t                  mask_x,
-					 int32_t                  mask_y,
-					 int32_t                  dest_x,
-					 int32_t                  dest_y,
-					 int32_t                  width,
-					 int32_t                  height);
+					 pixman_composite_info_t *info);
 typedef pixman_bool_t (*pixman_blt_func_t) (pixman_implementation_t *imp,
 					    uint32_t *               src_bits,
 					    uint32_t *               dst_bits,
@@ -408,8 +463,8 @@ typedef pixman_bool_t (*pixman_blt_func_t) (pixman_implementation_t *imp,
 					    int                      dst_bpp,
 					    int                      src_x,
 					    int                      src_y,
-					    int                      dst_x,
-					    int                      dst_y,
+					    int                      dest_x,
+					    int                      dest_y,
 					    int                      width,
 					    int                      height);
 typedef pixman_bool_t (*pixman_fill_func_t) (pixman_implementation_t *imp,
@@ -420,10 +475,10 @@ typedef pixman_bool_t (*pixman_fill_func_t) (pixman_implementation_t *imp,
 					     int                      y,
 					     int                      width,
 					     int                      height,
-					     uint32_t                 xor);
+					     uint32_t                 filler);
 
 void _pixman_setup_combiner_functions_32 (pixman_implementation_t *imp);
-void _pixman_setup_combiner_functions_64 (pixman_implementation_t *imp);
+void _pixman_setup_combiner_functions_float (pixman_implementation_t *imp);
 
 typedef struct
 {
@@ -440,50 +495,45 @@ typedef struct
 struct pixman_implementation_t
 {
     pixman_implementation_t *	toplevel;
-    pixman_implementation_t *	delegate;
+    pixman_implementation_t *	fallback;
     const pixman_fast_path_t *	fast_paths;
-    
+    const pixman_iter_info_t *  iter_info;
+
     pixman_blt_func_t		blt;
     pixman_fill_func_t		fill;
 
     pixman_combine_32_func_t	combine_32[PIXMAN_N_OPERATORS];
     pixman_combine_32_func_t	combine_32_ca[PIXMAN_N_OPERATORS];
-    pixman_combine_64_func_t	combine_64[PIXMAN_N_OPERATORS];
-    pixman_combine_64_func_t	combine_64_ca[PIXMAN_N_OPERATORS];
+    pixman_combine_float_func_t	combine_float[PIXMAN_N_OPERATORS];
+    pixman_combine_float_func_t	combine_float_ca[PIXMAN_N_OPERATORS];
 };
 
+uint32_t
+_pixman_image_get_solid (pixman_implementation_t *imp,
+			 pixman_image_t *         image,
+                         pixman_format_code_t     format);
+
 pixman_implementation_t *
-_pixman_implementation_create (pixman_implementation_t *delegate,
+_pixman_implementation_create (pixman_implementation_t *fallback,
 			       const pixman_fast_path_t *fast_paths);
 
 void
-_pixman_implementation_combine_32 (pixman_implementation_t *imp,
-                                   pixman_op_t              op,
-                                   uint32_t *               dest,
-                                   const uint32_t *         src,
-                                   const uint32_t *         mask,
-                                   int                      width);
-void
-_pixman_implementation_combine_64 (pixman_implementation_t *imp,
-                                   pixman_op_t              op,
-                                   uint64_t *               dest,
-                                   const uint64_t *         src,
-                                   const uint64_t *         mask,
-                                   int                      width);
-void
-_pixman_implementation_combine_32_ca (pixman_implementation_t *imp,
-                                      pixman_op_t              op,
-                                      uint32_t *               dest,
-                                      const uint32_t *         src,
-                                      const uint32_t *         mask,
-                                      int                      width);
-void
-_pixman_implementation_combine_64_ca (pixman_implementation_t *imp,
-                                      pixman_op_t              op,
-                                      uint64_t *               dest,
-                                      const uint64_t *         src,
-                                      const uint64_t *         mask,
-                                      int                      width);
+_pixman_implementation_lookup_composite (pixman_implementation_t  *toplevel,
+					 pixman_op_t               op,
+					 pixman_format_code_t      src_format,
+					 uint32_t                  src_flags,
+					 pixman_format_code_t      mask_format,
+					 uint32_t                  mask_flags,
+					 pixman_format_code_t      dest_format,
+					 uint32_t                  dest_flags,
+					 pixman_implementation_t **out_imp,
+					 pixman_composite_func_t  *out_func);
+
+pixman_combine_32_func_t
+_pixman_implementation_lookup_combiner (pixman_implementation_t *imp,
+					pixman_op_t		 op,
+					pixman_bool_t		 component_alpha,
+					pixman_bool_t		 wide);
 
 pixman_bool_t
 _pixman_implementation_blt (pixman_implementation_t *imp,
@@ -495,8 +545,8 @@ _pixman_implementation_blt (pixman_implementation_t *imp,
                             int                      dst_bpp,
                             int                      src_x,
                             int                      src_y,
-                            int                      dst_x,
-                            int                      dst_y,
+                            int                      dest_x,
+                            int                      dest_y,
                             int                      width,
                             int                      height);
 
@@ -509,48 +559,108 @@ _pixman_implementation_fill (pixman_implementation_t *imp,
                              int                      y,
                              int                      width,
                              int                      height,
-                             uint32_t                 xor);
+                             uint32_t                 filler);
+
+void
+_pixman_implementation_iter_init (pixman_implementation_t       *imp,
+                                  pixman_iter_t                 *iter,
+                                  pixman_image_t                *image,
+                                  int                            x,
+                                  int                            y,
+                                  int                            width,
+                                  int                            height,
+                                  uint8_t                       *buffer,
+                                  iter_flags_t                   flags,
+                                  uint32_t                       image_flags);
 
 /* Specific implementations */
 pixman_implementation_t *
 _pixman_implementation_create_general (void);
 
 pixman_implementation_t *
-_pixman_implementation_create_fast_path (void);
+_pixman_implementation_create_fast_path (pixman_implementation_t *fallback);
+
+pixman_implementation_t *
+_pixman_implementation_create_noop (pixman_implementation_t *fallback);
 
-#ifdef USE_MMX
+#if defined USE_X86_MMX || defined USE_ARM_IWMMXT || defined USE_LOONGSON_MMI
 pixman_implementation_t *
-_pixman_implementation_create_mmx (void);
+_pixman_implementation_create_mmx (pixman_implementation_t *fallback);
 #endif
 
 #ifdef USE_SSE2
 pixman_implementation_t *
-_pixman_implementation_create_sse2 (void);
+_pixman_implementation_create_sse2 (pixman_implementation_t *fallback);
+#endif
+
+#ifdef USE_SSSE3
+pixman_implementation_t *
+_pixman_implementation_create_ssse3 (pixman_implementation_t *fallback);
 #endif
 
 #ifdef USE_ARM_SIMD
 pixman_implementation_t *
-_pixman_implementation_create_arm_simd (void);
+_pixman_implementation_create_arm_simd (pixman_implementation_t *fallback);
 #endif
 
 #ifdef USE_ARM_NEON
 pixman_implementation_t *
-_pixman_implementation_create_arm_neon (void);
+_pixman_implementation_create_arm_neon (pixman_implementation_t *fallback);
+#endif
+
+#ifdef USE_MIPS_DSPR2
+pixman_implementation_t *
+_pixman_implementation_create_mips_dspr2 (pixman_implementation_t *fallback);
 #endif
 
 #ifdef USE_VMX
 pixman_implementation_t *
-_pixman_implementation_create_vmx (void);
+_pixman_implementation_create_vmx (pixman_implementation_t *fallback);
 #endif
 
+pixman_bool_t
+_pixman_implementation_disabled (const char *name);
+
+pixman_implementation_t *
+_pixman_x86_get_implementations (pixman_implementation_t *imp);
+
+pixman_implementation_t *
+_pixman_arm_get_implementations (pixman_implementation_t *imp);
+
+pixman_implementation_t *
+_pixman_ppc_get_implementations (pixman_implementation_t *imp);
+
+pixman_implementation_t *
+_pixman_mips_get_implementations (pixman_implementation_t *imp);
+
 pixman_implementation_t *
 _pixman_choose_implementation (void);
 
+pixman_bool_t
+_pixman_disabled (const char *name);
 
 
 /*
  * Utilities
  */
+pixman_bool_t
+_pixman_compute_composite_region32 (pixman_region32_t * region,
+				    pixman_image_t *    src_image,
+				    pixman_image_t *    mask_image,
+				    pixman_image_t *    dest_image,
+				    int32_t             src_x,
+				    int32_t             src_y,
+				    int32_t             mask_x,
+				    int32_t             mask_y,
+				    int32_t             dest_x,
+				    int32_t             dest_y,
+				    int32_t             width,
+				    int32_t             height);
+uint32_t *
+_pixman_iter_get_scanline_noop (pixman_iter_t *iter, const uint32_t *mask);
+
+void
+_pixman_iter_init_bits_stride (pixman_iter_t *iter, const pixman_iter_info_t *info);
 
 /* These "formats" all have depth 0, so they
  * will never clash with any real ones
@@ -570,41 +680,66 @@ _pixman_choose_implementation (void);
 #define FAST_PATH_NO_PAD_REPEAT			(1 <<  3)
 #define FAST_PATH_NO_REFLECT_REPEAT		(1 <<  4)
 #define FAST_PATH_NO_ACCESSORS			(1 <<  5)
-#define FAST_PATH_NO_WIDE_FORMAT		(1 <<  6)
-#define FAST_PATH_COVERS_CLIP			(1 <<  7)
+#define FAST_PATH_NARROW_FORMAT			(1 <<  6)
 #define FAST_PATH_COMPONENT_ALPHA		(1 <<  8)
+#define FAST_PATH_SAMPLES_OPAQUE		(1 <<  7)
 #define FAST_PATH_UNIFIED_ALPHA			(1 <<  9)
 #define FAST_PATH_SCALE_TRANSFORM		(1 << 10)
 #define FAST_PATH_NEAREST_FILTER		(1 << 11)
-#define FAST_PATH_SIMPLE_REPEAT			(1 << 12)
+#define FAST_PATH_HAS_TRANSFORM			(1 << 12)
 #define FAST_PATH_IS_OPAQUE			(1 << 13)
-#define FAST_PATH_NEEDS_WORKAROUND		(1 << 14)
+#define FAST_PATH_NO_NORMAL_REPEAT		(1 << 14)
 #define FAST_PATH_NO_NONE_REPEAT		(1 << 15)
-#define FAST_PATH_SAMPLES_COVER_CLIP		(1 << 16)
-#define FAST_PATH_16BIT_SAFE			(1 << 17)
-#define FAST_PATH_X_UNIT_POSITIVE		(1 << 18)
+#define FAST_PATH_X_UNIT_POSITIVE		(1 << 16)
+#define FAST_PATH_AFFINE_TRANSFORM		(1 << 17)
+#define FAST_PATH_Y_UNIT_ZERO			(1 << 18)
+#define FAST_PATH_BILINEAR_FILTER		(1 << 19)
+#define FAST_PATH_ROTATE_90_TRANSFORM		(1 << 20)
+#define FAST_PATH_ROTATE_180_TRANSFORM		(1 << 21)
+#define FAST_PATH_ROTATE_270_TRANSFORM		(1 << 22)
+#define FAST_PATH_SAMPLES_COVER_CLIP_NEAREST	(1 << 23)
+#define FAST_PATH_SAMPLES_COVER_CLIP_BILINEAR	(1 << 24)
+#define FAST_PATH_BITS_IMAGE			(1 << 25)
+#define FAST_PATH_SEPARABLE_CONVOLUTION_FILTER  (1 << 26)
+
+#define FAST_PATH_PAD_REPEAT						\
+    (FAST_PATH_NO_NONE_REPEAT		|				\
+     FAST_PATH_NO_NORMAL_REPEAT		|				\
+     FAST_PATH_NO_REFLECT_REPEAT)
+
+#define FAST_PATH_NORMAL_REPEAT						\
+    (FAST_PATH_NO_NONE_REPEAT		|				\
+     FAST_PATH_NO_PAD_REPEAT		|				\
+     FAST_PATH_NO_REFLECT_REPEAT)
 
-#define _FAST_PATH_STANDARD_FLAGS					\
-    (FAST_PATH_ID_TRANSFORM		|				\
-     FAST_PATH_NO_ALPHA_MAP		|				\
-     FAST_PATH_NO_CONVOLUTION_FILTER	|				\
+#define FAST_PATH_NONE_REPEAT						\
+    (FAST_PATH_NO_NORMAL_REPEAT		|				\
      FAST_PATH_NO_PAD_REPEAT		|				\
-     FAST_PATH_NO_REFLECT_REPEAT	|				\
+     FAST_PATH_NO_REFLECT_REPEAT)
+
+#define FAST_PATH_REFLECT_REPEAT					\
+    (FAST_PATH_NO_NONE_REPEAT		|				\
+     FAST_PATH_NO_NORMAL_REPEAT		|				\
+     FAST_PATH_NO_PAD_REPEAT)
+
+#define FAST_PATH_STANDARD_FLAGS					\
+    (FAST_PATH_NO_CONVOLUTION_FILTER	|				\
      FAST_PATH_NO_ACCESSORS		|				\
-     FAST_PATH_NO_WIDE_FORMAT		|				\
-     FAST_PATH_COVERS_CLIP)
-
-#define FAST_PATH_STD_SRC_FLAGS						\
-    _FAST_PATH_STANDARD_FLAGS
-#define FAST_PATH_STD_MASK_U_FLAGS					\
-    (_FAST_PATH_STANDARD_FLAGS		|				\
-     FAST_PATH_UNIFIED_ALPHA)
-#define FAST_PATH_STD_MASK_CA_FLAGS					\
-    (_FAST_PATH_STANDARD_FLAGS		|				\
-     FAST_PATH_COMPONENT_ALPHA)
+     FAST_PATH_NO_ALPHA_MAP		|				\
+     FAST_PATH_NARROW_FORMAT)
+
 #define FAST_PATH_STD_DEST_FLAGS					\
     (FAST_PATH_NO_ACCESSORS		|				\
-     FAST_PATH_NO_WIDE_FORMAT)
+     FAST_PATH_NO_ALPHA_MAP		|				\
+     FAST_PATH_NARROW_FORMAT)
+
+#define SOURCE_FLAGS(format)						\
+    (FAST_PATH_STANDARD_FLAGS |						\
+     ((PIXMAN_ ## format == PIXMAN_solid) ?				\
+      0 : (FAST_PATH_SAMPLES_COVER_CLIP_NEAREST | FAST_PATH_NEAREST_FILTER | FAST_PATH_ID_TRANSFORM)))
+
+#define MASK_FLAGS(format, extra)					\
+    ((PIXMAN_ ## format == PIXMAN_null) ? 0 : (SOURCE_FLAGS (format) | extra))
 
 #define FAST_PATH(op, src, src_flags, mask, mask_flags, dest, dest_flags, func) \
     PIXMAN_OP_ ## op,							\
@@ -618,19 +753,37 @@ _pixman_choose_implementation (void);
 
 #define PIXMAN_STD_FAST_PATH(op, src, mask, dest, func)			\
     { FAST_PATH (							\
-	  op,								\
-	  src, FAST_PATH_STD_SRC_FLAGS,					\
-	  mask, (PIXMAN_ ## mask) ? FAST_PATH_STD_MASK_U_FLAGS : 0,	\
-	  dest, FAST_PATH_STD_DEST_FLAGS,				\
-	  func) }
+	    op,								\
+	    src,  SOURCE_FLAGS (src),					\
+	    mask, MASK_FLAGS (mask, FAST_PATH_UNIFIED_ALPHA),		\
+	    dest, FAST_PATH_STD_DEST_FLAGS,				\
+	    func) }
 
 #define PIXMAN_STD_FAST_PATH_CA(op, src, mask, dest, func)		\
     { FAST_PATH (							\
-	  op,								\
-	  src, FAST_PATH_STD_SRC_FLAGS,					\
-	  mask, FAST_PATH_STD_MASK_CA_FLAGS,				\
-	  dest, FAST_PATH_STD_DEST_FLAGS,				\
-	  func) }
+	    op,								\
+	    src,  SOURCE_FLAGS (src),					\
+	    mask, MASK_FLAGS (mask, FAST_PATH_COMPONENT_ALPHA),		\
+	    dest, FAST_PATH_STD_DEST_FLAGS,				\
+	    func) }
+
+extern pixman_implementation_t *global_implementation;
+
+static force_inline pixman_implementation_t *
+get_implementation (void)
+{
+#ifndef TOOLCHAIN_SUPPORTS_ATTRIBUTE_CONSTRUCTOR
+    if (!global_implementation)
+	global_implementation = _pixman_choose_implementation ();
+#endif
+    return global_implementation;
+}
+
+/* This function is exported for the sake of the test suite and not part
+ * of the ABI.
+ */
+PIXMAN_EXPORT pixman_implementation_t *
+_pixman_internal_only_get_implementation (void);
 
 /* Memory allocation helpers */
 void *
@@ -639,24 +792,29 @@ pixman_malloc_ab (unsigned int n, unsigned int b);
 void *
 pixman_malloc_abc (unsigned int a, unsigned int b, unsigned int c);
 
+void *
+pixman_malloc_ab_plus_c (unsigned int a, unsigned int b, unsigned int c);
+
+pixman_bool_t
+_pixman_multiply_overflows_size (size_t a, size_t b);
+
 pixman_bool_t
-pixman_multiply_overflows_int (unsigned int a, unsigned int b);
+_pixman_multiply_overflows_int (unsigned int a, unsigned int b);
 
 pixman_bool_t
-pixman_addition_overflows_int (unsigned int a, unsigned int b);
+_pixman_addition_overflows_int (unsigned int a, unsigned int b);
 
 /* Compositing utilities */
 void
-pixman_expand (uint64_t *           dst,
-               const uint32_t *     src,
-               pixman_format_code_t format,
-               int                  width);
+pixman_expand_to_float (argb_t               *dst,
+			const uint32_t       *src,
+			pixman_format_code_t  format,
+			int                   width);
 
 void
-pixman_contract (uint32_t *      dst,
-                 const uint64_t *src,
-                 int             width);
-
+pixman_contract_from_float (uint32_t     *dst,
+			    const argb_t *src,
+			    int           width);
 
 /* Region Helpers */
 pixman_bool_t
@@ -667,6 +825,50 @@ pixman_bool_t
 pixman_region16_copy_from_region32 (pixman_region16_t *dst,
                                     pixman_region32_t *src);
 
+/* Doubly linked lists */
+typedef struct pixman_link_t pixman_link_t;
+struct pixman_link_t
+{
+    pixman_link_t *next;
+    pixman_link_t *prev;
+};
+
+typedef struct pixman_list_t pixman_list_t;
+struct pixman_list_t
+{
+    pixman_link_t *head;
+    pixman_link_t *tail;
+};
+
+static force_inline void
+pixman_list_init (pixman_list_t *list)
+{
+    list->head = (pixman_link_t *)list;
+    list->tail = (pixman_link_t *)list;
+}
+
+static force_inline void
+pixman_list_prepend (pixman_list_t *list, pixman_link_t *link)
+{
+    link->next = list->head;
+    link->prev = (pixman_link_t *)list;
+    list->head->prev = link;
+    list->head = link;
+}
+
+static force_inline void
+pixman_list_unlink (pixman_link_t *link)
+{
+    link->prev->next = link->next;
+    link->next->prev = link->prev;
+}
+
+static force_inline void
+pixman_list_move_to_front (pixman_list_t *list, pixman_link_t *link)
+{
+    pixman_list_unlink (link);
+    pixman_list_prepend (list, link);
+}
 
 /* Misc macros */
 
@@ -696,29 +898,62 @@ pixman_region16_copy_from_region32 (pixman_region16_t *dst,
 
 #define CLIP(v, low, high) ((v) < (low) ? (low) : ((v) > (high) ? (high) : (v)))
 
+#define FLOAT_IS_ZERO(f)     (-FLT_MIN < (f) && (f) < FLT_MIN)
+
 /* Conversion between 8888 and 0565 */
 
-#define CONVERT_8888_TO_0565(s)						\
-    ((((s) >> 3) & 0x001f) |						\
-     (((s) >> 5) & 0x07e0) |						\
-     (((s) >> 8) & 0xf800))
+static force_inline uint16_t
+convert_8888_to_0565 (uint32_t s)
+{
+    /* The following code can be compiled into just 4 instructions on ARM */
+    uint32_t a, b;
+    a = (s >> 3) & 0x1F001F;
+    b = s & 0xFC00;
+    a |= a >> 5;
+    a |= b >> 5;
+    return (uint16_t)a;
+}
 
-#define CONVERT_0565_TO_0888(s)						\
-    (((((s) << 3) & 0xf8) | (((s) >> 2) & 0x7)) |			\
-     ((((s) << 5) & 0xfc00) | (((s) >> 1) & 0x300)) |			\
-     ((((s) << 8) & 0xf80000) | (((s) << 3) & 0x70000)))
+static force_inline uint32_t
+convert_0565_to_0888 (uint16_t s)
+{
+    return (((((s) << 3) & 0xf8) | (((s) >> 2) & 0x7)) |
+            ((((s) << 5) & 0xfc00) | (((s) >> 1) & 0x300)) |
+            ((((s) << 8) & 0xf80000) | (((s) << 3) & 0x70000)));
+}
 
-#define CONVERT_0565_TO_8888(s) (CONVERT_0565_TO_0888(s) | 0xff000000)
+static force_inline uint32_t
+convert_0565_to_8888 (uint16_t s)
+{
+    return convert_0565_to_0888 (s) | 0xff000000;
+}
 
 /* Trivial versions that are useful in macros */
-#define CONVERT_8888_TO_8888(s) (s)
-#define CONVERT_0565_TO_0565(s) (s)
+
+static force_inline uint32_t
+convert_8888_to_8888 (uint32_t s)
+{
+    return s;
+}
+
+static force_inline uint32_t
+convert_x888_to_8888 (uint32_t s)
+{
+    return s | 0xff000000;
+}
+
+static force_inline uint16_t
+convert_0565_to_0565 (uint16_t s)
+{
+    return s;
+}
 
 #define PIXMAN_FORMAT_IS_WIDE(f)					\
     (PIXMAN_FORMAT_A (f) > 8 ||						\
      PIXMAN_FORMAT_R (f) > 8 ||						\
      PIXMAN_FORMAT_G (f) > 8 ||						\
-     PIXMAN_FORMAT_B (f) > 8)
+     PIXMAN_FORMAT_B (f) > 8 ||						\
+     PIXMAN_FORMAT_TYPE (f) == PIXMAN_TYPE_ARGB_SRGB)
 
 #ifdef WORDS_BIGENDIAN
 #   define SCREEN_SHIFT_LEFT(x,n)	((x) << (n))
@@ -728,12 +963,61 @@ pixman_region16_copy_from_region32 (pixman_region16_t *dst,
 #   define SCREEN_SHIFT_RIGHT(x,n)	((x) << (n))
 #endif
 
+static force_inline uint32_t
+unorm_to_unorm (uint32_t val, int from_bits, int to_bits)
+{
+    uint32_t result;
+
+    if (from_bits == 0)
+	return 0;
+
+    /* Delete any extra bits */
+    val &= ((1 << from_bits) - 1);
+
+    if (from_bits >= to_bits)
+	return val >> (from_bits - to_bits);
+
+    /* Start out with the high bit of val in the high bit of result. */
+    result = val << (to_bits - from_bits);
+
+    /* Copy the bits in result, doubling the number of bits each time, until
+     * we fill all to_bits. Unrolled manually because from_bits and to_bits
+     * are usually known statically, so the compiler can turn all of this
+     * into a few shifts.
+     */
+#define REPLICATE()							\
+    do									\
+    {									\
+	if (from_bits < to_bits)					\
+	{								\
+	    result |= result >> from_bits;				\
+									\
+	    from_bits *= 2;						\
+	}								\
+    }									\
+    while (0)
+
+    REPLICATE();
+    REPLICATE();
+    REPLICATE();
+    REPLICATE();
+    REPLICATE();
+
+    return result;
+}
+
+uint16_t pixman_float_to_unorm (float f, int n_bits);
+float pixman_unorm_to_float (uint16_t u, int n_bits);
+
 /*
  * Various debugging code
  */
 
 #undef DEBUG
 
+#define COMPILE_TIME_ASSERT(x)						\
+    do { typedef int compile_time_assertion [(x)?1:-1]; } while (0)
+
 /* Turn on debugging depending on what type of release this is
  */
 #if (((PIXMAN_VERSION_MICRO % 2) == 0) && ((PIXMAN_VERSION_MINOR % 2) == 1))
@@ -751,15 +1035,13 @@ pixman_region16_copy_from_region32 (pixman_region16_t *dst,
 
 #endif
 
-#ifdef DEBUG
-
 void
 _pixman_log_error (const char *function, const char *message);
 
 #define return_if_fail(expr)                                            \
     do                                                                  \
     {                                                                   \
-	if (!(expr))							\
+	if (unlikely (!(expr)))                                         \
 	{								\
 	    _pixman_log_error (FUNC, "The expression " # expr " was false"); \
 	    return;							\
@@ -770,7 +1052,7 @@ _pixman_log_error (const char *function, const char *message);
 #define return_val_if_fail(expr, retval)                                \
     do                                                                  \
     {                                                                   \
-	if (!(expr))                                                    \
+	if (unlikely (!(expr)))                                         \
 	{								\
 	    _pixman_log_error (FUNC, "The expression " # expr " was false"); \
 	    return (retval);						\
@@ -781,38 +1063,31 @@ _pixman_log_error (const char *function, const char *message);
 #define critical_if_fail(expr)						\
     do									\
     {									\
-	if (!(expr))							\
+	if (unlikely (!(expr)))                                         \
 	    _pixman_log_error (FUNC, "The expression " # expr " was false"); \
     }									\
     while (0)
 
+/*
+ * Matrix
+ */
 
-#else
-
-#define _pixman_log_error(f,m) do { } while (0)				\
+typedef struct { pixman_fixed_48_16_t v[3]; } pixman_vector_48_16_t;
 
-#define return_if_fail(expr)						\
-    do                                                                  \
-    {                                                                   \
-	if (!(expr))							\
-	    return;							\
-    }                                                                   \
-    while (0)
+pixman_bool_t
+pixman_transform_point_31_16 (const pixman_transform_t    *t,
+                              const pixman_vector_48_16_t *v,
+                              pixman_vector_48_16_t       *result);
 
-#define return_val_if_fail(expr, retval)                                \
-    do                                                                  \
-    {                                                                   \
-	if (!(expr))							\
-	    return (retval);						\
-    }                                                                   \
-    while (0)
+void
+pixman_transform_point_31_16_3d (const pixman_transform_t    *t,
+                                 const pixman_vector_48_16_t *v,
+                                 pixman_vector_48_16_t       *result);
 
-#define critical_if_fail(expr)						\
-    do									\
-    {									\
-    }									\
-    while (0)
-#endif
+void
+pixman_transform_point_31_16_affine (const pixman_transform_t    *t,
+                                     const pixman_vector_48_16_t *v,
+                                     pixman_vector_48_16_t       *result);
 
 /*
  * Timers
@@ -823,10 +1098,11 @@ _pixman_log_error (const char *function, const char *message);
 static inline uint64_t
 oil_profile_stamp_rdtsc (void)
 {
-    uint64_t ts;
+    uint32_t hi, lo;
+
+    __asm__ __volatile__ ("rdtsc\n" : "=a" (lo), "=d" (hi));
 
-    __asm__ __volatile__ ("rdtsc\n" : "=A" (ts));
-    return ts;
+    return lo | (((uint64_t)hi) << 32);
 }
 
 #define OIL_STAMP oil_profile_stamp_rdtsc
@@ -865,6 +1141,13 @@ void pixman_timer_register (pixman_timer_t *timer);
     timer ## tname.total += OIL_STAMP () - begin ## tname;		\
     }
 
+#else
+
+#define TIMER_BEGIN(tname)
+#define TIMER_END(tname)
+
 #endif /* PIXMAN_TIMERS */
 
+#endif /* __ASSEMBLER__ */
+
 #endif /* PIXMAN_PRIVATE_H */
diff --git a/pixman/pixman/pixman-radial-gradient.c b/pixman/pixman/pixman-radial-gradient.c
index 022157b9b..6a217963d 100644
--- a/pixman/pixman/pixman-radial-gradient.c
+++ b/pixman/pixman/pixman-radial-gradient.c
@@ -1,3 +1,4 @@
+/* -*- Mode: c; c-basic-offset: 4; tab-width: 8; indent-tabs-mode: t; -*- */
 /*
  *
  * Copyright © 2000 Keith Packard, member of The XFree86 Project, Inc.
@@ -33,296 +34,395 @@
 #include <math.h>
 #include "pixman-private.h"
 
-static void
-radial_gradient_get_scanline_32 (pixman_image_t *image,
-                                 int             x,
-                                 int             y,
-                                 int             width,
-                                 uint32_t *      buffer,
-                                 const uint32_t *mask,
-                                 uint32_t        mask_bits)
+static inline pixman_fixed_32_32_t
+dot (pixman_fixed_48_16_t x1,
+     pixman_fixed_48_16_t y1,
+     pixman_fixed_48_16_t z1,
+     pixman_fixed_48_16_t x2,
+     pixman_fixed_48_16_t y2,
+     pixman_fixed_48_16_t z2)
 {
     /*
-     * In the radial gradient problem we are given two circles (c₁,r₁) and
-     * (c₂,r₂) that define the gradient itself. Then, for any point p, we
-     * must compute the value(s) of t within [0.0, 1.0] representing the
-     * circle(s) that would color the point.
-     *
-     * There are potentially two values of t since the point p can be
-     * colored by both sides of the circle, (which happens whenever one
-     * circle is not entirely contained within the other).
-     *
-     * If we solve for a value of t that is outside of [0.0, 1.0] then we
-     * use the extend mode (NONE, REPEAT, REFLECT, or PAD) to map to a
-     * value within [0.0, 1.0].
+     * Exact computation, assuming that the input values can
+     * be represented as pixman_fixed_16_16_t
+     */
+    return x1 * x2 + y1 * y2 + z1 * z2;
+}
+
+static inline double
+fdot (double x1,
+      double y1,
+      double z1,
+      double x2,
+      double y2,
+      double z2)
+{
+    /*
+     * Error can be unbound in some special cases.
+     * Using clever dot product algorithms (for example compensated
+     * dot product) would improve this but make the code much less
+     * obvious
+     */
+    return x1 * x2 + y1 * y2 + z1 * z2;
+}
+
+static uint32_t
+radial_compute_color (double                    a,
+		      double                    b,
+		      double                    c,
+		      double                    inva,
+		      double                    dr,
+		      double                    mindr,
+		      pixman_gradient_walker_t *walker,
+		      pixman_repeat_t           repeat)
+{
+    /*
+     * In this function error propagation can lead to bad results:
+     *  - discr can have an unbound error (if b*b-a*c is very small),
+     *    potentially making it the opposite sign of what it should have been
+     *    (thus clearing a pixel that would have been colored or vice-versa)
+     *    or propagating the error to sqrtdiscr;
+     *    if discr has the wrong sign or b is very small, this can lead to bad
+     *    results
      *
-     * Here is an illustration of the problem:
+     *  - the algorithm used to compute the solutions of the quadratic
+     *    equation is not numerically stable (but saves one division compared
+     *    to the numerically stable one);
+     *    this can be a problem if a*c is much smaller than b*b
      *
-     *              p₂
-     *           p  •
-     *           •   ╲
-     *        ·       ╲r₂
-     *  p₁ ·           ╲
-     *  •              θ╲
-     *   ╲             ╌╌•
-     *    ╲r₁        ·   c₂
-     *    θ╲    ·
-     *    ╌╌•
-     *      c₁
+     *  - the above problems are worse if a is small (as inva becomes bigger)
+     */
+    double discr;
+
+    if (a == 0)
+    {
+	double t;
+
+	if (b == 0)
+	    return 0;
+
+	t = pixman_fixed_1 / 2 * c / b;
+	if (repeat == PIXMAN_REPEAT_NONE)
+	{
+	    if (0 <= t && t <= pixman_fixed_1)
+		return _pixman_gradient_walker_pixel (walker, t);
+	}
+	else
+	{
+	    if (t * dr >= mindr)
+		return _pixman_gradient_walker_pixel (walker, t);
+	}
+
+	return 0;
+    }
+
+    discr = fdot (b, a, 0, b, -c, 0);
+    if (discr >= 0)
+    {
+	double sqrtdiscr, t0, t1;
+
+	sqrtdiscr = sqrt (discr);
+	t0 = (b + sqrtdiscr) * inva;
+	t1 = (b - sqrtdiscr) * inva;
+
+	/*
+	 * The root that must be used is the biggest one that belongs
+	 * to the valid range ([0,1] for PIXMAN_REPEAT_NONE, any
+	 * solution that results in a positive radius otherwise).
+	 *
+	 * If a > 0, t0 is the biggest solution, so if it is valid, it
+	 * is the correct result.
+	 *
+	 * If a < 0, only one of the solutions can be valid, so the
+	 * order in which they are tested is not important.
+	 */
+	if (repeat == PIXMAN_REPEAT_NONE)
+	{
+	    if (0 <= t0 && t0 <= pixman_fixed_1)
+		return _pixman_gradient_walker_pixel (walker, t0);
+	    else if (0 <= t1 && t1 <= pixman_fixed_1)
+		return _pixman_gradient_walker_pixel (walker, t1);
+	}
+	else
+	{
+	    if (t0 * dr >= mindr)
+		return _pixman_gradient_walker_pixel (walker, t0);
+	    else if (t1 * dr >= mindr)
+		return _pixman_gradient_walker_pixel (walker, t1);
+	}
+    }
+
+    return 0;
+}
+
+static uint32_t *
+radial_get_scanline_narrow (pixman_iter_t *iter, const uint32_t *mask)
+{
+    /*
+     * Implementation of radial gradients following the PDF specification.
+     * See section 8.7.4.5.4 Type 3 (Radial) Shadings of the PDF Reference
+     * Manual (PDF 32000-1:2008 at the time of this writing).
      *
-     * Given (c₁,r₁), (c₂,r₂) and p, we must find an angle θ such that two
-     * points p₁ and p₂ on the two circles are collinear with p. Then, the
-     * desired value of t is the ratio of the length of p₁p to the length
-     * of p₁p₂.
+     * In the radial gradient problem we are given two circles (c₁,r₁) and
+     * (c₂,r₂) that define the gradient itself.
      *
-     * So, we have six unknown values: (p₁x, p₁y), (p₂x, p₂y), θ and t.
-     * We can also write six equations that constrain the problem:
+     * Mathematically the gradient can be defined as the family of circles
      *
-     * Point p₁ is a distance r₁ from c₁ at an angle of θ:
+     *     ((1-t)·c₁ + t·(c₂), (1-t)·r₁ + t·r₂)
      *
-     *	1. p₁x = c₁x + r₁·cos θ
-     *	2. p₁y = c₁y + r₁·sin θ
+     * excluding those circles whose radius would be < 0. When a point
+     * belongs to more than one circle, the one with a bigger t is the only
+     * one that contributes to its color. When a point does not belong
+     * to any of the circles, it is transparent black, i.e. RGBA (0, 0, 0, 0).
+     * Further limitations on the range of values for t are imposed when
+     * the gradient is not repeated, namely t must belong to [0,1].
      *
-     * Point p₂ is a distance r₂ from c₂ at an angle of θ:
+     * The graphical result is the same as drawing the valid (radius > 0)
+     * circles with increasing t in [-inf, +inf] (or in [0,1] if the gradient
+     * is not repeated) using SOURCE operator composition.
      *
-     *	3. p₂x = c₂x + r2·cos θ
-     *	4. p₂y = c₂y + r2·sin θ
+     * It looks like a cone pointing towards the viewer if the ending circle
+     * is smaller than the starting one, a cone pointing inside the page if
+     * the starting circle is the smaller one and like a cylinder if they
+     * have the same radius.
      *
-     * Point p lies at a fraction t along the line segment p₁p₂:
+     * What we actually do is, given the point whose color we are interested
+     * in, compute the t values for that point, solving for t in:
      *
-     *	5. px = t·p₂x + (1-t)·p₁x
-     *	6. py = t·p₂y + (1-t)·p₁y
+     *     length((1-t)·c₁ + t·(c₂) - p) = (1-t)·r₁ + t·r₂
      *
-     * To solve, first subtitute 1-4 into 5 and 6:
+     * Let's rewrite it in a simpler way, by defining some auxiliary
+     * variables:
      *
-     * px = t·(c₂x + r₂·cos θ) + (1-t)·(c₁x + r₁·cos θ)
-     * py = t·(c₂y + r₂·sin θ) + (1-t)·(c₁y + r₁·sin θ)
+     *     cd = c₂ - c₁
+     *     pd = p - c₁
+     *     dr = r₂ - r₁
+     *     length(t·cd - pd) = r₁ + t·dr
      *
-     * Then solve each for cos θ and sin θ expressed as a function of t:
+     * which actually means
      *
-     * cos θ = (-(c₂x - c₁x)·t + (px - c₁x)) / ((r₂-r₁)·t + r₁)
-     * sin θ = (-(c₂y - c₁y)·t + (py - c₁y)) / ((r₂-r₁)·t + r₁)
+     *     hypot(t·cdx - pdx, t·cdy - pdy) = r₁ + t·dr
      *
-     * To simplify this a bit, we define new variables for several of the
-     * common terms as shown below:
+     * or
      *
-     *              p₂
-     *           p  •
-     *           •   ╲
-     *        ·  ┆    ╲r₂
-     *  p₁ ·     ┆     ╲
-     *  •     pdy┆      ╲
-     *   ╲       ┆       •c₂
-     *    ╲r₁    ┆   ·   ┆
-     *     ╲    ·┆       ┆cdy
-     *      •╌╌╌╌┴╌╌╌╌╌╌╌┘
-     *    c₁  pdx   cdx
+     *     ⎷((t·cdx - pdx)² + (t·cdy - pdy)²) = r₁ + t·dr.
      *
-     * cdx = (c₂x - c₁x)
-     * cdy = (c₂y - c₁y)
-     *  dr =  r₂-r₁
-     * pdx =  px - c₁x
-     * pdy =  py - c₁y
+     * If we impose (as stated earlier) that r₁ + t·dr >= 0, it becomes:
      *
-     * Note that cdx, cdy, and dr do not depend on point p at all, so can
-     * be pre-computed for the entire gradient. The simplifed equations
-     * are now:
+     *     (t·cdx - pdx)² + (t·cdy - pdy)² = (r₁ + t·dr)²
      *
-     * cos θ = (-cdx·t + pdx) / (dr·t + r₁)
-     * sin θ = (-cdy·t + pdy) / (dr·t + r₁)
+     * where we can actually expand the squares and solve for t:
      *
-     * Finally, to get a single function of t and eliminate the last
-     * unknown θ, we use the identity sin²θ + cos²θ = 1. First, square
-     * each equation, (we knew a quadratic was coming since it must be
-     * possible to obtain two solutions in some cases):
+     *     t²cdx² - 2t·cdx·pdx + pdx² + t²cdy² - 2t·cdy·pdy + pdy² =
+     *       = r₁² + 2·r₁·t·dr + t²·dr²
      *
-     * cos²θ = (cdx²t² - 2·cdx·pdx·t + pdx²) / (dr²·t² + 2·r₁·dr·t + r₁²)
-     * sin²θ = (cdy²t² - 2·cdy·pdy·t + pdy²) / (dr²·t² + 2·r₁·dr·t + r₁²)
+     *     (cdx² + cdy² - dr²)t² - 2(cdx·pdx + cdy·pdy + r₁·dr)t +
+     *         (pdx² + pdy² - r₁²) = 0
      *
-     * Then add both together, set the result equal to 1, and express as a
-     * standard quadratic equation in t of the form At² + Bt + C = 0
+     *     A = cdx² + cdy² - dr²
+     *     B = pdx·cdx + pdy·cdy + r₁·dr
+     *     C = pdx² + pdy² - r₁²
+     *     At² - 2Bt + C = 0
      *
-     * (cdx² + cdy² - dr²)·t² - 2·(cdx·pdx + cdy·pdy + r₁·dr)·t + (pdx² + pdy² - r₁²) = 0
+     * The solutions (unless the equation degenerates because of A = 0) are:
      *
-     * In other words:
+     *     t = (B ± ⎷(B² - A·C)) / A
      *
-     * A = cdx² + cdy² - dr²
-     * B = -2·(pdx·cdx + pdy·cdy + r₁·dr)
-     * C = pdx² + pdy² - r₁²
+     * The solution we are going to prefer is the bigger one, unless the
+     * radius associated to it is negative (or it falls outside the valid t
+     * range).
      *
-     * And again, notice that A does not depend on p, so can be
-     * precomputed. From here we just use the quadratic formula to solve
-     * for t:
+     * Additional observations (useful for optimizations):
+     * A does not depend on p
      *
-     * t = (-2·B ± ⎷(B² - 4·A·C)) / 2·A
+     * A < 0 <=> one of the two circles completely contains the other one
+     *   <=> for every p, the radiuses associated with the two t solutions
+     *       have opposite sign
      */
+    pixman_image_t *image = iter->image;
+    int x = iter->x;
+    int y = iter->y;
+    int width = iter->width;
+    uint32_t *buffer = iter->buffer;
 
     gradient_t *gradient = (gradient_t *)image;
-    source_image_t *source = (source_image_t *)image;
     radial_gradient_t *radial = (radial_gradient_t *)image;
     uint32_t *end = buffer + width;
     pixman_gradient_walker_t walker;
-    pixman_bool_t affine = TRUE;
-    double cx = 1.;
-    double cy = 0.;
-    double cz = 0.;
-    double rx = x + 0.5;
-    double ry = y + 0.5;
-    double rz = 1.;
+    pixman_vector_t v, unit;
+
+    /* reference point is the center of the pixel */
+    v.vector[0] = pixman_int_to_fixed (x) + pixman_fixed_1 / 2;
+    v.vector[1] = pixman_int_to_fixed (y) + pixman_fixed_1 / 2;
+    v.vector[2] = pixman_fixed_1;
 
-    _pixman_gradient_walker_init (&walker, gradient, source->common.repeat);
+    _pixman_gradient_walker_init (&walker, gradient, image->common.repeat);
+
+    if (image->common.transform)
+    {
+	if (!pixman_transform_point_3d (image->common.transform, &v))
+	    return iter->buffer;
 
-    if (source->common.transform)
+	unit.vector[0] = image->common.transform->matrix[0][0];
+	unit.vector[1] = image->common.transform->matrix[1][0];
+	unit.vector[2] = image->common.transform->matrix[2][0];
+    }
+    else
     {
-	pixman_vector_t v;
-	/* reference point is the center of the pixel */
-	v.vector[0] = pixman_int_to_fixed (x) + pixman_fixed_1 / 2;
-	v.vector[1] = pixman_int_to_fixed (y) + pixman_fixed_1 / 2;
-	v.vector[2] = pixman_fixed_1;
-	
-	if (!pixman_transform_point_3d (source->common.transform, &v))
-	    return;
-
-	cx = source->common.transform->matrix[0][0] / 65536.;
-	cy = source->common.transform->matrix[1][0] / 65536.;
-	cz = source->common.transform->matrix[2][0] / 65536.;
-	
-	rx = v.vector[0] / 65536.;
-	ry = v.vector[1] / 65536.;
-	rz = v.vector[2] / 65536.;
-
-	affine =
-	    source->common.transform->matrix[2][0] == 0 &&
-	    v.vector[2] == pixman_fixed_1;
+	unit.vector[0] = pixman_fixed_1;
+	unit.vector[1] = 0;
+	unit.vector[2] = 0;
     }
 
-    if (affine)
+    if (unit.vector[2] == 0 && v.vector[2] == pixman_fixed_1)
     {
-	/* When computing t over a scanline, we notice that some expressions
-	 * are constant so we can compute them just once. Given:
+	/*
+	 * Given:
 	 *
-	 * t = (-2·B ± ⎷(B² - 4·A·C)) / 2·A
+	 * t = (B ± ⎷(B² - A·C)) / A
 	 *
 	 * where
 	 *
-	 * A = cdx² + cdy² - dr² [precomputed as radial->A]
-	 * B = -2·(pdx·cdx + pdy·cdy + r₁·dr)
+	 * A = cdx² + cdy² - dr²
+	 * B = pdx·cdx + pdy·cdy + r₁·dr
 	 * C = pdx² + pdy² - r₁²
+	 * det = B² - A·C
 	 *
 	 * Since we have an affine transformation, we know that (pdx, pdy)
 	 * increase linearly with each pixel,
 	 *
-	 * pdx = pdx₀ + n·cx,
-	 * pdy = pdy₀ + n·cy,
-	 *
-	 * we can then express B in terms of an linear increment along
-	 * the scanline:
+	 * pdx = pdx₀ + n·ux,
+	 * pdy = pdy₀ + n·uy,
 	 *
-	 * B = B₀ + n·cB, with
-	 * B₀ = -2·(pdx₀·cdx + pdy₀·cdy + r₁·dr) and
-	 * cB = -2·(cx·cdx + cy·cdy)
-	 *
-	 * Thus we can replace the full evaluation of B per-pixel (4 multiplies,
-	 * 2 additions) with a single addition.
+	 * we can then express B, C and det through multiple differentiation.
+	 */
+	pixman_fixed_32_32_t b, db, c, dc, ddc;
+
+	/* warning: this computation may overflow */
+	v.vector[0] -= radial->c1.x;
+	v.vector[1] -= radial->c1.y;
+
+	/*
+	 * B and C are computed and updated exactly.
+	 * If fdot was used instead of dot, in the worst case it would
+	 * lose 11 bits of precision in each of the multiplication and
+	 * summing up would zero out all the bit that were preserved,
+	 * thus making the result 0 instead of the correct one.
+	 * This would mean a worst case of unbound relative error or
+	 * about 2^10 absolute error
 	 */
-	double r1   = radial->c1.radius / 65536.;
-	double r1sq = r1 * r1;
-	double pdx  = rx - radial->c1.x / 65536.;
-	double pdy  = ry - radial->c1.y / 65536.;
-	double A = radial->A;
-	double invA = -65536. / (2. * A);
-	double A4 = -4. * A;
-	double B  = -2. * (pdx*radial->cdx + pdy*radial->cdy + r1*radial->dr);
-	double cB = -2. *  (cx*radial->cdx +  cy*radial->cdy);
-	pixman_bool_t invert = A * radial->dr < 0;
+	b = dot (v.vector[0], v.vector[1], radial->c1.radius,
+		 radial->delta.x, radial->delta.y, radial->delta.radius);
+	db = dot (unit.vector[0], unit.vector[1], 0,
+		  radial->delta.x, radial->delta.y, 0);
+
+	c = dot (v.vector[0], v.vector[1],
+		 -((pixman_fixed_48_16_t) radial->c1.radius),
+		 v.vector[0], v.vector[1], radial->c1.radius);
+	dc = dot (2 * (pixman_fixed_48_16_t) v.vector[0] + unit.vector[0],
+		  2 * (pixman_fixed_48_16_t) v.vector[1] + unit.vector[1],
+		  0,
+		  unit.vector[0], unit.vector[1], 0);
+	ddc = 2 * dot (unit.vector[0], unit.vector[1], 0,
+		       unit.vector[0], unit.vector[1], 0);
 
 	while (buffer < end)
 	{
-	    if (!mask || *mask++ & mask_bits)
+	    if (!mask || *mask++)
 	    {
-		pixman_fixed_48_16_t t;
-		double det = B * B + A4 * (pdx * pdx + pdy * pdy - r1sq);
-		if (det <= 0.)
-		    t = (pixman_fixed_48_16_t) (B * invA);
-		else if (invert)
-		    t = (pixman_fixed_48_16_t) ((B + sqrt (det)) * invA);
-		else
-		    t = (pixman_fixed_48_16_t) ((B - sqrt (det)) * invA);
-
-		*buffer = _pixman_gradient_walker_pixel (&walker, t);
+		*buffer = radial_compute_color (radial->a, b, c,
+						radial->inva,
+						radial->delta.radius,
+						radial->mindr,
+						&walker,
+						image->common.repeat);
 	    }
-	    ++buffer;
 
-	    pdx += cx;
-	    pdy += cy;
-	    B += cB;
+	    b += db;
+	    c += dc;
+	    dc += ddc;
+	    ++buffer;
 	}
     }
     else
     {
 	/* projective */
+	/* Warning:
+	 * error propagation guarantees are much looser than in the affine case
+	 */
 	while (buffer < end)
 	{
-	    if (!mask || *mask++ & mask_bits)
+	    if (!mask || *mask++)
 	    {
-		double pdx, pdy;
-		double B, C;
-		double det;
-		double c1x = radial->c1.x / 65536.0;
-		double c1y = radial->c1.y / 65536.0;
-		double r1  = radial->c1.radius / 65536.0;
-		pixman_fixed_48_16_t t;
-		double x, y;
-
-		if (rz != 0)
-		{
-		    x = rx / rz;
-		    y = ry / rz;
-		}
-		else
+		if (v.vector[2] != 0)
 		{
-		    x = y = 0.;
-		}
+		    double pdx, pdy, invv2, b, c;
 
-		pdx = x - c1x;
-		pdy = y - c1y;
+		    invv2 = 1. * pixman_fixed_1 / v.vector[2];
 
-		B = -2 * (pdx * radial->cdx +
-			  pdy * radial->cdy +
-			  r1 * radial->dr);
-		C = (pdx * pdx + pdy * pdy - r1 * r1);
+		    pdx = v.vector[0] * invv2 - radial->c1.x;
+		    /*    / pixman_fixed_1 */
 
-		det = (B * B) - (4 * radial->A * C);
-		if (det < 0.0)
-		    det = 0.0;
+		    pdy = v.vector[1] * invv2 - radial->c1.y;
+		    /*    / pixman_fixed_1 */
 
-		if (radial->A * radial->dr < 0)
-		    t = (pixman_fixed_48_16_t) ((-B - sqrt (det)) / (2.0 * radial->A) * 65536);
-		else
-		    t = (pixman_fixed_48_16_t) ((-B + sqrt (det)) / (2.0 * radial->A) * 65536);
+		    b = fdot (pdx, pdy, radial->c1.radius,
+			      radial->delta.x, radial->delta.y,
+			      radial->delta.radius);
+		    /*  / pixman_fixed_1 / pixman_fixed_1 */
 
-		*buffer = _pixman_gradient_walker_pixel (&walker, t);
+		    c = fdot (pdx, pdy, -radial->c1.radius,
+			      pdx, pdy, radial->c1.radius);
+		    /*  / pixman_fixed_1 / pixman_fixed_1 */
+
+		    *buffer = radial_compute_color (radial->a, b, c,
+						    radial->inva,
+						    radial->delta.radius,
+						    radial->mindr,
+						    &walker,
+						    image->common.repeat);
+		}
+		else
+		{
+		    *buffer = 0;
+		}
 	    }
-	    
+
 	    ++buffer;
 
-	    rx += cx;
-	    ry += cy;
-	    rz += cz;
+	    v.vector[0] += unit.vector[0];
+	    v.vector[1] += unit.vector[1];
+	    v.vector[2] += unit.vector[2];
 	}
     }
+
+    iter->y++;
+    return iter->buffer;
 }
 
-static void
-radial_gradient_property_changed (pixman_image_t *image)
+static uint32_t *
+radial_get_scanline_wide (pixman_iter_t *iter, const uint32_t *mask)
 {
-    image->common.get_scanline_32 = radial_gradient_get_scanline_32;
-    image->common.get_scanline_64 = _pixman_image_get_scanline_generic_64;
+    uint32_t *buffer = radial_get_scanline_narrow (iter, NULL);
+
+    pixman_expand_to_float (
+	(argb_t *)buffer, buffer, PIXMAN_a8r8g8b8, iter->width);
+
+    return buffer;
+}
+
+void
+_pixman_radial_gradient_iter_init (pixman_image_t *image, pixman_iter_t *iter)
+{
+    if (iter->iter_flags & ITER_NARROW)
+	iter->get_scanline = radial_get_scanline_narrow;
+    else
+	iter->get_scanline = radial_get_scanline_wide;
 }
 
 PIXMAN_EXPORT pixman_image_t *
-pixman_image_create_radial_gradient (pixman_point_fixed_t *        inner,
-                                     pixman_point_fixed_t *        outer,
+pixman_image_create_radial_gradient (const pixman_point_fixed_t *  inner,
+                                     const pixman_point_fixed_t *  outer,
                                      pixman_fixed_t                inner_radius,
                                      pixman_fixed_t                outer_radius,
                                      const pixman_gradient_stop_t *stops,
@@ -331,8 +431,6 @@ pixman_image_create_radial_gradient (pixman_point_fixed_t *        inner,
     pixman_image_t *image;
     radial_gradient_t *radial;
 
-    return_val_if_fail (n_stops >= 2, NULL);
-
     image = _pixman_image_allocate ();
 
     if (!image)
@@ -354,15 +452,20 @@ pixman_image_create_radial_gradient (pixman_point_fixed_t *        inner,
     radial->c2.x = outer->x;
     radial->c2.y = outer->y;
     radial->c2.radius = outer_radius;
-    radial->cdx = pixman_fixed_to_double (radial->c2.x - radial->c1.x);
-    radial->cdy = pixman_fixed_to_double (radial->c2.y - radial->c1.y);
-    radial->dr = pixman_fixed_to_double (radial->c2.radius - radial->c1.radius);
-    radial->A = (radial->cdx * radial->cdx +
-		 radial->cdy * radial->cdy -
-		 radial->dr  * radial->dr);
 
-    image->common.property_changed = radial_gradient_property_changed;
+    /* warning: this computations may overflow */
+    radial->delta.x = radial->c2.x - radial->c1.x;
+    radial->delta.y = radial->c2.y - radial->c1.y;
+    radial->delta.radius = radial->c2.radius - radial->c1.radius;
+
+    /* computed exactly, then cast to double -> every bit of the double
+       representation is correct (53 bits) */
+    radial->a = dot (radial->delta.x, radial->delta.y, -radial->delta.radius,
+		     radial->delta.x, radial->delta.y, radial->delta.radius);
+    if (radial->a != 0)
+	radial->inva = 1. * pixman_fixed_1 / radial->a;
+
+    radial->mindr = -1. * pixman_fixed_1 * radial->c1.radius;
 
     return image;
 }
-
diff --git a/pixman/pixman/pixman-region.c b/pixman/pixman/pixman-region.c
index a6a400581..59bc9c797 100644
--- a/pixman/pixman/pixman-region.c
+++ b/pixman/pixman/pixman-region.c
@@ -102,7 +102,11 @@
 
 static const box_type_t PREFIX (_empty_box_) = { 0, 0, 0, 0 };
 static const region_data_type_t PREFIX (_empty_data_) = { 0, 0 };
+#if defined (__llvm__) && !defined (__clang__)
+static const volatile region_data_type_t PREFIX (_broken_data_) = { 0, 0 };
+#else
 static const region_data_type_t PREFIX (_broken_data_) = { 0, 0 };
+#endif
 
 static box_type_t *pixman_region_empty_box =
     (box_type_t *)&PREFIX (_empty_box_);
@@ -198,7 +202,7 @@ PIXREGION_SZOF (size_t n)
     return size + sizeof(region_data_type_t);
 }
 
-static void *
+static region_data_type_t *
 alloc_data (size_t n)
 {
     size_t sz = PIXREGION_SZOF (n);
@@ -738,8 +742,7 @@ typedef pixman_bool_t (*overlap_proc_ptr) (region_type_t *region,
 					   box_type_t *   r2,
 					   box_type_t *   r2_end,
 					   int            y1,
-					   int            y2,
-					   int *          overlap);
+					   int            y2);
 
 static pixman_bool_t
 pixman_op (region_type_t *  new_reg,               /* Place to store result	    */
@@ -750,10 +753,10 @@ pixman_op (region_type_t *  new_reg,               /* Place to store result
 	   int              append_non1,           /* Append non-overlapping bands  
 						    * in region 1 ?
 						    */
-	   int              append_non2,           /* Append non-overlapping bands
+	   int              append_non2            /* Append non-overlapping bands
 						    * in region 2 ?
 						    */
-	   int *            overlap)
+    )
 {
     box_type_t *r1;                 /* Pointer into first region     */
     box_type_t *r2;                 /* Pointer into 2d region	     */
@@ -824,8 +827,7 @@ pixman_op (region_type_t *  new_reg,               /* Place to store result
     {
         if (!pixman_rect_alloc (new_reg, new_size))
         {
-            if (old_data)
-		free (old_data);
+            free (old_data);
             return FALSE;
 	}
     }
@@ -932,8 +934,7 @@ pixman_op (region_type_t *  new_reg,               /* Place to store result
             if (!(*overlap_func)(new_reg,
                                  r1, r1_band_end,
                                  r2, r2_band_end,
-                                 ytop, ybot,
-                                 overlap))
+                                 ytop, ybot))
 	    {
 		goto bail;
 	    }
@@ -1001,8 +1002,7 @@ pixman_op (region_type_t *  new_reg,               /* Place to store result
         APPEND_REGIONS (new_reg, r2_band_end, r2_end);
     }
 
-    if (old_data)
-	free (old_data);
+    free (old_data);
 
     if (!(numRects = new_reg->data->numRects))
     {
@@ -1023,8 +1023,7 @@ pixman_op (region_type_t *  new_reg,               /* Place to store result
     return TRUE;
 
 bail:
-    if (old_data)
-	free (old_data);
+    free (old_data);
 
     return pixman_break (new_reg);
 }
@@ -1112,8 +1111,7 @@ pixman_region_intersect_o (region_type_t *region,
                            box_type_t *   r2,
                            box_type_t *   r2_end,
                            int            y1,
-                           int            y2,
-                           int *          overlap)
+                           int            y2)
 {
     int x1;
     int x2;
@@ -1209,13 +1207,9 @@ PREFIX (_intersect) (region_type_t *     new_reg,
     else
     {
         /* General purpose intersection */
-        int overlap; /* result ignored */
 
-        if (!pixman_op (new_reg, reg1, reg2, pixman_region_intersect_o, FALSE, FALSE,
-                        &overlap))
-	{
+        if (!pixman_op (new_reg, reg1, reg2, pixman_region_intersect_o, FALSE, FALSE))
 	    return FALSE;
-	}
 	
         pixman_set_extents (new_reg);
     }
@@ -1230,9 +1224,6 @@ PREFIX (_intersect) (region_type_t *     new_reg,
         if (r->x1 <= x2)						\
 	{								\
             /* Merge with current rectangle */				\
-            if (r->x1 < x2)						\
-		*overlap = TRUE;					\
-									\
             if (x2 < r->x2)						\
 		x2 = r->x2;						\
 	}								\
@@ -1272,8 +1263,7 @@ pixman_region_union_o (region_type_t *region,
 		       box_type_t *   r2,
 		       box_type_t *   r2_end,
 		       int            y1,
-		       int            y2,
-		       int *          overlap)
+		       int            y2)
 {
     box_type_t *next_rect;
     int x1;            /* left and right side of current union */
@@ -1329,6 +1319,24 @@ pixman_region_union_o (region_type_t *region,
     return TRUE;
 }
 
+PIXMAN_EXPORT pixman_bool_t
+PREFIX(_intersect_rect) (region_type_t *dest,
+			 region_type_t *source,
+			 int x, int y,
+			 unsigned int width,
+			 unsigned int height)
+{
+    region_type_t region;
+
+    region.data = NULL;
+    region.extents.x1 = x;
+    region.extents.y1 = y;
+    region.extents.x2 = x + width;
+    region.extents.y2 = y + height;
+
+    return PREFIX(_intersect) (dest, source, &region);
+}
+
 /* Convenience function for performing union of region with a
  * single rectangle
  */
@@ -1364,8 +1372,6 @@ PREFIX (_union) (region_type_t *new_reg,
                  region_type_t *reg1,
                  region_type_t *reg2)
 {
-    int overlap; /* result ignored */
-
     /* Return TRUE if some overlap
      * between reg1, reg2
      */
@@ -1431,7 +1437,7 @@ PREFIX (_union) (region_type_t *new_reg,
 	return TRUE;
     }
 
-    if (!pixman_op (new_reg, reg1, reg2, pixman_region_union_o, TRUE, TRUE, &overlap))
+    if (!pixman_op (new_reg, reg1, reg2, pixman_region_union_o, TRUE, TRUE))
 	return FALSE;
 
     new_reg->extents.x1 = MIN (reg1->extents.x1, reg2->extents.x1);
@@ -1498,9 +1504,7 @@ quick_sort_rects (
                 r++;
                 i++;
 	    }
-
-            while (i != numRects && (r->y1 < y1 || (r->y1 == y1 && r->x1 < x1)))
-		;
+	    while (i != numRects && (r->y1 < y1 || (r->y1 == y1 && r->x1 < x1)));
 
 	    r = &(rects[j]);
             do
@@ -1561,8 +1565,7 @@ quick_sort_rects (
  */
 
 static pixman_bool_t
-validate (region_type_t * badreg,
-          int *           overlap)
+validate (region_type_t * badreg)
 {
     /* Descriptor for regions under construction  in Step 2. */
     typedef struct
@@ -1587,7 +1590,6 @@ validate (region_type_t * badreg,
     region_type_t *hreg;            /* ri[j_half].reg			    */
     pixman_bool_t ret = TRUE;
 
-    *overlap = FALSE;
     if (!badreg->data)
     {
         GOOD (badreg);
@@ -1661,9 +1663,6 @@ validate (region_type_t * badreg,
                 if (box->x1 <= ri_box->x2)
                 {
                     /* Merge it with ri_box */
-                    if (box->x1 < ri_box->x2)
-			*overlap = TRUE;
-
                     if (box->x2 > ri_box->x2)
 			ri_box->x2 = box->x2;
 		}
@@ -1767,7 +1766,7 @@ validate (region_type_t * badreg,
             reg = &ri[j].reg;
             hreg = &ri[j + half].reg;
 
-            if (!pixman_op (reg, reg, hreg, pixman_region_union_o, TRUE, TRUE, overlap))
+            if (!pixman_op (reg, reg, hreg, pixman_region_union_o, TRUE, TRUE))
 		ret = FALSE;
 
             if (hreg->extents.x1 < reg->extents.x1)
@@ -1835,8 +1834,7 @@ pixman_region_subtract_o (region_type_t * region,
 			  box_type_t *    r2,
 			  box_type_t *    r2_end,
 			  int             y1,
-			  int             y2,
-			  int *           overlap)
+			  int             y2)
 {
     box_type_t *        next_rect;
     int x1;
@@ -1860,7 +1858,7 @@ pixman_region_subtract_o (region_type_t * region,
         else if (r2->x1 <= x1)
         {
             /*
-	     * Subtrahend preceeds minuend: nuke left edge of minuend.
+	     * Subtrahend precedes minuend: nuke left edge of minuend.
 	     */
             x1 = r2->x2;
             if (x1 >= r1->x2)
@@ -1960,8 +1958,6 @@ PREFIX (_subtract) (region_type_t *reg_d,
                     region_type_t *reg_m,
                     region_type_t *reg_s)
 {
-    int overlap; /* result ignored */
-
     GOOD (reg_m);
     GOOD (reg_s);
     GOOD (reg_d);
@@ -1986,9 +1982,9 @@ PREFIX (_subtract) (region_type_t *reg_d,
     }
 
     /* Add those rectangles in region 1 that aren't in region 2,
-       do yucky substraction for overlaps, and
+       do yucky subtraction for overlaps, and
        just throw away rectangles in region 2 that aren't in region 1 */
-    if (!pixman_op (reg_d, reg_m, reg_s, pixman_region_subtract_o, TRUE, FALSE, &overlap))
+    if (!pixman_op (reg_d, reg_m, reg_s, pixman_region_subtract_o, TRUE, FALSE))
 	return FALSE;
 
     /*
@@ -2022,15 +2018,13 @@ PREFIX (_subtract) (region_type_t *reg_d,
  *
  *-----------------------------------------------------------------------
  */
-pixman_bool_t
-PIXMAN_EXPORT PREFIX (_inverse) (region_type_t *new_reg,  /* Destination region */
-                                 region_type_t *reg1,     /* Region to invert */
-                                 box_type_t *   inv_rect) /* Bounding box for inversion */
+PIXMAN_EXPORT pixman_bool_t
+PREFIX (_inverse) (region_type_t *new_reg,  /* Destination region */
+		   region_type_t *reg1,     /* Region to invert */
+		   box_type_t *   inv_rect) /* Bounding box for inversion */
 {
     region_type_t inv_reg; /* Quick and dirty region made from the
 			    * bounding box */
-    int overlap;           /* result ignored */
-
     GOOD (reg1);
     GOOD (new_reg);
     
@@ -2048,12 +2042,12 @@ PIXMAN_EXPORT PREFIX (_inverse) (region_type_t *new_reg,  /* Destination region
     }
 
     /* Add those rectangles in region 1 that aren't in region 2,
-     * do yucky substraction for overlaps, and
+     * do yucky subtraction for overlaps, and
      * just throw away rectangles in region 2 that aren't in region 1
      */
     inv_reg.extents = *inv_rect;
     inv_reg.data = (region_data_type_t *)NULL;
-    if (!pixman_op (new_reg, &inv_reg, reg1, pixman_region_subtract_o, TRUE, FALSE, &overlap))
+    if (!pixman_op (new_reg, &inv_reg, reg1, pixman_region_subtract_o, TRUE, FALSE))
 	return FALSE;
 
     /*
@@ -2068,6 +2062,40 @@ PIXMAN_EXPORT PREFIX (_inverse) (region_type_t *new_reg,  /* Destination region
     return TRUE;
 }
 
+/* In time O(log n), locate the first box whose y2 is greater than y.
+ * Return @end if no such box exists.
+ */
+static box_type_t *
+find_box_for_y (box_type_t *begin, box_type_t *end, int y)
+{
+    box_type_t *mid;
+
+    if (end == begin)
+	return end;
+
+    if (end - begin == 1)
+    {
+	if (begin->y2 > y)
+	    return begin;
+	else
+	    return end;
+    }
+
+    mid = begin + (end - begin) / 2;
+    if (mid->y2 > y)
+    {
+	/* If no box is found in [begin, mid], the function
+	 * will return @mid, which is then known to be the
+	 * correct answer.
+	 */
+	return find_box_for_y (begin, mid, y);
+    }
+    else
+    {
+	return find_box_for_y (mid, end, y);
+    }
+}
+
 /*
  *   rect_in(region, rect)
  *   This routine takes a pointer to a region and a pointer to a box
@@ -2084,10 +2112,9 @@ PIXMAN_EXPORT PREFIX (_inverse) (region_type_t *new_reg,  /* Destination region
  *   partially in the region) or is outside the region (we reached a band
  *   that doesn't overlap the box at all and part_in is false)
  */
-
-pixman_region_overlap_t
-PIXMAN_EXPORT PREFIX (_contains_rectangle) (region_type_t *  region,
-                                            box_type_t *     prect)
+PIXMAN_EXPORT pixman_region_overlap_t
+PREFIX (_contains_rectangle) (region_type_t *  region,
+			      box_type_t *     prect)
 {
     box_type_t *     pbox;
     box_type_t *     pbox_end;
@@ -2121,12 +2148,15 @@ PIXMAN_EXPORT PREFIX (_contains_rectangle) (region_type_t *  region,
 
     /* can stop when both part_out and part_in are TRUE, or we reach prect->y2 */
     for (pbox = PIXREGION_BOXPTR (region), pbox_end = pbox + numRects;
-         pbox != pbox_end;
-         pbox++)
+	 pbox != pbox_end;
+	 pbox++)
     {
-
-        if (pbox->y2 <= y)
-	    continue;   /* getting up to speed or skipping remainder of band */
+	/* getting up to speed or skipping remainder of band */
+	if (pbox->y2 <= y)
+	{
+	    if ((pbox = find_box_for_y (pbox, pbox_end, y)) == pbox_end)
+		break;
+	}
 
         if (pbox->y1 > y)
         {
@@ -2194,7 +2224,7 @@ PIXMAN_EXPORT PREFIX (_contains_rectangle) (region_type_t *  region,
 PIXMAN_EXPORT void
 PREFIX (_translate) (region_type_t *region, int x, int y)
 {
-    int x1, x2, y1, y2;
+    overflow_int_t x1, x2, y1, y2;
     int nbox;
     box_type_t * pbox;
 
@@ -2204,7 +2234,7 @@ PREFIX (_translate) (region_type_t *region, int x, int y)
     region->extents.x2 = x2 = region->extents.x2 + x;
     region->extents.y2 = y2 = region->extents.y2 + y;
     
-    if (((x1 - SHRT_MIN) | (y1 - SHRT_MIN) | (SHRT_MAX - x2) | (SHRT_MAX - y2)) >= 0)
+    if (((x1 - PIXMAN_REGION_MIN) | (y1 - PIXMAN_REGION_MIN) | (PIXMAN_REGION_MAX - x2) | (PIXMAN_REGION_MAX - y2)) >= 0)
     {
         if (region->data && (nbox = region->data->numRects))
         {
@@ -2219,7 +2249,7 @@ PREFIX (_translate) (region_type_t *region, int x, int y)
         return;
     }
 
-    if (((x2 - SHRT_MIN) | (y2 - SHRT_MIN) | (SHRT_MAX - x1) | (SHRT_MAX - y1)) <= 0)
+    if (((x2 - PIXMAN_REGION_MIN) | (y2 - PIXMAN_REGION_MIN) | (PIXMAN_REGION_MAX - x1) | (PIXMAN_REGION_MAX - y1)) <= 0)
     {
         region->extents.x2 = region->extents.x1;
         region->extents.y2 = region->extents.y1;
@@ -2228,15 +2258,15 @@ PREFIX (_translate) (region_type_t *region, int x, int y)
         return;
     }
 
-    if (x1 < SHRT_MIN)
-	region->extents.x1 = SHRT_MIN;
-    else if (x2 > SHRT_MAX)
-	region->extents.x2 = SHRT_MAX;
+    if (x1 < PIXMAN_REGION_MIN)
+	region->extents.x1 = PIXMAN_REGION_MIN;
+    else if (x2 > PIXMAN_REGION_MAX)
+	region->extents.x2 = PIXMAN_REGION_MAX;
 
-    if (y1 < SHRT_MIN)
-	region->extents.y1 = SHRT_MIN;
-    else if (y2 > SHRT_MAX)
-	region->extents.y2 = SHRT_MAX;
+    if (y1 < PIXMAN_REGION_MIN)
+	region->extents.y1 = PIXMAN_REGION_MIN;
+    else if (y2 > PIXMAN_REGION_MAX)
+	region->extents.y2 = PIXMAN_REGION_MAX;
 
     if (region->data && (nbox = region->data->numRects))
     {
@@ -2249,22 +2279,22 @@ PREFIX (_translate) (region_type_t *region, int x, int y)
             pbox_out->x2 = x2 = pbox->x2 + x;
             pbox_out->y2 = y2 = pbox->y2 + y;
 
-            if (((x2 - SHRT_MIN) | (y2 - SHRT_MIN) |
-                 (SHRT_MAX - x1) | (SHRT_MAX - y1)) <= 0)
+            if (((x2 - PIXMAN_REGION_MIN) | (y2 - PIXMAN_REGION_MIN) |
+                 (PIXMAN_REGION_MAX - x1) | (PIXMAN_REGION_MAX - y1)) <= 0)
             {
                 region->data->numRects--;
                 continue;
 	    }
 
-            if (x1 < SHRT_MIN)
-		pbox_out->x1 = SHRT_MIN;
-            else if (x2 > SHRT_MAX)
-		pbox_out->x2 = SHRT_MAX;
+            if (x1 < PIXMAN_REGION_MIN)
+		pbox_out->x1 = PIXMAN_REGION_MIN;
+            else if (x2 > PIXMAN_REGION_MAX)
+		pbox_out->x2 = PIXMAN_REGION_MAX;
 
-            if (y1 < SHRT_MIN)
-		pbox_out->y1 = SHRT_MIN;
-            else if (y2 > SHRT_MAX)
-		pbox_out->y2 = SHRT_MAX;
+            if (y1 < PIXMAN_REGION_MIN)
+		pbox_out->y1 = PIXMAN_REGION_MIN;
+            else if (y2 > PIXMAN_REGION_MAX)
+		pbox_out->y2 = PIXMAN_REGION_MAX;
 
             pbox_out++;
 	}
@@ -2301,6 +2331,16 @@ PREFIX (_reset) (region_type_t *region, box_type_t *box)
     region->data = NULL;
 }
 
+PIXMAN_EXPORT void
+PREFIX (_clear) (region_type_t *region)
+{
+    GOOD (region);
+    FREE_DATA (region);
+
+    region->extents = *pixman_region_empty_box;
+    region->data = pixman_region_empty_data;
+}
+
 /* box is "return" value */
 PIXMAN_EXPORT int
 PREFIX (_contains_point) (region_type_t * region,
@@ -2324,13 +2364,13 @@ PREFIX (_contains_point) (region_type_t * region,
         return(TRUE);
     }
 
-    for (pbox = PIXREGION_BOXPTR (region), pbox_end = pbox + numRects;
-	 pbox != pbox_end;
-	 pbox++)
-    {
-        if (y >= pbox->y2)
-	    continue;           /* not there yet */
+    pbox = PIXREGION_BOXPTR (region);
+    pbox_end = pbox + numRects;
 
+    pbox = find_box_for_y (pbox, pbox_end, y);
+
+    for (;pbox != pbox_end; pbox++)
+    {
         if ((y < pbox->y1) || (x < pbox->x1))
 	    break;              /* missed it */
 
@@ -2510,7 +2550,7 @@ PREFIX (_init_rects) (region_type_t *region,
     /* Validate */
     region->extents.x1 = region->extents.x2 = 0;
 
-    return validate (region, &i);
+    return validate (region);
 }
 
 #define READ(_ptr) (*(_ptr))
@@ -2527,8 +2567,7 @@ bitmap_addrect (region_type_t *reg,
 	   ((r-1)->y1 == ry1) && ((r-1)->y2 == ry2) &&
 	   ((r-1)->x1 <= rx1) && ((r-1)->x2 >= rx2))))
     {
-	if (!reg->data ||
-	    reg->data->numRects == reg->data->size)
+	if (reg->data->numRects == reg->data->size)
 	{
 	    if (!pixman_rect_alloc (reg, 1))
 		return NULL;
@@ -2572,6 +2611,8 @@ PREFIX (_init_from_image) (region_type_t *region,
 
     PREFIX(_init) (region);
 
+    critical_if_fail (region->data);
+
     return_if_fail (image->type == BITS);
     return_if_fail (image->bits.format == PIXMAN_a1);
 
diff --git a/pixman/pixman/pixman-region16.c b/pixman/pixman/pixman-region16.c
index 46f5e26ea..d88d3380f 100644
--- a/pixman/pixman/pixman-region16.c
+++ b/pixman/pixman/pixman-region16.c
@@ -35,6 +35,7 @@
 typedef pixman_box16_t		box_type_t;
 typedef pixman_region16_data_t	region_data_type_t;
 typedef pixman_region16_t	region_type_t;
+typedef int32_t                 overflow_int_t;
 
 typedef struct {
     int x, y;
@@ -42,6 +43,9 @@ typedef struct {
 
 #define PREFIX(x) pixman_region##x
 
+#define PIXMAN_REGION_MAX INT16_MAX
+#define PIXMAN_REGION_MIN INT16_MIN
+
 #include "pixman-region.c"
 
 /* This function exists only to make it possible to preserve the X ABI -
diff --git a/pixman/pixman/pixman-region32.c b/pixman/pixman/pixman-region32.c
index aeee86cf9..abd6b1a93 100644
--- a/pixman/pixman/pixman-region32.c
+++ b/pixman/pixman/pixman-region32.c
@@ -33,6 +33,7 @@
 typedef pixman_box32_t		box_type_t;
 typedef pixman_region32_data_t	region_data_type_t;
 typedef pixman_region32_t	region_type_t;
+typedef int64_t                 overflow_int_t;
 
 typedef struct {
     int x, y;
@@ -40,4 +41,7 @@ typedef struct {
 
 #define PREFIX(x) pixman_region32##x
 
+#define PIXMAN_REGION_MAX INT32_MAX
+#define PIXMAN_REGION_MIN INT32_MIN
+
 #include "pixman-region.c"
diff --git a/pixman/pixman/pixman-solid-fill.c b/pixman/pixman/pixman-solid-fill.c
index 48c999a0e..5f9fef630 100644
--- a/pixman/pixman/pixman-solid-fill.c
+++ b/pixman/pixman/pixman-solid-fill.c
@@ -26,58 +26,6 @@
 #endif
 #include "pixman-private.h"
 
-static void
-solid_fill_get_scanline_32 (pixman_image_t *image,
-                            int             x,
-                            int             y,
-                            int             width,
-                            uint32_t *      buffer,
-                            const uint32_t *mask,
-                            uint32_t        mask_bits)
-{
-    uint32_t *end = buffer + width;
-    uint32_t color = image->solid.color_32;
-
-    while (buffer < end)
-	*(buffer++) = color;
-
-    return;
-}
-
-static void
-solid_fill_get_scanline_64 (pixman_image_t *image,
-			    int             x,
-			    int             y,
-			    int             width,
-			    uint32_t *      buffer,
-			    const uint32_t *mask,
-			    uint32_t        mask_bits)
-{
-    uint64_t *b = (uint64_t *)buffer;
-    uint64_t *e = b + width;
-    uint64_t color = image->solid.color_64;
-
-    while (b < e)
-	*(b++) = color;
-}
-
-static source_image_class_t
-solid_fill_classify (pixman_image_t *image,
-                     int             x,
-                     int             y,
-                     int             width,
-                     int             height)
-{
-    return (image->source.class = SOURCE_IMAGE_CLASS_HORIZONTAL);
-}
-
-static void
-solid_fill_property_changed (pixman_image_t *image)
-{
-    image->common.get_scanline_32 = solid_fill_get_scanline_32;
-    image->common.get_scanline_64 = solid_fill_get_scanline_64;
-}
-
 static uint32_t
 color_to_uint32 (const pixman_color_t *color)
 {
@@ -88,18 +36,21 @@ color_to_uint32 (const pixman_color_t *color)
         (color->blue >> 8);
 }
 
-static uint64_t
-color_to_uint64 (const pixman_color_t *color)
+static argb_t
+color_to_float (const pixman_color_t *color)
 {
-    return
-        ((uint64_t)color->alpha << 48) |
-        ((uint64_t)color->red << 32) |
-        ((uint64_t)color->green << 16) |
-        ((uint64_t)color->blue);
+    argb_t result;
+
+    result.a = pixman_unorm_to_float (color->alpha, 16);
+    result.r = pixman_unorm_to_float (color->red, 16);
+    result.g = pixman_unorm_to_float (color->green, 16);
+    result.b = pixman_unorm_to_float (color->blue, 16);
+
+    return result;
 }
 
 PIXMAN_EXPORT pixman_image_t *
-pixman_image_create_solid_fill (pixman_color_t *color)
+pixman_image_create_solid_fill (const pixman_color_t *color)
 {
     pixman_image_t *img = _pixman_image_allocate ();
 
@@ -109,11 +60,7 @@ pixman_image_create_solid_fill (pixman_color_t *color)
     img->type = SOLID;
     img->solid.color = *color;
     img->solid.color_32 = color_to_uint32 (color);
-    img->solid.color_64 = color_to_uint64 (color);
-
-    img->source.class = SOURCE_IMAGE_CLASS_UNKNOWN;
-    img->common.classify = solid_fill_classify;
-    img->common.property_changed = solid_fill_property_changed;
+    img->solid.color_float = color_to_float (color);
 
     return img;
 }
diff --git a/pixman/pixman/pixman-sse2.c b/pixman/pixman/pixman-sse2.c
index 946e7ba37..a6e780815 100644
--- a/pixman/pixman/pixman-sse2.c
+++ b/pixman/pixman/pixman-sse2.c
@@ -30,34 +30,14 @@
 #include <config.h>
 #endif
 
-#include <mmintrin.h>
+/* PSHUFD is slow on a lot of old processors, and new processors have SSSE3 */
+#define PSHUFD_IS_FAST 0
+
 #include <xmmintrin.h> /* for _mm_shuffle_pi16 and _MM_SHUFFLE */
 #include <emmintrin.h> /* for SSE2 intrinsics */
 #include "pixman-private.h"
 #include "pixman-combine32.h"
-
-#if defined(_MSC_VER) && defined(_M_AMD64)
-/* Windows 64 doesn't allow MMX to be used, so
- * the pixman-x64-mmx-emulation.h file contains
- * implementations of those MMX intrinsics that
- * are used in the SSE2 implementation.
- */
-#   include "pixman-x64-mmx-emulation.h"
-#endif
-
-#ifdef USE_SSE2
-
-/* --------------------------------------------------------------------
- * Locals
- */
-
-static __m64 mask_x0080;
-static __m64 mask_x00ff;
-static __m64 mask_x0101;
-static __m64 mask_x_alpha;
-
-static __m64 mask_x565_rgb;
-static __m64 mask_x565_unpack;
+#include "pixman-inlines.h"
 
 static __m128i mask_0080;
 static __m128i mask_00ff;
@@ -76,9 +56,9 @@ static __m128i mask_blue;
 static __m128i mask_565_fix_rb;
 static __m128i mask_565_fix_g;
 
-/* ----------------------------------------------------------------------
- * SSE2 Inlines
- */
+static __m128i mask_565_rb;
+static __m128i mask_565_pack_multiplier;
+
 static force_inline __m128i
 unpack_32_1x128 (uint32_t data)
 {
@@ -147,6 +127,29 @@ pack_2x128_128 (__m128i lo, __m128i hi)
 }
 
 static force_inline __m128i
+pack_565_2packedx128_128 (__m128i lo, __m128i hi)
+{
+    __m128i rb0 = _mm_and_si128 (lo, mask_565_rb);
+    __m128i rb1 = _mm_and_si128 (hi, mask_565_rb);
+
+    __m128i t0 = _mm_madd_epi16 (rb0, mask_565_pack_multiplier);
+    __m128i t1 = _mm_madd_epi16 (rb1, mask_565_pack_multiplier);
+
+    __m128i g0 = _mm_and_si128 (lo, mask_green);
+    __m128i g1 = _mm_and_si128 (hi, mask_green);
+
+    t0 = _mm_or_si128 (t0, g0);
+    t1 = _mm_or_si128 (t1, g1);
+
+    /* Simulates _mm_packus_epi32 */
+    t0 = _mm_slli_epi32 (t0, 16 - 5);
+    t1 = _mm_slli_epi32 (t1, 16 - 5);
+    t0 = _mm_srai_epi32 (t0, 16);
+    t1 = _mm_srai_epi32 (t1, 16);
+    return _mm_packs_epi32 (t0, t1);
+}
+
+static force_inline __m128i
 pack_565_2x128_128 (__m128i lo, __m128i hi)
 {
     __m128i data;
@@ -356,34 +359,6 @@ in_over_2x128 (__m128i* src_lo,
     over_2x128 (&s_lo, &s_hi, &a_lo, &a_hi, dst_lo, dst_hi);
 }
 
-static force_inline void
-cache_prefetch (__m128i* addr)
-{
-    _mm_prefetch ((void const*)addr, _MM_HINT_T0);
-}
-
-static force_inline void
-cache_prefetch_next (__m128i* addr)
-{
-    _mm_prefetch ((void const *)(addr + 4), _MM_HINT_T0); /* 64 bytes ahead */
-}
-
-/* prefetching NULL is very slow on some systems. don't do that. */
-
-static force_inline void
-maybe_prefetch (__m128i* addr)
-{
-    if (addr)
-	cache_prefetch (addr);
-}
-
-static force_inline void
-maybe_prefetch_next (__m128i* addr)
-{
-    if (addr)
-	cache_prefetch_next (addr);
-}
-
 /* load 4 pixels from a 16-byte boundary aligned address */
 static force_inline __m128i
 load_128_aligned (__m128i* src)
@@ -424,146 +399,104 @@ save_128_unaligned (__m128i* dst,
     _mm_storeu_si128 (dst, data);
 }
 
-/* ------------------------------------------------------------------
- * MMX inlines
- */
-
-static force_inline __m64
-load_32_1x64 (uint32_t data)
-{
-    return _mm_cvtsi32_si64 (data);
-}
-
-static force_inline __m64
-unpack_32_1x64 (uint32_t data)
-{
-    return _mm_unpacklo_pi8 (load_32_1x64 (data), _mm_setzero_si64 ());
-}
-
-static force_inline __m64
-expand_alpha_1x64 (__m64 data)
+static force_inline __m128i
+load_32_1x128 (uint32_t data)
 {
-    return _mm_shuffle_pi16 (data, _MM_SHUFFLE (3, 3, 3, 3));
+    return _mm_cvtsi32_si128 (data);
 }
 
-static force_inline __m64
-expand_alpha_rev_1x64 (__m64 data)
+static force_inline __m128i
+expand_alpha_rev_1x128 (__m128i data)
 {
-    return _mm_shuffle_pi16 (data, _MM_SHUFFLE (0, 0, 0, 0));
+    return _mm_shufflelo_epi16 (data, _MM_SHUFFLE (0, 0, 0, 0));
 }
 
-static force_inline __m64
-expand_pixel_8_1x64 (uint8_t data)
+static force_inline __m128i
+expand_pixel_8_1x128 (uint8_t data)
 {
-    return _mm_shuffle_pi16 (
-	unpack_32_1x64 ((uint32_t)data), _MM_SHUFFLE (0, 0, 0, 0));
+    return _mm_shufflelo_epi16 (
+	unpack_32_1x128 ((uint32_t)data), _MM_SHUFFLE (0, 0, 0, 0));
 }
 
-static force_inline __m64
-pix_multiply_1x64 (__m64 data,
-                   __m64 alpha)
+static force_inline __m128i
+pix_multiply_1x128 (__m128i data,
+		    __m128i alpha)
 {
-    return _mm_mulhi_pu16 (_mm_adds_pu16 (_mm_mullo_pi16 (data, alpha),
-                                          mask_x0080),
-                           mask_x0101);
+    return _mm_mulhi_epu16 (_mm_adds_epu16 (_mm_mullo_epi16 (data, alpha),
+					    mask_0080),
+			    mask_0101);
 }
 
-static force_inline __m64
-pix_add_multiply_1x64 (__m64* src,
-                       __m64* alpha_dst,
-                       __m64* dst,
-                       __m64* alpha_src)
+static force_inline __m128i
+pix_add_multiply_1x128 (__m128i* src,
+			__m128i* alpha_dst,
+			__m128i* dst,
+			__m128i* alpha_src)
 {
-    __m64 t1 = pix_multiply_1x64 (*src, *alpha_dst);
-    __m64 t2 = pix_multiply_1x64 (*dst, *alpha_src);
+    __m128i t1 = pix_multiply_1x128 (*src, *alpha_dst);
+    __m128i t2 = pix_multiply_1x128 (*dst, *alpha_src);
 
-    return _mm_adds_pu8 (t1, t2);
+    return _mm_adds_epu8 (t1, t2);
 }
 
-static force_inline __m64
-negate_1x64 (__m64 data)
+static force_inline __m128i
+negate_1x128 (__m128i data)
 {
-    return _mm_xor_si64 (data, mask_x00ff);
+    return _mm_xor_si128 (data, mask_00ff);
 }
 
-static force_inline __m64
-invert_colors_1x64 (__m64 data)
+static force_inline __m128i
+invert_colors_1x128 (__m128i data)
 {
-    return _mm_shuffle_pi16 (data, _MM_SHUFFLE (3, 0, 1, 2));
+    return _mm_shufflelo_epi16 (data, _MM_SHUFFLE (3, 0, 1, 2));
 }
 
-static force_inline __m64
-over_1x64 (__m64 src, __m64 alpha, __m64 dst)
+static force_inline __m128i
+over_1x128 (__m128i src, __m128i alpha, __m128i dst)
 {
-    return _mm_adds_pu8 (src, pix_multiply_1x64 (dst, negate_1x64 (alpha)));
+    return _mm_adds_epu8 (src, pix_multiply_1x128 (dst, negate_1x128 (alpha)));
 }
 
-static force_inline __m64
-in_over_1x64 (__m64* src, __m64* alpha, __m64* mask, __m64* dst)
+static force_inline __m128i
+in_over_1x128 (__m128i* src, __m128i* alpha, __m128i* mask, __m128i* dst)
 {
-    return over_1x64 (pix_multiply_1x64 (*src, *mask),
-                      pix_multiply_1x64 (*alpha, *mask),
-                      *dst);
+    return over_1x128 (pix_multiply_1x128 (*src, *mask),
+		       pix_multiply_1x128 (*alpha, *mask),
+		       *dst);
 }
 
-static force_inline __m64
-over_rev_non_pre_1x64 (__m64 src, __m64 dst)
+static force_inline __m128i
+over_rev_non_pre_1x128 (__m128i src, __m128i dst)
 {
-    __m64 alpha = expand_alpha_1x64 (src);
+    __m128i alpha = expand_alpha_1x128 (src);
 
-    return over_1x64 (pix_multiply_1x64 (invert_colors_1x64 (src),
-                                         _mm_or_si64 (alpha, mask_x_alpha)),
-                      alpha,
-                      dst);
+    return over_1x128 (pix_multiply_1x128 (invert_colors_1x128 (src),
+					   _mm_or_si128 (alpha, mask_alpha)),
+		       alpha,
+		       dst);
 }
 
 static force_inline uint32_t
-pack_1x64_32 (__m64 data)
+pack_1x128_32 (__m128i data)
 {
-    return _mm_cvtsi64_si32 (_mm_packs_pu16 (data, _mm_setzero_si64 ()));
+    return _mm_cvtsi128_si32 (_mm_packus_epi16 (data, _mm_setzero_si128 ()));
 }
 
-/* Expand 16 bits positioned at @pos (0-3) of a mmx register into
- *
- *    00RR00GG00BB
- *
- * --- Expanding 565 in the low word ---
- *
- * m = (m << (32 - 3)) | (m << (16 - 5)) | m;
- * m = m & (01f0003f001f);
- * m = m * (008404100840);
- * m = m >> 8;
- *
- * Note the trick here - the top word is shifted by another nibble to
- * avoid it bumping into the middle word
- */
-static force_inline __m64
-expand565_16_1x64 (uint16_t pixel)
+static force_inline __m128i
+expand565_16_1x128 (uint16_t pixel)
 {
-    __m64 p;
-    __m64 t1, t2;
+    __m128i m = _mm_cvtsi32_si128 (pixel);
 
-    p = _mm_cvtsi32_si64 ((uint32_t) pixel);
+    m = unpack_565_to_8888 (m);
 
-    t1 = _mm_slli_si64 (p, 36 - 11);
-    t2 = _mm_slli_si64 (p, 16 - 5);
-
-    p = _mm_or_si64 (t1, p);
-    p = _mm_or_si64 (t2, p);
-    p = _mm_and_si64 (p, mask_x565_rgb);
-    p = _mm_mullo_pi16 (p, mask_x565_unpack);
-
-    return _mm_srli_pi16 (p, 8);
+    return _mm_unpacklo_epi8 (m, _mm_setzero_si128 ());
 }
 
-/* ----------------------------------------------------------------------------
- * Compose Core transformations
- */
 static force_inline uint32_t
 core_combine_over_u_pixel_sse2 (uint32_t src, uint32_t dst)
 {
     uint8_t a;
-    __m64 ms;
+    __m128i xmms;
 
     a = src >> 24;
 
@@ -573,9 +506,10 @@ core_combine_over_u_pixel_sse2 (uint32_t src, uint32_t dst)
     }
     else if (src)
     {
-	ms = unpack_32_1x64 (src);
-	return pack_1x64_32 (
-	    over_1x64 (ms, expand_alpha_1x64 (ms), unpack_32_1x64 (dst)));
+	xmms = unpack_32_1x128 (src);
+	return pack_1x128_32 (
+	    over_1x128 (xmms, expand_alpha_1x128 (xmms),
+			unpack_32_1x128 (dst)));
     }
 
     return dst;
@@ -588,15 +522,15 @@ combine1 (const uint32_t *ps, const uint32_t *pm)
 
     if (pm)
     {
-	__m64 ms, mm;
+	__m128i ms, mm;
 
-	mm = unpack_32_1x64 (*pm);
-	mm = expand_alpha_1x64 (mm);
+	mm = unpack_32_1x128 (*pm);
+	mm = expand_alpha_1x128 (mm);
 
-	ms = unpack_32_1x64 (s);
-	ms = pix_multiply_1x64 (ms, mm);
+	ms = unpack_32_1x128 (s);
+	ms = pix_multiply_1x128 (ms, mm);
 
-	s = pack_1x64_32 (ms);
+	s = pack_1x128_32 (ms);
     }
 
     return s;
@@ -637,101 +571,182 @@ combine4 (const __m128i *ps, const __m128i *pm)
 }
 
 static force_inline void
-core_combine_over_u_sse2 (uint32_t*       pd,
-                          const uint32_t* ps,
-                          const uint32_t* pm,
-                          int             w)
+core_combine_over_u_sse2_mask (uint32_t *	  pd,
+			       const uint32_t*    ps,
+			       const uint32_t*    pm,
+			       int                w)
 {
     uint32_t s, d;
 
-    __m128i xmm_dst_lo, xmm_dst_hi;
-    __m128i xmm_src_lo, xmm_src_hi;
-    __m128i xmm_alpha_lo, xmm_alpha_hi;
+    /* Align dst on a 16-byte boundary */
+    while (w && ((uintptr_t)pd & 15))
+    {
+	d = *pd;
+	s = combine1 (ps, pm);
+
+	if (s)
+	    *pd = core_combine_over_u_pixel_sse2 (s, d);
+	pd++;
+	ps++;
+	pm++;
+	w--;
+    }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
+    while (w >= 4)
+    {
+	__m128i mask = load_128_unaligned ((__m128i *)pm);
 
-    /* Align dst on a 16-byte boundary */
-    while (w && ((unsigned long)pd & 15))
+	if (!is_zero (mask))
+	{
+	    __m128i src;
+	    __m128i src_hi, src_lo;
+	    __m128i mask_hi, mask_lo;
+	    __m128i alpha_hi, alpha_lo;
+
+	    src = load_128_unaligned ((__m128i *)ps);
+
+	    if (is_opaque (_mm_and_si128 (src, mask)))
+	    {
+		save_128_aligned ((__m128i *)pd, src);
+	    }
+	    else
+	    {
+		__m128i dst = load_128_aligned ((__m128i *)pd);
+		__m128i dst_hi, dst_lo;
+
+		unpack_128_2x128 (mask, &mask_lo, &mask_hi);
+		unpack_128_2x128 (src, &src_lo, &src_hi);
+
+		expand_alpha_2x128 (mask_lo, mask_hi, &mask_lo, &mask_hi);
+		pix_multiply_2x128 (&src_lo, &src_hi,
+				    &mask_lo, &mask_hi,
+				    &src_lo, &src_hi);
+
+		unpack_128_2x128 (dst, &dst_lo, &dst_hi);
+
+		expand_alpha_2x128 (src_lo, src_hi,
+				    &alpha_lo, &alpha_hi);
+
+		over_2x128 (&src_lo, &src_hi, &alpha_lo, &alpha_hi,
+			    &dst_lo, &dst_hi);
+
+		save_128_aligned (
+		    (__m128i *)pd,
+		    pack_2x128_128 (dst_lo, dst_hi));
+	    }
+	}
+
+	pm += 4;
+	ps += 4;
+	pd += 4;
+	w -= 4;
+    }
+    while (w)
     {
 	d = *pd;
 	s = combine1 (ps, pm);
 
-	*pd++ = core_combine_over_u_pixel_sse2 (s, d);
+	if (s)
+	    *pd = core_combine_over_u_pixel_sse2 (s, d);
+	pd++;
 	ps++;
-	if (pm)
-	    pm++;
+	pm++;
+
 	w--;
     }
+}
+
+static force_inline void
+core_combine_over_u_sse2_no_mask (uint32_t *	  pd,
+				  const uint32_t*    ps,
+				  int                w)
+{
+    uint32_t s, d;
+
+    /* Align dst on a 16-byte boundary */
+    while (w && ((uintptr_t)pd & 15))
+    {
+	d = *pd;
+	s = *ps;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
+	if (s)
+	    *pd = core_combine_over_u_pixel_sse2 (s, d);
+	pd++;
+	ps++;
+	w--;
+    }
 
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	maybe_prefetch_next ((__m128i*)pm);
+	__m128i src;
+	__m128i src_hi, src_lo, dst_hi, dst_lo;
+	__m128i alpha_hi, alpha_lo;
 
-	/* I'm loading unaligned because I'm not sure about
-	 * the address alignment.
-	 */
-	xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
+	src = load_128_unaligned ((__m128i *)ps);
 
-	if (is_opaque (xmm_src_hi))
-	{
-	    save_128_aligned ((__m128i*)pd, xmm_src_hi);
-	}
-	else if (!is_zero (xmm_src_hi))
+	if (!is_zero (src))
 	{
-	    xmm_dst_hi = load_128_aligned ((__m128i*) pd);
-
-	    unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
-	    unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+	    if (is_opaque (src))
+	    {
+		save_128_aligned ((__m128i *)pd, src);
+	    }
+	    else
+	    {
+		__m128i dst = load_128_aligned ((__m128i *)pd);
 
-	    expand_alpha_2x128 (
-		xmm_src_lo, xmm_src_hi, &xmm_alpha_lo, &xmm_alpha_hi);
+		unpack_128_2x128 (src, &src_lo, &src_hi);
+		unpack_128_2x128 (dst, &dst_lo, &dst_hi);
 
-	    over_2x128 (&xmm_src_lo, &xmm_src_hi,
-			&xmm_alpha_lo, &xmm_alpha_hi,
-			&xmm_dst_lo, &xmm_dst_hi);
+		expand_alpha_2x128 (src_lo, src_hi,
+				    &alpha_lo, &alpha_hi);
+		over_2x128 (&src_lo, &src_hi, &alpha_lo, &alpha_hi,
+			    &dst_lo, &dst_hi);
 
-	    /* rebuid the 4 pixel data and save*/
-	    save_128_aligned ((__m128i*)pd,
-			      pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+		save_128_aligned (
+		    (__m128i *)pd,
+		    pack_2x128_128 (dst_lo, dst_hi));
+	    }
 	}
 
-	w -= 4;
 	ps += 4;
 	pd += 4;
-	if (pm)
-	    pm += 4;
+	w -= 4;
     }
-
     while (w)
     {
 	d = *pd;
-	s = combine1 (ps, pm);
+	s = *ps;
 
-	*pd++ = core_combine_over_u_pixel_sse2 (s, d);
+	if (s)
+	    *pd = core_combine_over_u_pixel_sse2 (s, d);
+	pd++;
 	ps++;
-	if (pm)
-	    pm++;
 
 	w--;
     }
 }
 
 static force_inline void
-core_combine_over_reverse_u_sse2 (uint32_t*       pd,
-                                  const uint32_t* ps,
-                                  const uint32_t* pm,
-                                  int             w)
+sse2_combine_over_u (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     uint32_t *               pd,
+                     const uint32_t *         ps,
+                     const uint32_t *         pm,
+                     int                      w)
+{
+    if (pm)
+	core_combine_over_u_sse2_mask (pd, ps, pm, w);
+    else
+	core_combine_over_u_sse2_no_mask (pd, ps, w);
+}
+
+static void
+sse2_combine_over_reverse_u (pixman_implementation_t *imp,
+                             pixman_op_t              op,
+                             uint32_t *               pd,
+                             const uint32_t *         ps,
+                             const uint32_t *         pm,
+                             int                      w)
 {
     uint32_t s, d;
 
@@ -739,14 +754,9 @@ core_combine_over_reverse_u_sse2 (uint32_t*       pd,
     __m128i xmm_src_lo, xmm_src_hi;
     __m128i xmm_alpha_lo, xmm_alpha_hi;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
     /* Align dst on a 16-byte boundary */
     while (w &&
-           ((unsigned long)pd & 15))
+           ((uintptr_t)pd & 15))
     {
 	d = *pd;
 	s = combine1 (ps, pm);
@@ -758,18 +768,8 @@ core_combine_over_reverse_u_sse2 (uint32_t*       pd,
 	    pm++;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	maybe_prefetch_next ((__m128i*)pm);
-
 	/* I'm loading unaligned because I'm not sure
 	 * about the address alignment.
 	 */
@@ -812,7 +812,7 @@ core_combine_over_reverse_u_sse2 (uint32_t*       pd,
 }
 
 static force_inline uint32_t
-core_combine_in_u_pixelsse2 (uint32_t src, uint32_t dst)
+core_combine_in_u_pixel_sse2 (uint32_t src, uint32_t dst)
 {
     uint32_t maska = src >> 24;
 
@@ -822,54 +822,41 @@ core_combine_in_u_pixelsse2 (uint32_t src, uint32_t dst)
     }
     else if (maska != 0xff)
     {
-	return pack_1x64_32 (
-	    pix_multiply_1x64 (unpack_32_1x64 (dst),
-			       expand_alpha_1x64 (unpack_32_1x64 (src))));
+	return pack_1x128_32 (
+	    pix_multiply_1x128 (unpack_32_1x128 (dst),
+				expand_alpha_1x128 (unpack_32_1x128 (src))));
     }
 
     return dst;
 }
 
-static force_inline void
-core_combine_in_u_sse2 (uint32_t*       pd,
-                        const uint32_t* ps,
-                        const uint32_t* pm,
-                        int             w)
+static void
+sse2_combine_in_u (pixman_implementation_t *imp,
+                   pixman_op_t              op,
+                   uint32_t *               pd,
+                   const uint32_t *         ps,
+                   const uint32_t *         pm,
+                   int                      w)
 {
     uint32_t s, d;
 
     __m128i xmm_src_lo, xmm_src_hi;
     __m128i xmm_dst_lo, xmm_dst_hi;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
-    while (w && ((unsigned long) pd & 15))
+    while (w && ((uintptr_t)pd & 15))
     {
 	s = combine1 (ps, pm);
 	d = *pd;
 
-	*pd++ = core_combine_in_u_pixelsse2 (d, s);
+	*pd++ = core_combine_in_u_pixel_sse2 (d, s);
 	w--;
 	ps++;
 	if (pm)
 	    pm++;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	maybe_prefetch_next ((__m128i*)pm);
-
 	xmm_dst_hi = load_128_aligned ((__m128i*) pd);
 	xmm_src_hi = combine4 ((__m128i*) ps, (__m128i*) pm);
 
@@ -896,7 +883,7 @@ core_combine_in_u_sse2 (uint32_t*       pd,
 	s = combine1 (ps, pm);
 	d = *pd;
 
-	*pd++ = core_combine_in_u_pixelsse2 (d, s);
+	*pd++ = core_combine_in_u_pixel_sse2 (d, s);
 	w--;
 	ps++;
 	if (pm)
@@ -904,46 +891,33 @@ core_combine_in_u_sse2 (uint32_t*       pd,
     }
 }
 
-static force_inline void
-core_combine_reverse_in_u_sse2 (uint32_t*       pd,
-                                const uint32_t* ps,
-                                const uint32_t *pm,
-                                int             w)
+static void
+sse2_combine_in_reverse_u (pixman_implementation_t *imp,
+                           pixman_op_t              op,
+                           uint32_t *               pd,
+                           const uint32_t *         ps,
+                           const uint32_t *         pm,
+                           int                      w)
 {
     uint32_t s, d;
 
     __m128i xmm_src_lo, xmm_src_hi;
     __m128i xmm_dst_lo, xmm_dst_hi;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
-    while (w && ((unsigned long) pd & 15))
+    while (w && ((uintptr_t)pd & 15))
     {
 	s = combine1 (ps, pm);
 	d = *pd;
 
-	*pd++ = core_combine_in_u_pixelsse2 (s, d);
+	*pd++ = core_combine_in_u_pixel_sse2 (s, d);
 	ps++;
 	w--;
 	if (pm)
 	    pm++;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	maybe_prefetch_next ((__m128i*)pm);
-
 	xmm_dst_hi = load_128_aligned ((__m128i*) pd);
 	xmm_src_hi = combine4 ((__m128i*) ps, (__m128i*)pm);
 
@@ -970,7 +944,7 @@ core_combine_reverse_in_u_sse2 (uint32_t*       pd,
 	s = combine1 (ps, pm);
 	d = *pd;
 
-	*pd++ = core_combine_in_u_pixelsse2 (s, d);
+	*pd++ = core_combine_in_u_pixel_sse2 (s, d);
 	w--;
 	ps++;
 	if (pm)
@@ -978,26 +952,23 @@ core_combine_reverse_in_u_sse2 (uint32_t*       pd,
     }
 }
 
-static force_inline void
-core_combine_reverse_out_u_sse2 (uint32_t*       pd,
-                                 const uint32_t* ps,
-                                 const uint32_t* pm,
-                                 int             w)
+static void
+sse2_combine_out_reverse_u (pixman_implementation_t *imp,
+                            pixman_op_t              op,
+                            uint32_t *               pd,
+                            const uint32_t *         ps,
+                            const uint32_t *         pm,
+                            int                      w)
 {
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
-    while (w && ((unsigned long) pd & 15))
+    while (w && ((uintptr_t)pd & 15))
     {
 	uint32_t s = combine1 (ps, pm);
 	uint32_t d = *pd;
 
-	*pd++ = pack_1x64_32 (
-	    pix_multiply_1x64 (
-		unpack_32_1x64 (d), negate_1x64 (
-		    expand_alpha_1x64 (unpack_32_1x64 (s)))));
+	*pd++ = pack_1x128_32 (
+	    pix_multiply_1x128 (
+		unpack_32_1x128 (d), negate_1x128 (
+		    expand_alpha_1x128 (unpack_32_1x128 (s)))));
 
 	if (pm)
 	    pm++;
@@ -1005,21 +976,11 @@ core_combine_reverse_out_u_sse2 (uint32_t*       pd,
 	w--;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
 	__m128i xmm_src_lo, xmm_src_hi;
 	__m128i xmm_dst_lo, xmm_dst_hi;
 
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	maybe_prefetch_next ((__m128i*)pm);
-
 	xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
 	xmm_dst_hi = load_128_aligned ((__m128i*) pd);
 
@@ -1049,10 +1010,10 @@ core_combine_reverse_out_u_sse2 (uint32_t*       pd,
 	uint32_t s = combine1 (ps, pm);
 	uint32_t d = *pd;
 
-	*pd++ = pack_1x64_32 (
-	    pix_multiply_1x64 (
-		unpack_32_1x64 (d), negate_1x64 (
-		    expand_alpha_1x64 (unpack_32_1x64 (s)))));
+	*pd++ = pack_1x128_32 (
+	    pix_multiply_1x128 (
+		unpack_32_1x128 (d), negate_1x128 (
+		    expand_alpha_1x128 (unpack_32_1x128 (s)))));
 	ps++;
 	if (pm)
 	    pm++;
@@ -1060,47 +1021,34 @@ core_combine_reverse_out_u_sse2 (uint32_t*       pd,
     }
 }
 
-static force_inline void
-core_combine_out_u_sse2 (uint32_t*       pd,
-                         const uint32_t* ps,
-                         const uint32_t* pm,
-                         int             w)
+static void
+sse2_combine_out_u (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               pd,
+                    const uint32_t *         ps,
+                    const uint32_t *         pm,
+                    int                      w)
 {
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
-    while (w && ((unsigned long) pd & 15))
+    while (w && ((uintptr_t)pd & 15))
     {
 	uint32_t s = combine1 (ps, pm);
 	uint32_t d = *pd;
 
-	*pd++ = pack_1x64_32 (
-	    pix_multiply_1x64 (
-		unpack_32_1x64 (s), negate_1x64 (
-		    expand_alpha_1x64 (unpack_32_1x64 (d)))));
+	*pd++ = pack_1x128_32 (
+	    pix_multiply_1x128 (
+		unpack_32_1x128 (s), negate_1x128 (
+		    expand_alpha_1x128 (unpack_32_1x128 (d)))));
 	w--;
 	ps++;
 	if (pm)
 	    pm++;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
 	__m128i xmm_src_lo, xmm_src_hi;
 	__m128i xmm_dst_lo, xmm_dst_hi;
 
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	maybe_prefetch_next ((__m128i*)pm);
-
 	xmm_src_hi = combine4 ((__m128i*) ps, (__m128i*)pm);
 	xmm_dst_hi = load_128_aligned ((__m128i*) pd);
 
@@ -1129,10 +1077,10 @@ core_combine_out_u_sse2 (uint32_t*       pd,
 	uint32_t s = combine1 (ps, pm);
 	uint32_t d = *pd;
 
-	*pd++ = pack_1x64_32 (
-	    pix_multiply_1x64 (
-		unpack_32_1x64 (s), negate_1x64 (
-		    expand_alpha_1x64 (unpack_32_1x64 (d)))));
+	*pd++ = pack_1x128_32 (
+	    pix_multiply_1x128 (
+		unpack_32_1x128 (s), negate_1x128 (
+		    expand_alpha_1x128 (unpack_32_1x128 (d)))));
 	w--;
 	ps++;
 	if (pm)
@@ -1144,20 +1092,22 @@ static force_inline uint32_t
 core_combine_atop_u_pixel_sse2 (uint32_t src,
                                 uint32_t dst)
 {
-    __m64 s = unpack_32_1x64 (src);
-    __m64 d = unpack_32_1x64 (dst);
+    __m128i s = unpack_32_1x128 (src);
+    __m128i d = unpack_32_1x128 (dst);
 
-    __m64 sa = negate_1x64 (expand_alpha_1x64 (s));
-    __m64 da = expand_alpha_1x64 (d);
+    __m128i sa = negate_1x128 (expand_alpha_1x128 (s));
+    __m128i da = expand_alpha_1x128 (d);
 
-    return pack_1x64_32 (pix_add_multiply_1x64 (&s, &da, &d, &sa));
+    return pack_1x128_32 (pix_add_multiply_1x128 (&s, &da, &d, &sa));
 }
 
-static force_inline void
-core_combine_atop_u_sse2 (uint32_t*       pd,
-                          const uint32_t* ps,
-                          const uint32_t* pm,
-                          int             w)
+static void
+sse2_combine_atop_u (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     uint32_t *               pd,
+                     const uint32_t *         ps,
+                     const uint32_t *         pm,
+                     int                      w)
 {
     uint32_t s, d;
 
@@ -1166,12 +1116,7 @@ core_combine_atop_u_sse2 (uint32_t*       pd,
     __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
     __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
-    while (w && ((unsigned long) pd & 15))
+    while (w && ((uintptr_t)pd & 15))
     {
 	s = combine1 (ps, pm);
 	d = *pd;
@@ -1183,18 +1128,8 @@ core_combine_atop_u_sse2 (uint32_t*       pd,
 	    pm++;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	maybe_prefetch_next ((__m128i*)pm);
-
 	xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
 	xmm_dst_hi = load_128_aligned ((__m128i*) pd);
 
@@ -1241,20 +1176,22 @@ static force_inline uint32_t
 core_combine_reverse_atop_u_pixel_sse2 (uint32_t src,
                                         uint32_t dst)
 {
-    __m64 s = unpack_32_1x64 (src);
-    __m64 d = unpack_32_1x64 (dst);
+    __m128i s = unpack_32_1x128 (src);
+    __m128i d = unpack_32_1x128 (dst);
 
-    __m64 sa = expand_alpha_1x64 (s);
-    __m64 da = negate_1x64 (expand_alpha_1x64 (d));
+    __m128i sa = expand_alpha_1x128 (s);
+    __m128i da = negate_1x128 (expand_alpha_1x128 (d));
 
-    return pack_1x64_32 (pix_add_multiply_1x64 (&s, &da, &d, &sa));
+    return pack_1x128_32 (pix_add_multiply_1x128 (&s, &da, &d, &sa));
 }
 
-static force_inline void
-core_combine_reverse_atop_u_sse2 (uint32_t*       pd,
-                                  const uint32_t* ps,
-                                  const uint32_t* pm,
-                                  int             w)
+static void
+sse2_combine_atop_reverse_u (pixman_implementation_t *imp,
+                             pixman_op_t              op,
+                             uint32_t *               pd,
+                             const uint32_t *         ps,
+                             const uint32_t *         pm,
+                             int                      w)
 {
     uint32_t s, d;
 
@@ -1263,12 +1200,7 @@ core_combine_reverse_atop_u_sse2 (uint32_t*       pd,
     __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
     __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
-    while (w && ((unsigned long) pd & 15))
+    while (w && ((uintptr_t)pd & 15))
     {
 	s = combine1 (ps, pm);
 	d = *pd;
@@ -1280,18 +1212,8 @@ core_combine_reverse_atop_u_sse2 (uint32_t*       pd,
 	    pm++;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	maybe_prefetch_next ((__m128i*)pm);
-
 	xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
 	xmm_dst_hi = load_128_aligned ((__m128i*) pd);
 
@@ -1338,20 +1260,22 @@ static force_inline uint32_t
 core_combine_xor_u_pixel_sse2 (uint32_t src,
                                uint32_t dst)
 {
-    __m64 s = unpack_32_1x64 (src);
-    __m64 d = unpack_32_1x64 (dst);
+    __m128i s = unpack_32_1x128 (src);
+    __m128i d = unpack_32_1x128 (dst);
 
-    __m64 neg_d = negate_1x64 (expand_alpha_1x64 (d));
-    __m64 neg_s = negate_1x64 (expand_alpha_1x64 (s));
+    __m128i neg_d = negate_1x128 (expand_alpha_1x128 (d));
+    __m128i neg_s = negate_1x128 (expand_alpha_1x128 (s));
 
-    return pack_1x64_32 (pix_add_multiply_1x64 (&s, &neg_d, &d, &neg_s));
+    return pack_1x128_32 (pix_add_multiply_1x128 (&s, &neg_d, &d, &neg_s));
 }
 
-static force_inline void
-core_combine_xor_u_sse2 (uint32_t*       dst,
-                         const uint32_t* src,
-                         const uint32_t *mask,
-                         int             width)
+static void
+sse2_combine_xor_u (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dst,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
 {
     int w = width;
     uint32_t s, d;
@@ -1364,12 +1288,7 @@ core_combine_xor_u_sse2 (uint32_t*       dst,
     __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
     __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
-    while (w && ((unsigned long) pd & 15))
+    while (w && ((uintptr_t)pd & 15))
     {
 	s = combine1 (ps, pm);
 	d = *pd;
@@ -1381,18 +1300,8 @@ core_combine_xor_u_sse2 (uint32_t*       dst,
 	    pm++;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	maybe_prefetch_next ((__m128i*)pm);
-
 	xmm_src = combine4 ((__m128i*) ps, (__m128i*) pm);
 	xmm_dst = load_128_aligned ((__m128i*) pd);
 
@@ -1438,10 +1347,12 @@ core_combine_xor_u_sse2 (uint32_t*       dst,
 }
 
 static force_inline void
-core_combine_add_u_sse2 (uint32_t*       dst,
-                         const uint32_t* src,
-                         const uint32_t* mask,
-                         int             width)
+sse2_combine_add_u (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dst,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
 {
     int w = width;
     uint32_t s, d;
@@ -1449,12 +1360,7 @@ core_combine_add_u_sse2 (uint32_t*       dst,
     const uint32_t* ps = src;
     const uint32_t* pm = mask;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
-    while (w && (unsigned long)pd & 15)
+    while (w && (uintptr_t)pd & 15)
     {
 	s = combine1 (ps, pm);
 	d = *pd;
@@ -1462,25 +1368,15 @@ core_combine_add_u_sse2 (uint32_t*       dst,
 	ps++;
 	if (pm)
 	    pm++;
-	*pd++ = _mm_cvtsi64_si32 (
-	    _mm_adds_pu8 (_mm_cvtsi32_si64 (s), _mm_cvtsi32_si64 (d)));
+	*pd++ = _mm_cvtsi128_si32 (
+	    _mm_adds_epu8 (_mm_cvtsi32_si128 (s), _mm_cvtsi32_si128 (d)));
 	w--;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
 	__m128i s;
 
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	maybe_prefetch_next ((__m128i*)pm);
-
 	s = combine4 ((__m128i*)ps, (__m128i*)pm);
 
 	save_128_aligned (
@@ -1499,8 +1395,8 @@ core_combine_add_u_sse2 (uint32_t*       dst,
 	d = *pd;
 
 	ps++;
-	*pd++ = _mm_cvtsi64_si32 (
-	    _mm_adds_pu8 (_mm_cvtsi32_si64 (s), _mm_cvtsi32_si64 (d)));
+	*pd++ = _mm_cvtsi128_si32 (
+	    _mm_adds_epu8 (_mm_cvtsi32_si128 (s), _mm_cvtsi32_si128 (d)));
 	if (pm)
 	    pm++;
     }
@@ -1510,37 +1406,34 @@ static force_inline uint32_t
 core_combine_saturate_u_pixel_sse2 (uint32_t src,
                                     uint32_t dst)
 {
-    __m64 ms = unpack_32_1x64 (src);
-    __m64 md = unpack_32_1x64 (dst);
+    __m128i ms = unpack_32_1x128 (src);
+    __m128i md = unpack_32_1x128 (dst);
     uint32_t sa = src >> 24;
     uint32_t da = ~dst >> 24;
 
     if (sa > da)
     {
-	ms = pix_multiply_1x64 (
-	    ms, expand_alpha_1x64 (unpack_32_1x64 (DIV_UN8 (da, sa) << 24)));
+	ms = pix_multiply_1x128 (
+	    ms, expand_alpha_1x128 (unpack_32_1x128 (DIV_UN8 (da, sa) << 24)));
     }
 
-    return pack_1x64_32 (_mm_adds_pu16 (md, ms));
+    return pack_1x128_32 (_mm_adds_epu16 (md, ms));
 }
 
-static force_inline void
-core_combine_saturate_u_sse2 (uint32_t *      pd,
-                              const uint32_t *ps,
-                              const uint32_t *pm,
-                              int             w)
+static void
+sse2_combine_saturate_u (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         uint32_t *               pd,
+                         const uint32_t *         ps,
+                         const uint32_t *         pm,
+                         int                      w)
 {
     uint32_t s, d;
 
     uint32_t pack_cmp;
     __m128i xmm_src, xmm_dst;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
-    while (w && (unsigned long)pd & 15)
+    while (w && (uintptr_t)pd & 15)
     {
 	s = combine1 (ps, pm);
 	d = *pd;
@@ -1552,18 +1445,8 @@ core_combine_saturate_u_sse2 (uint32_t *      pd,
 	    pm++;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    maybe_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	maybe_prefetch_next ((__m128i*)pm);
-
 	xmm_dst = load_128_aligned  ((__m128i*)pd);
 	xmm_src = combine4 ((__m128i*)ps, (__m128i*)pm);
 
@@ -1624,11 +1507,13 @@ core_combine_saturate_u_sse2 (uint32_t *      pd,
     }
 }
 
-static force_inline void
-core_combine_src_ca_sse2 (uint32_t*       pd,
-                          const uint32_t* ps,
-                          const uint32_t *pm,
-                          int             w)
+static void
+sse2_combine_src_ca (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     uint32_t *               pd,
+                     const uint32_t *         ps,
+                     const uint32_t *         pm,
+                     int                      w)
 {
     uint32_t s, m;
 
@@ -1636,32 +1521,17 @@ core_combine_src_ca_sse2 (uint32_t*       pd,
     __m128i xmm_mask_lo, xmm_mask_hi;
     __m128i xmm_dst_lo, xmm_dst_hi;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
-    while (w && (unsigned long)pd & 15)
+    while (w && (uintptr_t)pd & 15)
     {
 	s = *ps++;
 	m = *pm++;
-	*pd++ = pack_1x64_32 (
-	    pix_multiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (m)));
+	*pd++ = pack_1x128_32 (
+	    pix_multiply_1x128 (unpack_32_1x128 (s), unpack_32_1x128 (m)));
 	w--;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	cache_prefetch_next ((__m128i*)pm);
-
 	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
 	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
 
@@ -1685,8 +1555,8 @@ core_combine_src_ca_sse2 (uint32_t*       pd,
     {
 	s = *ps++;
 	m = *pm++;
-	*pd++ = pack_1x64_32 (
-	    pix_multiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (m)));
+	*pd++ = pack_1x128_32 (
+	    pix_multiply_1x128 (unpack_32_1x128 (s), unpack_32_1x128 (m)));
 	w--;
     }
 }
@@ -1696,19 +1566,21 @@ core_combine_over_ca_pixel_sse2 (uint32_t src,
                                  uint32_t mask,
                                  uint32_t dst)
 {
-    __m64 s = unpack_32_1x64 (src);
-    __m64 expAlpha = expand_alpha_1x64 (s);
-    __m64 unpk_mask = unpack_32_1x64 (mask);
-    __m64 unpk_dst  = unpack_32_1x64 (dst);
+    __m128i s = unpack_32_1x128 (src);
+    __m128i expAlpha = expand_alpha_1x128 (s);
+    __m128i unpk_mask = unpack_32_1x128 (mask);
+    __m128i unpk_dst  = unpack_32_1x128 (dst);
 
-    return pack_1x64_32 (in_over_1x64 (&s, &expAlpha, &unpk_mask, &unpk_dst));
+    return pack_1x128_32 (in_over_1x128 (&s, &expAlpha, &unpk_mask, &unpk_dst));
 }
 
-static force_inline void
-core_combine_over_ca_sse2 (uint32_t*       pd,
-                           const uint32_t* ps,
-                           const uint32_t *pm,
-                           int             w)
+static void
+sse2_combine_over_ca (pixman_implementation_t *imp,
+                      pixman_op_t              op,
+                      uint32_t *               pd,
+                      const uint32_t *         ps,
+                      const uint32_t *         pm,
+                      int                      w)
 {
     uint32_t s, m, d;
 
@@ -1717,12 +1589,7 @@ core_combine_over_ca_sse2 (uint32_t*       pd,
     __m128i xmm_dst_lo, xmm_dst_hi;
     __m128i xmm_mask_lo, xmm_mask_hi;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
-    while (w && (unsigned long)pd & 15)
+    while (w && (uintptr_t)pd & 15)
     {
 	s = *ps++;
 	m = *pm++;
@@ -1732,18 +1599,8 @@ core_combine_over_ca_sse2 (uint32_t*       pd,
 	w--;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	cache_prefetch_next ((__m128i*)pm);
-
 	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
 	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
 	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
@@ -1785,19 +1642,21 @@ core_combine_over_reverse_ca_pixel_sse2 (uint32_t src,
                                          uint32_t mask,
                                          uint32_t dst)
 {
-    __m64 d = unpack_32_1x64 (dst);
+    __m128i d = unpack_32_1x128 (dst);
 
-    return pack_1x64_32 (
-	over_1x64 (d, expand_alpha_1x64 (d),
-		   pix_multiply_1x64 (unpack_32_1x64 (src),
-				      unpack_32_1x64 (mask))));
+    return pack_1x128_32 (
+	over_1x128 (d, expand_alpha_1x128 (d),
+		    pix_multiply_1x128 (unpack_32_1x128 (src),
+					unpack_32_1x128 (mask))));
 }
 
-static force_inline void
-core_combine_over_reverse_ca_sse2 (uint32_t*       pd,
-                                   const uint32_t* ps,
-                                   const uint32_t *pm,
-                                   int             w)
+static void
+sse2_combine_over_reverse_ca (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              uint32_t *               pd,
+                              const uint32_t *         ps,
+                              const uint32_t *         pm,
+                              int                      w)
 {
     uint32_t s, m, d;
 
@@ -1806,12 +1665,7 @@ core_combine_over_reverse_ca_sse2 (uint32_t*       pd,
     __m128i xmm_dst_lo, xmm_dst_hi;
     __m128i xmm_mask_lo, xmm_mask_hi;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
-    while (w && (unsigned long)pd & 15)
+    while (w && (uintptr_t)pd & 15)
     {
 	s = *ps++;
 	m = *pm++;
@@ -1821,18 +1675,8 @@ core_combine_over_reverse_ca_sse2 (uint32_t*       pd,
 	w--;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	cache_prefetch_next ((__m128i*)pm);
-
 	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
 	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
 	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
@@ -1871,11 +1715,13 @@ core_combine_over_reverse_ca_sse2 (uint32_t*       pd,
     }
 }
 
-static force_inline void
-core_combine_in_ca_sse2 (uint32_t *      pd,
-                         const uint32_t *ps,
-                         const uint32_t *pm,
-                         int             w)
+static void
+sse2_combine_in_ca (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               pd,
+                    const uint32_t *         ps,
+                    const uint32_t *         pm,
+                    int                      w)
 {
     uint32_t s, m, d;
 
@@ -1884,37 +1730,22 @@ core_combine_in_ca_sse2 (uint32_t *      pd,
     __m128i xmm_dst_lo, xmm_dst_hi;
     __m128i xmm_mask_lo, xmm_mask_hi;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
-    while (w && (unsigned long)pd & 15)
+    while (w && (uintptr_t)pd & 15)
     {
 	s = *ps++;
 	m = *pm++;
 	d = *pd;
 
-	*pd++ = pack_1x64_32 (
-	    pix_multiply_1x64 (
-		pix_multiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (m)),
-		expand_alpha_1x64 (unpack_32_1x64 (d))));
+	*pd++ = pack_1x128_32 (
+	    pix_multiply_1x128 (
+		pix_multiply_1x128 (unpack_32_1x128 (s), unpack_32_1x128 (m)),
+		expand_alpha_1x128 (unpack_32_1x128 (d))));
 
 	w--;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	cache_prefetch_next ((__m128i*)pm);
-
 	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
 	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
 	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
@@ -1949,21 +1780,23 @@ core_combine_in_ca_sse2 (uint32_t *      pd,
 	m = *pm++;
 	d = *pd;
 
-	*pd++ = pack_1x64_32 (
-	    pix_multiply_1x64 (
-		pix_multiply_1x64 (
-		    unpack_32_1x64 (s), unpack_32_1x64 (m)),
-		expand_alpha_1x64 (unpack_32_1x64 (d))));
+	*pd++ = pack_1x128_32 (
+	    pix_multiply_1x128 (
+		pix_multiply_1x128 (
+		    unpack_32_1x128 (s), unpack_32_1x128 (m)),
+		expand_alpha_1x128 (unpack_32_1x128 (d))));
 
 	w--;
     }
 }
 
-static force_inline void
-core_combine_in_reverse_ca_sse2 (uint32_t *      pd,
-                                 const uint32_t *ps,
-                                 const uint32_t *pm,
-                                 int             w)
+static void
+sse2_combine_in_reverse_ca (pixman_implementation_t *imp,
+                            pixman_op_t              op,
+                            uint32_t *               pd,
+                            const uint32_t *         ps,
+                            const uint32_t *         pm,
+                            int                      w)
 {
     uint32_t s, m, d;
 
@@ -1972,37 +1805,22 @@ core_combine_in_reverse_ca_sse2 (uint32_t *      pd,
     __m128i xmm_dst_lo, xmm_dst_hi;
     __m128i xmm_mask_lo, xmm_mask_hi;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
-    while (w && (unsigned long)pd & 15)
+    while (w && (uintptr_t)pd & 15)
     {
 	s = *ps++;
 	m = *pm++;
 	d = *pd;
 
-	*pd++ = pack_1x64_32 (
-	    pix_multiply_1x64 (
-		unpack_32_1x64 (d),
-		pix_multiply_1x64 (unpack_32_1x64 (m),
-				   expand_alpha_1x64 (unpack_32_1x64 (s)))));
+	*pd++ = pack_1x128_32 (
+	    pix_multiply_1x128 (
+		unpack_32_1x128 (d),
+		pix_multiply_1x128 (unpack_32_1x128 (m),
+				   expand_alpha_1x128 (unpack_32_1x128 (s)))));
 	w--;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	cache_prefetch_next ((__m128i*)pm);
-
 	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
 	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
 	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
@@ -2036,20 +1854,22 @@ core_combine_in_reverse_ca_sse2 (uint32_t *      pd,
 	m = *pm++;
 	d = *pd;
 
-	*pd++ = pack_1x64_32 (
-	    pix_multiply_1x64 (
-		unpack_32_1x64 (d),
-		pix_multiply_1x64 (unpack_32_1x64 (m),
-				   expand_alpha_1x64 (unpack_32_1x64 (s)))));
+	*pd++ = pack_1x128_32 (
+	    pix_multiply_1x128 (
+		unpack_32_1x128 (d),
+		pix_multiply_1x128 (unpack_32_1x128 (m),
+				   expand_alpha_1x128 (unpack_32_1x128 (s)))));
 	w--;
     }
 }
 
-static force_inline void
-core_combine_out_ca_sse2 (uint32_t *      pd,
-                          const uint32_t *ps,
-                          const uint32_t *pm,
-                          int             w)
+static void
+sse2_combine_out_ca (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     uint32_t *               pd,
+                     const uint32_t *         ps,
+                     const uint32_t *         pm,
+                     int                      w)
 {
     uint32_t s, m, d;
 
@@ -2058,37 +1878,22 @@ core_combine_out_ca_sse2 (uint32_t *      pd,
     __m128i xmm_dst_lo, xmm_dst_hi;
     __m128i xmm_mask_lo, xmm_mask_hi;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
-    while (w && (unsigned long)pd & 15)
+    while (w && (uintptr_t)pd & 15)
     {
 	s = *ps++;
 	m = *pm++;
 	d = *pd;
 
-	*pd++ = pack_1x64_32 (
-	    pix_multiply_1x64 (
-		pix_multiply_1x64 (
-		    unpack_32_1x64 (s), unpack_32_1x64 (m)),
-		negate_1x64 (expand_alpha_1x64 (unpack_32_1x64 (d)))));
+	*pd++ = pack_1x128_32 (
+	    pix_multiply_1x128 (
+		pix_multiply_1x128 (
+		    unpack_32_1x128 (s), unpack_32_1x128 (m)),
+		negate_1x128 (expand_alpha_1x128 (unpack_32_1x128 (d)))));
 	w--;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	cache_prefetch_next ((__m128i*)pm);
-
 	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
 	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
 	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
@@ -2124,21 +1929,23 @@ core_combine_out_ca_sse2 (uint32_t *      pd,
 	m = *pm++;
 	d = *pd;
 
-	*pd++ = pack_1x64_32 (
-	    pix_multiply_1x64 (
-		pix_multiply_1x64 (
-		    unpack_32_1x64 (s), unpack_32_1x64 (m)),
-		negate_1x64 (expand_alpha_1x64 (unpack_32_1x64 (d)))));
+	*pd++ = pack_1x128_32 (
+	    pix_multiply_1x128 (
+		pix_multiply_1x128 (
+		    unpack_32_1x128 (s), unpack_32_1x128 (m)),
+		negate_1x128 (expand_alpha_1x128 (unpack_32_1x128 (d)))));
 
 	w--;
     }
 }
 
-static force_inline void
-core_combine_out_reverse_ca_sse2 (uint32_t *      pd,
-                                  const uint32_t *ps,
-                                  const uint32_t *pm,
-                                  int             w)
+static void
+sse2_combine_out_reverse_ca (pixman_implementation_t *imp,
+                             pixman_op_t              op,
+                             uint32_t *               pd,
+                             const uint32_t *         ps,
+                             const uint32_t *         pm,
+                             int                      w)
 {
     uint32_t s, m, d;
 
@@ -2147,38 +1954,23 @@ core_combine_out_reverse_ca_sse2 (uint32_t *      pd,
     __m128i xmm_dst_lo, xmm_dst_hi;
     __m128i xmm_mask_lo, xmm_mask_hi;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
-    while (w && (unsigned long)pd & 15)
+    while (w && (uintptr_t)pd & 15)
     {
 	s = *ps++;
 	m = *pm++;
 	d = *pd;
 
-	*pd++ = pack_1x64_32 (
-	    pix_multiply_1x64 (
-		unpack_32_1x64 (d),
-		negate_1x64 (pix_multiply_1x64 (
-				 unpack_32_1x64 (m),
-				 expand_alpha_1x64 (unpack_32_1x64 (s))))));
+	*pd++ = pack_1x128_32 (
+	    pix_multiply_1x128 (
+		unpack_32_1x128 (d),
+		negate_1x128 (pix_multiply_1x128 (
+				 unpack_32_1x128 (m),
+				 expand_alpha_1x128 (unpack_32_1x128 (s))))));
 	w--;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	cache_prefetch_next ((__m128i*)pm);
-
 	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
 	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
 	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
@@ -2216,12 +2008,12 @@ core_combine_out_reverse_ca_sse2 (uint32_t *      pd,
 	m = *pm++;
 	d = *pd;
 
-	*pd++ = pack_1x64_32 (
-	    pix_multiply_1x64 (
-		unpack_32_1x64 (d),
-		negate_1x64 (pix_multiply_1x64 (
-				 unpack_32_1x64 (m),
-				 expand_alpha_1x64 (unpack_32_1x64 (s))))));
+	*pd++ = pack_1x128_32 (
+	    pix_multiply_1x128 (
+		unpack_32_1x128 (d),
+		negate_1x128 (pix_multiply_1x128 (
+				 unpack_32_1x128 (m),
+				 expand_alpha_1x128 (unpack_32_1x128 (s))))));
 	w--;
     }
 }
@@ -2231,23 +2023,25 @@ core_combine_atop_ca_pixel_sse2 (uint32_t src,
                                  uint32_t mask,
                                  uint32_t dst)
 {
-    __m64 m = unpack_32_1x64 (mask);
-    __m64 s = unpack_32_1x64 (src);
-    __m64 d = unpack_32_1x64 (dst);
-    __m64 sa = expand_alpha_1x64 (s);
-    __m64 da = expand_alpha_1x64 (d);
+    __m128i m = unpack_32_1x128 (mask);
+    __m128i s = unpack_32_1x128 (src);
+    __m128i d = unpack_32_1x128 (dst);
+    __m128i sa = expand_alpha_1x128 (s);
+    __m128i da = expand_alpha_1x128 (d);
 
-    s = pix_multiply_1x64 (s, m);
-    m = negate_1x64 (pix_multiply_1x64 (m, sa));
+    s = pix_multiply_1x128 (s, m);
+    m = negate_1x128 (pix_multiply_1x128 (m, sa));
 
-    return pack_1x64_32 (pix_add_multiply_1x64 (&d, &m, &s, &da));
+    return pack_1x128_32 (pix_add_multiply_1x128 (&d, &m, &s, &da));
 }
 
-static force_inline void
-core_combine_atop_ca_sse2 (uint32_t *      pd,
-                           const uint32_t *ps,
-                           const uint32_t *pm,
-                           int             w)
+static void
+sse2_combine_atop_ca (pixman_implementation_t *imp,
+                      pixman_op_t              op,
+                      uint32_t *               pd,
+                      const uint32_t *         ps,
+                      const uint32_t *         pm,
+                      int                      w)
 {
     uint32_t s, m, d;
 
@@ -2257,12 +2051,7 @@ core_combine_atop_ca_sse2 (uint32_t *      pd,
     __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
     __m128i xmm_mask_lo, xmm_mask_hi;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
-    while (w && (unsigned long)pd & 15)
+    while (w && (uintptr_t)pd & 15)
     {
 	s = *ps++;
 	m = *pm++;
@@ -2272,18 +2061,8 @@ core_combine_atop_ca_sse2 (uint32_t *      pd,
 	w--;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	cache_prefetch_next ((__m128i*)pm);
-
 	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
 	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
 	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
@@ -2336,24 +2115,26 @@ core_combine_reverse_atop_ca_pixel_sse2 (uint32_t src,
                                          uint32_t mask,
                                          uint32_t dst)
 {
-    __m64 m = unpack_32_1x64 (mask);
-    __m64 s = unpack_32_1x64 (src);
-    __m64 d = unpack_32_1x64 (dst);
+    __m128i m = unpack_32_1x128 (mask);
+    __m128i s = unpack_32_1x128 (src);
+    __m128i d = unpack_32_1x128 (dst);
 
-    __m64 da = negate_1x64 (expand_alpha_1x64 (d));
-    __m64 sa = expand_alpha_1x64 (s);
+    __m128i da = negate_1x128 (expand_alpha_1x128 (d));
+    __m128i sa = expand_alpha_1x128 (s);
 
-    s = pix_multiply_1x64 (s, m);
-    m = pix_multiply_1x64 (m, sa);
+    s = pix_multiply_1x128 (s, m);
+    m = pix_multiply_1x128 (m, sa);
 
-    return pack_1x64_32 (pix_add_multiply_1x64 (&d, &m, &s, &da));
+    return pack_1x128_32 (pix_add_multiply_1x128 (&d, &m, &s, &da));
 }
 
-static force_inline void
-core_combine_reverse_atop_ca_sse2 (uint32_t *      pd,
-                                   const uint32_t *ps,
-                                   const uint32_t *pm,
-                                   int             w)
+static void
+sse2_combine_atop_reverse_ca (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              uint32_t *               pd,
+                              const uint32_t *         ps,
+                              const uint32_t *         pm,
+                              int                      w)
 {
     uint32_t s, m, d;
 
@@ -2363,12 +2144,7 @@ core_combine_reverse_atop_ca_sse2 (uint32_t *      pd,
     __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
     __m128i xmm_mask_lo, xmm_mask_hi;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
-    while (w && (unsigned long)pd & 15)
+    while (w && (uintptr_t)pd & 15)
     {
 	s = *ps++;
 	m = *pm++;
@@ -2378,18 +2154,8 @@ core_combine_reverse_atop_ca_sse2 (uint32_t *      pd,
 	w--;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	cache_prefetch_next ((__m128i*)pm);
-
 	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
 	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
 	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
@@ -2443,26 +2209,28 @@ core_combine_xor_ca_pixel_sse2 (uint32_t src,
                                 uint32_t mask,
                                 uint32_t dst)
 {
-    __m64 a = unpack_32_1x64 (mask);
-    __m64 s = unpack_32_1x64 (src);
-    __m64 d = unpack_32_1x64 (dst);
+    __m128i a = unpack_32_1x128 (mask);
+    __m128i s = unpack_32_1x128 (src);
+    __m128i d = unpack_32_1x128 (dst);
 
-    __m64 alpha_dst = negate_1x64 (pix_multiply_1x64 (
-				       a, expand_alpha_1x64 (s)));
-    __m64 dest      = pix_multiply_1x64 (s, a);
-    __m64 alpha_src = negate_1x64 (expand_alpha_1x64 (d));
+    __m128i alpha_dst = negate_1x128 (pix_multiply_1x128 (
+				       a, expand_alpha_1x128 (s)));
+    __m128i dest      = pix_multiply_1x128 (s, a);
+    __m128i alpha_src = negate_1x128 (expand_alpha_1x128 (d));
 
-    return pack_1x64_32 (pix_add_multiply_1x64 (&d,
+    return pack_1x128_32 (pix_add_multiply_1x128 (&d,
                                                 &alpha_dst,
                                                 &dest,
                                                 &alpha_src));
 }
 
-static force_inline void
-core_combine_xor_ca_sse2 (uint32_t *      pd,
-                          const uint32_t *ps,
-                          const uint32_t *pm,
-                          int             w)
+static void
+sse2_combine_xor_ca (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     uint32_t *               pd,
+                     const uint32_t *         ps,
+                     const uint32_t *         pm,
+                     int                      w)
 {
     uint32_t s, m, d;
 
@@ -2472,12 +2240,7 @@ core_combine_xor_ca_sse2 (uint32_t *      pd,
     __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
     __m128i xmm_mask_lo, xmm_mask_hi;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
-    while (w && (unsigned long)pd & 15)
+    while (w && (uintptr_t)pd & 15)
     {
 	s = *ps++;
 	m = *pm++;
@@ -2487,18 +2250,8 @@ core_combine_xor_ca_sse2 (uint32_t *      pd,
 	w--;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	cache_prefetch_next ((__m128i*)pm);
-
 	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
 	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
 	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
@@ -2549,11 +2302,13 @@ core_combine_xor_ca_sse2 (uint32_t *      pd,
     }
 }
 
-static force_inline void
-core_combine_add_ca_sse2 (uint32_t *      pd,
-                          const uint32_t *ps,
-                          const uint32_t *pm,
-                          int             w)
+static void
+sse2_combine_add_ca (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     uint32_t *               pd,
+                     const uint32_t *         ps,
+                     const uint32_t *         pm,
+                     int                      w)
 {
     uint32_t s, m, d;
 
@@ -2561,36 +2316,21 @@ core_combine_add_ca_sse2 (uint32_t *      pd,
     __m128i xmm_dst_lo, xmm_dst_hi;
     __m128i xmm_mask_lo, xmm_mask_hi;
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
-    while (w && (unsigned long)pd & 15)
+    while (w && (uintptr_t)pd & 15)
     {
 	s = *ps++;
 	m = *pm++;
 	d = *pd;
 
-	*pd++ = pack_1x64_32 (
-	    _mm_adds_pu8 (pix_multiply_1x64 (unpack_32_1x64 (s),
-					     unpack_32_1x64 (m)),
-			  unpack_32_1x64 (d)));
+	*pd++ = pack_1x128_32 (
+	    _mm_adds_epu8 (pix_multiply_1x128 (unpack_32_1x128 (s),
+					       unpack_32_1x128 (m)),
+			   unpack_32_1x128 (d)));
 	w--;
     }
 
-    /* call prefetch hint to optimize cache load*/
-    cache_prefetch ((__m128i*)ps);
-    cache_prefetch ((__m128i*)pd);
-    cache_prefetch ((__m128i*)pm);
-
     while (w >= 4)
     {
-	/* fill cache line with next memory */
-	cache_prefetch_next ((__m128i*)ps);
-	cache_prefetch_next ((__m128i*)pd);
-	cache_prefetch_next ((__m128i*)pm);
-
 	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
 	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
 	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
@@ -2620,36 +2360,20 @@ core_combine_add_ca_sse2 (uint32_t *      pd,
 	m = *pm++;
 	d = *pd;
 
-	*pd++ = pack_1x64_32 (
-	    _mm_adds_pu8 (pix_multiply_1x64 (unpack_32_1x64 (s),
-					     unpack_32_1x64 (m)),
-			  unpack_32_1x64 (d)));
+	*pd++ = pack_1x128_32 (
+	    _mm_adds_epu8 (pix_multiply_1x128 (unpack_32_1x128 (s),
+					       unpack_32_1x128 (m)),
+			   unpack_32_1x128 (d)));
 	w--;
     }
 }
 
-/* ---------------------------------------------------
- * fb_compose_setup_sSE2
- */
-static force_inline __m64
-create_mask_16_64 (uint16_t mask)
-{
-    return _mm_set1_pi16 (mask);
-}
-
 static force_inline __m128i
 create_mask_16_128 (uint16_t mask)
 {
     return _mm_set1_epi16 (mask);
 }
 
-static force_inline __m64
-create_mask_2x32_64 (uint32_t mask0,
-                     uint32_t mask1)
-{
-    return _mm_set_pi32 (mask0, mask1);
-}
-
 /* Work around a code generation bug in Sun Studio 12. */
 #if defined(__SUNPRO_C) && (__SUNPRO_C >= 0x590)
 # define create_mask_2x32_128(mask0, mask1)				\
@@ -2663,291 +2387,11 @@ create_mask_2x32_128 (uint32_t mask0,
 }
 #endif
 
-/* SSE2 code patch for fbcompose.c */
-
-static void
-sse2_combine_over_u (pixman_implementation_t *imp,
-                     pixman_op_t              op,
-                     uint32_t *               dst,
-                     const uint32_t *         src,
-                     const uint32_t *         mask,
-                     int                      width)
-{
-    core_combine_over_u_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_over_reverse_u (pixman_implementation_t *imp,
-                             pixman_op_t              op,
-                             uint32_t *               dst,
-                             const uint32_t *         src,
-                             const uint32_t *         mask,
-                             int                      width)
-{
-    core_combine_over_reverse_u_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_in_u (pixman_implementation_t *imp,
-                   pixman_op_t              op,
-                   uint32_t *               dst,
-                   const uint32_t *         src,
-                   const uint32_t *         mask,
-                   int                      width)
-{
-    core_combine_in_u_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_in_reverse_u (pixman_implementation_t *imp,
-                           pixman_op_t              op,
-                           uint32_t *               dst,
-                           const uint32_t *         src,
-                           const uint32_t *         mask,
-                           int                      width)
-{
-    core_combine_reverse_in_u_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_out_u (pixman_implementation_t *imp,
-                    pixman_op_t              op,
-                    uint32_t *               dst,
-                    const uint32_t *         src,
-                    const uint32_t *         mask,
-                    int                      width)
-{
-    core_combine_out_u_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_out_reverse_u (pixman_implementation_t *imp,
-                            pixman_op_t              op,
-                            uint32_t *               dst,
-                            const uint32_t *         src,
-                            const uint32_t *         mask,
-                            int                      width)
-{
-    core_combine_reverse_out_u_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_atop_u (pixman_implementation_t *imp,
-                     pixman_op_t              op,
-                     uint32_t *               dst,
-                     const uint32_t *         src,
-                     const uint32_t *         mask,
-                     int                      width)
-{
-    core_combine_atop_u_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_atop_reverse_u (pixman_implementation_t *imp,
-                             pixman_op_t              op,
-                             uint32_t *               dst,
-                             const uint32_t *         src,
-                             const uint32_t *         mask,
-                             int                      width)
-{
-    core_combine_reverse_atop_u_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_xor_u (pixman_implementation_t *imp,
-                    pixman_op_t              op,
-                    uint32_t *               dst,
-                    const uint32_t *         src,
-                    const uint32_t *         mask,
-                    int                      width)
-{
-    core_combine_xor_u_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_add_u (pixman_implementation_t *imp,
-                    pixman_op_t              op,
-                    uint32_t *               dst,
-                    const uint32_t *         src,
-                    const uint32_t *         mask,
-                    int                      width)
-{
-    core_combine_add_u_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_saturate_u (pixman_implementation_t *imp,
-                         pixman_op_t              op,
-                         uint32_t *               dst,
-                         const uint32_t *         src,
-                         const uint32_t *         mask,
-                         int                      width)
-{
-    core_combine_saturate_u_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_src_ca (pixman_implementation_t *imp,
-                     pixman_op_t              op,
-                     uint32_t *               dst,
-                     const uint32_t *         src,
-                     const uint32_t *         mask,
-                     int                      width)
-{
-    core_combine_src_ca_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_over_ca (pixman_implementation_t *imp,
-                      pixman_op_t              op,
-                      uint32_t *               dst,
-                      const uint32_t *         src,
-                      const uint32_t *         mask,
-                      int                      width)
-{
-    core_combine_over_ca_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_over_reverse_ca (pixman_implementation_t *imp,
-                              pixman_op_t              op,
-                              uint32_t *               dst,
-                              const uint32_t *         src,
-                              const uint32_t *         mask,
-                              int                      width)
-{
-    core_combine_over_reverse_ca_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_in_ca (pixman_implementation_t *imp,
-                    pixman_op_t              op,
-                    uint32_t *               dst,
-                    const uint32_t *         src,
-                    const uint32_t *         mask,
-                    int                      width)
-{
-    core_combine_in_ca_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_in_reverse_ca (pixman_implementation_t *imp,
-                            pixman_op_t              op,
-                            uint32_t *               dst,
-                            const uint32_t *         src,
-                            const uint32_t *         mask,
-                            int                      width)
-{
-    core_combine_in_reverse_ca_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_out_ca (pixman_implementation_t *imp,
-                     pixman_op_t              op,
-                     uint32_t *               dst,
-                     const uint32_t *         src,
-                     const uint32_t *         mask,
-                     int                      width)
-{
-    core_combine_out_ca_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_out_reverse_ca (pixman_implementation_t *imp,
-                             pixman_op_t              op,
-                             uint32_t *               dst,
-                             const uint32_t *         src,
-                             const uint32_t *         mask,
-                             int                      width)
-{
-    core_combine_out_reverse_ca_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_atop_ca (pixman_implementation_t *imp,
-                      pixman_op_t              op,
-                      uint32_t *               dst,
-                      const uint32_t *         src,
-                      const uint32_t *         mask,
-                      int                      width)
-{
-    core_combine_atop_ca_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_atop_reverse_ca (pixman_implementation_t *imp,
-                              pixman_op_t              op,
-                              uint32_t *               dst,
-                              const uint32_t *         src,
-                              const uint32_t *         mask,
-                              int                      width)
-{
-    core_combine_reverse_atop_ca_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_xor_ca (pixman_implementation_t *imp,
-                     pixman_op_t              op,
-                     uint32_t *               dst,
-                     const uint32_t *         src,
-                     const uint32_t *         mask,
-                     int                      width)
-{
-    core_combine_xor_ca_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-static void
-sse2_combine_add_ca (pixman_implementation_t *imp,
-                     pixman_op_t              op,
-                     uint32_t *               dst,
-                     const uint32_t *         src,
-                     const uint32_t *         mask,
-                     int                      width)
-{
-    core_combine_add_ca_sse2 (dst, src, mask, width);
-    _mm_empty ();
-}
-
-/* -------------------------------------------------------------------
- * composite_over_n_8888
- */
-
 static void
 sse2_composite_over_n_8888 (pixman_implementation_t *imp,
-                            pixman_op_t              op,
-                            pixman_image_t *         src_image,
-                            pixman_image_t *         mask_image,
-                            pixman_image_t *         dst_image,
-                            int32_t                  src_x,
-                            int32_t                  src_y,
-                            int32_t                  mask_x,
-                            int32_t                  mask_y,
-                            int32_t                  dest_x,
-                            int32_t                  dest_y,
-                            int32_t                  width,
-                            int32_t                  height)
+                            pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t src;
     uint32_t    *dst_line, *dst, d;
     int32_t w;
@@ -2955,13 +2399,13 @@ sse2_composite_over_n_8888 (pixman_implementation_t *imp,
     __m128i xmm_src, xmm_alpha;
     __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     if (src == 0)
 	return;
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
 
     xmm_src = expand_pixel_32_1x128 (src);
     xmm_alpha = expand_alpha_1x128 (xmm_src);
@@ -2970,28 +2414,20 @@ sse2_composite_over_n_8888 (pixman_implementation_t *imp,
     {
 	dst = dst_line;
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)dst);
-
 	dst_line += dst_stride;
 	w = width;
 
-	while (w && (unsigned long)dst & 15)
+	while (w && (uintptr_t)dst & 15)
 	{
 	    d = *dst;
-	    *dst++ = pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmm_src),
-	                                      _mm_movepi64_pi64 (xmm_alpha),
-	                                      unpack_32_1x64 (d)));
+	    *dst++ = pack_1x128_32 (over_1x128 (xmm_src,
+						xmm_alpha,
+						unpack_32_1x128 (d)));
 	    w--;
 	}
 
-	cache_prefetch ((__m128i*)dst);
-
 	while (w >= 4)
 	{
-	    /* fill cache line with next memory */
-	    cache_prefetch_next ((__m128i*)dst);
-
 	    xmm_dst = load_128_aligned ((__m128i*)dst);
 
 	    unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
@@ -3011,34 +2447,20 @@ sse2_composite_over_n_8888 (pixman_implementation_t *imp,
 	while (w)
 	{
 	    d = *dst;
-	    *dst++ = pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmm_src),
-	                                      _mm_movepi64_pi64 (xmm_alpha),
-	                                      unpack_32_1x64 (d)));
+	    *dst++ = pack_1x128_32 (over_1x128 (xmm_src,
+						xmm_alpha,
+						unpack_32_1x128 (d)));
 	    w--;
 	}
 
     }
-    _mm_empty ();
 }
 
-/* ---------------------------------------------------------------------
- * composite_over_n_0565
- */
 static void
 sse2_composite_over_n_0565 (pixman_implementation_t *imp,
-                            pixman_op_t              op,
-                            pixman_image_t *         src_image,
-                            pixman_image_t *         mask_image,
-                            pixman_image_t *         dst_image,
-                            int32_t                  src_x,
-                            int32_t                  src_y,
-                            int32_t                  mask_x,
-                            int32_t                  mask_y,
-                            int32_t                  dest_x,
-                            int32_t                  dest_y,
-                            int32_t                  width,
-                            int32_t                  height)
+                            pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t src;
     uint16_t    *dst_line, *dst, d;
     int32_t w;
@@ -3046,13 +2468,13 @@ sse2_composite_over_n_0565 (pixman_implementation_t *imp,
     __m128i xmm_src, xmm_alpha;
     __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     if (src == 0)
 	return;
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
 
     xmm_src = expand_pixel_32_1x128 (src);
     xmm_alpha = expand_alpha_1x128 (xmm_src);
@@ -3061,31 +2483,22 @@ sse2_composite_over_n_0565 (pixman_implementation_t *imp,
     {
 	dst = dst_line;
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)dst);
-
 	dst_line += dst_stride;
 	w = width;
 
-	while (w && (unsigned long)dst & 15)
+	while (w && (uintptr_t)dst & 15)
 	{
 	    d = *dst;
 
 	    *dst++ = pack_565_32_16 (
-		pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmm_src),
-					 _mm_movepi64_pi64 (xmm_alpha),
-					 expand565_16_1x64 (d))));
+		pack_1x128_32 (over_1x128 (xmm_src,
+					   xmm_alpha,
+					   expand565_16_1x128 (d))));
 	    w--;
 	}
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)dst);
-
 	while (w >= 8)
 	{
-	    /* fill cache line with next memory */
-	    cache_prefetch_next ((__m128i*)dst);
-
 	    xmm_dst = load_128_aligned ((__m128i*)dst);
 
 	    unpack_565_128_4x128 (xmm_dst,
@@ -3111,61 +2524,43 @@ sse2_composite_over_n_0565 (pixman_implementation_t *imp,
 	{
 	    d = *dst;
 	    *dst++ = pack_565_32_16 (
-		pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmm_src),
-					 _mm_movepi64_pi64 (xmm_alpha),
-					 expand565_16_1x64 (d))));
+		pack_1x128_32 (over_1x128 (xmm_src, xmm_alpha,
+					   expand565_16_1x128 (d))));
 	}
     }
 
-    _mm_empty ();
 }
 
-/* ------------------------------
- * composite_add_n_8888_8888_ca
- */
 static void
 sse2_composite_add_n_8888_8888_ca (pixman_implementation_t *imp,
-				   pixman_op_t              op,
-				   pixman_image_t *         src_image,
-				   pixman_image_t *         mask_image,
-				   pixman_image_t *         dst_image,
-				   int32_t                  src_x,
-				   int32_t                  src_y,
-				   int32_t                  mask_x,
-				   int32_t                  mask_y,
-				   int32_t                  dest_x,
-				   int32_t                  dest_y,
-				   int32_t                  width,
-				   int32_t                  height)
+				   pixman_composite_info_t *info)
 {
-    uint32_t src, srca;
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint32_t src;
     uint32_t    *dst_line, d;
     uint32_t    *mask_line, m;
     uint32_t pack_cmp;
     int dst_stride, mask_stride;
 
-    __m128i xmm_src, xmm_alpha;
+    __m128i xmm_src;
     __m128i xmm_dst;
     __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
 
-    __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
+    __m128i mmx_src, mmx_mask, mmx_dest;
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
-    srca = src >> 24;
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     if (src == 0)
 	return;
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (
 	mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
 
     xmm_src = _mm_unpacklo_epi8 (
 	create_mask_2x32_128 (src, src), _mm_setzero_si128 ());
-    xmm_alpha = expand_alpha_1x128 (xmm_src);
-    mmx_src   = _mm_movepi64_pi64 (xmm_src);
-    mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
+    mmx_src   = xmm_src;
 
     while (height--)
     {
@@ -3176,11 +2571,7 @@ sse2_composite_add_n_8888_8888_ca (pixman_implementation_t *imp,
 	dst_line += dst_stride;
 	mask_line += mask_stride;
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)pd);
-	cache_prefetch ((__m128i*)pm);
-
-	while (w && (unsigned long)pd & 15)
+	while (w && (uintptr_t)pd & 15)
 	{
 	    m = *pm++;
 
@@ -3188,27 +2579,20 @@ sse2_composite_add_n_8888_8888_ca (pixman_implementation_t *imp,
 	    {
 		d = *pd;
 
-		mmx_mask = unpack_32_1x64 (m);
-		mmx_dest = unpack_32_1x64 (d);
+		mmx_mask = unpack_32_1x128 (m);
+		mmx_dest = unpack_32_1x128 (d);
 
-		*pd = pack_1x64_32 (
-		    _mm_adds_pu8 (pix_multiply_1x64 (mmx_mask, mmx_src), mmx_dest));
+		*pd = pack_1x128_32 (
+		    _mm_adds_epu8 (pix_multiply_1x128 (mmx_mask, mmx_src),
+				   mmx_dest));
 	    }
 
 	    pd++;
 	    w--;
 	}
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)pd);
-	cache_prefetch ((__m128i*)pm);
-
 	while (w >= 4)
 	{
-	    /* fill cache line with next memory */
-	    cache_prefetch_next ((__m128i*)pd);
-	    cache_prefetch_next ((__m128i*)pm);
-
 	    xmm_mask = load_128_unaligned ((__m128i*)pm);
 
 	    pack_cmp =
@@ -3244,11 +2628,12 @@ sse2_composite_add_n_8888_8888_ca (pixman_implementation_t *imp,
 	    {
 		d = *pd;
 
-		mmx_mask = unpack_32_1x64 (m);
-		mmx_dest = unpack_32_1x64 (d);
+		mmx_mask = unpack_32_1x128 (m);
+		mmx_dest = unpack_32_1x128 (d);
 
-		*pd = pack_1x64_32 (
-		    _mm_adds_pu8 (pix_multiply_1x64 (mmx_mask, mmx_src), mmx_dest));
+		*pd = pack_1x128_32 (
+		    _mm_adds_epu8 (pix_multiply_1x128 (mmx_mask, mmx_src),
+				   mmx_dest));
 	    }
 
 	    pd++;
@@ -3256,28 +2641,13 @@ sse2_composite_add_n_8888_8888_ca (pixman_implementation_t *imp,
 	}
     }
 
-    _mm_empty ();
 }
 
-/* ---------------------------------------------------------------------------
- * composite_over_n_8888_8888_ca
- */
-
 static void
 sse2_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
-                                    pixman_op_t              op,
-                                    pixman_image_t *         src_image,
-                                    pixman_image_t *         mask_image,
-                                    pixman_image_t *         dst_image,
-                                    int32_t                  src_x,
-                                    int32_t                  src_y,
-                                    int32_t                  mask_x,
-                                    int32_t                  mask_y,
-                                    int32_t                  dest_x,
-                                    int32_t                  dest_y,
-                                    int32_t                  width,
-                                    int32_t                  height)
+                                    pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t src;
     uint32_t    *dst_line, d;
     uint32_t    *mask_line, m;
@@ -3288,23 +2658,23 @@ sse2_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
     __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
     __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
 
-    __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
+    __m128i mmx_src, mmx_alpha, mmx_mask, mmx_dest;
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     if (src == 0)
 	return;
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (
 	mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
 
     xmm_src = _mm_unpacklo_epi8 (
 	create_mask_2x32_128 (src, src), _mm_setzero_si128 ());
     xmm_alpha = expand_alpha_1x128 (xmm_src);
-    mmx_src   = _mm_movepi64_pi64 (xmm_src);
-    mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
+    mmx_src   = xmm_src;
+    mmx_alpha = xmm_alpha;
 
     while (height--)
     {
@@ -3315,21 +2685,17 @@ sse2_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
 	dst_line += dst_stride;
 	mask_line += mask_stride;
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)pd);
-	cache_prefetch ((__m128i*)pm);
-
-	while (w && (unsigned long)pd & 15)
+	while (w && (uintptr_t)pd & 15)
 	{
 	    m = *pm++;
 
 	    if (m)
 	    {
 		d = *pd;
-		mmx_mask = unpack_32_1x64 (m);
-		mmx_dest = unpack_32_1x64 (d);
+		mmx_mask = unpack_32_1x128 (m);
+		mmx_dest = unpack_32_1x128 (d);
 
-		*pd = pack_1x64_32 (in_over_1x64 (&mmx_src,
+		*pd = pack_1x128_32 (in_over_1x128 (&mmx_src,
 		                                  &mmx_alpha,
 		                                  &mmx_mask,
 		                                  &mmx_dest));
@@ -3339,16 +2705,8 @@ sse2_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
 	    w--;
 	}
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)pd);
-	cache_prefetch ((__m128i*)pm);
-
 	while (w >= 4)
 	{
-	    /* fill cache line with next memory */
-	    cache_prefetch_next ((__m128i*)pd);
-	    cache_prefetch_next ((__m128i*)pm);
-
 	    xmm_mask = load_128_unaligned ((__m128i*)pm);
 
 	    pack_cmp =
@@ -3384,11 +2742,11 @@ sse2_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
 	    if (m)
 	    {
 		d = *pd;
-		mmx_mask = unpack_32_1x64 (m);
-		mmx_dest = unpack_32_1x64 (d);
+		mmx_mask = unpack_32_1x128 (m);
+		mmx_dest = unpack_32_1x128 (d);
 
-		*pd = pack_1x64_32 (
-		    in_over_1x64 (&mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest));
+		*pd = pack_1x128_32 (
+		    in_over_1x128 (&mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest));
 	    }
 
 	    pd++;
@@ -3396,28 +2754,13 @@ sse2_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
 	}
     }
 
-    _mm_empty ();
 }
 
-/*---------------------------------------------------------------------
- * composite_over_8888_n_8888
- */
-
 static void
 sse2_composite_over_8888_n_8888 (pixman_implementation_t *imp,
-                                 pixman_op_t              op,
-                                 pixman_image_t *         src_image,
-                                 pixman_image_t *         mask_image,
-                                 pixman_image_t *         dst_image,
-                                 int32_t                  src_x,
-                                 int32_t                  src_y,
-                                 int32_t                  mask_x,
-                                 int32_t                  mask_y,
-                                 int32_t                  dest_x,
-                                 int32_t                  dest_y,
-                                 int32_t                  width,
-                                 int32_t                  height)
+                                 pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t    *dst_line, *dst;
     uint32_t    *src_line, *src;
     uint32_t mask;
@@ -3430,11 +2773,11 @@ sse2_composite_over_8888_n_8888 (pixman_implementation_t *imp,
     __m128i xmm_alpha_lo, xmm_alpha_hi;
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (
 	src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
 
-    mask = _pixman_image_get_solid (mask_image, PIXMAN_a8r8g8b8);
+    mask = _pixman_image_get_solid (imp, mask_image, PIXMAN_a8r8g8b8);
 
     xmm_mask = create_mask_16_128 (mask >> 24);
 
@@ -3446,52 +2789,48 @@ sse2_composite_over_8888_n_8888 (pixman_implementation_t *imp,
 	src_line += src_stride;
 	w = width;
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)dst);
-	cache_prefetch ((__m128i*)src);
-
-	while (w && (unsigned long)dst & 15)
+	while (w && (uintptr_t)dst & 15)
 	{
 	    uint32_t s = *src++;
-	    uint32_t d = *dst;
-
-	    __m64 ms = unpack_32_1x64 (s);
-	    __m64 alpha    = expand_alpha_1x64 (ms);
-	    __m64 dest     = _mm_movepi64_pi64 (xmm_mask);
-	    __m64 alpha_dst = unpack_32_1x64 (d);
-
-	    *dst++ = pack_1x64_32 (
-		in_over_1x64 (&ms, &alpha, &dest, &alpha_dst));
 
+	    if (s)
+	    {
+		uint32_t d = *dst;
+		
+		__m128i ms = unpack_32_1x128 (s);
+		__m128i alpha    = expand_alpha_1x128 (ms);
+		__m128i dest     = xmm_mask;
+		__m128i alpha_dst = unpack_32_1x128 (d);
+		
+		*dst = pack_1x128_32 (
+		    in_over_1x128 (&ms, &alpha, &dest, &alpha_dst));
+	    }
+	    dst++;
 	    w--;
 	}
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)dst);
-	cache_prefetch ((__m128i*)src);
-
 	while (w >= 4)
 	{
-	    /* fill cache line with next memory */
-	    cache_prefetch_next ((__m128i*)dst);
-	    cache_prefetch_next ((__m128i*)src);
-
 	    xmm_src = load_128_unaligned ((__m128i*)src);
-	    xmm_dst = load_128_aligned ((__m128i*)dst);
-
-	    unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
-	    unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
-	    expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
-				&xmm_alpha_lo, &xmm_alpha_hi);
-
-	    in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
-			   &xmm_alpha_lo, &xmm_alpha_hi,
-			   &xmm_mask, &xmm_mask,
-			   &xmm_dst_lo, &xmm_dst_hi);
-
-	    save_128_aligned (
-		(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
 
+	    if (!is_zero (xmm_src))
+	    {
+		xmm_dst = load_128_aligned ((__m128i*)dst);
+		
+		unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
+		unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+		expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
+				    &xmm_alpha_lo, &xmm_alpha_hi);
+		
+		in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
+			       &xmm_alpha_lo, &xmm_alpha_hi,
+			       &xmm_mask, &xmm_mask,
+			       &xmm_dst_lo, &xmm_dst_hi);
+		
+		save_128_aligned (
+		    (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+	    }
+		
 	    dst += 4;
 	    src += 4;
 	    w -= 4;
@@ -3500,41 +2839,141 @@ sse2_composite_over_8888_n_8888 (pixman_implementation_t *imp,
 	while (w)
 	{
 	    uint32_t s = *src++;
-	    uint32_t d = *dst;
 
-	    __m64 ms = unpack_32_1x64 (s);
-	    __m64 alpha = expand_alpha_1x64 (ms);
-	    __m64 mask  = _mm_movepi64_pi64 (xmm_mask);
-	    __m64 dest  = unpack_32_1x64 (d);
+	    if (s)
+	    {
+		uint32_t d = *dst;
+		
+		__m128i ms = unpack_32_1x128 (s);
+		__m128i alpha = expand_alpha_1x128 (ms);
+		__m128i mask  = xmm_mask;
+		__m128i dest  = unpack_32_1x128 (d);
+		
+		*dst = pack_1x128_32 (
+		    in_over_1x128 (&ms, &alpha, &mask, &dest));
+	    }
+
+	    dst++;
+	    w--;
+	}
+    }
+
+}
+
+static void
+sse2_composite_src_x888_0565 (pixman_implementation_t *imp,
+                              pixman_composite_info_t *info)
+{
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint16_t    *dst_line, *dst;
+    uint32_t    *src_line, *src, s;
+    int dst_stride, src_stride;
+    int32_t w;
+
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w && (uintptr_t)dst & 15)
+	{
+	    s = *src++;
+	    *dst = convert_8888_to_0565 (s);
+	    dst++;
+	    w--;
+	}
+
+	while (w >= 8)
+	{
+	    __m128i xmm_src0 = load_128_unaligned ((__m128i *)src + 0);
+	    __m128i xmm_src1 = load_128_unaligned ((__m128i *)src + 1);
+
+	    save_128_aligned ((__m128i*)dst, pack_565_2packedx128_128 (xmm_src0, xmm_src1));
+
+	    w -= 8;
+	    src += 8;
+	    dst += 8;
+	}
+
+	while (w)
+	{
+	    s = *src++;
+	    *dst = convert_8888_to_0565 (s);
+	    dst++;
+	    w--;
+	}
+    }
+}
+
+static void
+sse2_composite_src_x888_8888 (pixman_implementation_t *imp,
+			      pixman_composite_info_t *info)
+{
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint32_t    *dst_line, *dst;
+    uint32_t    *src_line, *src;
+    int32_t w;
+    int dst_stride, src_stride;
+
+
+    PIXMAN_IMAGE_GET_LINE (
+	dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w && (uintptr_t)dst & 15)
+	{
+	    *dst++ = *src++ | 0xff000000;
+	    w--;
+	}
 
-	    *dst++ = pack_1x64_32 (
-		in_over_1x64 (&ms, &alpha, &mask, &dest));
+	while (w >= 16)
+	{
+	    __m128i xmm_src1, xmm_src2, xmm_src3, xmm_src4;
+	    
+	    xmm_src1 = load_128_unaligned ((__m128i*)src + 0);
+	    xmm_src2 = load_128_unaligned ((__m128i*)src + 1);
+	    xmm_src3 = load_128_unaligned ((__m128i*)src + 2);
+	    xmm_src4 = load_128_unaligned ((__m128i*)src + 3);
+	    
+	    save_128_aligned ((__m128i*)dst + 0, _mm_or_si128 (xmm_src1, mask_ff000000));
+	    save_128_aligned ((__m128i*)dst + 1, _mm_or_si128 (xmm_src2, mask_ff000000));
+	    save_128_aligned ((__m128i*)dst + 2, _mm_or_si128 (xmm_src3, mask_ff000000));
+	    save_128_aligned ((__m128i*)dst + 3, _mm_or_si128 (xmm_src4, mask_ff000000));
+	    
+	    dst += 16;
+	    src += 16;
+	    w -= 16;
+	}
 
+	while (w)
+	{
+	    *dst++ = *src++ | 0xff000000;
 	    w--;
 	}
     }
 
-    _mm_empty ();
 }
 
-/* ---------------------------------------------------------------------
- * composite_over_x888_n_8888
- */
 static void
 sse2_composite_over_x888_n_8888 (pixman_implementation_t *imp,
-                                 pixman_op_t              op,
-                                 pixman_image_t *         src_image,
-                                 pixman_image_t *         mask_image,
-                                 pixman_image_t *         dst_image,
-                                 int32_t                  src_x,
-                                 int32_t                  src_y,
-                                 int32_t                  mask_x,
-                                 int32_t                  mask_y,
-                                 int32_t                  dest_x,
-                                 int32_t                  dest_y,
-                                 int32_t                  width,
-                                 int32_t                  height)
+                                 pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t    *dst_line, *dst;
     uint32_t    *src_line, *src;
     uint32_t mask;
@@ -3546,11 +2985,11 @@ sse2_composite_over_x888_n_8888 (pixman_implementation_t *imp,
     __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (
 	src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
 
-    mask = _pixman_image_get_solid (mask_image, PIXMAN_a8r8g8b8);
+    mask = _pixman_image_get_solid (imp, mask_image, PIXMAN_a8r8g8b8);
 
     xmm_mask = create_mask_16_128 (mask >> 24);
     xmm_alpha = mask_00ff;
@@ -3563,36 +3002,24 @@ sse2_composite_over_x888_n_8888 (pixman_implementation_t *imp,
 	src_line += src_stride;
 	w = width;
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)dst);
-	cache_prefetch ((__m128i*)src);
-
-	while (w && (unsigned long)dst & 15)
+	while (w && (uintptr_t)dst & 15)
 	{
 	    uint32_t s = (*src++) | 0xff000000;
 	    uint32_t d = *dst;
 
-	    __m64 src   = unpack_32_1x64 (s);
-	    __m64 alpha = _mm_movepi64_pi64 (xmm_alpha);
-	    __m64 mask  = _mm_movepi64_pi64 (xmm_mask);
-	    __m64 dest  = unpack_32_1x64 (d);
+	    __m128i src   = unpack_32_1x128 (s);
+	    __m128i alpha = xmm_alpha;
+	    __m128i mask  = xmm_mask;
+	    __m128i dest  = unpack_32_1x128 (d);
 
-	    *dst++ = pack_1x64_32 (
-		in_over_1x64 (&src, &alpha, &mask, &dest));
+	    *dst++ = pack_1x128_32 (
+		in_over_1x128 (&src, &alpha, &mask, &dest));
 
 	    w--;
 	}
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)dst);
-	cache_prefetch ((__m128i*)src);
-
 	while (w >= 4)
 	{
-	    /* fill cache line with next memory */
-	    cache_prefetch_next ((__m128i*)dst);
-	    cache_prefetch_next ((__m128i*)src);
-
 	    xmm_src = _mm_or_si128 (
 		load_128_unaligned ((__m128i*)src), mask_ff000000);
 	    xmm_dst = load_128_aligned ((__m128i*)dst);
@@ -3619,45 +3046,31 @@ sse2_composite_over_x888_n_8888 (pixman_implementation_t *imp,
 	    uint32_t s = (*src++) | 0xff000000;
 	    uint32_t d = *dst;
 
-	    __m64 src  = unpack_32_1x64 (s);
-	    __m64 alpha = _mm_movepi64_pi64 (xmm_alpha);
-	    __m64 mask  = _mm_movepi64_pi64 (xmm_mask);
-	    __m64 dest  = unpack_32_1x64 (d);
+	    __m128i src  = unpack_32_1x128 (s);
+	    __m128i alpha = xmm_alpha;
+	    __m128i mask  = xmm_mask;
+	    __m128i dest  = unpack_32_1x128 (d);
 
-	    *dst++ = pack_1x64_32 (
-		in_over_1x64 (&src, &alpha, &mask, &dest));
+	    *dst++ = pack_1x128_32 (
+		in_over_1x128 (&src, &alpha, &mask, &dest));
 
 	    w--;
 	}
     }
 
-    _mm_empty ();
 }
 
-/* --------------------------------------------------------------------
- * composite_over_8888_8888
- */
 static void
 sse2_composite_over_8888_8888 (pixman_implementation_t *imp,
-                               pixman_op_t              op,
-                               pixman_image_t *         src_image,
-                               pixman_image_t *         mask_image,
-                               pixman_image_t *         dst_image,
-                               int32_t                  src_x,
-                               int32_t                  src_y,
-                               int32_t                  mask_x,
-                               int32_t                  mask_y,
-                               int32_t                  dest_x,
-                               int32_t                  dest_y,
-                               int32_t                  width,
-                               int32_t                  height)
+                               pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     int dst_stride, src_stride;
     uint32_t    *dst_line, *dst;
     uint32_t    *src_line, *src;
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (
 	src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
 
@@ -3666,44 +3079,30 @@ sse2_composite_over_8888_8888 (pixman_implementation_t *imp,
 
     while (height--)
     {
-	core_combine_over_u_sse2 (dst, src, NULL, width);
+	sse2_combine_over_u (imp, op, dst, src, NULL, width);
 
 	dst += dst_stride;
 	src += src_stride;
     }
-    _mm_empty ();
 }
 
-/* ------------------------------------------------------------------
- * composite_over_8888_0565
- */
 static force_inline uint16_t
 composite_over_8888_0565pixel (uint32_t src, uint16_t dst)
 {
-    __m64 ms;
+    __m128i ms;
 
-    ms = unpack_32_1x64 (src);
+    ms = unpack_32_1x128 (src);
     return pack_565_32_16 (
-	pack_1x64_32 (
-	    over_1x64 (
-		ms, expand_alpha_1x64 (ms), expand565_16_1x64 (dst))));
+	pack_1x128_32 (
+	    over_1x128 (
+		ms, expand_alpha_1x128 (ms), expand565_16_1x128 (dst))));
 }
 
 static void
 sse2_composite_over_8888_0565 (pixman_implementation_t *imp,
-                               pixman_op_t              op,
-                               pixman_image_t *         src_image,
-                               pixman_image_t *         mask_image,
-                               pixman_image_t *         dst_image,
-                               int32_t                  src_x,
-                               int32_t                  src_y,
-                               int32_t                  mask_x,
-                               int32_t                  mask_y,
-                               int32_t                  dest_x,
-                               int32_t                  dest_y,
-                               int32_t                  width,
-                               int32_t                  height)
+                               pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint16_t    *dst_line, *dst, d;
     uint32_t    *src_line, *src, s;
     int dst_stride, src_stride;
@@ -3714,35 +3113,22 @@ sse2_composite_over_8888_0565 (pixman_implementation_t *imp,
     __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (
 	src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
 
-#if 0
-    /* FIXME
-     *
-     * I copy the code from MMX one and keep the fixme.
-     * If it's a problem there, probably is a problem here.
-     */
-    assert (src_image->drawable == mask_image->drawable);
-#endif
-
     while (height--)
     {
 	dst = dst_line;
 	src = src_line;
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)src);
-	cache_prefetch ((__m128i*)dst);
-
 	dst_line += dst_stride;
 	src_line += src_stride;
 	w = width;
 
 	/* Align dst on a 16-byte boundary */
 	while (w &&
-	       ((unsigned long)dst & 15))
+	       ((uintptr_t)dst & 15))
 	{
 	    s = *src++;
 	    d = *dst;
@@ -3751,17 +3137,9 @@ sse2_composite_over_8888_0565 (pixman_implementation_t *imp,
 	    w--;
 	}
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)src);
-	cache_prefetch ((__m128i*)dst);
-
 	/* It's a 8 pixel loop */
 	while (w >= 8)
 	{
-	    /* fill cache line with next memory */
-	    cache_prefetch_next ((__m128i*)src);
-	    cache_prefetch_next ((__m128i*)dst);
-
 	    /* I'm loading unaligned because I'm not sure
 	     * about the address alignment.
 	     */
@@ -3811,28 +3189,13 @@ sse2_composite_over_8888_0565 (pixman_implementation_t *imp,
 	}
     }
 
-    _mm_empty ();
 }
 
-/* -----------------------------------------------------------------
- * composite_over_n_8_8888
- */
-
 static void
 sse2_composite_over_n_8_8888 (pixman_implementation_t *imp,
-                              pixman_op_t              op,
-                              pixman_image_t *         src_image,
-                              pixman_image_t *         mask_image,
-                              pixman_image_t *         dst_image,
-                              int32_t                  src_x,
-                              int32_t                  src_y,
-                              int32_t                  mask_x,
-                              int32_t                  mask_y,
-                              int32_t                  dest_x,
-                              int32_t                  dest_y,
-                              int32_t                  width,
-                              int32_t                  height)
+                              pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t src, srca;
     uint32_t *dst_line, *dst;
     uint8_t *mask_line, *mask;
@@ -3844,24 +3207,24 @@ sse2_composite_over_n_8_8888 (pixman_implementation_t *imp,
     __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
     __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
 
-    __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
+    __m128i mmx_src, mmx_alpha, mmx_mask, mmx_dest;
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     srca = src >> 24;
     if (src == 0)
 	return;
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (
 	mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
 
     xmm_def = create_mask_2x32_128 (src, src);
     xmm_src = expand_pixel_32_1x128 (src);
     xmm_alpha = expand_alpha_1x128 (xmm_src);
-    mmx_src   = _mm_movepi64_pi64 (xmm_src);
-    mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
+    mmx_src   = xmm_src;
+    mmx_alpha = xmm_alpha;
 
     while (height--)
     {
@@ -3871,21 +3234,17 @@ sse2_composite_over_n_8_8888 (pixman_implementation_t *imp,
 	mask_line += mask_stride;
 	w = width;
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)mask);
-	cache_prefetch ((__m128i*)dst);
-
-	while (w && (unsigned long)dst & 15)
+	while (w && (uintptr_t)dst & 15)
 	{
 	    uint8_t m = *mask++;
 
 	    if (m)
 	    {
 		d = *dst;
-		mmx_mask = expand_pixel_8_1x64 (m);
-		mmx_dest = unpack_32_1x64 (d);
+		mmx_mask = expand_pixel_8_1x128 (m);
+		mmx_dest = unpack_32_1x128 (d);
 
-		*dst = pack_1x64_32 (in_over_1x64 (&mmx_src,
+		*dst = pack_1x128_32 (in_over_1x128 (&mmx_src,
 		                                   &mmx_alpha,
 		                                   &mmx_mask,
 		                                   &mmx_dest));
@@ -3895,16 +3254,8 @@ sse2_composite_over_n_8_8888 (pixman_implementation_t *imp,
 	    dst++;
 	}
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)mask);
-	cache_prefetch ((__m128i*)dst);
-
 	while (w >= 4)
 	{
-	    /* fill cache line with next memory */
-	    cache_prefetch_next ((__m128i*)mask);
-	    cache_prefetch_next ((__m128i*)dst);
-
 	    m = *((uint32_t*)mask);
 
 	    if (srca == 0xff && m == 0xffffffff)
@@ -3945,10 +3296,10 @@ sse2_composite_over_n_8_8888 (pixman_implementation_t *imp,
 	    if (m)
 	    {
 		d = *dst;
-		mmx_mask = expand_pixel_8_1x64 (m);
-		mmx_dest = unpack_32_1x64 (d);
+		mmx_mask = expand_pixel_8_1x128 (m);
+		mmx_dest = unpack_32_1x128 (d);
 
-		*dst = pack_1x64_32 (in_over_1x64 (&mmx_src,
+		*dst = pack_1x128_32 (in_over_1x128 (&mmx_src,
 		                                   &mmx_alpha,
 		                                   &mmx_mask,
 		                                   &mmx_dest));
@@ -3959,49 +3310,63 @@ sse2_composite_over_n_8_8888 (pixman_implementation_t *imp,
 	}
     }
 
-    _mm_empty ();
 }
 
-/* ----------------------------------------------------------------
- * composite_over_n_8_8888
- */
-
-pixman_bool_t
-pixman_fill_sse2 (uint32_t *bits,
-                  int       stride,
-                  int       bpp,
-                  int       x,
-                  int       y,
-                  int       width,
-                  int       height,
-                  uint32_t  data)
+#if defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
+__attribute__((__force_align_arg_pointer__))
+#endif
+static pixman_bool_t
+sse2_fill (pixman_implementation_t *imp,
+           uint32_t *               bits,
+           int                      stride,
+           int                      bpp,
+           int                      x,
+           int                      y,
+           int                      width,
+           int                      height,
+           uint32_t		    filler)
 {
     uint32_t byte_width;
-    uint8_t         *byte_line;
+    uint8_t *byte_line;
 
     __m128i xmm_def;
 
-    if (bpp != 16 && bpp != 32)
-	return FALSE;
+    if (bpp == 8)
+    {
+	uint8_t b;
+	uint16_t w;
+
+	stride = stride * (int) sizeof (uint32_t) / 1;
+	byte_line = (uint8_t *)(((uint8_t *)bits) + stride * y + x);
+	byte_width = width;
+	stride *= 1;
 
-    if (bpp == 16)
+	b = filler & 0xff;
+	w = (b << 8) | b;
+	filler = (w << 16) | w;
+    }
+    else if (bpp == 16)
     {
 	stride = stride * (int) sizeof (uint32_t) / 2;
 	byte_line = (uint8_t *)(((uint16_t *)bits) + stride * y + x);
 	byte_width = 2 * width;
 	stride *= 2;
-        data = (data & 0xffff) * 0x00010001;
+
+        filler = (filler & 0xffff) * 0x00010001;
     }
-    else
+    else if (bpp == 32)
     {
 	stride = stride * (int) sizeof (uint32_t) / 4;
 	byte_line = (uint8_t *)(((uint32_t *)bits) + stride * y + x);
 	byte_width = 4 * width;
 	stride *= 4;
     }
+    else
+    {
+	return FALSE;
+    }
 
-    cache_prefetch ((__m128i*)byte_line);
-    xmm_def = create_mask_2x32_128 (data, data);
+    xmm_def = create_mask_2x32_128 (filler, filler);
 
     while (height--)
     {
@@ -4010,30 +3375,30 @@ pixman_fill_sse2 (uint32_t *bits,
 	byte_line += stride;
 	w = byte_width;
 
+	if (w >= 1 && ((uintptr_t)d & 1))
+	{
+	    *(uint8_t *)d = filler;
+	    w -= 1;
+	    d += 1;
+	}
 
-	cache_prefetch_next ((__m128i*)d);
-
-	while (w >= 2 && ((unsigned long)d & 3))
+	while (w >= 2 && ((uintptr_t)d & 3))
 	{
-	    *(uint16_t *)d = data;
+	    *(uint16_t *)d = filler;
 	    w -= 2;
 	    d += 2;
 	}
 
-	while (w >= 4 && ((unsigned long)d & 15))
+	while (w >= 4 && ((uintptr_t)d & 15))
 	{
-	    *(uint32_t *)d = data;
+	    *(uint32_t *)d = filler;
 
 	    w -= 4;
 	    d += 4;
 	}
 
-	cache_prefetch_next ((__m128i*)d);
-
 	while (w >= 128)
 	{
-	    cache_prefetch (((__m128i*)d) + 12);
-
 	    save_128_aligned ((__m128i*)(d),     xmm_def);
 	    save_128_aligned ((__m128i*)(d + 16),  xmm_def);
 	    save_128_aligned ((__m128i*)(d + 32),  xmm_def);
@@ -4049,8 +3414,6 @@ pixman_fill_sse2 (uint32_t *bits,
 
 	if (w >= 64)
 	{
-	    cache_prefetch (((__m128i*)d) + 8);
-
 	    save_128_aligned ((__m128i*)(d),     xmm_def);
 	    save_128_aligned ((__m128i*)(d + 16),  xmm_def);
 	    save_128_aligned ((__m128i*)(d + 32),  xmm_def);
@@ -4060,8 +3423,6 @@ pixman_fill_sse2 (uint32_t *bits,
 	    w -= 64;
 	}
 
-	cache_prefetch_next ((__m128i*)d);
-
 	if (w >= 32)
 	{
 	    save_128_aligned ((__m128i*)(d),     xmm_def);
@@ -4079,11 +3440,9 @@ pixman_fill_sse2 (uint32_t *bits,
 	    w -= 16;
 	}
 
-	cache_prefetch_next ((__m128i*)d);
-
 	while (w >= 4)
 	{
-	    *(uint32_t *)d = data;
+	    *(uint32_t *)d = filler;
 
 	    w -= 4;
 	    d += 4;
@@ -4091,31 +3450,27 @@ pixman_fill_sse2 (uint32_t *bits,
 
 	if (w >= 2)
 	{
-	    *(uint16_t *)d = data;
+	    *(uint16_t *)d = filler;
 	    w -= 2;
 	    d += 2;
 	}
+
+	if (w >= 1)
+	{
+	    *(uint8_t *)d = filler;
+	    w -= 1;
+	    d += 1;
+	}
     }
 
-    _mm_empty ();
     return TRUE;
 }
 
 static void
 sse2_composite_src_n_8_8888 (pixman_implementation_t *imp,
-                             pixman_op_t              op,
-                             pixman_image_t *         src_image,
-                             pixman_image_t *         mask_image,
-                             pixman_image_t *         dst_image,
-                             int32_t                  src_x,
-                             int32_t                  src_y,
-                             int32_t                  mask_x,
-                             int32_t                  mask_y,
-                             int32_t                  dest_x,
-                             int32_t                  dest_y,
-                             int32_t                  width,
-                             int32_t                  height)
+                             pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t src, srca;
     uint32_t    *dst_line, *dst;
     uint8_t     *mask_line, *mask;
@@ -4126,19 +3481,19 @@ sse2_composite_src_n_8_8888 (pixman_implementation_t *imp,
     __m128i xmm_src, xmm_def;
     __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     srca = src >> 24;
     if (src == 0)
     {
-	pixman_fill_sse2 (dst_image->bits.bits, dst_image->bits.rowstride,
-	                  PIXMAN_FORMAT_BPP (dst_image->bits.format),
-	                  dest_x, dest_y, width, height, 0);
+	sse2_fill (imp, dest_image->bits.bits, dest_image->bits.rowstride,
+		   PIXMAN_FORMAT_BPP (dest_image->bits.format),
+		   dest_x, dest_y, width, height, 0);
 	return;
     }
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (
 	mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
 
@@ -4153,19 +3508,14 @@ sse2_composite_src_n_8_8888 (pixman_implementation_t *imp,
 	mask_line += mask_stride;
 	w = width;
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)mask);
-	cache_prefetch ((__m128i*)dst);
-
-	while (w && (unsigned long)dst & 15)
+	while (w && (uintptr_t)dst & 15)
 	{
 	    uint8_t m = *mask++;
 
 	    if (m)
 	    {
-		*dst = pack_1x64_32 (
-		    pix_multiply_1x64 (
-			_mm_movepi64_pi64 (xmm_src), expand_pixel_8_1x64 (m)));
+		*dst = pack_1x128_32 (
+		    pix_multiply_1x128 (xmm_src, expand_pixel_8_1x128 (m)));
 	    }
 	    else
 	    {
@@ -4176,16 +3526,8 @@ sse2_composite_src_n_8_8888 (pixman_implementation_t *imp,
 	    dst++;
 	}
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)mask);
-	cache_prefetch ((__m128i*)dst);
-
 	while (w >= 4)
 	{
-	    /* fill cache line with next memory */
-	    cache_prefetch_next ((__m128i*)mask);
-	    cache_prefetch_next ((__m128i*)dst);
-
 	    m = *((uint32_t*)mask);
 
 	    if (srca == 0xff && m == 0xffffffff)
@@ -4226,9 +3568,9 @@ sse2_composite_src_n_8_8888 (pixman_implementation_t *imp,
 
 	    if (m)
 	    {
-		*dst = pack_1x64_32 (
-		    pix_multiply_1x64 (
-			_mm_movepi64_pi64 (xmm_src), expand_pixel_8_1x64 (m)));
+		*dst = pack_1x128_32 (
+		    pix_multiply_1x128 (
+			xmm_src, expand_pixel_8_1x128 (m)));
 	    }
 	    else
 	    {
@@ -4240,55 +3582,39 @@ sse2_composite_src_n_8_8888 (pixman_implementation_t *imp,
 	}
     }
 
-    _mm_empty ();
 }
 
-/*-----------------------------------------------------------------------
- * composite_over_n_8_0565
- */
-
 static void
 sse2_composite_over_n_8_0565 (pixman_implementation_t *imp,
-                              pixman_op_t              op,
-                              pixman_image_t *         src_image,
-                              pixman_image_t *         mask_image,
-                              pixman_image_t *         dst_image,
-                              int32_t                  src_x,
-                              int32_t                  src_y,
-                              int32_t                  mask_x,
-                              int32_t                  mask_y,
-                              int32_t                  dest_x,
-                              int32_t                  dest_y,
-                              int32_t                  width,
-                              int32_t                  height)
+                              pixman_composite_info_t *info)
 {
-    uint32_t src, srca;
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint32_t src;
     uint16_t    *dst_line, *dst, d;
     uint8_t     *mask_line, *mask;
     int dst_stride, mask_stride;
     int32_t w;
     uint32_t m;
-    __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
+    __m128i mmx_src, mmx_alpha, mmx_mask, mmx_dest;
 
     __m128i xmm_src, xmm_alpha;
     __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
     __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
-    srca = src >> 24;
     if (src == 0)
 	return;
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (
 	mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
 
     xmm_src = expand_pixel_32_1x128 (src);
     xmm_alpha = expand_alpha_1x128 (xmm_src);
-    mmx_src = _mm_movepi64_pi64 (xmm_src);
-    mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
+    mmx_src = xmm_src;
+    mmx_alpha = xmm_alpha;
 
     while (height--)
     {
@@ -4298,23 +3624,19 @@ sse2_composite_over_n_8_0565 (pixman_implementation_t *imp,
 	mask_line += mask_stride;
 	w = width;
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)mask);
-	cache_prefetch ((__m128i*)dst);
-
-	while (w && (unsigned long)dst & 15)
+	while (w && (uintptr_t)dst & 15)
 	{
 	    m = *mask++;
 
 	    if (m)
 	    {
 		d = *dst;
-		mmx_mask = expand_alpha_rev_1x64 (unpack_32_1x64 (m));
-		mmx_dest = expand565_16_1x64 (d);
+		mmx_mask = expand_alpha_rev_1x128 (unpack_32_1x128 (m));
+		mmx_dest = expand565_16_1x128 (d);
 
 		*dst = pack_565_32_16 (
-		    pack_1x64_32 (
-			in_over_1x64 (
+		    pack_1x128_32 (
+			in_over_1x128 (
 			    &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
 	    }
 
@@ -4322,16 +3644,8 @@ sse2_composite_over_n_8_0565 (pixman_implementation_t *imp,
 	    dst++;
 	}
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)mask);
-	cache_prefetch ((__m128i*)dst);
-
 	while (w >= 8)
 	{
-	    /* fill cache line with next memory */
-	    cache_prefetch_next ((__m128i*)mask);
-	    cache_prefetch_next ((__m128i*)dst);
-
 	    xmm_dst = load_128_aligned ((__m128i*) dst);
 	    unpack_565_128_4x128 (xmm_dst,
 				  &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
@@ -4390,12 +3704,12 @@ sse2_composite_over_n_8_0565 (pixman_implementation_t *imp,
 	    if (m)
 	    {
 		d = *dst;
-		mmx_mask = expand_alpha_rev_1x64 (unpack_32_1x64 (m));
-		mmx_dest = expand565_16_1x64 (d);
+		mmx_mask = expand_alpha_rev_1x128 (unpack_32_1x128 (m));
+		mmx_dest = expand565_16_1x128 (d);
 
 		*dst = pack_565_32_16 (
-		    pack_1x64_32 (
-			in_over_1x64 (
+		    pack_1x128_32 (
+			in_over_1x128 (
 			    &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
 	    }
 
@@ -4404,52 +3718,28 @@ sse2_composite_over_n_8_0565 (pixman_implementation_t *imp,
 	}
     }
 
-    _mm_empty ();
 }
 
-/* -----------------------------------------------------------------------
- * composite_over_pixbuf_0565
- */
-
 static void
 sse2_composite_over_pixbuf_0565 (pixman_implementation_t *imp,
-                                 pixman_op_t              op,
-                                 pixman_image_t *         src_image,
-                                 pixman_image_t *         mask_image,
-                                 pixman_image_t *         dst_image,
-                                 int32_t                  src_x,
-                                 int32_t                  src_y,
-                                 int32_t                  mask_x,
-                                 int32_t                  mask_y,
-                                 int32_t                  dest_x,
-                                 int32_t                  dest_y,
-                                 int32_t                  width,
-                                 int32_t                  height)
+                                 pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint16_t    *dst_line, *dst, d;
     uint32_t    *src_line, *src, s;
     int dst_stride, src_stride;
     int32_t w;
     uint32_t opaque, zero;
 
-    __m64 ms;
+    __m128i ms;
     __m128i xmm_src, xmm_src_lo, xmm_src_hi;
     __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (
 	src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
 
-#if 0
-    /* FIXME
-     *
-     * I copy the code from MMX one and keep the fixme.
-     * If it's a problem there, probably is a problem here.
-     */
-    assert (src_image->drawable == mask_image->drawable);
-#endif
-
     while (height--)
     {
 	dst = dst_line;
@@ -4458,33 +3748,21 @@ sse2_composite_over_pixbuf_0565 (pixman_implementation_t *imp,
 	src_line += src_stride;
 	w = width;
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)src);
-	cache_prefetch ((__m128i*)dst);
-
-	while (w && (unsigned long)dst & 15)
+	while (w && (uintptr_t)dst & 15)
 	{
 	    s = *src++;
 	    d = *dst;
 
-	    ms = unpack_32_1x64 (s);
+	    ms = unpack_32_1x128 (s);
 
 	    *dst++ = pack_565_32_16 (
-		pack_1x64_32 (
-		    over_rev_non_pre_1x64 (ms, expand565_16_1x64 (d))));
+		pack_1x128_32 (
+		    over_rev_non_pre_1x128 (ms, expand565_16_1x128 (d))));
 	    w--;
 	}
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)src);
-	cache_prefetch ((__m128i*)dst);
-
 	while (w >= 8)
 	{
-	    /* fill cache line with next memory */
-	    cache_prefetch_next ((__m128i*)src);
-	    cache_prefetch_next ((__m128i*)dst);
-
 	    /* First round */
 	    xmm_src = load_128_unaligned ((__m128i*)src);
 	    xmm_dst = load_128_aligned  ((__m128i*)dst);
@@ -4541,37 +3819,22 @@ sse2_composite_over_pixbuf_0565 (pixman_implementation_t *imp,
 	    s = *src++;
 	    d = *dst;
 
-	    ms = unpack_32_1x64 (s);
+	    ms = unpack_32_1x128 (s);
 
 	    *dst++ = pack_565_32_16 (
-		pack_1x64_32 (
-		    over_rev_non_pre_1x64 (ms, expand565_16_1x64 (d))));
+		pack_1x128_32 (
+		    over_rev_non_pre_1x128 (ms, expand565_16_1x128 (d))));
 	    w--;
 	}
     }
 
-    _mm_empty ();
 }
 
-/* -------------------------------------------------------------------------
- * composite_over_pixbuf_8888
- */
-
 static void
 sse2_composite_over_pixbuf_8888 (pixman_implementation_t *imp,
-                                 pixman_op_t              op,
-                                 pixman_image_t *         src_image,
-                                 pixman_image_t *         mask_image,
-                                 pixman_image_t *         dst_image,
-                                 int32_t                  src_x,
-                                 int32_t                  src_y,
-                                 int32_t                  mask_x,
-                                 int32_t                  mask_y,
-                                 int32_t                  dest_x,
-                                 int32_t                  dest_y,
-                                 int32_t                  width,
-                                 int32_t                  height)
+                                 pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t    *dst_line, *dst, d;
     uint32_t    *src_line, *src, s;
     int dst_stride, src_stride;
@@ -4582,19 +3845,10 @@ sse2_composite_over_pixbuf_8888 (pixman_implementation_t *imp,
     __m128i xmm_dst_lo, xmm_dst_hi;
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (
 	src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
 
-#if 0
-    /* FIXME
-     *
-     * I copy the code from MMX one and keep the fixme.
-     * If it's a problem there, probably is a problem here.
-     */
-    assert (src_image->drawable == mask_image->drawable);
-#endif
-
     while (height--)
     {
 	dst = dst_line;
@@ -4603,32 +3857,20 @@ sse2_composite_over_pixbuf_8888 (pixman_implementation_t *imp,
 	src_line += src_stride;
 	w = width;
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)src);
-	cache_prefetch ((__m128i*)dst);
-
-	while (w && (unsigned long)dst & 15)
+	while (w && (uintptr_t)dst & 15)
 	{
 	    s = *src++;
 	    d = *dst;
 
-	    *dst++ = pack_1x64_32 (
-		over_rev_non_pre_1x64 (
-		    unpack_32_1x64 (s), unpack_32_1x64 (d)));
+	    *dst++ = pack_1x128_32 (
+		over_rev_non_pre_1x128 (
+		    unpack_32_1x128 (s), unpack_32_1x128 (d)));
 
 	    w--;
 	}
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)src);
-	cache_prefetch ((__m128i*)dst);
-
 	while (w >= 4)
 	{
-	    /* fill cache line with next memory */
-	    cache_prefetch_next ((__m128i*)src);
-	    cache_prefetch_next ((__m128i*)dst);
-
 	    xmm_src_hi = load_128_unaligned ((__m128i*)src);
 
 	    opaque = is_opaque (xmm_src_hi);
@@ -4667,36 +3909,21 @@ sse2_composite_over_pixbuf_8888 (pixman_implementation_t *imp,
 	    s = *src++;
 	    d = *dst;
 
-	    *dst++ = pack_1x64_32 (
-		over_rev_non_pre_1x64 (
-		    unpack_32_1x64 (s), unpack_32_1x64 (d)));
+	    *dst++ = pack_1x128_32 (
+		over_rev_non_pre_1x128 (
+		    unpack_32_1x128 (s), unpack_32_1x128 (d)));
 
 	    w--;
 	}
     }
 
-    _mm_empty ();
 }
 
-/* -------------------------------------------------------------------------------------------------
- * composite_over_n_8888_0565_ca
- */
-
 static void
 sse2_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
-                                    pixman_op_t              op,
-                                    pixman_image_t *         src_image,
-                                    pixman_image_t *         mask_image,
-                                    pixman_image_t *         dst_image,
-                                    int32_t                  src_x,
-                                    int32_t                  src_y,
-                                    int32_t                  mask_x,
-                                    int32_t                  mask_y,
-                                    int32_t                  dest_x,
-                                    int32_t                  dest_y,
-                                    int32_t                  width,
-                                    int32_t                  height)
+                                    pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t src;
     uint16_t    *dst_line, *dst, d;
     uint32_t    *mask_line, *mask, m;
@@ -4708,22 +3935,22 @@ sse2_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
     __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
     __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
 
-    __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
+    __m128i mmx_src, mmx_alpha, mmx_mask, mmx_dest;
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     if (src == 0)
 	return;
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (
 	mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
 
     xmm_src = expand_pixel_32_1x128 (src);
     xmm_alpha = expand_alpha_1x128 (xmm_src);
-    mmx_src = _mm_movepi64_pi64 (xmm_src);
-    mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
+    mmx_src = xmm_src;
+    mmx_alpha = xmm_alpha;
 
     while (height--)
     {
@@ -4733,23 +3960,19 @@ sse2_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
 	mask_line += mask_stride;
 	dst_line += dst_stride;
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)mask);
-	cache_prefetch ((__m128i*)dst);
-
-	while (w && ((unsigned long)dst & 15))
+	while (w && ((uintptr_t)dst & 15))
 	{
 	    m = *(uint32_t *) mask;
 
 	    if (m)
 	    {
 		d = *dst;
-		mmx_mask = unpack_32_1x64 (m);
-		mmx_dest = expand565_16_1x64 (d);
+		mmx_mask = unpack_32_1x128 (m);
+		mmx_dest = expand565_16_1x128 (d);
 
 		*dst = pack_565_32_16 (
-		    pack_1x64_32 (
-			in_over_1x64 (
+		    pack_1x128_32 (
+			in_over_1x128 (
 			    &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
 	    }
 
@@ -4758,16 +3981,8 @@ sse2_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
 	    mask++;
 	}
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)mask);
-	cache_prefetch ((__m128i*)dst);
-
 	while (w >= 8)
 	{
-	    /* fill cache line with next memory */
-	    cache_prefetch_next ((__m128i*)mask);
-	    cache_prefetch_next ((__m128i*)dst);
-
 	    /* First round */
 	    xmm_mask = load_128_unaligned ((__m128i*)mask);
 	    xmm_dst = load_128_aligned ((__m128i*)dst);
@@ -4821,12 +4036,12 @@ sse2_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
 	    if (m)
 	    {
 		d = *dst;
-		mmx_mask = unpack_32_1x64 (m);
-		mmx_dest = expand565_16_1x64 (d);
+		mmx_mask = unpack_32_1x128 (m);
+		mmx_dest = expand565_16_1x128 (d);
 
 		*dst = pack_565_32_16 (
-		    pack_1x64_32 (
-			in_over_1x64 (
+		    pack_1x128_32 (
+			in_over_1x128 (
 			    &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
 	    }
 
@@ -4836,34 +4051,18 @@ sse2_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
 	}
     }
 
-    _mm_empty ();
 }
 
-/* -----------------------------------------------------------------------
- * composite_in_n_8_8
- */
-
 static void
 sse2_composite_in_n_8_8 (pixman_implementation_t *imp,
-                         pixman_op_t              op,
-                         pixman_image_t *         src_image,
-                         pixman_image_t *         mask_image,
-                         pixman_image_t *         dst_image,
-                         int32_t                  src_x,
-                         int32_t                  src_y,
-                         int32_t                  mask_x,
-                         int32_t                  mask_y,
-                         int32_t                  dest_x,
-                         int32_t                  dest_y,
-                         int32_t                  width,
-                         int32_t                  height)
+                         pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint8_t     *dst_line, *dst;
     uint8_t     *mask_line, *mask;
     int dst_stride, mask_stride;
     uint32_t d, m;
     uint32_t src;
-    uint8_t sa;
     int32_t w;
 
     __m128i xmm_alpha;
@@ -4871,13 +4070,11 @@ sse2_composite_in_n_8_8 (pixman_implementation_t *imp,
     __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (
 	mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
-
-    sa = src >> 24;
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     xmm_alpha = expand_alpha_1x128 (expand_pixel_32_1x128 (src));
 
@@ -4889,33 +4086,21 @@ sse2_composite_in_n_8_8 (pixman_implementation_t *imp,
 	mask_line += mask_stride;
 	w = width;
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)mask);
-	cache_prefetch ((__m128i*)dst);
-
-	while (w && ((unsigned long)dst & 15))
+	while (w && ((uintptr_t)dst & 15))
 	{
 	    m = (uint32_t) *mask++;
 	    d = (uint32_t) *dst;
 
-	    *dst++ = (uint8_t) pack_1x64_32 (
-		pix_multiply_1x64 (
-		    pix_multiply_1x64 (_mm_movepi64_pi64 (xmm_alpha),
-				       unpack_32_1x64 (m)),
-		    unpack_32_1x64 (d)));
+	    *dst++ = (uint8_t) pack_1x128_32 (
+		pix_multiply_1x128 (
+		    pix_multiply_1x128 (xmm_alpha,
+				       unpack_32_1x128 (m)),
+		    unpack_32_1x128 (d)));
 	    w--;
 	}
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)mask);
-	cache_prefetch ((__m128i*)dst);
-
 	while (w >= 16)
 	{
-	    /* fill cache line with next memory */
-	    cache_prefetch_next ((__m128i*)mask);
-	    cache_prefetch_next ((__m128i*)dst);
-
 	    xmm_mask = load_128_unaligned ((__m128i*)mask);
 	    xmm_dst = load_128_aligned ((__m128i*)dst);
 
@@ -4943,37 +4128,104 @@ sse2_composite_in_n_8_8 (pixman_implementation_t *imp,
 	    m = (uint32_t) *mask++;
 	    d = (uint32_t) *dst;
 
-	    *dst++ = (uint8_t) pack_1x64_32 (
-		pix_multiply_1x64 (
-		    pix_multiply_1x64 (
-			_mm_movepi64_pi64 (xmm_alpha), unpack_32_1x64 (m)),
-		    unpack_32_1x64 (d)));
+	    *dst++ = (uint8_t) pack_1x128_32 (
+		pix_multiply_1x128 (
+		    pix_multiply_1x128 (
+			xmm_alpha, unpack_32_1x128 (m)),
+		    unpack_32_1x128 (d)));
 	    w--;
 	}
     }
 
-    _mm_empty ();
 }
 
-/* ---------------------------------------------------------------------------
- * composite_in_8_8
- */
+static void
+sse2_composite_in_n_8 (pixman_implementation_t *imp,
+		       pixman_composite_info_t *info)
+{
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint8_t     *dst_line, *dst;
+    int dst_stride;
+    uint32_t d;
+    uint32_t src;
+    int32_t w;
+
+    __m128i xmm_alpha;
+    __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
+
+    xmm_alpha = expand_alpha_1x128 (expand_pixel_32_1x128 (src));
+
+    src = src >> 24;
+
+    if (src == 0xff)
+	return;
+
+    if (src == 0x00)
+    {
+	pixman_fill (dest_image->bits.bits, dest_image->bits.rowstride,
+		     8, dest_x, dest_y, width, height, src);
+
+	return;
+    }
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	w = width;
+
+	while (w && ((uintptr_t)dst & 15))
+	{
+	    d = (uint32_t) *dst;
+
+	    *dst++ = (uint8_t) pack_1x128_32 (
+		pix_multiply_1x128 (
+		    xmm_alpha,
+		    unpack_32_1x128 (d)));
+	    w--;
+	}
+
+	while (w >= 16)
+	{
+	    xmm_dst = load_128_aligned ((__m128i*)dst);
+
+	    unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+	    
+	    pix_multiply_2x128 (&xmm_alpha, &xmm_alpha,
+				&xmm_dst_lo, &xmm_dst_hi,
+				&xmm_dst_lo, &xmm_dst_hi);
+
+	    save_128_aligned (
+		(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	    dst += 16;
+	    w -= 16;
+	}
+
+	while (w)
+	{
+	    d = (uint32_t) *dst;
+
+	    *dst++ = (uint8_t) pack_1x128_32 (
+		pix_multiply_1x128 (
+		    xmm_alpha,
+		    unpack_32_1x128 (d)));
+	    w--;
+	}
+    }
+
+}
 
 static void
 sse2_composite_in_8_8 (pixman_implementation_t *imp,
-                       pixman_op_t              op,
-                       pixman_image_t *         src_image,
-                       pixman_image_t *         mask_image,
-                       pixman_image_t *         dst_image,
-                       int32_t                  src_x,
-                       int32_t                  src_y,
-                       int32_t                  mask_x,
-                       int32_t                  mask_y,
-                       int32_t                  dest_x,
-                       int32_t                  dest_y,
-                       int32_t                  width,
-                       int32_t                  height)
+                       pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint8_t     *dst_line, *dst;
     uint8_t     *src_line, *src;
     int src_stride, dst_stride;
@@ -4984,7 +4236,7 @@ sse2_composite_in_8_8 (pixman_implementation_t *imp,
     __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (
 	src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
 
@@ -4996,31 +4248,19 @@ sse2_composite_in_8_8 (pixman_implementation_t *imp,
 	src_line += src_stride;
 	w = width;
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)src);
-	cache_prefetch ((__m128i*)dst);
-
-	while (w && ((unsigned long)dst & 15))
+	while (w && ((uintptr_t)dst & 15))
 	{
 	    s = (uint32_t) *src++;
 	    d = (uint32_t) *dst;
 
-	    *dst++ = (uint8_t) pack_1x64_32 (
-		pix_multiply_1x64 (
-		    unpack_32_1x64 (s), unpack_32_1x64 (d)));
+	    *dst++ = (uint8_t) pack_1x128_32 (
+		pix_multiply_1x128 (
+		    unpack_32_1x128 (s), unpack_32_1x128 (d)));
 	    w--;
 	}
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)src);
-	cache_prefetch ((__m128i*)dst);
-
 	while (w >= 16)
 	{
-	    /* fill cache line with next memory */
-	    cache_prefetch_next ((__m128i*)src);
-	    cache_prefetch_next ((__m128i*)dst);
-
 	    xmm_src = load_128_unaligned ((__m128i*)src);
 	    xmm_dst = load_128_aligned ((__m128i*)dst);
 
@@ -5044,40 +4284,24 @@ sse2_composite_in_8_8 (pixman_implementation_t *imp,
 	    s = (uint32_t) *src++;
 	    d = (uint32_t) *dst;
 
-	    *dst++ = (uint8_t) pack_1x64_32 (
-		pix_multiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (d)));
+	    *dst++ = (uint8_t) pack_1x128_32 (
+		pix_multiply_1x128 (unpack_32_1x128 (s), unpack_32_1x128 (d)));
 	    w--;
 	}
     }
 
-    _mm_empty ();
 }
 
-/* -------------------------------------------------------------------------
- * composite_add_n_8_8
- */
-
 static void
 sse2_composite_add_n_8_8 (pixman_implementation_t *imp,
-			  pixman_op_t              op,
-			  pixman_image_t *         src_image,
-			  pixman_image_t *         mask_image,
-			  pixman_image_t *         dst_image,
-			  int32_t                  src_x,
-			  int32_t                  src_y,
-			  int32_t                  mask_x,
-			  int32_t                  mask_y,
-			  int32_t                  dest_x,
-			  int32_t                  dest_y,
-			  int32_t                  width,
-			  int32_t                  height)
+			  pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint8_t     *dst_line, *dst;
     uint8_t     *mask_line, *mask;
     int dst_stride, mask_stride;
     int32_t w;
     uint32_t src;
-    uint8_t sa;
     uint32_t m, d;
 
     __m128i xmm_alpha;
@@ -5085,13 +4309,11 @@ sse2_composite_add_n_8_8 (pixman_implementation_t *imp,
     __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (
 	mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
 
-    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
-
-    sa = src >> 24;
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
 
     xmm_alpha = expand_alpha_1x128 (expand_pixel_32_1x128 (src));
 
@@ -5103,33 +4325,21 @@ sse2_composite_add_n_8_8 (pixman_implementation_t *imp,
 	mask_line += mask_stride;
 	w = width;
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)mask);
-	cache_prefetch ((__m128i*)dst);
-
-	while (w && ((unsigned long)dst & 15))
+	while (w && ((uintptr_t)dst & 15))
 	{
 	    m = (uint32_t) *mask++;
 	    d = (uint32_t) *dst;
 
-	    *dst++ = (uint8_t) pack_1x64_32 (
-		_mm_adds_pu16 (
-		    pix_multiply_1x64 (
-			_mm_movepi64_pi64 (xmm_alpha), unpack_32_1x64 (m)),
-		    unpack_32_1x64 (d)));
+	    *dst++ = (uint8_t) pack_1x128_32 (
+		_mm_adds_epu16 (
+		    pix_multiply_1x128 (
+			xmm_alpha, unpack_32_1x128 (m)),
+		    unpack_32_1x128 (d)));
 	    w--;
 	}
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)mask);
-	cache_prefetch ((__m128i*)dst);
-
 	while (w >= 16)
 	{
-	    /* fill cache line with next memory */
-	    cache_prefetch_next ((__m128i*)mask);
-	    cache_prefetch_next ((__m128i*)dst);
-
 	    xmm_mask = load_128_unaligned ((__m128i*)mask);
 	    xmm_dst = load_128_aligned ((__m128i*)dst);
 
@@ -5156,38 +4366,96 @@ sse2_composite_add_n_8_8 (pixman_implementation_t *imp,
 	    m = (uint32_t) *mask++;
 	    d = (uint32_t) *dst;
 
-	    *dst++ = (uint8_t) pack_1x64_32 (
-		_mm_adds_pu16 (
-		    pix_multiply_1x64 (
-			_mm_movepi64_pi64 (xmm_alpha), unpack_32_1x64 (m)),
-		    unpack_32_1x64 (d)));
+	    *dst++ = (uint8_t) pack_1x128_32 (
+		_mm_adds_epu16 (
+		    pix_multiply_1x128 (
+			xmm_alpha, unpack_32_1x128 (m)),
+		    unpack_32_1x128 (d)));
 
 	    w--;
 	}
     }
 
-    _mm_empty ();
 }
 
-/* ----------------------------------------------------------------------
- * composite_add_8000_8000
- */
+static void
+sse2_composite_add_n_8 (pixman_implementation_t *imp,
+			pixman_composite_info_t *info)
+{
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint8_t     *dst_line, *dst;
+    int dst_stride;
+    int32_t w;
+    uint32_t src;
+
+    __m128i xmm_src;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
+
+    src >>= 24;
+
+    if (src == 0x00)
+	return;
+
+    if (src == 0xff)
+    {
+	pixman_fill (dest_image->bits.bits, dest_image->bits.rowstride,
+		     8, dest_x, dest_y, width, height, 0xff);
+
+	return;
+    }
+
+    src = (src << 24) | (src << 16) | (src << 8) | src;
+    xmm_src = _mm_set_epi32 (src, src, src, src);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	w = width;
+
+	while (w && ((uintptr_t)dst & 15))
+	{
+	    *dst = (uint8_t)_mm_cvtsi128_si32 (
+		_mm_adds_epu8 (
+		    xmm_src,
+		    _mm_cvtsi32_si128 (*dst)));
+
+	    w--;
+	    dst++;
+	}
+
+	while (w >= 16)
+	{
+	    save_128_aligned (
+		(__m128i*)dst, _mm_adds_epu8 (xmm_src, load_128_aligned  ((__m128i*)dst)));
+
+	    dst += 16;
+	    w -= 16;
+	}
+
+	while (w)
+	{
+	    *dst = (uint8_t)_mm_cvtsi128_si32 (
+		_mm_adds_epu8 (
+		    xmm_src,
+		    _mm_cvtsi32_si128 (*dst)));
+
+	    w--;
+	    dst++;
+	}
+    }
+
+}
 
 static void
-sse2_composite_add_8000_8000 (pixman_implementation_t *imp,
-                              pixman_op_t              op,
-                              pixman_image_t *         src_image,
-                              pixman_image_t *         mask_image,
-                              pixman_image_t *         dst_image,
-                              int32_t                  src_x,
-                              int32_t                  src_y,
-                              int32_t                  mask_x,
-                              int32_t                  mask_y,
-                              int32_t                  dest_x,
-                              int32_t                  dest_y,
-                              int32_t                  width,
-                              int32_t                  height)
+sse2_composite_add_8_8 (pixman_implementation_t *imp,
+			pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint8_t     *dst_line, *dst;
     uint8_t     *src_line, *src;
     int dst_stride, src_stride;
@@ -5197,30 +4465,27 @@ sse2_composite_add_8000_8000 (pixman_implementation_t *imp,
     PIXMAN_IMAGE_GET_LINE (
 	src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
 
     while (height--)
     {
 	dst = dst_line;
 	src = src_line;
 
-	/* call prefetch hint to optimize cache load*/
-	cache_prefetch ((__m128i*)src);
-	cache_prefetch ((__m128i*)dst);
-
 	dst_line += dst_stride;
 	src_line += src_stride;
 	w = width;
 
 	/* Small head */
-	while (w && (unsigned long)dst & 3)
+	while (w && (uintptr_t)dst & 3)
 	{
 	    t = (*dst) + (*src++);
 	    *dst++ = t | (0 - (t >> 8));
 	    w--;
 	}
 
-	core_combine_add_u_sse2 ((uint32_t*)dst, (uint32_t*)src, NULL, w >> 2);
+	sse2_combine_add_u (imp, op,
+			    (uint32_t*)dst, (uint32_t*)src, NULL, w >> 2);
 
 	/* Small tail */
 	dst += w & 0xfffc;
@@ -5236,27 +4501,13 @@ sse2_composite_add_8000_8000 (pixman_implementation_t *imp,
 	}
     }
 
-    _mm_empty ();
 }
 
-/* ---------------------------------------------------------------------
- * composite_add_8888_8888
- */
 static void
 sse2_composite_add_8888_8888 (pixman_implementation_t *imp,
-                              pixman_op_t              op,
-                              pixman_image_t *         src_image,
-                              pixman_image_t *         mask_image,
-                              pixman_image_t *         dst_image,
-                              int32_t                  src_x,
-                              int32_t                  src_y,
-                              int32_t                  mask_x,
-                              int32_t                  mask_y,
-                              int32_t                  dest_x,
-                              int32_t                  dest_y,
-                              int32_t                  width,
-                              int32_t                  height)
+                              pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t    *dst_line, *dst;
     uint32_t    *src_line, *src;
     int dst_stride, src_stride;
@@ -5264,7 +4515,7 @@ sse2_composite_add_8888_8888 (pixman_implementation_t *imp,
     PIXMAN_IMAGE_GET_LINE (
 	src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
 
     while (height--)
     {
@@ -5273,29 +4524,181 @@ sse2_composite_add_8888_8888 (pixman_implementation_t *imp,
 	src = src_line;
 	src_line += src_stride;
 
-	core_combine_add_u_sse2 (dst, src, NULL, width);
+	sse2_combine_add_u (imp, op, dst, src, NULL, width);
+    }
+}
+
+static void
+sse2_composite_add_n_8888 (pixman_implementation_t *imp,
+			   pixman_composite_info_t *info)
+{
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint32_t *dst_line, *dst, src;
+    int dst_stride;
+
+    __m128i xmm_src;
+
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
+    if (src == 0)
+	return;
+
+    if (src == ~0)
+    {
+	pixman_fill (dest_image->bits.bits, dest_image->bits.rowstride, 32,
+		     dest_x, dest_y, width, height, ~0);
+
+	return;
     }
 
-    _mm_empty ();
+    xmm_src = _mm_set_epi32 (src, src, src, src);
+    while (height--)
+    {
+	int w = width;
+	uint32_t d;
+
+	dst = dst_line;
+	dst_line += dst_stride;
+
+	while (w && (uintptr_t)dst & 15)
+	{
+	    d = *dst;
+	    *dst++ =
+		_mm_cvtsi128_si32 ( _mm_adds_epu8 (xmm_src, _mm_cvtsi32_si128 (d)));
+	    w--;
+	}
+
+	while (w >= 4)
+	{
+	    save_128_aligned
+		((__m128i*)dst,
+		 _mm_adds_epu8 (xmm_src, load_128_aligned ((__m128i*)dst)));
+
+	    dst += 4;
+	    w -= 4;
+	}
+
+	while (w--)
+	{
+	    d = *dst;
+	    *dst++ =
+		_mm_cvtsi128_si32 (_mm_adds_epu8 (xmm_src,
+						  _mm_cvtsi32_si128 (d)));
+	}
+    }
 }
 
-/* -------------------------------------------------------------------------------------------------
- * sse2_composite_copy_area
- */
+static void
+sse2_composite_add_n_8_8888 (pixman_implementation_t *imp,
+			     pixman_composite_info_t *info)
+{
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint32_t     *dst_line, *dst;
+    uint8_t     *mask_line, *mask;
+    int dst_stride, mask_stride;
+    int32_t w;
+    uint32_t src;
+
+    __m128i xmm_src;
+
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
+    if (src == 0)
+	return;
+    xmm_src = expand_pixel_32_1x128 (src);
+
+    PIXMAN_IMAGE_GET_LINE (
+	dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+	while (w && ((uintptr_t)dst & 15))
+	{
+	    uint8_t m = *mask++;
+	    if (m)
+	    {
+		*dst = pack_1x128_32
+		    (_mm_adds_epu16
+		     (pix_multiply_1x128 (xmm_src, expand_pixel_8_1x128 (m)),
+		      unpack_32_1x128 (*dst)));
+	    }
+	    dst++;
+	    w--;
+	}
+
+	while (w >= 4)
+	{
+	    uint32_t m = *(uint32_t*)mask;
+	    if (m)
+	    {
+		__m128i xmm_mask_lo, xmm_mask_hi;
+		__m128i xmm_dst_lo, xmm_dst_hi;
+
+		__m128i xmm_dst = load_128_aligned ((__m128i*)dst);
+		__m128i xmm_mask =
+		    _mm_unpacklo_epi8 (unpack_32_1x128(m),
+				       _mm_setzero_si128 ());
+
+		unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
+		unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+
+		expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
+					&xmm_mask_lo, &xmm_mask_hi);
+
+		pix_multiply_2x128 (&xmm_src, &xmm_src,
+				    &xmm_mask_lo, &xmm_mask_hi,
+				    &xmm_mask_lo, &xmm_mask_hi);
+
+		xmm_dst_lo = _mm_adds_epu16 (xmm_mask_lo, xmm_dst_lo);
+		xmm_dst_hi = _mm_adds_epu16 (xmm_mask_hi, xmm_dst_hi);
+
+		save_128_aligned (
+		    (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+	    }
+
+	    w -= 4;
+	    dst += 4;
+	    mask += 4;
+	}
+
+	while (w)
+	{
+	    uint8_t m = *mask++;
+	    if (m)
+	    {
+		*dst = pack_1x128_32
+		    (_mm_adds_epu16
+		     (pix_multiply_1x128 (xmm_src, expand_pixel_8_1x128 (m)),
+		      unpack_32_1x128 (*dst)));
+	    }
+	    dst++;
+	    w--;
+	}
+    }
+}
 
 static pixman_bool_t
-pixman_blt_sse2 (uint32_t *src_bits,
-                 uint32_t *dst_bits,
-                 int       src_stride,
-                 int       dst_stride,
-                 int       src_bpp,
-                 int       dst_bpp,
-                 int       src_x,
-                 int       src_y,
-                 int       dst_x,
-                 int       dst_y,
-                 int       width,
-                 int       height)
+sse2_blt (pixman_implementation_t *imp,
+          uint32_t *               src_bits,
+          uint32_t *               dst_bits,
+          int                      src_stride,
+          int                      dst_stride,
+          int                      src_bpp,
+          int                      dst_bpp,
+          int                      src_x,
+          int                      src_y,
+          int                      dest_x,
+          int                      dest_y,
+          int                      width,
+          int                      height)
 {
     uint8_t *   src_bytes;
     uint8_t *   dst_bytes;
@@ -5309,7 +4712,7 @@ pixman_blt_sse2 (uint32_t *src_bits,
 	src_stride = src_stride * (int) sizeof (uint32_t) / 2;
 	dst_stride = dst_stride * (int) sizeof (uint32_t) / 2;
 	src_bytes =(uint8_t *)(((uint16_t *)src_bits) + src_stride * (src_y) + (src_x));
-	dst_bytes = (uint8_t *)(((uint16_t *)dst_bits) + dst_stride * (dst_y) + (dst_x));
+	dst_bytes = (uint8_t *)(((uint16_t *)dst_bits) + dst_stride * (dest_y) + (dest_x));
 	byte_width = 2 * width;
 	src_stride *= 2;
 	dst_stride *= 2;
@@ -5319,7 +4722,7 @@ pixman_blt_sse2 (uint32_t *src_bits,
 	src_stride = src_stride * (int) sizeof (uint32_t) / 4;
 	dst_stride = dst_stride * (int) sizeof (uint32_t) / 4;
 	src_bytes = (uint8_t *)(((uint32_t *)src_bits) + src_stride * (src_y) + (src_x));
-	dst_bytes = (uint8_t *)(((uint32_t *)dst_bits) + dst_stride * (dst_y) + (dst_x));
+	dst_bytes = (uint8_t *)(((uint32_t *)dst_bits) + dst_stride * (dest_y) + (dest_x));
 	byte_width = 4 * width;
 	src_stride *= 4;
 	dst_stride *= 4;
@@ -5329,9 +4732,6 @@ pixman_blt_sse2 (uint32_t *src_bits,
 	return FALSE;
     }
 
-    cache_prefetch ((__m128i*)src_bytes);
-    cache_prefetch ((__m128i*)dst_bytes);
-
     while (height--)
     {
 	int w;
@@ -5341,10 +4741,7 @@ pixman_blt_sse2 (uint32_t *src_bits,
 	dst_bytes += dst_stride;
 	w = byte_width;
 
-	cache_prefetch_next ((__m128i*)s);
-	cache_prefetch_next ((__m128i*)d);
-
-	while (w >= 2 && ((unsigned long)d & 3))
+	while (w >= 2 && ((uintptr_t)d & 3))
 	{
 	    *(uint16_t *)d = *(uint16_t *)s;
 	    w -= 2;
@@ -5352,7 +4749,7 @@ pixman_blt_sse2 (uint32_t *src_bits,
 	    d += 2;
 	}
 
-	while (w >= 4 && ((unsigned long)d & 15))
+	while (w >= 4 && ((uintptr_t)d & 15))
 	{
 	    *(uint32_t *)d = *(uint32_t *)s;
 
@@ -5361,17 +4758,10 @@ pixman_blt_sse2 (uint32_t *src_bits,
 	    d += 4;
 	}
 
-	cache_prefetch_next ((__m128i*)s);
-	cache_prefetch_next ((__m128i*)d);
-
 	while (w >= 64)
 	{
 	    __m128i xmm0, xmm1, xmm2, xmm3;
 
-	    /* 128 bytes ahead */
-	    cache_prefetch (((__m128i*)s) + 8);
-	    cache_prefetch (((__m128i*)d) + 8);
-
 	    xmm0 = load_128_unaligned ((__m128i*)(s));
 	    xmm1 = load_128_unaligned ((__m128i*)(s + 16));
 	    xmm2 = load_128_unaligned ((__m128i*)(s + 32));
@@ -5387,9 +4777,6 @@ pixman_blt_sse2 (uint32_t *src_bits,
 	    w -= 64;
 	}
 
-	cache_prefetch_next ((__m128i*)s);
-	cache_prefetch_next ((__m128i*)d);
-
 	while (w >= 16)
 	{
 	    save_128_aligned ((__m128i*)d, load_128_unaligned ((__m128i*)s) );
@@ -5399,9 +4786,6 @@ pixman_blt_sse2 (uint32_t *src_bits,
 	    s += 16;
 	}
 
-	cache_prefetch_next ((__m128i*)s);
-	cache_prefetch_next ((__m128i*)d);
-
 	while (w >= 4)
 	{
 	    *(uint32_t *)d = *(uint32_t *)s;
@@ -5420,64 +4804,42 @@ pixman_blt_sse2 (uint32_t *src_bits,
 	}
     }
 
-    _mm_empty ();
-
     return TRUE;
 }
 
 static void
 sse2_composite_copy_area (pixman_implementation_t *imp,
-                          pixman_op_t              op,
-                          pixman_image_t *         src_image,
-                          pixman_image_t *         mask_image,
-                          pixman_image_t *         dst_image,
-                          int32_t                  src_x,
-                          int32_t                  src_y,
-                          int32_t                  mask_x,
-                          int32_t                  mask_y,
-                          int32_t                  dest_x,
-                          int32_t                  dest_y,
-                          int32_t                  width,
-                          int32_t                  height)
+                          pixman_composite_info_t *info)
 {
-    pixman_blt_sse2 (src_image->bits.bits,
-                     dst_image->bits.bits,
-                     src_image->bits.rowstride,
-                     dst_image->bits.rowstride,
-                     PIXMAN_FORMAT_BPP (src_image->bits.format),
-                     PIXMAN_FORMAT_BPP (dst_image->bits.format),
-                     src_x, src_y, dest_x, dest_y, width, height);
+    PIXMAN_COMPOSITE_ARGS (info);
+    sse2_blt (imp, src_image->bits.bits,
+	      dest_image->bits.bits,
+	      src_image->bits.rowstride,
+	      dest_image->bits.rowstride,
+	      PIXMAN_FORMAT_BPP (src_image->bits.format),
+	      PIXMAN_FORMAT_BPP (dest_image->bits.format),
+	      src_x, src_y, dest_x, dest_y, width, height);
 }
 
 static void
 sse2_composite_over_x888_8_8888 (pixman_implementation_t *imp,
-                                 pixman_op_t              op,
-                                 pixman_image_t *         src_image,
-                                 pixman_image_t *         mask_image,
-                                 pixman_image_t *         dst_image,
-                                 int32_t                  src_x,
-                                 int32_t                  src_y,
-                                 int32_t                  mask_x,
-                                 int32_t                  mask_y,
-                                 int32_t                  dest_x,
-                                 int32_t                  dest_y,
-                                 int32_t                  width,
-                                 int32_t                  height)
+                                 pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t    *src, *src_line, s;
     uint32_t    *dst, *dst_line, d;
     uint8_t         *mask, *mask_line;
     uint32_t m;
     int src_stride, mask_stride, dst_stride;
     int32_t w;
-    __m64 ms;
+    __m128i ms;
 
     __m128i xmm_src, xmm_src_lo, xmm_src_hi;
     __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
     __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (
 	mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
     PIXMAN_IMAGE_GET_LINE (
@@ -5494,44 +4856,30 @@ sse2_composite_over_x888_8_8888 (pixman_implementation_t *imp,
 
         w = width;
 
-        /* call prefetch hint to optimize cache load*/
-        cache_prefetch ((__m128i*)src);
-        cache_prefetch ((__m128i*)dst);
-        cache_prefetch ((__m128i*)mask);
-
-        while (w && (unsigned long)dst & 15)
+        while (w && (uintptr_t)dst & 15)
         {
             s = 0xff000000 | *src++;
             m = (uint32_t) *mask++;
             d = *dst;
-            ms = unpack_32_1x64 (s);
+            ms = unpack_32_1x128 (s);
 
             if (m != 0xff)
             {
-		__m64 ma = expand_alpha_rev_1x64 (unpack_32_1x64 (m));
-		__m64 md = unpack_32_1x64 (d);
+		__m128i ma = expand_alpha_rev_1x128 (unpack_32_1x128 (m));
+		__m128i md = unpack_32_1x128 (d);
 
-                ms = in_over_1x64 (&ms, &mask_x00ff, &ma, &md);
+                ms = in_over_1x128 (&ms, &mask_00ff, &ma, &md);
             }
 
-            *dst++ = pack_1x64_32 (ms);
+            *dst++ = pack_1x128_32 (ms);
             w--;
         }
 
-        /* call prefetch hint to optimize cache load*/
-        cache_prefetch ((__m128i*)src);
-        cache_prefetch ((__m128i*)dst);
-        cache_prefetch ((__m128i*)mask);
-
         while (w >= 4)
         {
-            /* fill cache line with next memory */
-            cache_prefetch_next ((__m128i*)src);
-            cache_prefetch_next ((__m128i*)dst);
-            cache_prefetch_next ((__m128i*)mask);
-
             m = *(uint32_t*) mask;
-            xmm_src = _mm_or_si128 (load_128_unaligned ((__m128i*)src), mask_ff000000);
+            xmm_src = _mm_or_si128 (
+		load_128_unaligned ((__m128i*)src), mask_ff000000);
 
             if (m == 0xffffffff)
             {
@@ -5547,9 +4895,12 @@ sse2_composite_over_x888_8_8888 (pixman_implementation_t *imp,
                 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
                 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
 
-                expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
+                expand_alpha_rev_2x128 (
+		    xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
 
-                in_over_2x128 (&xmm_src_lo, &xmm_src_hi, &mask_00ff, &mask_00ff, &xmm_mask_lo, &xmm_mask_hi, &xmm_dst_lo, &xmm_dst_hi);
+                in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
+			       &mask_00ff, &mask_00ff, &xmm_mask_lo, &xmm_mask_hi,
+			       &xmm_dst_lo, &xmm_dst_hi);
 
                 save_128_aligned ((__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
             }
@@ -5574,15 +4925,15 @@ sse2_composite_over_x888_8_8888 (pixman_implementation_t *imp,
                 }
                 else
                 {
-		    __m64 ma, md, ms;
+		    __m128i ma, md, ms;
 
                     d = *dst;
 
-		    ma = expand_alpha_rev_1x64 (unpack_32_1x64 (m));
-		    md = unpack_32_1x64 (d);
-		    ms = unpack_32_1x64 (s);
+		    ma = expand_alpha_rev_1x128 (unpack_32_1x128 (m));
+		    md = unpack_32_1x128 (d);
+		    ms = unpack_32_1x128 (s);
 
-                    *dst = pack_1x64_32 (in_over_1x64 (&ms, &mask_x00ff, &ma, &md));
+                    *dst = pack_1x128_32 (in_over_1x128 (&ms, &mask_00ff, &ma, &md));
                 }
 
             }
@@ -5593,24 +4944,13 @@ sse2_composite_over_x888_8_8888 (pixman_implementation_t *imp,
         }
     }
 
-    _mm_empty ();
 }
 
 static void
 sse2_composite_over_8888_8_8888 (pixman_implementation_t *imp,
-                                 pixman_op_t              op,
-                                 pixman_image_t *         src_image,
-                                 pixman_image_t *         mask_image,
-                                 pixman_image_t *         dst_image,
-                                 int32_t                  src_x,
-                                 int32_t                  src_y,
-                                 int32_t                  mask_x,
-                                 int32_t                  mask_y,
-                                 int32_t                  dest_x,
-                                 int32_t                  dest_y,
-                                 int32_t                  width,
-                                 int32_t                  height)
+                                 pixman_composite_info_t *info)
 {
+    PIXMAN_COMPOSITE_ARGS (info);
     uint32_t    *src, *src_line, s;
     uint32_t    *dst, *dst_line, d;
     uint8_t         *mask, *mask_line;
@@ -5623,7 +4963,7 @@ sse2_composite_over_8888_8_8888 (pixman_implementation_t *imp,
     __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
 
     PIXMAN_IMAGE_GET_LINE (
-	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+	dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
     PIXMAN_IMAGE_GET_LINE (
 	mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
     PIXMAN_IMAGE_GET_LINE (
@@ -5640,12 +4980,7 @@ sse2_composite_over_8888_8_8888 (pixman_implementation_t *imp,
 
         w = width;
 
-        /* call prefetch hint to optimize cache load*/
-        cache_prefetch ((__m128i *)src);
-        cache_prefetch ((__m128i *)dst);
-        cache_prefetch ((__m128i *)mask);
-
-        while (w && (unsigned long)dst & 15)
+        while (w && (uintptr_t)dst & 15)
         {
 	    uint32_t sa;
 
@@ -5663,15 +4998,15 @@ sse2_composite_over_8888_8_8888 (pixman_implementation_t *imp,
 		}
 		else
 		{
-		    __m64 ms, md, ma, msa;
+		    __m128i ms, md, ma, msa;
 
-		    ma = expand_alpha_rev_1x64 (load_32_1x64 (m));
-		    ms = unpack_32_1x64 (s);
-		    md = unpack_32_1x64 (d);
+		    ma = expand_alpha_rev_1x128 (load_32_1x128 (m));
+		    ms = unpack_32_1x128 (s);
+		    md = unpack_32_1x128 (d);
 
-		    msa = expand_alpha_rev_1x64 (load_32_1x64 (sa));
+		    msa = expand_alpha_rev_1x128 (load_32_1x128 (sa));
 
-		    *dst = pack_1x64_32 (in_over_1x64 (&ms, &msa, &ma, &md));
+		    *dst = pack_1x128_32 (in_over_1x128 (&ms, &msa, &ma, &md));
 		}
 	    }
 
@@ -5679,18 +5014,8 @@ sse2_composite_over_8888_8_8888 (pixman_implementation_t *imp,
             w--;
         }
 
-        /* call prefetch hint to optimize cache load*/
-        cache_prefetch ((__m128i *)src);
-        cache_prefetch ((__m128i *)dst);
-        cache_prefetch ((__m128i *)mask);
-
         while (w >= 4)
         {
-            /* fill cache line with next memory */
-            cache_prefetch_next ((__m128i *)src);
-            cache_prefetch_next ((__m128i *)dst);
-            cache_prefetch_next ((__m128i *)mask);
-
             m = *(uint32_t *) mask;
 
 	    if (m)
@@ -5745,26 +5070,1114 @@ sse2_composite_over_8888_8_8888 (pixman_implementation_t *imp,
 		}
 		else
 		{
-		    __m64 ms, md, ma, msa;
+		    __m128i ms, md, ma, msa;
+
+		    ma = expand_alpha_rev_1x128 (load_32_1x128 (m));
+		    ms = unpack_32_1x128 (s);
+		    md = unpack_32_1x128 (d);
+
+		    msa = expand_alpha_rev_1x128 (load_32_1x128 (sa));
+
+		    *dst = pack_1x128_32 (in_over_1x128 (&ms, &msa, &ma, &md));
+		}
+	    }
+
+	    dst++;
+            w--;
+        }
+    }
+
+}
+
+static void
+sse2_composite_over_reverse_n_8888 (pixman_implementation_t *imp,
+				    pixman_composite_info_t *info)
+{
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint32_t src;
+    uint32_t    *dst_line, *dst;
+    __m128i xmm_src;
+    __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_dsta_hi, xmm_dsta_lo;
+    int dst_stride;
+    int32_t w;
+
+    src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
+
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+
+    xmm_src = expand_pixel_32_1x128 (src);
+
+    while (height--)
+    {
+	dst = dst_line;
+
+	dst_line += dst_stride;
+	w = width;
+
+	while (w && (uintptr_t)dst & 15)
+	{
+	    __m128i vd;
+
+	    vd = unpack_32_1x128 (*dst);
+
+	    *dst = pack_1x128_32 (over_1x128 (vd, expand_alpha_1x128 (vd),
+					      xmm_src));
+	    w--;
+	    dst++;
+	}
+
+	while (w >= 4)
+	{
+	    __m128i tmp_lo, tmp_hi;
+
+	    xmm_dst = load_128_aligned ((__m128i*)dst);
+
+	    unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+	    expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi, &xmm_dsta_lo, &xmm_dsta_hi);
+
+	    tmp_lo = xmm_src;
+	    tmp_hi = xmm_src;
+
+	    over_2x128 (&xmm_dst_lo, &xmm_dst_hi,
+			&xmm_dsta_lo, &xmm_dsta_hi,
+			&tmp_lo, &tmp_hi);
+
+	    save_128_aligned (
+		(__m128i*)dst, pack_2x128_128 (tmp_lo, tmp_hi));
+
+	    w -= 4;
+	    dst += 4;
+	}
+
+	while (w)
+	{
+	    __m128i vd;
+
+	    vd = unpack_32_1x128 (*dst);
+
+	    *dst = pack_1x128_32 (over_1x128 (vd, expand_alpha_1x128 (vd),
+					      xmm_src));
+	    w--;
+	    dst++;
+	}
+
+    }
+
+}
+
+static void
+sse2_composite_over_8888_8888_8888 (pixman_implementation_t *imp,
+				    pixman_composite_info_t *info)
+{
+    PIXMAN_COMPOSITE_ARGS (info);
+    uint32_t    *src, *src_line, s;
+    uint32_t    *dst, *dst_line, d;
+    uint32_t    *mask, *mask_line;
+    uint32_t    m;
+    int src_stride, mask_stride, dst_stride;
+    int32_t w;
+
+    __m128i xmm_src, xmm_src_lo, xmm_src_hi, xmm_srca_lo, xmm_srca_hi;
+    __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+    while (height--)
+    {
+        src = src_line;
+        src_line += src_stride;
+        dst = dst_line;
+        dst_line += dst_stride;
+        mask = mask_line;
+        mask_line += mask_stride;
+
+        w = width;
+
+        while (w && (uintptr_t)dst & 15)
+        {
+	    uint32_t sa;
+
+            s = *src++;
+            m = (*mask++) >> 24;
+            d = *dst;
+
+	    sa = s >> 24;
+
+	    if (m)
+	    {
+		if (sa == 0xff && m == 0xff)
+		{
+		    *dst = s;
+		}
+		else
+		{
+		    __m128i ms, md, ma, msa;
 
-		    ma = expand_alpha_rev_1x64 (load_32_1x64 (m));
-		    ms = unpack_32_1x64 (s);
-		    md = unpack_32_1x64 (d);
+		    ma = expand_alpha_rev_1x128 (load_32_1x128 (m));
+		    ms = unpack_32_1x128 (s);
+		    md = unpack_32_1x128 (d);
 
-		    msa = expand_alpha_rev_1x64 (load_32_1x64 (sa));
+		    msa = expand_alpha_rev_1x128 (load_32_1x128 (sa));
 
-		    *dst = pack_1x64_32 (in_over_1x64 (&ms, &msa, &ma, &md));
+		    *dst = pack_1x128_32 (in_over_1x128 (&ms, &msa, &ma, &md));
 		}
 	    }
 
 	    dst++;
             w--;
         }
+
+        while (w >= 4)
+        {
+	    xmm_mask = load_128_unaligned ((__m128i*)mask);
+
+	    if (!is_transparent (xmm_mask))
+	    {
+		xmm_src = load_128_unaligned ((__m128i*)src);
+
+		if (is_opaque (xmm_mask) && is_opaque (xmm_src))
+		{
+		    save_128_aligned ((__m128i *)dst, xmm_src);
+		}
+		else
+		{
+		    xmm_dst = load_128_aligned ((__m128i *)dst);
+
+		    unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
+		    unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
+		    unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+
+		    expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi);
+		    expand_alpha_2x128 (xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
+
+		    in_over_2x128 (&xmm_src_lo, &xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi,
+				   &xmm_mask_lo, &xmm_mask_hi, &xmm_dst_lo, &xmm_dst_hi);
+
+		    save_128_aligned ((__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+		}
+	    }
+
+            src += 4;
+            dst += 4;
+            mask += 4;
+            w -= 4;
+        }
+
+        while (w)
+        {
+	    uint32_t sa;
+
+            s = *src++;
+            m = (*mask++) >> 24;
+            d = *dst;
+
+	    sa = s >> 24;
+
+	    if (m)
+	    {
+		if (sa == 0xff && m == 0xff)
+		{
+		    *dst = s;
+		}
+		else
+		{
+		    __m128i ms, md, ma, msa;
+
+		    ma = expand_alpha_rev_1x128 (load_32_1x128 (m));
+		    ms = unpack_32_1x128 (s);
+		    md = unpack_32_1x128 (d);
+
+		    msa = expand_alpha_rev_1x128 (load_32_1x128 (sa));
+
+		    *dst = pack_1x128_32 (in_over_1x128 (&ms, &msa, &ma, &md));
+		}
+	    }
+
+	    dst++;
+            w--;
+        }
+    }
+
+}
+
+/* A variant of 'sse2_combine_over_u' with minor tweaks */
+static force_inline void
+scaled_nearest_scanline_sse2_8888_8888_OVER (uint32_t*       pd,
+                                             const uint32_t* ps,
+                                             int32_t         w,
+                                             pixman_fixed_t  vx,
+                                             pixman_fixed_t  unit_x,
+                                             pixman_fixed_t  src_width_fixed,
+                                             pixman_bool_t   fully_transparent_src)
+{
+    uint32_t s, d;
+    const uint32_t* pm = NULL;
+
+    __m128i xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_alpha_lo, xmm_alpha_hi;
+
+    if (fully_transparent_src)
+	return;
+
+    /* Align dst on a 16-byte boundary */
+    while (w && ((uintptr_t)pd & 15))
+    {
+	d = *pd;
+	s = combine1 (ps + pixman_fixed_to_int (vx), pm);
+	vx += unit_x;
+	while (vx >= 0)
+	    vx -= src_width_fixed;
+
+	*pd++ = core_combine_over_u_pixel_sse2 (s, d);
+	if (pm)
+	    pm++;
+	w--;
+    }
+
+    while (w >= 4)
+    {
+	__m128i tmp;
+	uint32_t tmp1, tmp2, tmp3, tmp4;
+
+	tmp1 = *(ps + pixman_fixed_to_int (vx));
+	vx += unit_x;
+	while (vx >= 0)
+	    vx -= src_width_fixed;
+	tmp2 = *(ps + pixman_fixed_to_int (vx));
+	vx += unit_x;
+	while (vx >= 0)
+	    vx -= src_width_fixed;
+	tmp3 = *(ps + pixman_fixed_to_int (vx));
+	vx += unit_x;
+	while (vx >= 0)
+	    vx -= src_width_fixed;
+	tmp4 = *(ps + pixman_fixed_to_int (vx));
+	vx += unit_x;
+	while (vx >= 0)
+	    vx -= src_width_fixed;
+
+	tmp = _mm_set_epi32 (tmp4, tmp3, tmp2, tmp1);
+
+	xmm_src_hi = combine4 ((__m128i*)&tmp, (__m128i*)pm);
+
+	if (is_opaque (xmm_src_hi))
+	{
+	    save_128_aligned ((__m128i*)pd, xmm_src_hi);
+	}
+	else if (!is_zero (xmm_src_hi))
+	{
+	    xmm_dst_hi = load_128_aligned ((__m128i*) pd);
+
+	    unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	    unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+
+	    expand_alpha_2x128 (
+		xmm_src_lo, xmm_src_hi, &xmm_alpha_lo, &xmm_alpha_hi);
+
+	    over_2x128 (&xmm_src_lo, &xmm_src_hi,
+			&xmm_alpha_lo, &xmm_alpha_hi,
+			&xmm_dst_lo, &xmm_dst_hi);
+
+	    /* rebuid the 4 pixel data and save*/
+	    save_128_aligned ((__m128i*)pd,
+			      pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+	}
+
+	w -= 4;
+	pd += 4;
+	if (pm)
+	    pm += 4;
+    }
+
+    while (w)
+    {
+	d = *pd;
+	s = combine1 (ps + pixman_fixed_to_int (vx), pm);
+	vx += unit_x;
+	while (vx >= 0)
+	    vx -= src_width_fixed;
+
+	*pd++ = core_combine_over_u_pixel_sse2 (s, d);
+	if (pm)
+	    pm++;
+
+	w--;
+    }
+}
+
+FAST_NEAREST_MAINLOOP (sse2_8888_8888_cover_OVER,
+		       scaled_nearest_scanline_sse2_8888_8888_OVER,
+		       uint32_t, uint32_t, COVER)
+FAST_NEAREST_MAINLOOP (sse2_8888_8888_none_OVER,
+		       scaled_nearest_scanline_sse2_8888_8888_OVER,
+		       uint32_t, uint32_t, NONE)
+FAST_NEAREST_MAINLOOP (sse2_8888_8888_pad_OVER,
+		       scaled_nearest_scanline_sse2_8888_8888_OVER,
+		       uint32_t, uint32_t, PAD)
+FAST_NEAREST_MAINLOOP (sse2_8888_8888_normal_OVER,
+		       scaled_nearest_scanline_sse2_8888_8888_OVER,
+		       uint32_t, uint32_t, NORMAL)
+
+static force_inline void
+scaled_nearest_scanline_sse2_8888_n_8888_OVER (const uint32_t * mask,
+					       uint32_t *       dst,
+					       const uint32_t * src,
+					       int32_t          w,
+					       pixman_fixed_t   vx,
+					       pixman_fixed_t   unit_x,
+					       pixman_fixed_t   src_width_fixed,
+					       pixman_bool_t    zero_src)
+{
+    __m128i xmm_mask;
+    __m128i xmm_src, xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_alpha_lo, xmm_alpha_hi;
+
+    if (zero_src || (*mask >> 24) == 0)
+	return;
+
+    xmm_mask = create_mask_16_128 (*mask >> 24);
+
+    while (w && (uintptr_t)dst & 15)
+    {
+	uint32_t s = *(src + pixman_fixed_to_int (vx));
+	vx += unit_x;
+	while (vx >= 0)
+	    vx -= src_width_fixed;
+
+	if (s)
+	{
+	    uint32_t d = *dst;
+
+	    __m128i ms = unpack_32_1x128 (s);
+	    __m128i alpha     = expand_alpha_1x128 (ms);
+	    __m128i dest      = xmm_mask;
+	    __m128i alpha_dst = unpack_32_1x128 (d);
+
+	    *dst = pack_1x128_32 (
+		in_over_1x128 (&ms, &alpha, &dest, &alpha_dst));
+	}
+	dst++;
+	w--;
+    }
+
+    while (w >= 4)
+    {
+	uint32_t tmp1, tmp2, tmp3, tmp4;
+
+	tmp1 = *(src + pixman_fixed_to_int (vx));
+	vx += unit_x;
+	while (vx >= 0)
+	    vx -= src_width_fixed;
+	tmp2 = *(src + pixman_fixed_to_int (vx));
+	vx += unit_x;
+	while (vx >= 0)
+	    vx -= src_width_fixed;
+	tmp3 = *(src + pixman_fixed_to_int (vx));
+	vx += unit_x;
+	while (vx >= 0)
+	    vx -= src_width_fixed;
+	tmp4 = *(src + pixman_fixed_to_int (vx));
+	vx += unit_x;
+	while (vx >= 0)
+	    vx -= src_width_fixed;
+
+	xmm_src = _mm_set_epi32 (tmp4, tmp3, tmp2, tmp1);
+
+	if (!is_zero (xmm_src))
+	{
+	    xmm_dst = load_128_aligned ((__m128i*)dst);
+
+	    unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
+	    unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+	    expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
+			        &xmm_alpha_lo, &xmm_alpha_hi);
+
+	    in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
+			   &xmm_alpha_lo, &xmm_alpha_hi,
+			   &xmm_mask, &xmm_mask,
+			   &xmm_dst_lo, &xmm_dst_hi);
+
+	    save_128_aligned (
+		(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+	}
+
+	dst += 4;
+	w -= 4;
+    }
+
+    while (w)
+    {
+	uint32_t s = *(src + pixman_fixed_to_int (vx));
+	vx += unit_x;
+	while (vx >= 0)
+	    vx -= src_width_fixed;
+
+	if (s)
+	{
+	    uint32_t d = *dst;
+
+	    __m128i ms = unpack_32_1x128 (s);
+	    __m128i alpha = expand_alpha_1x128 (ms);
+	    __m128i mask  = xmm_mask;
+	    __m128i dest  = unpack_32_1x128 (d);
+
+	    *dst = pack_1x128_32 (
+		in_over_1x128 (&ms, &alpha, &mask, &dest));
+	}
+
+	dst++;
+	w--;
+    }
+
+}
+
+FAST_NEAREST_MAINLOOP_COMMON (sse2_8888_n_8888_cover_OVER,
+			      scaled_nearest_scanline_sse2_8888_n_8888_OVER,
+			      uint32_t, uint32_t, uint32_t, COVER, TRUE, TRUE)
+FAST_NEAREST_MAINLOOP_COMMON (sse2_8888_n_8888_pad_OVER,
+			      scaled_nearest_scanline_sse2_8888_n_8888_OVER,
+			      uint32_t, uint32_t, uint32_t, PAD, TRUE, TRUE)
+FAST_NEAREST_MAINLOOP_COMMON (sse2_8888_n_8888_none_OVER,
+			      scaled_nearest_scanline_sse2_8888_n_8888_OVER,
+			      uint32_t, uint32_t, uint32_t, NONE, TRUE, TRUE)
+FAST_NEAREST_MAINLOOP_COMMON (sse2_8888_n_8888_normal_OVER,
+			      scaled_nearest_scanline_sse2_8888_n_8888_OVER,
+			      uint32_t, uint32_t, uint32_t, NORMAL, TRUE, TRUE)
+
+#if PSHUFD_IS_FAST
+
+/***********************************************************************************/
+
+# define BILINEAR_DECLARE_VARIABLES						\
+    const __m128i xmm_wt = _mm_set_epi16 (wt, wt, wt, wt, wt, wt, wt, wt);	\
+    const __m128i xmm_wb = _mm_set_epi16 (wb, wb, wb, wb, wb, wb, wb, wb);	\
+    const __m128i xmm_addc = _mm_set_epi16 (0, 1, 0, 1, 0, 1, 0, 1);		\
+    const __m128i xmm_ux1 = _mm_set_epi16 (unit_x, -unit_x, unit_x, -unit_x,	\
+					   unit_x, -unit_x, unit_x, -unit_x);	\
+    const __m128i xmm_ux4 = _mm_set_epi16 (unit_x * 4, -unit_x * 4,		\
+					   unit_x * 4, -unit_x * 4,		\
+					   unit_x * 4, -unit_x * 4,		\
+					   unit_x * 4, -unit_x * 4);		\
+    const __m128i xmm_zero = _mm_setzero_si128 ();				\
+    __m128i xmm_x = _mm_set_epi16 (vx + unit_x * 3, -(vx + 1) - unit_x * 3,	\
+				   vx + unit_x * 2, -(vx + 1) - unit_x * 2,	\
+				   vx + unit_x * 1, -(vx + 1) - unit_x * 1,	\
+				   vx + unit_x * 0, -(vx + 1) - unit_x * 0);	\
+    __m128i xmm_wh_state;
+
+#define BILINEAR_INTERPOLATE_ONE_PIXEL_HELPER(pix, phase_)			\
+do {										\
+    int phase = phase_;								\
+    __m128i xmm_wh, xmm_a, xmm_b;						\
+    /* fetch 2x2 pixel block into sse2 registers */				\
+    __m128i tltr = _mm_loadl_epi64 ((__m128i *)&src_top[vx >> 16]);		\
+    __m128i blbr = _mm_loadl_epi64 ((__m128i *)&src_bottom[vx >> 16]);		\
+    vx += unit_x;								\
+    /* vertical interpolation */						\
+    xmm_a = _mm_mullo_epi16 (_mm_unpacklo_epi8 (tltr, xmm_zero), xmm_wt);	\
+    xmm_b = _mm_mullo_epi16 (_mm_unpacklo_epi8 (blbr, xmm_zero), xmm_wb);	\
+    xmm_a = _mm_add_epi16 (xmm_a, xmm_b);						\
+    /* calculate horizontal weights */						\
+    if (phase <= 0)								\
+    {										\
+	xmm_wh_state = _mm_add_epi16 (xmm_addc, _mm_srli_epi16 (xmm_x,		\
+					16 - BILINEAR_INTERPOLATION_BITS));	\
+	xmm_x = _mm_add_epi16 (xmm_x, (phase < 0) ? xmm_ux1 : xmm_ux4);		\
+	phase = 0;								\
+    }										\
+    xmm_wh = _mm_shuffle_epi32 (xmm_wh_state, _MM_SHUFFLE (phase, phase,	\
+							   phase, phase));	\
+    /* horizontal interpolation */						\
+    xmm_a = _mm_madd_epi16 (_mm_unpackhi_epi16 (_mm_shuffle_epi32 (		\
+		xmm_a, _MM_SHUFFLE (1, 0, 3, 2)), xmm_a), xmm_wh);		\
+    /* shift the result */							\
+    pix = _mm_srli_epi32 (xmm_a, BILINEAR_INTERPOLATION_BITS * 2);		\
+} while (0)
+
+#else /************************************************************************/
+
+# define BILINEAR_DECLARE_VARIABLES						\
+    const __m128i xmm_wt = _mm_set_epi16 (wt, wt, wt, wt, wt, wt, wt, wt);	\
+    const __m128i xmm_wb = _mm_set_epi16 (wb, wb, wb, wb, wb, wb, wb, wb);	\
+    const __m128i xmm_addc = _mm_set_epi16 (0, 1, 0, 1, 0, 1, 0, 1);		\
+    const __m128i xmm_ux1 = _mm_set_epi16 (unit_x, -unit_x, unit_x, -unit_x,	\
+					  unit_x, -unit_x, unit_x, -unit_x);	\
+    const __m128i xmm_ux4 = _mm_set_epi16 (unit_x * 4, -unit_x * 4,		\
+					   unit_x * 4, -unit_x * 4,		\
+					   unit_x * 4, -unit_x * 4,		\
+					   unit_x * 4, -unit_x * 4);		\
+    const __m128i xmm_zero = _mm_setzero_si128 ();				\
+    __m128i xmm_x = _mm_set_epi16 (vx, -(vx + 1), vx, -(vx + 1),		\
+				   vx, -(vx + 1), vx, -(vx + 1))
+
+#define BILINEAR_INTERPOLATE_ONE_PIXEL_HELPER(pix, phase)			\
+do {										\
+    __m128i xmm_wh, xmm_a, xmm_b;						\
+    /* fetch 2x2 pixel block into sse2 registers */				\
+    __m128i tltr = _mm_loadl_epi64 ((__m128i *)&src_top[vx >> 16]);		\
+    __m128i blbr = _mm_loadl_epi64 ((__m128i *)&src_bottom[vx >> 16]);		\
+    (void)xmm_ux4; /* suppress warning: unused variable 'xmm_ux4' */		\
+    vx += unit_x;								\
+    /* vertical interpolation */						\
+    xmm_a = _mm_mullo_epi16 (_mm_unpacklo_epi8 (tltr, xmm_zero), xmm_wt);	\
+    xmm_b = _mm_mullo_epi16 (_mm_unpacklo_epi8 (blbr, xmm_zero), xmm_wb);	\
+    xmm_a = _mm_add_epi16 (xmm_a, xmm_b);					\
+    /* calculate horizontal weights */						\
+    xmm_wh = _mm_add_epi16 (xmm_addc, _mm_srli_epi16 (xmm_x,			\
+					16 - BILINEAR_INTERPOLATION_BITS));	\
+    xmm_x = _mm_add_epi16 (xmm_x, xmm_ux1);					\
+    /* horizontal interpolation */						\
+    xmm_b = _mm_unpacklo_epi64 (/* any value is fine here */ xmm_b, xmm_a);	\
+    xmm_a = _mm_madd_epi16 (_mm_unpackhi_epi16 (xmm_b, xmm_a), xmm_wh);		\
+    /* shift the result */							\
+    pix = _mm_srli_epi32 (xmm_a, BILINEAR_INTERPOLATION_BITS * 2);		\
+} while (0)
+
+/***********************************************************************************/
+
+#endif
+
+#define BILINEAR_INTERPOLATE_ONE_PIXEL(pix);					\
+do {										\
+	__m128i xmm_pix;							\
+	BILINEAR_INTERPOLATE_ONE_PIXEL_HELPER (xmm_pix, -1);			\
+	xmm_pix = _mm_packs_epi32 (xmm_pix, xmm_pix);				\
+	xmm_pix = _mm_packus_epi16 (xmm_pix, xmm_pix);				\
+	pix = _mm_cvtsi128_si32 (xmm_pix);					\
+} while(0)
+
+#define BILINEAR_INTERPOLATE_FOUR_PIXELS(pix);					\
+do {										\
+	__m128i xmm_pix1, xmm_pix2, xmm_pix3, xmm_pix4;				\
+	BILINEAR_INTERPOLATE_ONE_PIXEL_HELPER (xmm_pix1, 0);			\
+	BILINEAR_INTERPOLATE_ONE_PIXEL_HELPER (xmm_pix2, 1);			\
+	BILINEAR_INTERPOLATE_ONE_PIXEL_HELPER (xmm_pix3, 2);			\
+	BILINEAR_INTERPOLATE_ONE_PIXEL_HELPER (xmm_pix4, 3);			\
+	xmm_pix1 = _mm_packs_epi32 (xmm_pix1, xmm_pix2);			\
+	xmm_pix3 = _mm_packs_epi32 (xmm_pix3, xmm_pix4);			\
+	pix = _mm_packus_epi16 (xmm_pix1, xmm_pix3);				\
+} while(0)
+
+#define BILINEAR_SKIP_ONE_PIXEL()						\
+do {										\
+    vx += unit_x;								\
+    xmm_x = _mm_add_epi16 (xmm_x, xmm_ux1);					\
+} while(0)
+
+#define BILINEAR_SKIP_FOUR_PIXELS()						\
+do {										\
+    vx += unit_x * 4;								\
+    xmm_x = _mm_add_epi16 (xmm_x, xmm_ux4);					\
+} while(0)
+
+/***********************************************************************************/
+
+static force_inline void
+scaled_bilinear_scanline_sse2_8888_8888_SRC (uint32_t *       dst,
+					     const uint32_t * mask,
+					     const uint32_t * src_top,
+					     const uint32_t * src_bottom,
+					     int32_t          w,
+					     int              wt,
+					     int              wb,
+					     pixman_fixed_t   vx_,
+					     pixman_fixed_t   unit_x_,
+					     pixman_fixed_t   max_vx,
+					     pixman_bool_t    zero_src)
+{
+    intptr_t vx = vx_;
+    intptr_t unit_x = unit_x_;
+    BILINEAR_DECLARE_VARIABLES;
+    uint32_t pix1, pix2;
+
+    while (w && ((uintptr_t)dst & 15))
+    {
+	BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
+	*dst++ = pix1;
+	w--;
+    }
+
+    while ((w -= 4) >= 0) {
+	__m128i xmm_src;
+	BILINEAR_INTERPOLATE_FOUR_PIXELS (xmm_src);
+	_mm_store_si128 ((__m128i *)dst, xmm_src);
+	dst += 4;
+    }
+
+    if (w & 2)
+    {
+	BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
+	BILINEAR_INTERPOLATE_ONE_PIXEL (pix2);
+	*dst++ = pix1;
+	*dst++ = pix2;
+    }
+
+    if (w & 1)
+    {
+	BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
+	*dst = pix1;
+    }
+
+}
+
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_cover_SRC,
+			       scaled_bilinear_scanline_sse2_8888_8888_SRC,
+			       uint32_t, uint32_t, uint32_t,
+			       COVER, FLAG_NONE)
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_pad_SRC,
+			       scaled_bilinear_scanline_sse2_8888_8888_SRC,
+			       uint32_t, uint32_t, uint32_t,
+			       PAD, FLAG_NONE)
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_none_SRC,
+			       scaled_bilinear_scanline_sse2_8888_8888_SRC,
+			       uint32_t, uint32_t, uint32_t,
+			       NONE, FLAG_NONE)
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_normal_SRC,
+			       scaled_bilinear_scanline_sse2_8888_8888_SRC,
+			       uint32_t, uint32_t, uint32_t,
+			       NORMAL, FLAG_NONE)
+
+static force_inline void
+scaled_bilinear_scanline_sse2_x888_8888_SRC (uint32_t *       dst,
+					     const uint32_t * mask,
+					     const uint32_t * src_top,
+					     const uint32_t * src_bottom,
+					     int32_t          w,
+					     int              wt,
+					     int              wb,
+					     pixman_fixed_t   vx_,
+					     pixman_fixed_t   unit_x_,
+					     pixman_fixed_t   max_vx,
+					     pixman_bool_t    zero_src)
+{
+    intptr_t vx = vx_;
+    intptr_t unit_x = unit_x_;
+    BILINEAR_DECLARE_VARIABLES;
+    uint32_t pix1, pix2;
+
+    while (w && ((uintptr_t)dst & 15))
+    {
+	BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
+	*dst++ = pix1 | 0xFF000000;
+	w--;
+    }
+
+    while ((w -= 4) >= 0) {
+	__m128i xmm_src;
+	BILINEAR_INTERPOLATE_FOUR_PIXELS (xmm_src);
+	_mm_store_si128 ((__m128i *)dst, _mm_or_si128 (xmm_src, mask_ff000000));
+	dst += 4;
+    }
+
+    if (w & 2)
+    {
+	BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
+	BILINEAR_INTERPOLATE_ONE_PIXEL (pix2);
+	*dst++ = pix1 | 0xFF000000;
+	*dst++ = pix2 | 0xFF000000;
+    }
+
+    if (w & 1)
+    {
+	BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
+	*dst = pix1 | 0xFF000000;
+    }
+}
+
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_x888_8888_cover_SRC,
+			       scaled_bilinear_scanline_sse2_x888_8888_SRC,
+			       uint32_t, uint32_t, uint32_t,
+			       COVER, FLAG_NONE)
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_x888_8888_pad_SRC,
+			       scaled_bilinear_scanline_sse2_x888_8888_SRC,
+			       uint32_t, uint32_t, uint32_t,
+			       PAD, FLAG_NONE)
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_x888_8888_normal_SRC,
+			       scaled_bilinear_scanline_sse2_x888_8888_SRC,
+			       uint32_t, uint32_t, uint32_t,
+			       NORMAL, FLAG_NONE)
+
+static force_inline void
+scaled_bilinear_scanline_sse2_8888_8888_OVER (uint32_t *       dst,
+					      const uint32_t * mask,
+					      const uint32_t * src_top,
+					      const uint32_t * src_bottom,
+					      int32_t          w,
+					      int              wt,
+					      int              wb,
+					      pixman_fixed_t   vx_,
+					      pixman_fixed_t   unit_x_,
+					      pixman_fixed_t   max_vx,
+					      pixman_bool_t    zero_src)
+{
+    intptr_t vx = vx_;
+    intptr_t unit_x = unit_x_;
+    BILINEAR_DECLARE_VARIABLES;
+    uint32_t pix1, pix2;
+
+    while (w && ((uintptr_t)dst & 15))
+    {
+	BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
+
+	if (pix1)
+	{
+	    pix2 = *dst;
+	    *dst = core_combine_over_u_pixel_sse2 (pix1, pix2);
+	}
+
+	w--;
+	dst++;
+    }
+
+    while (w  >= 4)
+    {
+	__m128i xmm_src;
+	__m128i xmm_src_hi, xmm_src_lo, xmm_dst_hi, xmm_dst_lo;
+	__m128i xmm_alpha_hi, xmm_alpha_lo;
+
+	BILINEAR_INTERPOLATE_FOUR_PIXELS (xmm_src);
+
+	if (!is_zero (xmm_src))
+	{
+	    if (is_opaque (xmm_src))
+	    {
+		save_128_aligned ((__m128i *)dst, xmm_src);
+	    }
+	    else
+	    {
+		__m128i xmm_dst = load_128_aligned ((__m128i *)dst);
+
+		unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
+		unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+
+		expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_alpha_lo, &xmm_alpha_hi);
+		over_2x128 (&xmm_src_lo, &xmm_src_hi, &xmm_alpha_lo, &xmm_alpha_hi,
+			    &xmm_dst_lo, &xmm_dst_hi);
+
+		save_128_aligned ((__m128i *)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+	    }
+	}
+
+	w -= 4;
+	dst += 4;
+    }
+
+    while (w)
+    {
+	BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
+
+	if (pix1)
+	{
+	    pix2 = *dst;
+	    *dst = core_combine_over_u_pixel_sse2 (pix1, pix2);
+	}
+
+	w--;
+	dst++;
+    }
+}
+
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_cover_OVER,
+			       scaled_bilinear_scanline_sse2_8888_8888_OVER,
+			       uint32_t, uint32_t, uint32_t,
+			       COVER, FLAG_NONE)
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_pad_OVER,
+			       scaled_bilinear_scanline_sse2_8888_8888_OVER,
+			       uint32_t, uint32_t, uint32_t,
+			       PAD, FLAG_NONE)
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_none_OVER,
+			       scaled_bilinear_scanline_sse2_8888_8888_OVER,
+			       uint32_t, uint32_t, uint32_t,
+			       NONE, FLAG_NONE)
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_normal_OVER,
+			       scaled_bilinear_scanline_sse2_8888_8888_OVER,
+			       uint32_t, uint32_t, uint32_t,
+			       NORMAL, FLAG_NONE)
+
+static force_inline void
+scaled_bilinear_scanline_sse2_8888_8_8888_OVER (uint32_t *       dst,
+						const uint8_t  * mask,
+						const uint32_t * src_top,
+						const uint32_t * src_bottom,
+						int32_t          w,
+						int              wt,
+						int              wb,
+						pixman_fixed_t   vx_,
+						pixman_fixed_t   unit_x_,
+						pixman_fixed_t   max_vx,
+						pixman_bool_t    zero_src)
+{
+    intptr_t vx = vx_;
+    intptr_t unit_x = unit_x_;
+    BILINEAR_DECLARE_VARIABLES;
+    uint32_t pix1, pix2;
+    uint32_t m;
+
+    while (w && ((uintptr_t)dst & 15))
+    {
+	uint32_t sa;
+
+	m = (uint32_t) *mask++;
+
+	if (m)
+	{
+	    BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
+	    sa = pix1 >> 24;
+
+	    if (sa == 0xff && m == 0xff)
+	    {
+		*dst = pix1;
+	    }
+	    else
+	    {
+		__m128i ms, md, ma, msa;
+
+		pix2 = *dst;
+		ma = expand_alpha_rev_1x128 (load_32_1x128 (m));
+		ms = unpack_32_1x128 (pix1);
+		md = unpack_32_1x128 (pix2);
+
+		msa = expand_alpha_rev_1x128 (load_32_1x128 (sa));
+
+		*dst = pack_1x128_32 (in_over_1x128 (&ms, &msa, &ma, &md));
+	    }
+	}
+	else
+	{
+	    BILINEAR_SKIP_ONE_PIXEL ();
+	}
+
+	w--;
+	dst++;
+    }
+
+    while (w >= 4)
+    {
+	__m128i xmm_src, xmm_src_lo, xmm_src_hi, xmm_srca_lo, xmm_srca_hi;
+	__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
+	__m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
+
+	m = *(uint32_t*)mask;
+
+	if (m)
+	{
+	    BILINEAR_INTERPOLATE_FOUR_PIXELS (xmm_src);
+
+	    if (m == 0xffffffff && is_opaque (xmm_src))
+	    {
+		save_128_aligned ((__m128i *)dst, xmm_src);
+	    }
+	    else
+	    {
+		xmm_dst = load_128_aligned ((__m128i *)dst);
+
+		xmm_mask = _mm_unpacklo_epi16 (unpack_32_1x128 (m), _mm_setzero_si128());
+
+		unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
+		unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
+		unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+
+		expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi);
+		expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
+
+		in_over_2x128 (&xmm_src_lo, &xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi,
+			       &xmm_mask_lo, &xmm_mask_hi, &xmm_dst_lo, &xmm_dst_hi);
+
+		save_128_aligned ((__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+	    }
+	}
+	else
+	{
+	    BILINEAR_SKIP_FOUR_PIXELS ();
+	}
+
+	w -= 4;
+	dst += 4;
+	mask += 4;
     }
 
-    _mm_empty ();
+    while (w)
+    {
+	uint32_t sa;
+
+	m = (uint32_t) *mask++;
+
+	if (m)
+	{
+	    BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
+	    sa = pix1 >> 24;
+
+	    if (sa == 0xff && m == 0xff)
+	    {
+		*dst = pix1;
+	    }
+	    else
+	    {
+		__m128i ms, md, ma, msa;
+
+		pix2 = *dst;
+		ma = expand_alpha_rev_1x128 (load_32_1x128 (m));
+		ms = unpack_32_1x128 (pix1);
+		md = unpack_32_1x128 (pix2);
+
+		msa = expand_alpha_rev_1x128 (load_32_1x128 (sa));
+
+		*dst = pack_1x128_32 (in_over_1x128 (&ms, &msa, &ma, &md));
+	    }
+	}
+	else
+	{
+	    BILINEAR_SKIP_ONE_PIXEL ();
+	}
+
+	w--;
+	dst++;
+    }
 }
 
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8_8888_cover_OVER,
+			       scaled_bilinear_scanline_sse2_8888_8_8888_OVER,
+			       uint32_t, uint8_t, uint32_t,
+			       COVER, FLAG_HAVE_NON_SOLID_MASK)
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8_8888_pad_OVER,
+			       scaled_bilinear_scanline_sse2_8888_8_8888_OVER,
+			       uint32_t, uint8_t, uint32_t,
+			       PAD, FLAG_HAVE_NON_SOLID_MASK)
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8_8888_none_OVER,
+			       scaled_bilinear_scanline_sse2_8888_8_8888_OVER,
+			       uint32_t, uint8_t, uint32_t,
+			       NONE, FLAG_HAVE_NON_SOLID_MASK)
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8_8888_normal_OVER,
+			       scaled_bilinear_scanline_sse2_8888_8_8888_OVER,
+			       uint32_t, uint8_t, uint32_t,
+			       NORMAL, FLAG_HAVE_NON_SOLID_MASK)
+
+static force_inline void
+scaled_bilinear_scanline_sse2_8888_n_8888_OVER (uint32_t *       dst,
+						const uint32_t * mask,
+						const uint32_t * src_top,
+						const uint32_t * src_bottom,
+						int32_t          w,
+						int              wt,
+						int              wb,
+						pixman_fixed_t   vx_,
+						pixman_fixed_t   unit_x_,
+						pixman_fixed_t   max_vx,
+						pixman_bool_t    zero_src)
+{
+    intptr_t vx = vx_;
+    intptr_t unit_x = unit_x_;
+    BILINEAR_DECLARE_VARIABLES;
+    uint32_t pix1;
+    __m128i xmm_mask;
+
+    if (zero_src || (*mask >> 24) == 0)
+	return;
+
+    xmm_mask = create_mask_16_128 (*mask >> 24);
+
+    while (w && ((uintptr_t)dst & 15))
+    {
+	BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
+	if (pix1)
+	{
+		uint32_t d = *dst;
+
+		__m128i ms = unpack_32_1x128 (pix1);
+		__m128i alpha     = expand_alpha_1x128 (ms);
+		__m128i dest      = xmm_mask;
+		__m128i alpha_dst = unpack_32_1x128 (d);
+
+		*dst = pack_1x128_32
+			(in_over_1x128 (&ms, &alpha, &dest, &alpha_dst));
+	}
+
+	dst++;
+	w--;
+    }
+
+    while (w >= 4)
+    {
+	__m128i xmm_src;
+	BILINEAR_INTERPOLATE_FOUR_PIXELS (xmm_src);
+
+	if (!is_zero (xmm_src))
+	{
+	    __m128i xmm_src_lo, xmm_src_hi;
+	    __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
+	    __m128i xmm_alpha_lo, xmm_alpha_hi;
+
+	    xmm_dst = load_128_aligned ((__m128i*)dst);
+
+	    unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
+	    unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+	    expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
+				&xmm_alpha_lo, &xmm_alpha_hi);
+
+	    in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
+			   &xmm_alpha_lo, &xmm_alpha_hi,
+			   &xmm_mask, &xmm_mask,
+			   &xmm_dst_lo, &xmm_dst_hi);
+
+	    save_128_aligned
+		((__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+	}
+
+	dst += 4;
+	w -= 4;
+    }
+
+    while (w)
+    {
+	BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
+	if (pix1)
+	{
+		uint32_t d = *dst;
+
+		__m128i ms = unpack_32_1x128 (pix1);
+		__m128i alpha     = expand_alpha_1x128 (ms);
+		__m128i dest      = xmm_mask;
+		__m128i alpha_dst = unpack_32_1x128 (d);
+
+		*dst = pack_1x128_32
+			(in_over_1x128 (&ms, &alpha, &dest, &alpha_dst));
+	}
+
+	dst++;
+	w--;
+    }
+}
+
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_n_8888_cover_OVER,
+			       scaled_bilinear_scanline_sse2_8888_n_8888_OVER,
+			       uint32_t, uint32_t, uint32_t,
+			       COVER, FLAG_HAVE_SOLID_MASK)
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_n_8888_pad_OVER,
+			       scaled_bilinear_scanline_sse2_8888_n_8888_OVER,
+			       uint32_t, uint32_t, uint32_t,
+			       PAD, FLAG_HAVE_SOLID_MASK)
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_n_8888_none_OVER,
+			       scaled_bilinear_scanline_sse2_8888_n_8888_OVER,
+			       uint32_t, uint32_t, uint32_t,
+			       NONE, FLAG_HAVE_SOLID_MASK)
+FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_n_8888_normal_OVER,
+			       scaled_bilinear_scanline_sse2_8888_n_8888_OVER,
+			       uint32_t, uint32_t, uint32_t,
+			       NORMAL, FLAG_HAVE_SOLID_MASK)
+
 static const pixman_fast_path_t sse2_fast_paths[] =
 {
     /* PIXMAN_OP_OVER */
@@ -5773,6 +6186,7 @@ static const pixman_fast_path_t sse2_fast_paths[] =
     PIXMAN_STD_FAST_PATH (OVER, solid, null, a8r8g8b8, sse2_composite_over_n_8888),
     PIXMAN_STD_FAST_PATH (OVER, solid, null, x8r8g8b8, sse2_composite_over_n_8888),
     PIXMAN_STD_FAST_PATH (OVER, solid, null, r5g6b5, sse2_composite_over_n_0565),
+    PIXMAN_STD_FAST_PATH (OVER, solid, null, b5g6r5, sse2_composite_over_n_0565),
     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, a8r8g8b8, sse2_composite_over_8888_8888),
     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, x8r8g8b8, sse2_composite_over_8888_8888),
     PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, a8b8g8r8, sse2_composite_over_8888_8888),
@@ -5783,6 +6197,7 @@ static const pixman_fast_path_t sse2_fast_paths[] =
     PIXMAN_STD_FAST_PATH (OVER, solid, a8, x8r8g8b8, sse2_composite_over_n_8_8888),
     PIXMAN_STD_FAST_PATH (OVER, solid, a8, a8b8g8r8, sse2_composite_over_n_8_8888),
     PIXMAN_STD_FAST_PATH (OVER, solid, a8, x8b8g8r8, sse2_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, a8r8g8b8, sse2_composite_over_8888_8888_8888),
     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8, x8r8g8b8, sse2_composite_over_8888_8_8888),
     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8, a8r8g8b8, sse2_composite_over_8888_8_8888),
     PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8, x8b8g8r8, sse2_composite_over_8888_8_8888),
@@ -5813,19 +6228,38 @@ static const pixman_fast_path_t sse2_fast_paths[] =
     PIXMAN_STD_FAST_PATH (OVER, rpixbuf, rpixbuf, b5g6r5, sse2_composite_over_pixbuf_0565),
     PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, null, x8r8g8b8, sse2_composite_copy_area),
     PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, null, x8b8g8r8, sse2_composite_copy_area),
+    
+    /* PIXMAN_OP_OVER_REVERSE */
+    PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8r8g8b8, sse2_composite_over_reverse_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8b8g8r8, sse2_composite_over_reverse_n_8888),
 
     /* PIXMAN_OP_ADD */
     PIXMAN_STD_FAST_PATH_CA (ADD, solid, a8r8g8b8, a8r8g8b8, sse2_composite_add_n_8888_8888_ca),
-    PIXMAN_STD_FAST_PATH (ADD, a8, null, a8, sse2_composite_add_8000_8000),
+    PIXMAN_STD_FAST_PATH (ADD, a8, null, a8, sse2_composite_add_8_8),
     PIXMAN_STD_FAST_PATH (ADD, a8r8g8b8, null, a8r8g8b8, sse2_composite_add_8888_8888),
     PIXMAN_STD_FAST_PATH (ADD, a8b8g8r8, null, a8b8g8r8, sse2_composite_add_8888_8888),
     PIXMAN_STD_FAST_PATH (ADD, solid, a8, a8, sse2_composite_add_n_8_8),
+    PIXMAN_STD_FAST_PATH (ADD, solid, null, a8, sse2_composite_add_n_8),
+    PIXMAN_STD_FAST_PATH (ADD, solid, null, x8r8g8b8, sse2_composite_add_n_8888),
+    PIXMAN_STD_FAST_PATH (ADD, solid, null, a8r8g8b8, sse2_composite_add_n_8888),
+    PIXMAN_STD_FAST_PATH (ADD, solid, null, x8b8g8r8, sse2_composite_add_n_8888),
+    PIXMAN_STD_FAST_PATH (ADD, solid, null, a8b8g8r8, sse2_composite_add_n_8888),
+    PIXMAN_STD_FAST_PATH (ADD, solid, a8, x8r8g8b8, sse2_composite_add_n_8_8888),
+    PIXMAN_STD_FAST_PATH (ADD, solid, a8, a8r8g8b8, sse2_composite_add_n_8_8888),
+    PIXMAN_STD_FAST_PATH (ADD, solid, a8, x8b8g8r8, sse2_composite_add_n_8_8888),
+    PIXMAN_STD_FAST_PATH (ADD, solid, a8, a8b8g8r8, sse2_composite_add_n_8_8888),
 
     /* PIXMAN_OP_SRC */
     PIXMAN_STD_FAST_PATH (SRC, solid, a8, a8r8g8b8, sse2_composite_src_n_8_8888),
     PIXMAN_STD_FAST_PATH (SRC, solid, a8, x8r8g8b8, sse2_composite_src_n_8_8888),
     PIXMAN_STD_FAST_PATH (SRC, solid, a8, a8b8g8r8, sse2_composite_src_n_8_8888),
     PIXMAN_STD_FAST_PATH (SRC, solid, a8, x8b8g8r8, sse2_composite_src_n_8_8888),
+    PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, r5g6b5, sse2_composite_src_x888_0565),
+    PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, b5g6r5, sse2_composite_src_x888_0565),
+    PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, r5g6b5, sse2_composite_src_x888_0565),
+    PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, b5g6r5, sse2_composite_src_x888_0565),
+    PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, a8r8g8b8, sse2_composite_src_x888_8888),
+    PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, a8b8g8r8, sse2_composite_src_x888_8888),
     PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, a8r8g8b8, sse2_composite_copy_area),
     PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, a8b8g8r8, sse2_composite_copy_area),
     PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, x8r8g8b8, sse2_composite_copy_area),
@@ -5838,73 +6272,218 @@ static const pixman_fast_path_t sse2_fast_paths[] =
     /* PIXMAN_OP_IN */
     PIXMAN_STD_FAST_PATH (IN, a8, null, a8, sse2_composite_in_8_8),
     PIXMAN_STD_FAST_PATH (IN, solid, a8, a8, sse2_composite_in_n_8_8),
+    PIXMAN_STD_FAST_PATH (IN, solid, null, a8, sse2_composite_in_n_8),
+
+    SIMPLE_NEAREST_FAST_PATH_COVER (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
+    SIMPLE_NEAREST_FAST_PATH_COVER (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_8888),
+    SIMPLE_NEAREST_FAST_PATH_COVER (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
+    SIMPLE_NEAREST_FAST_PATH_COVER (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_8888),
+    SIMPLE_NEAREST_FAST_PATH_NONE (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
+    SIMPLE_NEAREST_FAST_PATH_NONE (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_8888),
+    SIMPLE_NEAREST_FAST_PATH_NONE (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
+    SIMPLE_NEAREST_FAST_PATH_NONE (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_8888),
+    SIMPLE_NEAREST_FAST_PATH_PAD (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
+    SIMPLE_NEAREST_FAST_PATH_PAD (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_8888),
+    SIMPLE_NEAREST_FAST_PATH_PAD (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
+    SIMPLE_NEAREST_FAST_PATH_PAD (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_8888),
+    SIMPLE_NEAREST_FAST_PATH_NORMAL (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
+    SIMPLE_NEAREST_FAST_PATH_NORMAL (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_8888),
+    SIMPLE_NEAREST_FAST_PATH_NORMAL (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
+    SIMPLE_NEAREST_FAST_PATH_NORMAL (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_8888),
+
+    SIMPLE_NEAREST_SOLID_MASK_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_n_8888),
+    SIMPLE_NEAREST_SOLID_MASK_FAST_PATH (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_n_8888),
+    SIMPLE_NEAREST_SOLID_MASK_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_n_8888),
+    SIMPLE_NEAREST_SOLID_MASK_FAST_PATH (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_n_8888),
+    SIMPLE_NEAREST_SOLID_MASK_FAST_PATH_NORMAL (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_n_8888),
+    SIMPLE_NEAREST_SOLID_MASK_FAST_PATH_NORMAL (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_n_8888),
+    SIMPLE_NEAREST_SOLID_MASK_FAST_PATH_NORMAL (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_n_8888),
+    SIMPLE_NEAREST_SOLID_MASK_FAST_PATH_NORMAL (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_n_8888),
+
+    SIMPLE_BILINEAR_FAST_PATH (SRC, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
+    SIMPLE_BILINEAR_FAST_PATH (SRC, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
+    SIMPLE_BILINEAR_FAST_PATH (SRC, x8r8g8b8, x8r8g8b8, sse2_8888_8888),
+    SIMPLE_BILINEAR_FAST_PATH (SRC, a8b8g8r8, a8b8g8r8, sse2_8888_8888),
+    SIMPLE_BILINEAR_FAST_PATH (SRC, a8b8g8r8, x8b8g8r8, sse2_8888_8888),
+    SIMPLE_BILINEAR_FAST_PATH (SRC, x8b8g8r8, x8b8g8r8, sse2_8888_8888),
+
+    SIMPLE_BILINEAR_FAST_PATH_COVER  (SRC, x8r8g8b8, a8r8g8b8, sse2_x888_8888),
+    SIMPLE_BILINEAR_FAST_PATH_COVER  (SRC, x8b8g8r8, a8b8g8r8, sse2_x888_8888),
+    SIMPLE_BILINEAR_FAST_PATH_PAD    (SRC, x8r8g8b8, a8r8g8b8, sse2_x888_8888),
+    SIMPLE_BILINEAR_FAST_PATH_PAD    (SRC, x8b8g8r8, a8b8g8r8, sse2_x888_8888),
+    SIMPLE_BILINEAR_FAST_PATH_NORMAL (SRC, x8r8g8b8, a8r8g8b8, sse2_x888_8888),
+    SIMPLE_BILINEAR_FAST_PATH_NORMAL (SRC, x8b8g8r8, a8b8g8r8, sse2_x888_8888),
+
+    SIMPLE_BILINEAR_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
+    SIMPLE_BILINEAR_FAST_PATH (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_8888),
+    SIMPLE_BILINEAR_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
+    SIMPLE_BILINEAR_FAST_PATH (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_8888),
+
+    SIMPLE_BILINEAR_SOLID_MASK_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_n_8888),
+    SIMPLE_BILINEAR_SOLID_MASK_FAST_PATH (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_n_8888),
+    SIMPLE_BILINEAR_SOLID_MASK_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_n_8888),
+    SIMPLE_BILINEAR_SOLID_MASK_FAST_PATH (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_n_8888),
+
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_8_8888),
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_8_8888),
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_8_8888),
+    SIMPLE_BILINEAR_A8_MASK_FAST_PATH (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_8_8888),
 
     { PIXMAN_OP_NONE },
 };
 
-static pixman_bool_t
-sse2_blt (pixman_implementation_t *imp,
-          uint32_t *               src_bits,
-          uint32_t *               dst_bits,
-          int                      src_stride,
-          int                      dst_stride,
-          int                      src_bpp,
-          int                      dst_bpp,
-          int                      src_x,
-          int                      src_y,
-          int                      dst_x,
-          int                      dst_y,
-          int                      width,
-          int                      height)
+static uint32_t *
+sse2_fetch_x8r8g8b8 (pixman_iter_t *iter, const uint32_t *mask)
 {
-    if (!pixman_blt_sse2 (
-            src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
-            src_x, src_y, dst_x, dst_y, width, height))
+    int w = iter->width;
+    __m128i ff000000 = mask_ff000000;
+    uint32_t *dst = iter->buffer;
+    uint32_t *src = (uint32_t *)iter->bits;
+
+    iter->bits += iter->stride;
+
+    while (w && ((uintptr_t)dst) & 0x0f)
+    {
+	*dst++ = (*src++) | 0xff000000;
+	w--;
+    }
 
+    while (w >= 4)
     {
-	return _pixman_implementation_blt (
-	    imp->delegate,
-	    src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
-	    src_x, src_y, dst_x, dst_y, width, height);
+	save_128_aligned (
+	    (__m128i *)dst, _mm_or_si128 (
+		load_128_unaligned ((__m128i *)src), ff000000));
+
+	dst += 4;
+	src += 4;
+	w -= 4;
     }
 
-    return TRUE;
+    while (w)
+    {
+	*dst++ = (*src++) | 0xff000000;
+	w--;
+    }
+
+    return iter->buffer;
 }
 
-#if defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
-__attribute__((__force_align_arg_pointer__))
-#endif
-static pixman_bool_t
-sse2_fill (pixman_implementation_t *imp,
-           uint32_t *               bits,
-           int                      stride,
-           int                      bpp,
-           int                      x,
-           int                      y,
-           int                      width,
-           int                      height,
-           uint32_t xor)
+static uint32_t *
+sse2_fetch_r5g6b5 (pixman_iter_t *iter, const uint32_t *mask)
 {
-    if (!pixman_fill_sse2 (bits, stride, bpp, x, y, width, height, xor))
+    int w = iter->width;
+    uint32_t *dst = iter->buffer;
+    uint16_t *src = (uint16_t *)iter->bits;
+    __m128i ff000000 = mask_ff000000;
+
+    iter->bits += iter->stride;
+
+    while (w && ((uintptr_t)dst) & 0x0f)
     {
-	return _pixman_implementation_fill (
-	    imp->delegate, bits, stride, bpp, x, y, width, height, xor);
+	uint16_t s = *src++;
+
+	*dst++ = convert_0565_to_8888 (s);
+	w--;
     }
 
-    return TRUE;
+    while (w >= 8)
+    {
+	__m128i lo, hi, s;
+
+	s = _mm_loadu_si128 ((__m128i *)src);
+
+	lo = unpack_565_to_8888 (_mm_unpacklo_epi16 (s, _mm_setzero_si128 ()));
+	hi = unpack_565_to_8888 (_mm_unpackhi_epi16 (s, _mm_setzero_si128 ()));
+
+	save_128_aligned ((__m128i *)(dst + 0), _mm_or_si128 (lo, ff000000));
+	save_128_aligned ((__m128i *)(dst + 4), _mm_or_si128 (hi, ff000000));
+
+	dst += 8;
+	src += 8;
+	w -= 8;
+    }
+
+    while (w)
+    {
+	uint16_t s = *src++;
+
+	*dst++ = convert_0565_to_8888 (s);
+	w--;
+    }
+
+    return iter->buffer;
 }
 
+static uint32_t *
+sse2_fetch_a8 (pixman_iter_t *iter, const uint32_t *mask)
+{
+    int w = iter->width;
+    uint32_t *dst = iter->buffer;
+    uint8_t *src = iter->bits;
+    __m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6;
+
+    iter->bits += iter->stride;
+
+    while (w && (((uintptr_t)dst) & 15))
+    {
+        *dst++ = *(src++) << 24;
+        w--;
+    }
+
+    while (w >= 16)
+    {
+	xmm0 = _mm_loadu_si128((__m128i *)src);
+
+	xmm1 = _mm_unpacklo_epi8  (_mm_setzero_si128(), xmm0);
+	xmm2 = _mm_unpackhi_epi8  (_mm_setzero_si128(), xmm0);
+	xmm3 = _mm_unpacklo_epi16 (_mm_setzero_si128(), xmm1);
+	xmm4 = _mm_unpackhi_epi16 (_mm_setzero_si128(), xmm1);
+	xmm5 = _mm_unpacklo_epi16 (_mm_setzero_si128(), xmm2);
+	xmm6 = _mm_unpackhi_epi16 (_mm_setzero_si128(), xmm2);
+
+	_mm_store_si128(((__m128i *)(dst +  0)), xmm3);
+	_mm_store_si128(((__m128i *)(dst +  4)), xmm4);
+	_mm_store_si128(((__m128i *)(dst +  8)), xmm5);
+	_mm_store_si128(((__m128i *)(dst + 12)), xmm6);
+
+	dst += 16;
+	src += 16;
+	w -= 16;
+    }
+
+    while (w)
+    {
+	*dst++ = *(src++) << 24;
+	w--;
+    }
+
+    return iter->buffer;
+}
+
+#define IMAGE_FLAGS							\
+    (FAST_PATH_STANDARD_FLAGS | FAST_PATH_ID_TRANSFORM |		\
+     FAST_PATH_BITS_IMAGE | FAST_PATH_SAMPLES_COVER_CLIP_NEAREST)
+
+static const pixman_iter_info_t sse2_iters[] = 
+{
+    { PIXMAN_x8r8g8b8, IMAGE_FLAGS, ITER_NARROW,
+      _pixman_iter_init_bits_stride, sse2_fetch_x8r8g8b8, NULL
+    },
+    { PIXMAN_r5g6b5, IMAGE_FLAGS, ITER_NARROW,
+      _pixman_iter_init_bits_stride, sse2_fetch_r5g6b5, NULL
+    },
+    { PIXMAN_a8, IMAGE_FLAGS, ITER_NARROW,
+      _pixman_iter_init_bits_stride, sse2_fetch_a8, NULL
+    },
+    { PIXMAN_null },
+};
+
 #if defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
 __attribute__((__force_align_arg_pointer__))
 #endif
 pixman_implementation_t *
-_pixman_implementation_create_sse2 (void)
+_pixman_implementation_create_sse2 (pixman_implementation_t *fallback)
 {
-#ifdef USE_MMX
-    pixman_implementation_t *fallback = _pixman_implementation_create_mmx ();
-#else
-    pixman_implementation_t *fallback = _pixman_implementation_create_fast_path ();
-#endif
     pixman_implementation_t *imp = _pixman_implementation_create (fallback, sse2_fast_paths);
 
     /* SSE2 constants */
@@ -5923,21 +6502,10 @@ _pixman_implementation_create_sse2 (void)
     mask_ffff = create_mask_16_128 (0xffff);
     mask_ff000000 = create_mask_2x32_128 (0xff000000, 0xff000000);
     mask_alpha = create_mask_2x32_128 (0x00ff0000, 0x00000000);
-
-    /* MMX constants */
-    mask_x565_rgb = create_mask_2x32_64 (0x000001f0, 0x003f001f);
-    mask_x565_unpack = create_mask_2x32_64 (0x00000084, 0x04100840);
-
-    mask_x0080 = create_mask_16_64 (0x0080);
-    mask_x00ff = create_mask_16_64 (0x00ff);
-    mask_x0101 = create_mask_16_64 (0x0101);
-    mask_x_alpha = create_mask_2x32_64 (0x00ff0000, 0x00000000);
-
-    _mm_empty ();
+    mask_565_rb = create_mask_2x32_128 (0x00f800f8, 0x00f800f8);
+    mask_565_pack_multiplier = create_mask_2x32_128 (0x20000004, 0x20000004);
 
     /* Set up function pointers */
-
-    /* SSE code patch for fbcompose.c */
     imp->combine_32[PIXMAN_OP_OVER] = sse2_combine_over_u;
     imp->combine_32[PIXMAN_OP_OVER_REVERSE] = sse2_combine_over_reverse_u;
     imp->combine_32[PIXMAN_OP_IN] = sse2_combine_in_u;
@@ -5966,7 +6534,7 @@ _pixman_implementation_create_sse2 (void)
     imp->blt = sse2_blt;
     imp->fill = sse2_fill;
 
+    imp->iter_info = sse2_iters;
+
     return imp;
 }
-
-#endif /* USE_SSE2 */
diff --git a/pixman/pixman/pixman-ssse3.c b/pixman/pixman/pixman-ssse3.c
new file mode 100644
index 000000000..680d6b95a
--- /dev/null
+++ b/pixman/pixman/pixman-ssse3.c
@@ -0,0 +1,351 @@
+/*
+ * Copyright © 2013 Soren Sandmann Pedersen
+ * Copyright © 2013 Red Hat, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Author: Soren Sandmann (soren.sandmann@gmail.com)
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdlib.h>
+#include <mmintrin.h>
+#include <xmmintrin.h>
+#include <emmintrin.h>
+#include <tmmintrin.h>
+#include "pixman-private.h"
+#include "pixman-inlines.h"
+
+typedef struct
+{
+    int		y;
+    uint64_t *	buffer;
+} line_t;
+
+typedef struct
+{
+    line_t		lines[2];
+    pixman_fixed_t	y;
+    pixman_fixed_t	x;
+    uint64_t		data[1];
+} bilinear_info_t;
+
+static void
+ssse3_fetch_horizontal (bits_image_t *image, line_t *line,
+			int y, pixman_fixed_t x, pixman_fixed_t ux, int n)
+{
+    uint32_t *bits = image->bits + y * image->rowstride;
+    __m128i vx = _mm_set_epi16 (
+	- (x + 1), x, - (x + 1), x,
+	- (x + ux + 1), x + ux,  - (x + ux + 1), x + ux);
+    __m128i vux = _mm_set_epi16 (
+	- 2 * ux, 2 * ux, - 2 * ux, 2 * ux,
+	- 2 * ux, 2 * ux, - 2 * ux, 2 * ux);
+    __m128i vaddc = _mm_set_epi16 (1, 0, 1, 0, 1, 0, 1, 0);
+    __m128i *b = (__m128i *)line->buffer;
+    __m128i vrl0, vrl1;
+
+    while ((n -= 2) >= 0)
+    {
+	__m128i vw, vr, s;
+
+	vrl1 = _mm_loadl_epi64 (
+	    (__m128i *)(bits + pixman_fixed_to_int (x + ux)));
+	/* vrl1: R1, L1 */
+
+    final_pixel:
+	vrl0 = _mm_loadl_epi64 (
+	    (__m128i *)(bits + pixman_fixed_to_int (x)));
+	/* vrl0: R0, L0 */
+
+	/* The weights are based on vx which is a vector of 
+	 *
+	 *    - (x + 1), x, - (x + 1), x,
+	 *          - (x + ux + 1), x + ux, - (x + ux + 1), x + ux
+	 *
+	 * so the 16 bit weights end up like this:
+	 *
+	 *    iw0, w0, iw0, w0, iw1, w1, iw1, w1
+	 *
+	 * and after shifting and packing, we get these bytes:
+	 *
+	 *    iw0, w0, iw0, w0, iw1, w1, iw1, w1,
+	 *        iw0, w0, iw0, w0, iw1, w1, iw1, w1,
+	 *
+	 * which means the first and the second input pixel 
+	 * have to be interleaved like this:
+	 *
+	 *    la0, ra0, lr0, rr0, la1, ra1, lr1, rr1,
+	 *        lg0, rg0, lb0, rb0, lg1, rg1, lb1, rb1
+	 *
+	 * before maddubsw can be used.
+	 */
+
+	vw = _mm_add_epi16 (
+	    vaddc, _mm_srli_epi16 (vx, 16 - BILINEAR_INTERPOLATION_BITS));
+	/* vw: iw0, w0, iw0, w0, iw1, w1, iw1, w1
+	 */
+
+	vw = _mm_packus_epi16 (vw, vw);
+	/* vw: iw0, w0, iw0, w0, iw1, w1, iw1, w1,
+	 *         iw0, w0, iw0, w0, iw1, w1, iw1, w1
+	 */
+	vx = _mm_add_epi16 (vx, vux);
+
+	x += 2 * ux;
+
+	vr = _mm_unpacklo_epi16 (vrl1, vrl0);
+	/* vr: rar0, rar1, rgb0, rgb1, lar0, lar1, lgb0, lgb1 */
+
+	s = _mm_shuffle_epi32 (vr, _MM_SHUFFLE (1, 0, 3, 2));
+	/* s:  lar0, lar1, lgb0, lgb1, rar0, rar1, rgb0, rgb1 */
+
+	vr = _mm_unpackhi_epi8 (vr, s);
+	/* vr: la0, ra0, lr0, rr0, la1, ra1, lr1, rr1,
+	 *         lg0, rg0, lb0, rb0, lg1, rg1, lb1, rb1
+	 */
+
+	vr = _mm_maddubs_epi16 (vr, vw);
+
+	/* When the weight is 0, the inverse weight is
+	 * 128 which can't be represented in a signed byte.
+	 * As a result maddubsw computes the following:
+	 *
+	 *     r = l * -128 + r * 0
+	 *
+	 * rather than the desired
+	 *
+	 *     r = l * 128 + r * 0
+	 *
+	 * We fix this by taking the absolute value of the
+	 * result.
+	 */
+	vr = _mm_abs_epi16 (vr);
+
+	/* vr: A0, R0, A1, R1, G0, B0, G1, B1 */
+	_mm_store_si128 (b++, vr);
+    }
+
+    if (n == -1)
+    {
+	vrl1 = _mm_setzero_si128();
+	goto final_pixel;
+    }
+
+    line->y = y;
+}
+
+static uint32_t *
+ssse3_fetch_bilinear_cover (pixman_iter_t *iter, const uint32_t *mask)
+{
+    pixman_fixed_t fx, ux;
+    bilinear_info_t *info = iter->data;
+    line_t *line0, *line1;
+    int y0, y1;
+    int32_t dist_y;
+    __m128i vw;
+    int i;
+
+    fx = info->x;
+    ux = iter->image->common.transform->matrix[0][0];
+
+    y0 = pixman_fixed_to_int (info->y);
+    y1 = y0 + 1;
+
+    line0 = &info->lines[y0 & 0x01];
+    line1 = &info->lines[y1 & 0x01];
+
+    if (line0->y != y0)
+    {
+	ssse3_fetch_horizontal (
+	    &iter->image->bits, line0, y0, fx, ux, iter->width);
+    }
+
+    if (line1->y != y1)
+    {
+	ssse3_fetch_horizontal (
+	    &iter->image->bits, line1, y1, fx, ux, iter->width);
+    }
+
+    dist_y = pixman_fixed_to_bilinear_weight (info->y);
+    dist_y <<= (16 - BILINEAR_INTERPOLATION_BITS);
+
+    vw = _mm_set_epi16 (
+	dist_y, dist_y, dist_y, dist_y, dist_y, dist_y, dist_y, dist_y);
+
+    for (i = 0; i + 3 < iter->width; i += 4)
+    {
+	__m128i top0 = _mm_load_si128 ((__m128i *)(line0->buffer + i));
+	__m128i bot0 = _mm_load_si128 ((__m128i *)(line1->buffer + i));
+	__m128i top1 = _mm_load_si128 ((__m128i *)(line0->buffer + i + 2));
+	__m128i bot1 = _mm_load_si128 ((__m128i *)(line1->buffer + i + 2));
+	__m128i r0, r1, tmp, p;
+
+	r0 = _mm_mulhi_epu16 (
+	    _mm_sub_epi16 (bot0, top0), vw);
+	tmp = _mm_cmplt_epi16 (bot0, top0);
+	tmp = _mm_and_si128 (tmp, vw);
+	r0 = _mm_sub_epi16 (r0, tmp);
+	r0 = _mm_add_epi16 (r0, top0);
+	r0 = _mm_srli_epi16 (r0, BILINEAR_INTERPOLATION_BITS);
+	/* r0:  A0 R0 A1 R1 G0 B0 G1 B1 */
+	r0 = _mm_shuffle_epi32 (r0, _MM_SHUFFLE (2, 0, 3, 1));
+	/* r0:  A1 R1 G1 B1 A0 R0 G0 B0 */
+
+	r1 = _mm_mulhi_epu16 (
+	    _mm_sub_epi16 (bot1, top1), vw);
+	tmp = _mm_cmplt_epi16 (bot1, top1);
+	tmp = _mm_and_si128 (tmp, vw);
+	r1 = _mm_sub_epi16 (r1, tmp);
+	r1 = _mm_add_epi16 (r1, top1);
+	r1 = _mm_srli_epi16 (r1, BILINEAR_INTERPOLATION_BITS);
+	r1 = _mm_shuffle_epi32 (r1, _MM_SHUFFLE (2, 0, 3, 1));
+	/* r1: A3 R3 G3 B3 A2 R2 G2 B2 */
+
+	p = _mm_packus_epi16 (r0, r1);
+
+	_mm_storeu_si128 ((__m128i *)(iter->buffer + i), p);
+    }
+
+    while (i < iter->width)
+    {
+	__m128i top0 = _mm_load_si128 ((__m128i *)(line0->buffer + i));
+	__m128i bot0 = _mm_load_si128 ((__m128i *)(line1->buffer + i));
+	__m128i r0, tmp, p;
+
+	r0 = _mm_mulhi_epu16 (
+	    _mm_sub_epi16 (bot0, top0), vw);
+	tmp = _mm_cmplt_epi16 (bot0, top0);
+	tmp = _mm_and_si128 (tmp, vw);
+	r0 = _mm_sub_epi16 (r0, tmp);
+	r0 = _mm_add_epi16 (r0, top0);
+	r0 = _mm_srli_epi16 (r0, BILINEAR_INTERPOLATION_BITS);
+	/* r0:  A0 R0 A1 R1 G0 B0 G1 B1 */
+	r0 = _mm_shuffle_epi32 (r0, _MM_SHUFFLE (2, 0, 3, 1));
+	/* r0:  A1 R1 G1 B1 A0 R0 G0 B0 */
+
+	p = _mm_packus_epi16 (r0, r0);
+
+	if (iter->width - i == 1)
+	{
+	    *(uint32_t *)(iter->buffer + i) = _mm_cvtsi128_si32 (p);
+	    i++;
+	}
+	else
+	{
+	    _mm_storel_epi64 ((__m128i *)(iter->buffer + i), p);
+	    i += 2;
+	}
+    }
+    
+    info->y += iter->image->common.transform->matrix[1][1];
+
+    return iter->buffer;
+}
+
+static void
+ssse3_bilinear_cover_iter_fini (pixman_iter_t *iter)
+{
+    free (iter->data);
+}
+
+static void
+ssse3_bilinear_cover_iter_init (pixman_iter_t *iter, const pixman_iter_info_t *iter_info)
+{
+    int width = iter->width;
+    bilinear_info_t *info;
+    pixman_vector_t v;
+
+    /* Reference point is the center of the pixel */
+    v.vector[0] = pixman_int_to_fixed (iter->x) + pixman_fixed_1 / 2;
+    v.vector[1] = pixman_int_to_fixed (iter->y) + pixman_fixed_1 / 2;
+    v.vector[2] = pixman_fixed_1;
+
+    if (!pixman_transform_point_3d (iter->image->common.transform, &v))
+	goto fail;
+
+    info = malloc (sizeof (*info) + (2 * width - 1) * sizeof (uint64_t) + 64);
+    if (!info)
+	goto fail;
+
+    info->x = v.vector[0] - pixman_fixed_1 / 2;
+    info->y = v.vector[1] - pixman_fixed_1 / 2;
+
+#define ALIGN(addr)							\
+    ((void *)((((uintptr_t)(addr)) + 15) & (~15)))
+
+    /* It is safe to set the y coordinates to -1 initially
+     * because COVER_CLIP_BILINEAR ensures that we will only
+     * be asked to fetch lines in the [0, height) interval
+     */
+    info->lines[0].y = -1;
+    info->lines[0].buffer = ALIGN (&(info->data[0]));
+    info->lines[1].y = -1;
+    info->lines[1].buffer = ALIGN (info->lines[0].buffer + width);
+
+    iter->get_scanline = ssse3_fetch_bilinear_cover;
+    iter->fini = ssse3_bilinear_cover_iter_fini;
+
+    iter->data = info;
+    return;
+
+fail:
+    /* Something went wrong, either a bad matrix or OOM; in such cases,
+     * we don't guarantee any particular rendering.
+     */
+    _pixman_log_error (
+	FUNC, "Allocation failure or bad matrix, skipping rendering\n");
+    
+    iter->get_scanline = _pixman_iter_get_scanline_noop;
+    iter->fini = NULL;
+}
+
+static const pixman_iter_info_t ssse3_iters[] = 
+{
+    { PIXMAN_a8r8g8b8,
+      (FAST_PATH_STANDARD_FLAGS			|
+       FAST_PATH_SCALE_TRANSFORM		|
+       FAST_PATH_BILINEAR_FILTER		|
+       FAST_PATH_SAMPLES_COVER_CLIP_BILINEAR),
+      ITER_NARROW | ITER_SRC,
+      ssse3_bilinear_cover_iter_init,
+      NULL, NULL
+    },
+
+    { PIXMAN_null },
+};
+
+static const pixman_fast_path_t ssse3_fast_paths[] =
+{
+    { PIXMAN_OP_NONE },
+};
+
+pixman_implementation_t *
+_pixman_implementation_create_ssse3 (pixman_implementation_t *fallback)
+{
+    pixman_implementation_t *imp =
+	_pixman_implementation_create (fallback, ssse3_fast_paths);
+
+    imp->iter_info = ssse3_iters;
+
+    return imp;
+}
diff --git a/pixman/pixman/pixman-trap.c b/pixman/pixman/pixman-trap.c
index 8353992c5..91766fdbf 100644
--- a/pixman/pixman/pixman-trap.c
+++ b/pixman/pixman/pixman-trap.c
@@ -1,4 +1,5 @@
 /*
+ * Copyright © 2002 Keith Packard, member of The XFree86 Project, Inc.
  * Copyright © 2004 Keith Packard
  *
  * Permission to use, copy, modify, distribute, and sell this software and its
@@ -25,6 +26,7 @@
 #endif
 
 #include <stdio.h>
+#include <stdlib.h>
 #include "pixman-private.h"
 
 /*
@@ -137,7 +139,7 @@ _pixman_edge_multi_init (pixman_edge_t * e,
     if (ne > 0)
     {
 	int nx = ne / e->dy;
-	ne -= nx * e->dy;
+	ne -= nx * (pixman_fixed_48_16_t)e->dy;
 	stepx += nx * e->signdx;
     }
 
@@ -228,14 +230,13 @@ pixman_line_fixed_edge_init (pixman_edge_t *            e,
 }
 
 PIXMAN_EXPORT void
-pixman_add_traps (pixman_image_t * image,
-                  int16_t          x_off,
-                  int16_t          y_off,
-                  int              ntrap,
-                  pixman_trap_t *  traps)
+pixman_add_traps (pixman_image_t *     image,
+                  int16_t              x_off,
+                  int16_t              y_off,
+                  int                  ntrap,
+                  const pixman_trap_t *traps)
 {
     int bpp;
-    int width;
     int height;
 
     pixman_fixed_t x_off_fixed;
@@ -245,7 +246,6 @@ pixman_add_traps (pixman_image_t * image,
 
     _pixman_image_validate (image);
     
-    width = image->bits.width;
     height = image->bits.height;
     bpp = PIXMAN_FORMAT_BPP (image->bits.format);
 
@@ -349,10 +349,8 @@ pixman_rasterize_trapezoid (pixman_image_t *          image,
                             int                       y_off)
 {
     int bpp;
-    int width;
     int height;
 
-    pixman_fixed_t x_off_fixed;
     pixman_fixed_t y_off_fixed;
     pixman_edge_t l, r;
     pixman_fixed_t t, b;
@@ -364,11 +362,9 @@ pixman_rasterize_trapezoid (pixman_image_t *          image,
     if (!pixman_trapezoid_valid (trap))
 	return;
 
-    width = image->bits.width;
     height = image->bits.height;
     bpp = PIXMAN_FORMAT_BPP (image->bits.format);
 
-    x_off_fixed = pixman_int_to_fixed (x_off);
     y_off_fixed = pixman_int_to_fixed (y_off);
 
     t = trap->top + y_off_fixed;
@@ -390,3 +386,326 @@ pixman_rasterize_trapezoid (pixman_image_t *          image,
 	pixman_rasterize_edges (image, &l, &r, t, b);
     }
 }
+
+static const pixman_bool_t zero_src_has_no_effect[PIXMAN_N_OPERATORS] =
+{
+    FALSE,	/* Clear		0			0    */
+    FALSE,	/* Src			1			0    */
+    TRUE,	/* Dst			0			1    */
+    TRUE,	/* Over			1			1-Aa */
+    TRUE,	/* OverReverse		1-Ab			1    */
+    FALSE,	/* In			Ab			0    */
+    FALSE,	/* InReverse		0			Aa   */
+    FALSE,	/* Out			1-Ab			0    */
+    TRUE,	/* OutReverse		0			1-Aa */
+    TRUE,	/* Atop			Ab			1-Aa */
+    FALSE,	/* AtopReverse		1-Ab			Aa   */
+    TRUE,	/* Xor			1-Ab			1-Aa */
+    TRUE,	/* Add			1			1    */
+};
+
+static pixman_bool_t
+get_trap_extents (pixman_op_t op, pixman_image_t *dest,
+		  const pixman_trapezoid_t *traps, int n_traps,
+		  pixman_box32_t *box)
+{
+    int i;
+
+    /* When the operator is such that a zero source has an
+     * effect on the underlying image, we have to
+     * composite across the entire destination
+     */
+    if (!zero_src_has_no_effect [op])
+    {
+	box->x1 = 0;
+	box->y1 = 0;
+	box->x2 = dest->bits.width;
+	box->y2 = dest->bits.height;
+	return TRUE;
+    }
+    
+    box->x1 = INT32_MAX;
+    box->y1 = INT32_MAX;
+    box->x2 = INT32_MIN;
+    box->y2 = INT32_MIN;
+	
+    for (i = 0; i < n_traps; ++i)
+    {
+	const pixman_trapezoid_t *trap = &(traps[i]);
+	int y1, y2;
+	    
+	if (!pixman_trapezoid_valid (trap))
+	    continue;
+	    
+	y1 = pixman_fixed_to_int (trap->top);
+	if (y1 < box->y1)
+	    box->y1 = y1;
+	    
+	y2 = pixman_fixed_to_int (pixman_fixed_ceil (trap->bottom));
+	if (y2 > box->y2)
+	    box->y2 = y2;
+	    
+#define EXTEND_MIN(x)							\
+	if (pixman_fixed_to_int ((x)) < box->x1)			\
+	    box->x1 = pixman_fixed_to_int ((x));
+#define EXTEND_MAX(x)							\
+	if (pixman_fixed_to_int (pixman_fixed_ceil ((x))) > box->x2)	\
+	    box->x2 = pixman_fixed_to_int (pixman_fixed_ceil ((x)));
+	    
+#define EXTEND(x)							\
+	EXTEND_MIN(x);							\
+	EXTEND_MAX(x);
+	    
+	EXTEND(trap->left.p1.x);
+	EXTEND(trap->left.p2.x);
+	EXTEND(trap->right.p1.x);
+	EXTEND(trap->right.p2.x);
+    }
+	
+    if (box->x1 >= box->x2 || box->y1 >= box->y2)
+	return FALSE;
+
+    return TRUE;
+}
+
+/*
+ * pixman_composite_trapezoids()
+ *
+ * All the trapezoids are conceptually rendered to an infinitely big image.
+ * The (0, 0) coordinates of this image are then aligned with the (x, y)
+ * coordinates of the source image, and then both images are aligned with
+ * the (x, y) coordinates of the destination. Then these three images are
+ * composited across the entire destination.
+ */
+PIXMAN_EXPORT void
+pixman_composite_trapezoids (pixman_op_t		op,
+			     pixman_image_t *		src,
+			     pixman_image_t *		dst,
+			     pixman_format_code_t	mask_format,
+			     int			x_src,
+			     int			y_src,
+			     int			x_dst,
+			     int			y_dst,
+			     int			n_traps,
+			     const pixman_trapezoid_t *	traps)
+{
+    int i;
+
+    return_if_fail (PIXMAN_FORMAT_TYPE (mask_format) == PIXMAN_TYPE_A);
+    
+    if (n_traps <= 0)
+	return;
+
+    _pixman_image_validate (src);
+    _pixman_image_validate (dst);
+
+    if (op == PIXMAN_OP_ADD &&
+	(src->common.flags & FAST_PATH_IS_OPAQUE)		&&
+	(mask_format == dst->common.extended_format_code)	&&
+	!(dst->common.have_clip_region))
+    {
+	for (i = 0; i < n_traps; ++i)
+	{
+	    const pixman_trapezoid_t *trap = &(traps[i]);
+	    
+	    if (!pixman_trapezoid_valid (trap))
+		continue;
+	    
+	    pixman_rasterize_trapezoid (dst, trap, x_dst, y_dst);
+	}
+    }
+    else
+    {
+	pixman_image_t *tmp;
+	pixman_box32_t box;
+	int i;
+
+	if (!get_trap_extents (op, dst, traps, n_traps, &box))
+	    return;
+	
+	if (!(tmp = pixman_image_create_bits (
+		  mask_format, box.x2 - box.x1, box.y2 - box.y1, NULL, -1)))
+	    return;
+	
+	for (i = 0; i < n_traps; ++i)
+	{
+	    const pixman_trapezoid_t *trap = &(traps[i]);
+	    
+	    if (!pixman_trapezoid_valid (trap))
+		continue;
+	    
+	    pixman_rasterize_trapezoid (tmp, trap, - box.x1, - box.y1);
+	}
+	
+	pixman_image_composite (op, src, tmp, dst,
+				x_src + box.x1, y_src + box.y1,
+				0, 0,
+				x_dst + box.x1, y_dst + box.y1,
+				box.x2 - box.x1, box.y2 - box.y1);
+	
+	pixman_image_unref (tmp);
+    }
+}
+
+static int
+greater_y (const pixman_point_fixed_t *a, const pixman_point_fixed_t *b)
+{
+    if (a->y == b->y)
+	return a->x > b->x;
+    return a->y > b->y;
+}
+
+/*
+ * Note that the definition of this function is a bit odd because
+ * of the X coordinate space (y increasing downwards).
+ */
+static int
+clockwise (const pixman_point_fixed_t *ref,
+	   const pixman_point_fixed_t *a,
+	   const pixman_point_fixed_t *b)
+{
+    pixman_point_fixed_t	ad, bd;
+
+    ad.x = a->x - ref->x;
+    ad.y = a->y - ref->y;
+    bd.x = b->x - ref->x;
+    bd.y = b->y - ref->y;
+
+    return ((pixman_fixed_32_32_t) bd.y * ad.x -
+	    (pixman_fixed_32_32_t) ad.y * bd.x) < 0;
+}
+
+static void
+triangle_to_trapezoids (const pixman_triangle_t *tri, pixman_trapezoid_t *traps)
+{
+    const pixman_point_fixed_t *top, *left, *right, *tmp;
+
+    top = &tri->p1;
+    left = &tri->p2;
+    right = &tri->p3;
+
+    if (greater_y (top, left))
+    {
+	tmp = left;
+	left = top;
+	top = tmp;
+    }
+
+    if (greater_y (top, right))
+    {
+	tmp = right;
+	right = top;
+	top = tmp;
+    }
+
+    if (clockwise (top, right, left))
+    {
+	tmp = right;
+	right = left;
+	left = tmp;
+    }
+    
+    /*
+     * Two cases:
+     *
+     *		+		+
+     *	       / \             / \
+     *	      /   \           /	  \
+     *	     /     +         +	   \
+     *      /    --           --    \
+     *     /   --               --   \
+     *    / ---                   --- \
+     *	 +--                         --+
+     */
+
+    traps->top = top->y;
+    traps->left.p1 = *top;
+    traps->left.p2 = *left;
+    traps->right.p1 = *top;
+    traps->right.p2 = *right;
+
+    if (right->y < left->y)
+	traps->bottom = right->y;
+    else
+	traps->bottom = left->y;
+
+    traps++;
+
+    *traps = *(traps - 1);
+    
+    if (right->y < left->y)
+    {
+	traps->top = right->y;
+	traps->bottom = left->y;
+	traps->right.p1 = *right;
+	traps->right.p2 = *left;
+    }
+    else
+    {
+	traps->top = left->y;
+	traps->bottom = right->y;
+	traps->left.p1 = *left;
+	traps->left.p2 = *right;
+    }
+}
+
+static pixman_trapezoid_t *
+convert_triangles (int n_tris, const pixman_triangle_t *tris)
+{
+    pixman_trapezoid_t *traps;
+    int i;
+
+    if (n_tris <= 0)
+	return NULL;
+    
+    traps = pixman_malloc_ab (n_tris, 2 * sizeof (pixman_trapezoid_t));
+    if (!traps)
+	return NULL;
+
+    for (i = 0; i < n_tris; ++i)
+	triangle_to_trapezoids (&(tris[i]), traps + 2 * i);
+
+    return traps;
+}
+
+PIXMAN_EXPORT void
+pixman_composite_triangles (pixman_op_t			op,
+			    pixman_image_t *		src,
+			    pixman_image_t *		dst,
+			    pixman_format_code_t	mask_format,
+			    int				x_src,
+			    int				y_src,
+			    int				x_dst,
+			    int				y_dst,
+			    int				n_tris,
+			    const pixman_triangle_t *	tris)
+{
+    pixman_trapezoid_t *traps;
+
+    if ((traps = convert_triangles (n_tris, tris)))
+    {
+	pixman_composite_trapezoids (op, src, dst, mask_format,
+				     x_src, y_src, x_dst, y_dst,
+				     n_tris * 2, traps);
+	
+	free (traps);
+    }
+}
+
+PIXMAN_EXPORT void
+pixman_add_triangles (pixman_image_t          *image,
+		      int32_t	               x_off,
+		      int32_t	               y_off,
+		      int	               n_tris,
+		      const pixman_triangle_t *tris)
+{
+    pixman_trapezoid_t *traps;
+
+    if ((traps = convert_triangles (n_tris, tris)))
+    {
+	pixman_add_trapezoids (image, x_off, y_off,
+			       n_tris * 2, traps);
+
+	free (traps);
+    }
+}
diff --git a/pixman/pixman/pixman-utils.c b/pixman/pixman/pixman-utils.c
index 3ef88b753..4a3a835c4 100644
--- a/pixman/pixman/pixman-utils.c
+++ b/pixman/pixman/pixman-utils.c
@@ -31,20 +31,33 @@
 #include "pixman-private.h"
 
 pixman_bool_t
-pixman_multiply_overflows_int (unsigned int a,
-                               unsigned int b)
+_pixman_multiply_overflows_size (size_t a, size_t b)
+{
+    return a >= SIZE_MAX / b;
+}
+
+pixman_bool_t
+_pixman_multiply_overflows_int (unsigned int a, unsigned int b)
 {
     return a >= INT32_MAX / b;
 }
 
 pixman_bool_t
-pixman_addition_overflows_int (unsigned int a,
-                               unsigned int b)
+_pixman_addition_overflows_int (unsigned int a, unsigned int b)
 {
     return a > INT32_MAX - b;
 }
 
 void *
+pixman_malloc_ab_plus_c (unsigned int a, unsigned int b, unsigned int c)
+{
+    if (!b || a >= INT32_MAX / b || (a * b) > INT32_MAX - c)
+	return NULL;
+
+    return malloc (a * b + c);
+}
+
+void *
 pixman_malloc_ab (unsigned int a,
                   unsigned int b)
 {
@@ -67,61 +80,96 @@ pixman_malloc_abc (unsigned int a,
 	return malloc (a * b * c);
 }
 
-/*
- * Helper routine to expand a color component from 0 < n <= 8 bits to 16
- * bits by replication.
- */
-static inline uint64_t
-expand16 (const uint8_t val, int nbits)
+static force_inline uint16_t
+float_to_unorm (float f, int n_bits)
 {
-    /* Start out with the high bit of val in the high bit of result. */
-    uint16_t result = (uint16_t)val << (16 - nbits);
+    uint32_t u;
 
-    if (nbits == 0)
-	return 0;
+    if (f > 1.0)
+	f = 1.0;
+    if (f < 0.0)
+	f = 0.0;
 
-    /* Copy the bits in result, doubling the number of bits each time, until
-     * we fill all 16 bits.
-     */
-    while (nbits < 16)
-    {
-	result |= result >> nbits;
-	nbits *= 2;
-    }
+    u = f * (1 << n_bits);
+    u -= (u >> n_bits);
+
+    return u;
+}
+
+static force_inline float
+unorm_to_float (uint16_t u, int n_bits)
+{
+    uint32_t m = ((1 << n_bits) - 1);
 
-    return result;
+    return (u & m) * (1.f / (float)m);
 }
 
 /*
- * This function expands images from ARGB8 format to ARGB16.  To preserve
- * precision, it needs to know the original source format.  For example, if the
- * source was PIXMAN_x1r5g5b5 and the red component contained bits 12345, then
- * the expanded value is 12345123.  To correctly expand this to 16 bits, it
- * should be 1234512345123451 and not 1234512312345123.
+ * This function expands images from a8r8g8b8 to argb_t.  To preserve
+ * precision, it needs to know from which source format the a8r8g8b8 pixels
+ * originally came.
+ *
+ * For example, if the source was PIXMAN_x1r5g5b5 and the red component
+ * contained bits 12345, then the 8-bit value is 12345123.  To correctly
+ * expand this to floating point, it should be 12345 / 31.0 and not
+ * 12345123 / 255.0.
  */
 void
-pixman_expand (uint64_t *           dst,
-               const uint32_t *     src,
-               pixman_format_code_t format,
-               int                  width)
+pixman_expand_to_float (argb_t               *dst,
+			const uint32_t       *src,
+			pixman_format_code_t  format,
+			int                   width)
 {
+    static const float multipliers[16] = {
+	0.0f,
+	1.0f / ((1 <<  1) - 1),
+	1.0f / ((1 <<  2) - 1),
+	1.0f / ((1 <<  3) - 1),
+	1.0f / ((1 <<  4) - 1),
+	1.0f / ((1 <<  5) - 1),
+	1.0f / ((1 <<  6) - 1),
+	1.0f / ((1 <<  7) - 1),
+	1.0f / ((1 <<  8) - 1),
+	1.0f / ((1 <<  9) - 1),
+	1.0f / ((1 << 10) - 1),
+	1.0f / ((1 << 11) - 1),
+	1.0f / ((1 << 12) - 1),
+	1.0f / ((1 << 13) - 1),
+	1.0f / ((1 << 14) - 1),
+	1.0f / ((1 << 15) - 1),
+    };
+    int a_size, r_size, g_size, b_size;
+    int a_shift, r_shift, g_shift, b_shift;
+    float a_mul, r_mul, g_mul, b_mul;
+    uint32_t a_mask, r_mask, g_mask, b_mask;
+    int i;
+
+    if (!PIXMAN_FORMAT_VIS (format))
+	format = PIXMAN_a8r8g8b8;
+
     /*
      * Determine the sizes of each component and the masks and shifts
      * required to extract them from the source pixel.
      */
-    const int a_size = PIXMAN_FORMAT_A (format),
-              r_size = PIXMAN_FORMAT_R (format),
-              g_size = PIXMAN_FORMAT_G (format),
-              b_size = PIXMAN_FORMAT_B (format);
-    const int a_shift = 32 - a_size,
-              r_shift = 24 - r_size,
-              g_shift = 16 - g_size,
-              b_shift =  8 - b_size;
-    const uint8_t a_mask = ~(~0 << a_size),
-                  r_mask = ~(~0 << r_size),
-                  g_mask = ~(~0 << g_size),
-                  b_mask = ~(~0 << b_size);
-    int i;
+    a_size = PIXMAN_FORMAT_A (format);
+    r_size = PIXMAN_FORMAT_R (format);
+    g_size = PIXMAN_FORMAT_G (format);
+    b_size = PIXMAN_FORMAT_B (format);
+
+    a_shift = 32 - a_size;
+    r_shift = 24 - r_size;
+    g_shift = 16 - g_size;
+    b_shift =  8 - b_size;
+
+    a_mask = ((1 << a_size) - 1);
+    r_mask = ((1 << r_size) - 1);
+    g_mask = ((1 << g_size) - 1);
+    b_mask = ((1 << b_size) - 1);
+
+    a_mul = multipliers[a_size];
+    r_mul = multipliers[r_size];
+    g_mul = multipliers[g_size];
+    b_mul = multipliers[b_size];
 
     /* Start at the end so that we can do the expansion in place
      * when src == dst
@@ -129,44 +177,63 @@ pixman_expand (uint64_t *           dst,
     for (i = width - 1; i >= 0; i--)
     {
 	const uint32_t pixel = src[i];
-	const uint8_t a = (pixel >> a_shift) & a_mask,
-	              r = (pixel >> r_shift) & r_mask,
-	              g = (pixel >> g_shift) & g_mask,
-	              b = (pixel >> b_shift) & b_mask;
-	const uint64_t a16 = a_size ? expand16 (a, a_size) : 0xffff,
-	               r16 = expand16 (r, r_size),
-	               g16 = expand16 (g, g_size),
-	               b16 = expand16 (b, b_size);
-
-	dst[i] = a16 << 48 | r16 << 32 | g16 << 16 | b16;
+
+	dst[i].a = a_mask? ((pixel >> a_shift) & a_mask) * a_mul : 1.0f;
+	dst[i].r = ((pixel >> r_shift) & r_mask) * r_mul;
+	dst[i].g = ((pixel >> g_shift) & g_mask) * g_mul;
+	dst[i].b = ((pixel >> b_shift) & b_mask) * b_mul;
     }
 }
 
-/*
- * Contracting is easier than expanding.  We just need to truncate the
- * components.
- */
+uint16_t
+pixman_float_to_unorm (float f, int n_bits)
+{
+    return float_to_unorm (f, n_bits);
+}
+
+float
+pixman_unorm_to_float (uint16_t u, int n_bits)
+{
+    return unorm_to_float (u, n_bits);
+}
+
 void
-pixman_contract (uint32_t *      dst,
-                 const uint64_t *src,
-                 int             width)
+pixman_contract_from_float (uint32_t     *dst,
+			    const argb_t *src,
+			    int           width)
 {
     int i;
 
-    /* Start at the beginning so that we can do the contraction in
-     * place when src == dst
-     */
-    for (i = 0; i < width; i++)
+    for (i = 0; i < width; ++i)
     {
-	const uint8_t a = src[i] >> 56,
-	              r = src[i] >> 40,
-	              g = src[i] >> 24,
-	              b = src[i] >> 8;
+	uint8_t a, r, g, b;
+
+	a = float_to_unorm (src[i].a, 8);
+	r = float_to_unorm (src[i].r, 8);
+	g = float_to_unorm (src[i].g, 8);
+	b = float_to_unorm (src[i].b, 8);
 
-	dst[i] = a << 24 | r << 16 | g << 8 | b;
+	dst[i] = (a << 24) | (r << 16) | (g << 8) | (b << 0);
     }
 }
 
+uint32_t *
+_pixman_iter_get_scanline_noop (pixman_iter_t *iter, const uint32_t *mask)
+{
+    return iter->buffer;
+}
+
+void
+_pixman_iter_init_bits_stride (pixman_iter_t *iter, const pixman_iter_info_t *info)
+{
+    pixman_image_t *image = iter->image;
+    uint8_t *b = (uint8_t *)image->bits.bits;
+    int s = image->bits.rowstride * 4;
+
+    iter->bits = b + s * iter->y + iter->x * PIXMAN_FORMAT_BPP (info->format) / 8;
+    iter->stride = s;
+}
+
 #define N_TMP_BOXES (16)
 
 pixman_bool_t
@@ -236,7 +303,14 @@ pixman_region32_copy_from_region16 (pixman_region32_t *dst,
     return retval;
 }
 
-#ifdef DEBUG
+/* This function is exported for the sake of the test suite and not part
+ * of the ABI.
+ */
+PIXMAN_EXPORT pixman_implementation_t *
+_pixman_internal_only_get_implementation (void)
+{
+    return get_implementation ();
+}
 
 void
 _pixman_log_error (const char *function, const char *message)
@@ -254,5 +328,3 @@ _pixman_log_error (const char *function, const char *message)
 	n_messages++;
     }
 }
-
-#endif
diff --git a/pixman/pixman/pixman-vmx.c b/pixman/pixman/pixman-vmx.c
index e811cf733..c33631c0e 100644
--- a/pixman/pixman/pixman-vmx.c
+++ b/pixman/pixman/pixman-vmx.c
@@ -25,7 +25,9 @@
  * Based on fbmmx.c by Owen Taylor, Søren Sandmann and Nicholas Miell
  */
 
+#ifdef HAVE_CONFIG_H
 #include <config.h>
+#endif
 #include "pixman-private.h"
 #include "pixman-combine32.h"
 #include <altivec.h>
@@ -132,15 +134,11 @@ over (vector unsigned int src,
     source ## _mask = vec_lvsl (0, source);
 
 #define COMPUTE_SHIFT_MASKS(dest, source)				\
-    dest ## _mask = vec_lvsl (0, dest);					\
-    source ## _mask = vec_lvsl (0, source);				\
-    store_mask = vec_lvsr (0, dest);
+    source ## _mask = vec_lvsl (0, source);
 
 #define COMPUTE_SHIFT_MASKC(dest, source, mask)				\
     mask ## _mask = vec_lvsl (0, mask);					\
-    dest ## _mask = vec_lvsl (0, dest);					\
-    source ## _mask = vec_lvsl (0, source);				\
-    store_mask = vec_lvsr (0, dest);
+    source ## _mask = vec_lvsl (0, source);
 
 /* notice you have to declare temp vars...
  * Note: tmp3 and tmp4 must remain untouched!
@@ -149,23 +147,17 @@ over (vector unsigned int src,
 #define LOAD_VECTORS(dest, source)			  \
     tmp1 = (typeof(tmp1))vec_ld (0, source);		  \
     tmp2 = (typeof(tmp2))vec_ld (15, source);		  \
-    tmp3 = (typeof(tmp3))vec_ld (0, dest);		  \
     v ## source = (typeof(v ## source))			  \
 	vec_perm (tmp1, tmp2, source ## _mask);		  \
-    tmp4 = (typeof(tmp4))vec_ld (15, dest);		  \
-    v ## dest = (typeof(v ## dest))			  \
-	vec_perm (tmp3, tmp4, dest ## _mask);
+    v ## dest = (typeof(v ## dest))vec_ld (0, dest);
 
 #define LOAD_VECTORSC(dest, source, mask)		  \
     tmp1 = (typeof(tmp1))vec_ld (0, source);		  \
     tmp2 = (typeof(tmp2))vec_ld (15, source);		  \
-    tmp3 = (typeof(tmp3))vec_ld (0, dest);		  \
     v ## source = (typeof(v ## source))			  \
 	vec_perm (tmp1, tmp2, source ## _mask);		  \
-    tmp4 = (typeof(tmp4))vec_ld (15, dest);		  \
     tmp1 = (typeof(tmp1))vec_ld (0, mask);		  \
-    v ## dest = (typeof(v ## dest))			  \
-	vec_perm (tmp3, tmp4, dest ## _mask);		  \
+    v ## dest = (typeof(v ## dest))vec_ld (0, dest);	  \
     tmp2 = (typeof(tmp2))vec_ld (15, mask);		  \
     v ## mask = (typeof(v ## mask))			  \
 	vec_perm (tmp1, tmp2, mask ## _mask);
@@ -176,11 +168,7 @@ over (vector unsigned int src,
                                 splat_alpha (v ## mask));
 
 #define STORE_VECTOR(dest)						\
-    edges = vec_perm (tmp4, tmp3, dest ## _mask);			\
-    tmp3 = vec_perm ((vector unsigned char)v ## dest, edges, store_mask); \
-    tmp1 = vec_perm (edges, (vector unsigned char)v ## dest, store_mask); \
-    vec_st ((vector unsigned int) tmp3, 15, dest);			\
-    vec_st ((vector unsigned int) tmp1, 0, dest);
+    vec_st ((vector unsigned int) v ## dest, 0, dest);
 
 static void
 vmx_combine_over_u_no_mask (uint32_t *      dest,
@@ -189,8 +177,19 @@ vmx_combine_over_u_no_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+	uint32_t ia = ALPHA_8 (~s);
+
+	UN8x4_MUL_UN8_ADD_UN8x4 (d, ia, s);
+
+	*dest++ = d;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKS (dest, src);
 
@@ -228,8 +227,23 @@ vmx_combine_over_u_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, mask_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask, mask_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t m = ALPHA_8 (*mask++);
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+	uint32_t ia;
+
+	UN8x4_MUL_UN8 (s, m);
+
+	ia = ALPHA_8 (~s);
+
+	UN8x4_MUL_UN8_ADD_UN8x4 (d, ia, s);
+	*dest++ = d;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -284,8 +298,18 @@ vmx_combine_over_reverse_u_no_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+	uint32_t ia = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8_ADD_UN8x4 (s, ia, d);
+	*dest++ = s;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKS (dest, src);
 
@@ -322,8 +346,21 @@ vmx_combine_over_reverse_u_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, mask_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask, mask_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t m = ALPHA_8 (*mask++);
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+	uint32_t ia = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8 (s, m);
+
+	UN8x4_MUL_UN8_ADD_UN8x4 (s, ia, d);
+	*dest++ = s;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -377,8 +414,17 @@ vmx_combine_in_u_no_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t s = *src++;
+	uint32_t a = ALPHA_8 (*dest);
+
+	UN8x4_MUL_UN8 (s, a);
+	*dest++ = s;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKS (dest, src);
 
@@ -413,8 +459,20 @@ vmx_combine_in_u_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, mask_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask, mask_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t m = ALPHA_8 (*mask++);
+	uint32_t s = *src++;
+	uint32_t a = ALPHA_8 (*dest);
+
+	UN8x4_MUL_UN8 (s, m);
+	UN8x4_MUL_UN8 (s, a);
+
+	*dest++ = s;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -466,8 +524,18 @@ vmx_combine_in_reverse_u_no_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t d = *dest;
+	uint32_t a = ALPHA_8 (*src++);
+
+	UN8x4_MUL_UN8 (d, a);
+
+	*dest++ = d;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKS (dest, src);
 
@@ -503,8 +571,21 @@ vmx_combine_in_reverse_u_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, mask_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask, mask_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t m = ALPHA_8 (*mask++);
+	uint32_t d = *dest;
+	uint32_t a = *src++;
+
+	UN8x4_MUL_UN8 (a, m);
+	a = ALPHA_8 (a);
+	UN8x4_MUL_UN8 (d, a);
+
+	*dest++ = d;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -557,8 +638,18 @@ vmx_combine_out_u_no_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t s = *src++;
+	uint32_t a = ALPHA_8 (~(*dest));
+
+	UN8x4_MUL_UN8 (s, a);
+
+	*dest++ = s;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKS (dest, src);
 
@@ -594,8 +685,20 @@ vmx_combine_out_u_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, mask_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask, mask_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t m = ALPHA_8 (*mask++);
+	uint32_t s = *src++;
+	uint32_t a = ALPHA_8 (~(*dest));
+
+	UN8x4_MUL_UN8 (s, m);
+	UN8x4_MUL_UN8 (s, a);
+
+	*dest++ = s;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -647,8 +750,18 @@ vmx_combine_out_reverse_u_no_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t d = *dest;
+	uint32_t a = ALPHA_8 (~(*src++));
+
+	UN8x4_MUL_UN8 (d, a);
+
+	*dest++ = d;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKS (dest, src);
 
@@ -685,8 +798,21 @@ vmx_combine_out_reverse_u_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, mask_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask, mask_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t m = ALPHA_8 (*mask++);
+	uint32_t d = *dest;
+	uint32_t a = *src++;
+
+	UN8x4_MUL_UN8 (a, m);
+	a = ALPHA_8 (~a);
+	UN8x4_MUL_UN8 (d, a);
+
+	*dest++ = d;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -739,8 +865,20 @@ vmx_combine_atop_u_no_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+	uint32_t dest_a = ALPHA_8 (d);
+	uint32_t src_ia = ALPHA_8 (~s);
+
+	UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8 (s, dest_a, d, src_ia);
+
+	*dest++ = s;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKS (dest, src);
 
@@ -779,8 +917,25 @@ vmx_combine_atop_u_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, mask_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask, mask_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t m = ALPHA_8 (*mask++);
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+	uint32_t dest_a = ALPHA_8 (d);
+	uint32_t src_ia;
+
+	UN8x4_MUL_UN8 (s, m);
+
+	src_ia = ALPHA_8 (~s);
+
+	UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8 (s, dest_a, d, src_ia);
+
+	*dest++ = s;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -838,8 +993,20 @@ vmx_combine_atop_reverse_u_no_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+	uint32_t src_a = ALPHA_8 (s);
+	uint32_t dest_ia = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8 (s, dest_ia, d, src_a);
+
+	*dest++ = s;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKS (dest, src);
 
@@ -878,8 +1045,25 @@ vmx_combine_atop_reverse_u_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, mask_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask, mask_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t m = ALPHA_8 (*mask++);
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+	uint32_t src_a;
+	uint32_t dest_ia = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8 (s, m);
+
+	src_a = ALPHA_8 (s);
+
+	UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8 (s, dest_ia, d, src_a);
+
+	*dest++ = s;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -937,8 +1121,20 @@ vmx_combine_xor_u_no_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+	uint32_t src_ia = ALPHA_8 (~s);
+	uint32_t dest_ia = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8 (s, dest_ia, d, src_ia);
+
+	*dest++ = s;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKS (dest, src);
 
@@ -977,8 +1173,25 @@ vmx_combine_xor_u_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, mask_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask, mask_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t m = ALPHA_8 (*mask++);
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+	uint32_t src_ia;
+	uint32_t dest_ia = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8 (s, m);
+
+	src_ia = ALPHA_8 (~s);
+
+	UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8 (s, dest_ia, d, src_ia);
+
+	*dest++ = s;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -1036,8 +1249,18 @@ vmx_combine_add_u_no_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+
+	UN8x4_ADD_UN8x4 (d, s);
+
+	*dest++ = d;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKS (dest, src);
     /* printf ("%s\n",__PRETTY_FUNCTION__); */
@@ -1072,8 +1295,20 @@ vmx_combine_add_u_mask (uint32_t *      dest,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, src_mask, mask_mask, store_mask;
+    vector unsigned char tmp1, tmp2, src_mask, mask_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t m = ALPHA_8 (*mask++);
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+
+	UN8x4_MUL_UN8 (s, m);
+	UN8x4_ADD_UN8x4 (d, s);
+
+	*dest++ = d;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -1128,8 +1363,18 @@ vmx_combine_src_ca (pixman_implementation_t *imp,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, mask_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, mask_mask, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t a = *mask++;
+	uint32_t s = *src++;
+
+	UN8x4_MUL_UN8x4 (s, a);
+
+	*dest++ = s;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -1168,8 +1413,22 @@ vmx_combine_over_ca (pixman_implementation_t *imp,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, mask_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, mask_mask, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t a = *mask++;
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+	uint32_t sa = ALPHA_8 (s);
+
+	UN8x4_MUL_UN8x4 (s, a);
+	UN8x4_MUL_UN8 (a, sa);
+	UN8x4_MUL_UN8x4_ADD_UN8x4 (d, ~a, s);
+
+	*dest++ = d;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -1212,8 +1471,21 @@ vmx_combine_over_reverse_ca (pixman_implementation_t *imp,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, mask_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, mask_mask, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t a = *mask++;
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+	uint32_t ida = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8x4 (s, a);
+	UN8x4_MUL_UN8_ADD_UN8x4 (s, ida, d);
+
+	*dest++ = s;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -1255,8 +1527,20 @@ vmx_combine_in_ca (pixman_implementation_t *imp,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, mask_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, mask_mask, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t a = *mask++;
+	uint32_t s = *src++;
+	uint32_t da = ALPHA_8 (*dest);
+
+	UN8x4_MUL_UN8x4 (s, a);
+	UN8x4_MUL_UN8 (s, da);
+
+	*dest++ = s;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -1297,8 +1581,20 @@ vmx_combine_in_reverse_ca (pixman_implementation_t *imp,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, mask_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, mask_mask, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t a = *mask++;
+	uint32_t d = *dest;
+	uint32_t sa = ALPHA_8 (*src++);
+
+	UN8x4_MUL_UN8 (a, sa);
+	UN8x4_MUL_UN8x4 (d, a);
+
+	*dest++ = d;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -1340,8 +1636,21 @@ vmx_combine_out_ca (pixman_implementation_t *imp,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, mask_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, mask_mask, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t a = *mask++;
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+	uint32_t da = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8x4 (s, a);
+	UN8x4_MUL_UN8 (s, da);
+
+	*dest++ = s;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -1384,8 +1693,21 @@ vmx_combine_out_reverse_ca (pixman_implementation_t *imp,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, mask_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, mask_mask, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t a = *mask++;
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+	uint32_t sa = ALPHA_8 (s);
+
+	UN8x4_MUL_UN8 (a, sa);
+	UN8x4_MUL_UN8x4 (d, ~a);
+
+	*dest++ = d;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -1428,8 +1750,23 @@ vmx_combine_atop_ca (pixman_implementation_t *imp,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask, vsrca;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, mask_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, mask_mask, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t a = *mask++;
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+	uint32_t sa = ALPHA_8 (s);
+	uint32_t da = ALPHA_8 (d);
+
+	UN8x4_MUL_UN8x4 (s, a);
+	UN8x4_MUL_UN8 (a, sa);
+	UN8x4_MUL_UN8x4_ADD_UN8x4_MUL_UN8 (d, ~a, s, da);
+
+	*dest++ = d;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -1479,8 +1816,23 @@ vmx_combine_atop_reverse_ca (pixman_implementation_t *imp,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, mask_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, mask_mask, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t a = *mask++;
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+	uint32_t sa = ALPHA_8 (s);
+	uint32_t da = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8x4 (s, a);
+	UN8x4_MUL_UN8 (a, sa);
+	UN8x4_MUL_UN8x4_ADD_UN8x4_MUL_UN8 (d, a, s, da);
+
+	*dest++ = d;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -1527,8 +1879,23 @@ vmx_combine_xor_ca (pixman_implementation_t *imp,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, mask_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, mask_mask, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t a = *mask++;
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+	uint32_t sa = ALPHA_8 (s);
+	uint32_t da = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8x4 (s, a);
+	UN8x4_MUL_UN8 (a, sa);
+	UN8x4_MUL_UN8x4_ADD_UN8x4_MUL_UN8 (d, ~a, s, da);
+
+	*dest++ = d;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -1575,8 +1942,20 @@ vmx_combine_add_ca (pixman_implementation_t *imp,
 {
     int i;
     vector unsigned int vdest, vsrc, vmask;
-    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
-	dest_mask, mask_mask, src_mask, store_mask;
+    vector unsigned char tmp1, tmp2, mask_mask, src_mask;
+
+    while (width && ((uintptr_t)dest & 15))
+    {
+	uint32_t a = *mask++;
+	uint32_t s = *src++;
+	uint32_t d = *dest;
+
+	UN8x4_MUL_UN8x4 (s, a);
+	UN8x4_ADD_UN8x4 (s, d);
+
+	*dest++ = s;
+	width--;
+    }
 
     COMPUTE_SHIFT_MASKC (dest, src, mask);
 
@@ -1613,10 +1992,9 @@ static const pixman_fast_path_t vmx_fast_paths[] =
 };
 
 pixman_implementation_t *
-_pixman_implementation_create_vmx (void)
+_pixman_implementation_create_vmx (pixman_implementation_t *fallback)
 {
-    pixman_implementation_t *fast = _pixman_implementation_create_fast_path ();
-    pixman_implementation_t *imp = _pixman_implementation_create (fast, vmx_fast_paths);
+    pixman_implementation_t *imp = _pixman_implementation_create (fallback, vmx_fast_paths);
 
     /* Set up function pointers */
 
diff --git a/pixman/pixman/pixman-x64-mmx-emulation.h b/pixman/pixman/pixman-x64-mmx-emulation.h
deleted file mode 100644
index 378019cf2..000000000
--- a/pixman/pixman/pixman-x64-mmx-emulation.h
+++ /dev/null
@@ -1,263 +0,0 @@
-#ifndef MMX_X64_H_INCLUDED
-#define MMX_X64_H_INCLUDED
-
-/* Implementation of x64 MMX substitition functions, before
- * pixman is reimplemented not to use __m64 type on Visual C++
- *
- * Copyright (C)2009 by George Yohng
- * Released in public domain.
- */
-
-#include <intrin.h>
-
-#define M64C(a) (*(const __m64 *)(&a))
-#define M64U(a) (*(const unsigned long long *)(&a))
-
-__inline __m64
-_m_from_int (int a)
-{
-    long long i64 = a;
-
-    return M64C (i64);
-}
-
-__inline __m64
-_mm_setzero_si64 ()
-{
-    long long i64 = 0;
-
-    return M64C (i64);
-}
-
-__inline __m64
-_mm_set_pi32 (int i1,   int i0)
-{
-    unsigned long long i64 = ((unsigned)i0) + (((unsigned long long)(unsigned)i1) << 32);
-
-    return M64C (i64);
-}
-
-__inline void
-_m_empty ()
-{
-}
-
-__inline __m64
-_mm_set1_pi16 (short w)
-{
-    unsigned long long i64 = ((unsigned long long)(unsigned short)(w)) * 0x0001000100010001ULL;
-
-    return M64C (i64);
-}
-
-__inline int
-_m_to_int (__m64 m)
-{
-    return m.m64_i32[0];
-}
-
-__inline __m64
-_mm_movepi64_pi64 (__m128i a)
-{
-    return M64C (a.m128i_i64[0]);
-}
-
-__inline __m64
-_m_pand (__m64 a, __m64 b)
-{
-    unsigned long long i64 = M64U (a) & M64U (b);
-
-    return M64C (i64);
-}
-
-__inline __m64
-_m_por (__m64 a, __m64 b)
-{
-    unsigned long long i64 = M64U (a) | M64U (b);
-
-    return M64C (i64);
-}
-
-__inline __m64
-_m_pxor (__m64 a, __m64 b)
-{
-    unsigned long long i64 = M64U (a) ^ M64U (b);
-
-    return M64C (i64);
-}
-
-__inline __m64
-_m_pmulhuw (__m64 a, __m64 b)        /* unoptimized */
-{
-    unsigned short d[4] =
-    {
-	(unsigned short)((((unsigned)a.m64_u16[0]) * b.m64_u16[0]) >> 16),
-	(unsigned short)((((unsigned)a.m64_u16[1]) * b.m64_u16[1]) >> 16),
-	(unsigned short)((((unsigned)a.m64_u16[2]) * b.m64_u16[2]) >> 16),
-	(unsigned short)((((unsigned)a.m64_u16[3]) * b.m64_u16[3]) >> 16)
-    };
-
-    return M64C (d[0]);
-}
-
-__inline __m64
-_m_pmullw2 (__m64 a, __m64 b)        /* unoptimized */
-{
-    unsigned short d[4] =
-    {
-	(unsigned short)((((unsigned)a.m64_u16[0]) * b.m64_u16[0])),
-	(unsigned short)((((unsigned)a.m64_u16[1]) * b.m64_u16[1])),
-	(unsigned short)((((unsigned)a.m64_u16[2]) * b.m64_u16[2])),
-	(unsigned short)((((unsigned)a.m64_u16[3]) * b.m64_u16[3]))
-    };
-
-    return M64C (d[0]);
-}
-
-__inline __m64
-_m_pmullw (__m64 a, __m64 b)        /* unoptimized */
-{
-    unsigned long long x =
-	((unsigned long long)(unsigned short)((((unsigned)a.m64_u16[0]) * b.m64_u16[0])))  +
-	(((unsigned long long)(unsigned short)((((unsigned)a.m64_u16[1]) * b.m64_u16[1]))) << 16)  +
-	(((unsigned long long)(unsigned short)((((unsigned)a.m64_u16[2]) * b.m64_u16[2]))) << 32)  +
-	(((unsigned long long)(unsigned short)((((unsigned)a.m64_u16[3]) * b.m64_u16[3]))) << 48);
-
-    return M64C (x);
-}
-
-__inline __m64
-_m_paddusb (__m64 a, __m64 b)        /* unoptimized */
-{
-    unsigned long long x = (M64U (a) & 0x00FF00FF00FF00FFULL) +
-                           (M64U (b) & 0x00FF00FF00FF00FFULL);
-
-    unsigned long long y = ((M64U (a) >> 8) & 0x00FF00FF00FF00FFULL) +
-                           ((M64U (b) >> 8) & 0x00FF00FF00FF00FFULL);
-
-    x |= ((x & 0xFF00FF00FF00FF00ULL) >> 8) * 0xFF;
-    y |= ((y & 0xFF00FF00FF00FF00ULL) >> 8) * 0xFF;
-
-    x = (x & 0x00FF00FF00FF00FFULL) | ((y & 0x00FF00FF00FF00FFULL) << 8);
-
-    return M64C (x);
-}
-
-__inline __m64
-_m_paddusw (__m64 a, __m64 b)        /* unoptimized */
-{
-    unsigned long long x = (M64U (a) & 0x0000FFFF0000FFFFULL) +
-                           (M64U (b) & 0x0000FFFF0000FFFFULL);
-
-    unsigned long long y = ((M64U (a) >> 16) & 0x0000FFFF0000FFFFULL) +
-                           ((M64U (b) >> 16) & 0x0000FFFF0000FFFFULL);
-
-    x |= ((x & 0xFFFF0000FFFF0000) >> 16) * 0xFFFF;
-    y |= ((y & 0xFFFF0000FFFF0000) >> 16) * 0xFFFF;
-
-    x = (x & 0x0000FFFF0000FFFFULL) | ((y & 0x0000FFFF0000FFFFULL) << 16);
-
-    return M64C (x);
-}
-
-__inline __m64
-_m_pshufw (__m64 a, int n)         /* unoptimized */
-{
-    unsigned short d[4] =
-    {
-	a.m64_u16[n & 3],
-	a.m64_u16[(n >> 2) & 3],
-	a.m64_u16[(n >> 4) & 3],
-	a.m64_u16[(n >> 6) & 3]
-    };
-
-    return M64C (d[0]);
-}
-
-__inline unsigned char
-sat16 (unsigned short d)
-{
-    if (d > 0xFF) return 0xFF;
-    else return d & 0xFF;
-}
-
-__inline __m64
-_m_packuswb (__m64 m1, __m64 m2)          /* unoptimized */
-{
-    unsigned char d[8] =
-    {
-	sat16 (m1.m64_u16[0]),
-	sat16 (m1.m64_u16[1]),
-	sat16 (m1.m64_u16[2]),
-	sat16 (m1.m64_u16[3]),
-	sat16 (m2.m64_u16[0]),
-	sat16 (m2.m64_u16[1]),
-	sat16 (m2.m64_u16[2]),
-	sat16 (m2.m64_u16[3])
-    };
-
-    return M64C (d[0]);
-}
-
-__inline __m64 _m_punpcklbw (__m64 m1, __m64 m2)          /* unoptimized */
-{
-    unsigned char d[8] =
-    {
-	m1.m64_u8[0],
-	m2.m64_u8[0],
-	m1.m64_u8[1],
-	m2.m64_u8[1],
-	m1.m64_u8[2],
-	m2.m64_u8[2],
-	m1.m64_u8[3],
-	m2.m64_u8[3],
-    };
-
-    return M64C (d[0]);
-}
-
-__inline __m64 _m_punpckhbw (__m64 m1, __m64 m2)          /* unoptimized */
-{
-    unsigned char d[8] =
-    {
-	m1.m64_u8[4],
-	m2.m64_u8[4],
-	m1.m64_u8[5],
-	m2.m64_u8[5],
-	m1.m64_u8[6],
-	m2.m64_u8[6],
-	m1.m64_u8[7],
-	m2.m64_u8[7],
-    };
-
-    return M64C (d[0]);
-}
-
-__inline __m64 _m_psrlwi (__m64 a, int n)       /* unoptimized */
-{
-    unsigned short d[4] =
-    {
-	a.m64_u16[0] >> n,
-	a.m64_u16[1] >> n,
-	a.m64_u16[2] >> n,
-	a.m64_u16[3] >> n
-    };
-
-    return M64C (d[0]);
-}
-
-__inline __m64 _m_psrlqi (__m64 m, int n)
-{
-    unsigned long long x = M64U (m) >> n;
-
-    return M64C (x);
-}
-
-__inline __m64 _m_psllqi (__m64 m, int n)
-{
-    unsigned long long x = M64U (m) << n;
-
-    return M64C (x);
-}
-
-#endif /* MMX_X64_H_INCLUDED */
diff --git a/pixman/pixman/pixman-x86.c b/pixman/pixman/pixman-x86.c
new file mode 100644
index 000000000..05297c476
--- /dev/null
+++ b/pixman/pixman/pixman-x86.c
@@ -0,0 +1,248 @@
+/*
+ * Copyright © 2000 SuSE, Inc.
+ * Copyright © 2007 Red Hat, Inc.
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of SuSE not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  SuSE makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as is"
+ * without express or implied warranty.
+ *
+ * SuSE DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL SuSE
+ * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "pixman-private.h"
+
+#if defined(USE_X86_MMX) || defined (USE_SSE2) || defined (USE_SSSE3)
+
+/* The CPU detection code needs to be in a file not compiled with
+ * "-mmmx -msse", as gcc would generate CMOV instructions otherwise
+ * that would lead to SIGILL instructions on old CPUs that don't have
+ * it.
+ */
+
+typedef enum
+{
+    X86_MMX			= (1 << 0),
+    X86_MMX_EXTENSIONS		= (1 << 1),
+    X86_SSE			= (1 << 2) | X86_MMX_EXTENSIONS,
+    X86_SSE2			= (1 << 3),
+    X86_CMOV			= (1 << 4),
+    X86_SSSE3			= (1 << 5)
+} cpu_features_t;
+
+#ifdef HAVE_GETISAX
+
+#include <sys/auxv.h>
+
+static cpu_features_t
+detect_cpu_features (void)
+{
+    cpu_features_t features = 0;
+    unsigned int result = 0;
+
+    if (getisax (&result, 1))
+    {
+	if (result & AV_386_CMOV)
+	    features |= X86_CMOV;
+	if (result & AV_386_MMX)
+	    features |= X86_MMX;
+	if (result & AV_386_AMD_MMX)
+	    features |= X86_MMX_EXTENSIONS;
+	if (result & AV_386_SSE)
+	    features |= X86_SSE;
+	if (result & AV_386_SSE2)
+	    features |= X86_SSE2;
+	if (result & AV_386_SSSE3)
+	    features |= X86_SSSE3;
+    }
+
+    return features;
+}
+
+#else
+
+#define _PIXMAN_X86_64							\
+    (defined(__amd64__) || defined(__x86_64__) || defined(_M_AMD64))
+
+static pixman_bool_t
+have_cpuid (void)
+{
+#if _PIXMAN_X86_64 || defined (_MSC_VER)
+
+    return TRUE;
+
+#elif defined (__GNUC__)
+    uint32_t result;
+
+    __asm__ volatile (
+        "pushf"				"\n\t"
+        "pop %%eax"			"\n\t"
+        "mov %%eax, %%ecx"		"\n\t"
+        "xor $0x00200000, %%eax"	"\n\t"
+        "push %%eax"			"\n\t"
+        "popf"				"\n\t"
+        "pushf"				"\n\t"
+        "pop %%eax"			"\n\t"
+        "xor %%ecx, %%eax"		"\n\t"
+	"mov %%eax, %0"			"\n\t"
+	: "=r" (result)
+	:
+	: "%eax", "%ecx");
+
+    return !!result;
+
+#else
+#error "Unknown compiler"
+#endif
+}
+
+static void
+pixman_cpuid (uint32_t feature,
+	      uint32_t *a, uint32_t *b, uint32_t *c, uint32_t *d)
+{
+#if defined (__GNUC__)
+
+#if _PIXMAN_X86_64
+    __asm__ volatile (
+        "cpuid"				"\n\t"
+	: "=a" (*a), "=b" (*b), "=c" (*c), "=d" (*d)
+	: "a" (feature));
+#else
+    /* On x86-32 we need to be careful about the handling of %ebx
+     * and %esp. We can't declare either one as clobbered
+     * since they are special registers (%ebx is the "PIC
+     * register" holding an offset to global data, %esp the
+     * stack pointer), so we need to make sure that %ebx is
+     * preserved, and that %esp has its original value when
+     * accessing the output operands.
+     */
+    __asm__ volatile (
+	"xchg %%ebx, %1"		"\n\t"
+	"cpuid"				"\n\t"
+	"xchg %%ebx, %1"		"\n\t"
+	: "=a" (*a), "=r" (*b), "=c" (*c), "=d" (*d)
+	: "a" (feature));
+#endif
+
+#elif defined (_MSC_VER)
+    int info[4];
+
+    __cpuid (info, feature);
+
+    *a = info[0];
+    *b = info[1];
+    *c = info[2];
+    *d = info[3];
+#else
+#error Unknown compiler
+#endif
+}
+
+static cpu_features_t
+detect_cpu_features (void)
+{
+    uint32_t a, b, c, d;
+    cpu_features_t features = 0;
+
+    if (!have_cpuid())
+	return features;
+
+    /* Get feature bits */
+    pixman_cpuid (0x01, &a, &b, &c, &d);
+    if (d & (1 << 15))
+	features |= X86_CMOV;
+    if (d & (1 << 23))
+	features |= X86_MMX;
+    if (d & (1 << 25))
+	features |= X86_SSE;
+    if (d & (1 << 26))
+	features |= X86_SSE2;
+    if (c & (1 << 9))
+	features |= X86_SSSE3;
+
+    /* Check for AMD specific features */
+    if ((features & X86_MMX) && !(features & X86_SSE))
+    {
+	char vendor[13];
+
+	/* Get vendor string */
+	memset (vendor, 0, sizeof vendor);
+
+	pixman_cpuid (0x00, &a, &b, &c, &d);
+	memcpy (vendor + 0, &b, 4);
+	memcpy (vendor + 4, &d, 4);
+	memcpy (vendor + 8, &c, 4);
+
+	if (strcmp (vendor, "AuthenticAMD") == 0 ||
+	    strcmp (vendor, "Geode by NSC") == 0)
+	{
+	    pixman_cpuid (0x80000000, &a, &b, &c, &d);
+	    if (a >= 0x80000001)
+	    {
+		pixman_cpuid (0x80000001, &a, &b, &c, &d);
+
+		if (d & (1 << 22))
+		    features |= X86_MMX_EXTENSIONS;
+	    }
+	}
+    }
+
+    return features;
+}
+
+#endif
+
+static pixman_bool_t
+have_feature (cpu_features_t feature)
+{
+    static pixman_bool_t initialized;
+    static cpu_features_t features;
+
+    if (!initialized)
+    {
+	features = detect_cpu_features();
+	initialized = TRUE;
+    }
+
+    return (features & feature) == feature;
+}
+
+#endif
+
+pixman_implementation_t *
+_pixman_x86_get_implementations (pixman_implementation_t *imp)
+{
+#define MMX_BITS  (X86_MMX | X86_MMX_EXTENSIONS)
+#define SSE2_BITS (X86_MMX | X86_MMX_EXTENSIONS | X86_SSE | X86_SSE2)
+#define SSSE3_BITS (X86_SSE | X86_SSE2 | X86_SSSE3)
+
+#ifdef USE_X86_MMX
+    if (!_pixman_disabled ("mmx") && have_feature (MMX_BITS))
+	imp = _pixman_implementation_create_mmx (imp);
+#endif
+
+#ifdef USE_SSE2
+    if (!_pixman_disabled ("sse2") && have_feature (SSE2_BITS))
+	imp = _pixman_implementation_create_sse2 (imp);
+#endif
+
+#ifdef USE_SSSE3
+    if (!_pixman_disabled ("ssse3") && have_feature (SSSE3_BITS))
+	imp = _pixman_implementation_create_ssse3 (imp);
+#endif
+
+    return imp;
+}
diff --git a/pixman/pixman/pixman.c b/pixman/pixman/pixman.c
index 548242ba0..9555ceaaf 100644
--- a/pixman/pixman/pixman.c
+++ b/pixman/pixman/pixman.c
@@ -30,7 +30,15 @@
 
 #include <stdlib.h>
 
-static pixman_implementation_t *imp;
+pixman_implementation_t *global_implementation;
+
+#ifdef TOOLCHAIN_SUPPORTS_ATTRIBUTE_CONSTRUCTOR
+static void __attribute__((constructor))
+pixman_constructor (void)
+{
+    global_implementation = _pixman_choose_implementation ();
+}
+#endif
 
 typedef struct operator_info_t operator_info_t;
 
@@ -130,65 +138,18 @@ optimize_operator (pixman_op_t     op,
 		   uint32_t        dst_flags)
 {
     pixman_bool_t is_source_opaque, is_dest_opaque;
-    int opaqueness;
 
-    is_source_opaque = ((src_flags & mask_flags) & FAST_PATH_IS_OPAQUE) != 0;
-    is_dest_opaque = (dst_flags & FAST_PATH_IS_OPAQUE) != 0;
-
-    opaqueness = ((is_dest_opaque << 1) | is_source_opaque);
-
-    return operator_table[op].opaque_info[opaqueness];
-}
+#define OPAQUE_SHIFT 13
+    
+    COMPILE_TIME_ASSERT (FAST_PATH_IS_OPAQUE == (1 << OPAQUE_SHIFT));
+    
+    is_dest_opaque = (dst_flags & FAST_PATH_IS_OPAQUE);
+    is_source_opaque = ((src_flags & mask_flags) & FAST_PATH_IS_OPAQUE);
 
-static void
-apply_workaround (pixman_image_t *image,
-		  int32_t *       x,
-		  int32_t *       y,
-		  uint32_t **     save_bits,
-		  int *           save_dx,
-		  int *           save_dy)
-{
-    if (image && (image->common.flags & FAST_PATH_NEEDS_WORKAROUND))
-    {
-	/* Some X servers generate images that point to the
-	 * wrong place in memory, but then set the clip region
-	 * to point to the right place. Because of an old bug
-	 * in pixman, this would actually work.
-	 *
-	 * Here we try and undo the damage
-	 */
-	int bpp = PIXMAN_FORMAT_BPP (image->bits.format) / 8;
-	pixman_box32_t *extents;
-	uint8_t *t;
-	int dx, dy;
-	
-	extents = pixman_region32_extents (&(image->common.clip_region));
-	dx = extents->x1;
-	dy = extents->y1;
-	
-	*save_bits = image->bits.bits;
-	
-	*x -= dx;
-	*y -= dy;
-	pixman_region32_translate (&(image->common.clip_region), -dx, -dy);
-	
-	t = (uint8_t *)image->bits.bits;
-	t += dy * image->bits.rowstride * 4 + dx * bpp;
-	image->bits.bits = (uint32_t *)t;
-	
-	*save_dx = dx;
-	*save_dy = dy;
-    }
-}
+    is_dest_opaque >>= OPAQUE_SHIFT - 1;
+    is_source_opaque >>= OPAQUE_SHIFT;
 
-static void
-unapply_workaround (pixman_image_t *image, uint32_t *bits, int dx, int dy)
-{
-    if (image && (image->common.flags & FAST_PATH_NEEDS_WORKAROUND))
-    {
-	image->bits.bits = bits;
-	pixman_region32_translate (&image->common.clip_region, dx, dy);
-    }
+    return operator_table[op].opaque_info[is_dest_opaque | is_source_opaque];
 }
 
 /*
@@ -263,19 +224,19 @@ clip_source_image (pixman_region32_t * region,
  * returns FALSE if the final region is empty.  Indistinguishable from
  * an allocation failure, but rendering ignores those anyways.
  */
-static pixman_bool_t
-pixman_compute_composite_region32 (pixman_region32_t * region,
-                                   pixman_image_t *    src_image,
-                                   pixman_image_t *    mask_image,
-                                   pixman_image_t *    dst_image,
-                                   int32_t             src_x,
-                                   int32_t             src_y,
-                                   int32_t             mask_x,
-                                   int32_t             mask_y,
-                                   int32_t             dest_x,
-                                   int32_t             dest_y,
-                                   int32_t             width,
-                                   int32_t             height)
+pixman_bool_t
+_pixman_compute_composite_region32 (pixman_region32_t * region,
+				    pixman_image_t *    src_image,
+				    pixman_image_t *    mask_image,
+				    pixman_image_t *    dest_image,
+				    int32_t             src_x,
+				    int32_t             src_y,
+				    int32_t             mask_x,
+				    int32_t             mask_y,
+				    int32_t             dest_x,
+				    int32_t             dest_y,
+				    int32_t             width,
+				    int32_t             height)
 {
     region->extents.x1 = dest_x;
     region->extents.x2 = dest_x + width;
@@ -284,8 +245,8 @@ pixman_compute_composite_region32 (pixman_region32_t * region,
 
     region->extents.x1 = MAX (region->extents.x1, 0);
     region->extents.y1 = MAX (region->extents.y1, 0);
-    region->extents.x2 = MIN (region->extents.x2, dst_image->bits.width);
-    region->extents.y2 = MIN (region->extents.y2, dst_image->bits.height);
+    region->extents.x2 = MIN (region->extents.x2, dest_image->bits.width);
+    region->extents.y2 = MIN (region->extents.y2, dest_image->bits.height);
 
     region->data = 0;
 
@@ -293,38 +254,47 @@ pixman_compute_composite_region32 (pixman_region32_t * region,
     if (region->extents.x1 >= region->extents.x2 ||
         region->extents.y1 >= region->extents.y2)
     {
-	pixman_region32_init (region);
+	region->extents.x1 = 0;
+	region->extents.x2 = 0;
+	region->extents.y1 = 0;
+	region->extents.y2 = 0;
 	return FALSE;
     }
 
-    if (dst_image->common.have_clip_region)
+    if (dest_image->common.have_clip_region)
     {
-	if (!clip_general_image (region, &dst_image->common.clip_region, 0, 0))
-	{
-	    pixman_region32_fini (region);
+	if (!clip_general_image (region, &dest_image->common.clip_region, 0, 0))
 	    return FALSE;
-	}
     }
 
-    if (dst_image->common.alpha_map && dst_image->common.alpha_map->common.have_clip_region)
+    if (dest_image->common.alpha_map)
     {
-	if (!clip_general_image (region, &dst_image->common.alpha_map->common.clip_region,
-	                         -dst_image->common.alpha_origin_x,
-	                         -dst_image->common.alpha_origin_y))
+	if (!pixman_region32_intersect_rect (region, region,
+					     dest_image->common.alpha_origin_x,
+					     dest_image->common.alpha_origin_y,
+					     dest_image->common.alpha_map->width,
+					     dest_image->common.alpha_map->height))
 	{
-	    pixman_region32_fini (region);
 	    return FALSE;
 	}
+	if (!pixman_region32_not_empty (region))
+	    return FALSE;
+	if (dest_image->common.alpha_map->common.have_clip_region)
+	{
+	    if (!clip_general_image (region, &dest_image->common.alpha_map->common.clip_region,
+				     -dest_image->common.alpha_origin_x,
+				     -dest_image->common.alpha_origin_y))
+	    {
+		return FALSE;
+	    }
+	}
     }
 
     /* clip against src */
     if (src_image->common.have_clip_region)
     {
 	if (!clip_source_image (region, src_image, dest_x - src_x, dest_y - src_y))
-	{
-	    pixman_region32_fini (region);
 	    return FALSE;
-	}
     }
     if (src_image->common.alpha_map && src_image->common.alpha_map->common.have_clip_region)
     {
@@ -332,7 +302,6 @@ pixman_compute_composite_region32 (pixman_region32_t * region,
 	                        dest_x - (src_x - src_image->common.alpha_origin_x),
 	                        dest_y - (src_y - src_image->common.alpha_origin_y)))
 	{
-	    pixman_region32_fini (region);
 	    return FALSE;
 	}
     }
@@ -340,17 +309,14 @@ pixman_compute_composite_region32 (pixman_region32_t * region,
     if (mask_image && mask_image->common.have_clip_region)
     {
 	if (!clip_source_image (region, mask_image, dest_x - mask_x, dest_y - mask_y))
-	{
-	    pixman_region32_fini (region);
 	    return FALSE;
-	}
+
 	if (mask_image->common.alpha_map && mask_image->common.alpha_map->common.have_clip_region)
 	{
 	    if (!clip_source_image (region, (pixman_image_t *)mask_image->common.alpha_map,
 	                            dest_x - (mask_x - mask_image->common.alpha_origin_x),
 	                            dest_y - (mask_y - mask_image->common.alpha_origin_y)))
 	    {
-		pixman_region32_fini (region);
 		return FALSE;
 	    }
 	}
@@ -359,257 +325,297 @@ pixman_compute_composite_region32 (pixman_region32_t * region,
     return TRUE;
 }
 
-static void
-walk_region_internal (pixman_implementation_t *imp,
-                      pixman_op_t              op,
-                      pixman_image_t *         src_image,
-                      pixman_image_t *         mask_image,
-                      pixman_image_t *         dst_image,
-                      int32_t                  src_x,
-                      int32_t                  src_y,
-                      int32_t                  mask_x,
-                      int32_t                  mask_y,
-                      int32_t                  dest_x,
-                      int32_t                  dest_y,
-                      int32_t                  width,
-                      int32_t                  height,
-                      pixman_bool_t            src_repeat,
-                      pixman_bool_t            mask_repeat,
-                      pixman_region32_t *      region,
-                      pixman_composite_func_t  composite_rect)
+typedef struct
 {
-    int w, h, w_this, h_this;
-    int x_msk, y_msk, x_src, y_src, x_dst, y_dst;
-    int src_dy = src_y - dest_y;
-    int src_dx = src_x - dest_x;
-    int mask_dy = mask_y - dest_y;
-    int mask_dx = mask_x - dest_x;
-    const pixman_box32_t *pbox;
-    int n;
+    pixman_fixed_48_16_t	x1;
+    pixman_fixed_48_16_t	y1;
+    pixman_fixed_48_16_t	x2;
+    pixman_fixed_48_16_t	y2;
+} box_48_16_t;
+
+static pixman_bool_t
+compute_transformed_extents (pixman_transform_t *transform,
+			     const pixman_box32_t *extents,
+			     box_48_16_t *transformed)
+{
+    pixman_fixed_48_16_t tx1, ty1, tx2, ty2;
+    pixman_fixed_t x1, y1, x2, y2;
+    int i;
 
-    pbox = pixman_region32_rectangles (region, &n);
+    x1 = pixman_int_to_fixed (extents->x1) + pixman_fixed_1 / 2;
+    y1 = pixman_int_to_fixed (extents->y1) + pixman_fixed_1 / 2;
+    x2 = pixman_int_to_fixed (extents->x2) - pixman_fixed_1 / 2;
+    y2 = pixman_int_to_fixed (extents->y2) - pixman_fixed_1 / 2;
 
-    /* Fast path for non-repeating sources */
-    if (!src_repeat && !mask_repeat)
-    {
-       while (n--)
-       {
-           (*composite_rect) (imp, op,
-                              src_image, mask_image, dst_image,
-                              pbox->x1 + src_dx,
-                              pbox->y1 + src_dy,
-                              pbox->x1 + mask_dx,
-                              pbox->y1 + mask_dy,
-                              pbox->x1,
-                              pbox->y1,
-                              pbox->x2 - pbox->x1,
-                              pbox->y2 - pbox->y1);
-           
-           pbox++;
-       }
-
-       return;
-    }
-    
-    while (n--)
+    if (!transform)
     {
-	h = pbox->y2 - pbox->y1;
-	y_src = pbox->y1 + src_dy;
-	y_msk = pbox->y1 + mask_dy;
-	y_dst = pbox->y1;
+	transformed->x1 = x1;
+	transformed->y1 = y1;
+	transformed->x2 = x2;
+	transformed->y2 = y2;
 
-	while (h)
-	{
-	    h_this = h;
-	    w = pbox->x2 - pbox->x1;
-	    x_src = pbox->x1 + src_dx;
-	    x_msk = pbox->x1 + mask_dx;
-	    x_dst = pbox->x1;
+	return TRUE;
+    }
 
-	    if (mask_repeat)
-	    {
-		y_msk = MOD (y_msk, mask_image->bits.height);
-		if (h_this > mask_image->bits.height - y_msk)
-		    h_this = mask_image->bits.height - y_msk;
-	    }
+    tx1 = ty1 = INT64_MAX;
+    tx2 = ty2 = INT64_MIN;
 
-	    if (src_repeat)
-	    {
-		y_src = MOD (y_src, src_image->bits.height);
-		if (h_this > src_image->bits.height - y_src)
-		    h_this = src_image->bits.height - y_src;
-	    }
+    for (i = 0; i < 4; ++i)
+    {
+	pixman_fixed_48_16_t tx, ty;
+	pixman_vector_t v;
 
-	    while (w)
-	    {
-		w_this = w;
-
-		if (mask_repeat)
-		{
-		    x_msk = MOD (x_msk, mask_image->bits.width);
-		    if (w_this > mask_image->bits.width - x_msk)
-			w_this = mask_image->bits.width - x_msk;
-		}
-
-		if (src_repeat)
-		{
-		    x_src = MOD (x_src, src_image->bits.width);
-		    if (w_this > src_image->bits.width - x_src)
-			w_this = src_image->bits.width - x_src;
-		}
-
-		(*composite_rect) (imp, op,
-				   src_image, mask_image, dst_image,
-				   x_src, y_src, x_msk, y_msk, x_dst, y_dst,
-				   w_this, h_this);
-		w -= w_this;
-
-		x_src += w_this;
-		x_msk += w_this;
-		x_dst += w_this;
-	    }
+	v.vector[0] = (i & 0x01)? x1 : x2;
+	v.vector[1] = (i & 0x02)? y1 : y2;
+	v.vector[2] = pixman_fixed_1;
 
-	    h -= h_this;
-	    y_src += h_this;
-	    y_msk += h_this;
-	    y_dst += h_this;
-	}
+	if (!pixman_transform_point (transform, &v))
+	    return FALSE;
 
-	pbox++;
+	tx = (pixman_fixed_48_16_t)v.vector[0];
+	ty = (pixman_fixed_48_16_t)v.vector[1];
+
+	if (tx < tx1)
+	    tx1 = tx;
+	if (ty < ty1)
+	    ty1 = ty;
+	if (tx > tx2)
+	    tx2 = tx;
+	if (ty > ty2)
+	    ty2 = ty;
     }
+
+    transformed->x1 = tx1;
+    transformed->y1 = ty1;
+    transformed->x2 = tx2;
+    transformed->y2 = ty2;
+
+    return TRUE;
 }
 
 #define IS_16BIT(x) (((x) >= INT16_MIN) && ((x) <= INT16_MAX))
+#define ABS(f)      (((f) < 0)?  (-(f)) : (f))
+#define IS_16_16(f) (((f) >= pixman_min_fixed_48_16 && ((f) <= pixman_max_fixed_48_16)))
 
-static force_inline uint32_t
-compute_src_extents_flags (pixman_image_t *image,
-			   pixman_box32_t *extents,
-			   int             x,
-			   int             y)
+static pixman_bool_t
+analyze_extent (pixman_image_t       *image,
+		const pixman_box32_t *extents,
+		uint32_t             *flags)
 {
-    pixman_box16_t extents16;
-    uint32_t flags;
-
-    flags = FAST_PATH_COVERS_CLIP;
-
-    if (image->common.type != BITS)
-	return flags;
+    pixman_transform_t *transform;
+    pixman_fixed_t x_off, y_off;
+    pixman_fixed_t width, height;
+    pixman_fixed_t *params;
+    box_48_16_t transformed;
+    pixman_box32_t exp_extents;
+
+    if (!image)
+	return TRUE;
 
-    if (image->common.repeat == PIXMAN_REPEAT_NONE &&
-	(x > extents->x1 || y > extents->y1 ||
-	 x + image->bits.width < extents->x2 ||
-	 y + image->bits.height < extents->y2))
+    /* Some compositing functions walk one step
+     * outside the destination rectangle, so we
+     * check here that the expanded-by-one source
+     * extents in destination space fits in 16 bits
+     */
+    if (!IS_16BIT (extents->x1 - 1)		||
+	!IS_16BIT (extents->y1 - 1)		||
+	!IS_16BIT (extents->x2 + 1)		||
+	!IS_16BIT (extents->y2 + 1))
     {
-	flags &= ~FAST_PATH_COVERS_CLIP;
+	return FALSE;
     }
 
-    if (IS_16BIT (extents->x1 - x) &&
-	IS_16BIT (extents->y1 - y) &&
-	IS_16BIT (extents->x2 - x) &&
-	IS_16BIT (extents->y2 - y))
+    transform = image->common.transform;
+    if (image->common.type == BITS)
     {
-	extents16.x1 = extents->x1 - x;
-	extents16.y1 = extents->y1 - y;
-	extents16.x2 = extents->x2 - x;
-	extents16.y2 = extents->y2 - y;
+	/* During repeat mode calculations we might convert the
+	 * width/height of an image to fixed 16.16, so we need
+	 * them to be smaller than 16 bits.
+	 */
+	if (image->bits.width >= 0x7fff	|| image->bits.height >= 0x7fff)
+	    return FALSE;
 
-	if (!image->common.transform ||
-	    pixman_transform_bounds (image->common.transform, &extents16))
+	if ((image->common.flags & FAST_PATH_ID_TRANSFORM) == FAST_PATH_ID_TRANSFORM &&
+	    extents->x1 >= 0 &&
+	    extents->y1 >= 0 &&
+	    extents->x2 <= image->bits.width &&
+	    extents->y2 <= image->bits.height)
 	{
-	    if (extents16.x1 >= 0  && extents16.y1 >= 0 &&
-		extents16.x2 <= image->bits.width &&
-		extents16.y2 <= image->bits.height)
-	    {
-		flags |= FAST_PATH_SAMPLES_COVER_CLIP;
-	    }
+	    *flags |= FAST_PATH_SAMPLES_COVER_CLIP_NEAREST;
+	    return TRUE;
+	}
+
+	switch (image->common.filter)
+	{
+	case PIXMAN_FILTER_CONVOLUTION:
+	    params = image->common.filter_params;
+	    x_off = - pixman_fixed_e - ((params[0] - pixman_fixed_1) >> 1);
+	    y_off = - pixman_fixed_e - ((params[1] - pixman_fixed_1) >> 1);
+	    width = params[0];
+	    height = params[1];
+	    break;
+
+	case PIXMAN_FILTER_SEPARABLE_CONVOLUTION:
+	    params = image->common.filter_params;
+	    x_off = - pixman_fixed_e - ((params[0] - pixman_fixed_1) >> 1);
+	    y_off = - pixman_fixed_e - ((params[1] - pixman_fixed_1) >> 1);
+	    width = params[0];
+	    height = params[1];
+	    break;
+	    
+	case PIXMAN_FILTER_GOOD:
+	case PIXMAN_FILTER_BEST:
+	case PIXMAN_FILTER_BILINEAR:
+	    x_off = - pixman_fixed_1 / 2;
+	    y_off = - pixman_fixed_1 / 2;
+	    width = pixman_fixed_1;
+	    height = pixman_fixed_1;
+	    break;
+
+	case PIXMAN_FILTER_FAST:
+	case PIXMAN_FILTER_NEAREST:
+	    x_off = - pixman_fixed_e;
+	    y_off = - pixman_fixed_e;
+	    width = 0;
+	    height = 0;
+	    break;
+
+	default:
+	    return FALSE;
 	}
     }
+    else
+    {
+	x_off = 0;
+	y_off = 0;
+	width = 0;
+	height = 0;
+    }
+
+    if (!compute_transformed_extents (transform, extents, &transformed))
+	return FALSE;
+
+    /* Expand the source area by a tiny bit so account of different rounding that
+     * may happen during sampling. Note that (8 * pixman_fixed_e) is very far from
+     * 0.5 so this won't cause the area computed to be overly pessimistic.
+     */
+    transformed.x1 -= 8 * pixman_fixed_e;
+    transformed.y1 -= 8 * pixman_fixed_e;
+    transformed.x2 += 8 * pixman_fixed_e;
+    transformed.y2 += 8 * pixman_fixed_e;
 
-    if (IS_16BIT (extents->x1 - x - 1) &&
-	IS_16BIT (extents->y1 - y - 1) &&
-	IS_16BIT (extents->x2 - x + 1) &&
-	IS_16BIT (extents->y2 - y + 1))
+    if (image->common.type == BITS)
     {
-	extents16.x1 = extents->x1 - x - 1;
-	extents16.y1 = extents->y1 - y - 1;
-	extents16.x2 = extents->x2 - x + 1;
-	extents16.y2 = extents->y2 - y + 1;
-
-	if (/* src space expanded by one in dest space fits in 16 bit */
-	    (!image->common.transform ||
-	     pixman_transform_bounds (image->common.transform, &extents16)) &&
-	    /* And src image size can be used as 16.16 fixed point */
-	    image->bits.width < 0x7fff &&
-	    image->bits.height < 0x7fff)
+	if (pixman_fixed_to_int (transformed.x1) >= 0			&&
+	    pixman_fixed_to_int (transformed.y1) >= 0			&&
+	    pixman_fixed_to_int (transformed.x2) < image->bits.width	&&
+	    pixman_fixed_to_int (transformed.y2) < image->bits.height)
 	{
-	    /* Then we're "16bit safe" */
-	    flags |= FAST_PATH_16BIT_SAFE;
+	    *flags |= FAST_PATH_SAMPLES_COVER_CLIP_NEAREST;
+	}
+
+	if (pixman_fixed_to_int (transformed.x1 - pixman_fixed_1 / 2) >= 0		  &&
+	    pixman_fixed_to_int (transformed.y1 - pixman_fixed_1 / 2) >= 0		  &&
+	    pixman_fixed_to_int (transformed.x2 + pixman_fixed_1 / 2) < image->bits.width &&
+	    pixman_fixed_to_int (transformed.y2 + pixman_fixed_1 / 2) < image->bits.height)
+	{
+	    *flags |= FAST_PATH_SAMPLES_COVER_CLIP_BILINEAR;
 	}
     }
 
-    return flags;
-}
+    /* Check we don't overflow when the destination extents are expanded by one.
+     * This ensures that compositing functions can simply walk the source space
+     * using 16.16 variables without worrying about overflow.
+     */
+    exp_extents = *extents;
+    exp_extents.x1 -= 1;
+    exp_extents.y1 -= 1;
+    exp_extents.x2 += 1;
+    exp_extents.y2 += 1;
 
-#define N_CACHED_FAST_PATHS 8
+    if (!compute_transformed_extents (transform, &exp_extents, &transformed))
+	return FALSE;
+    
+    if (!IS_16_16 (transformed.x1 + x_off - 8 * pixman_fixed_e)	||
+	!IS_16_16 (transformed.y1 + y_off - 8 * pixman_fixed_e)	||
+	!IS_16_16 (transformed.x2 + x_off + 8 * pixman_fixed_e + width)	||
+	!IS_16_16 (transformed.y2 + y_off + 8 * pixman_fixed_e + height))
+    {
+	return FALSE;
+    }
 
-typedef struct
-{
-    pixman_fast_path_t cache [N_CACHED_FAST_PATHS];
-} cache_t;
-
-PIXMAN_DEFINE_THREAD_LOCAL (cache_t, fast_path_cache);
-
-static void
-do_composite (pixman_implementation_t *imp,
-	      pixman_op_t	       op,
-	      pixman_image_t	      *src,
-	      pixman_image_t	      *mask,
-	      pixman_image_t	      *dest,
-	      int		       src_x,
-	      int		       src_y,
-	      int		       mask_x,
-	      int		       mask_y,
-	      int		       dest_x,
-	      int		       dest_y,
-	      int		       width,
-	      int		       height)
+    return TRUE;
+}
+
+/*
+ * Work around GCC bug causing crashes in Mozilla with SSE2
+ *
+ * When using -msse, gcc generates movdqa instructions assuming that
+ * the stack is 16 byte aligned. Unfortunately some applications, such
+ * as Mozilla and Mono, end up aligning the stack to 4 bytes, which
+ * causes the movdqa instructions to fail.
+ *
+ * The __force_align_arg_pointer__ makes gcc generate a prologue that
+ * realigns the stack pointer to 16 bytes.
+ *
+ * On x86-64 this is not necessary because the standard ABI already
+ * calls for a 16 byte aligned stack.
+ *
+ * See https://bugs.freedesktop.org/show_bug.cgi?id=15693
+ */
+#if defined (USE_SSE2) && defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
+__attribute__((__force_align_arg_pointer__))
+#endif
+PIXMAN_EXPORT void
+pixman_image_composite32 (pixman_op_t      op,
+                          pixman_image_t * src,
+                          pixman_image_t * mask,
+                          pixman_image_t * dest,
+                          int32_t          src_x,
+                          int32_t          src_y,
+                          int32_t          mask_x,
+                          int32_t          mask_y,
+                          int32_t          dest_x,
+                          int32_t          dest_y,
+                          int32_t          width,
+                          int32_t          height)
 {
     pixman_format_code_t src_format, mask_format, dest_format;
-    uint32_t src_flags, mask_flags, dest_flags;
     pixman_region32_t region;
-    pixman_box32_t *extents;
-    uint32_t *src_bits;
-    int src_dx, src_dy;
-    uint32_t *mask_bits;
-    int mask_dx, mask_dy;
-    uint32_t *dest_bits;
-    int dest_dx, dest_dy;
-    pixman_bool_t need_workaround;
-    const pixman_fast_path_t *info;
-    cache_t *cache;
-    int i;
+    pixman_box32_t extents;
+    pixman_implementation_t *imp;
+    pixman_composite_func_t func;
+    pixman_composite_info_t info;
+    const pixman_box32_t *pbox;
+    int n;
+
+    _pixman_image_validate (src);
+    if (mask)
+	_pixman_image_validate (mask);
+    _pixman_image_validate (dest);
 
     src_format = src->common.extended_format_code;
-    src_flags = src->common.flags;
+    info.src_flags = src->common.flags;
 
-    if (mask)
+    if (mask && !(mask->common.flags & FAST_PATH_IS_OPAQUE))
     {
 	mask_format = mask->common.extended_format_code;
-	mask_flags = mask->common.flags;
+	info.mask_flags = mask->common.flags;
     }
     else
     {
 	mask_format = PIXMAN_null;
-	mask_flags = FAST_PATH_IS_OPAQUE;
+	info.mask_flags = FAST_PATH_IS_OPAQUE | FAST_PATH_NO_ALPHA_MAP;
     }
 
     dest_format = dest->common.extended_format_code;
-    dest_flags = dest->common.flags;
+    info.dest_flags = dest->common.flags;
 
     /* Check for pixbufs */
     if ((mask_format == PIXMAN_a8r8g8b8 || mask_format == PIXMAN_a8b8g8r8) &&
 	(src->type == BITS && src->bits.bits == mask->bits.bits)	   &&
 	(src->common.repeat == mask->common.repeat)			   &&
+	(info.src_flags & info.mask_flags & FAST_PATH_ID_TRANSFORM)	   &&
 	(src_x == mask_x && src_y == mask_y))
     {
 	if (src_format == PIXMAN_x8b8g8r8)
@@ -618,142 +624,92 @@ do_composite (pixman_implementation_t *imp,
 	    src_format = mask_format = PIXMAN_rpixbuf;
     }
 
-    /* Check for workaround */
-    need_workaround = (src_flags | mask_flags | dest_flags) & FAST_PATH_NEEDS_WORKAROUND;
-
-    if (need_workaround)
-    {
-	apply_workaround (src, &src_x, &src_y, &src_bits, &src_dx, &src_dy);
-	apply_workaround (mask, &mask_x, &mask_y, &mask_bits, &mask_dx, &mask_dy);
-	apply_workaround (dest, &dest_x, &dest_y, &dest_bits, &dest_dx, &dest_dy);
-    }
-
     pixman_region32_init (&region);
-    
-    if (!pixman_compute_composite_region32 (
+
+    if (!_pixman_compute_composite_region32 (
 	    &region, src, mask, dest,
 	    src_x, src_y, mask_x, mask_y, dest_x, dest_y, width, height))
     {
 	goto out;
     }
-    
-    extents = pixman_region32_extents (&region);
-    
-    src_flags |= compute_src_extents_flags (src, extents, dest_x - src_x, dest_y - src_y);
 
-    if (mask)
-	mask_flags |= compute_src_extents_flags (mask, extents, dest_x - mask_x, dest_y - mask_y);
+    extents = *pixman_region32_extents (&region);
 
-    /*
-     * Check if we can replace our operator by a simpler one
-     * if the src or dest are opaque. The output operator should be
-     * mathematically equivalent to the source.
-     */
-    op = optimize_operator (op, src_flags, mask_flags, dest_flags);
-    if (op == PIXMAN_OP_DST)
+    extents.x1 -= dest_x - src_x;
+    extents.y1 -= dest_y - src_y;
+    extents.x2 -= dest_x - src_x;
+    extents.y2 -= dest_y - src_y;
+
+    if (!analyze_extent (src, &extents, &info.src_flags))
 	goto out;
 
-    /* Check cache for fast paths */
-    cache = PIXMAN_GET_THREAD_LOCAL (fast_path_cache);
+    extents.x1 -= src_x - mask_x;
+    extents.y1 -= src_y - mask_y;
+    extents.x2 -= src_x - mask_x;
+    extents.y2 -= src_y - mask_y;
 
-    for (i = 0; i < N_CACHED_FAST_PATHS; ++i)
-    {
-	info = &(cache->cache[i]);
+    if (!analyze_extent (mask, &extents, &info.mask_flags))
+	goto out;
 
-	/* Note that we check for equality here, not whether
-	 * the cached fast path matches. This is to prevent
-	 * us from selecting an overly general fast path
-	 * when a more specific one would work.
-	 */
-	if (info->op == op			&&
-	    info->src_format == src_format	&&
-	    info->mask_format == mask_format	&&
-	    info->dest_format == dest_format	&&
-	    info->src_flags == src_flags	&&
-	    info->mask_flags == mask_flags	&&
-	    info->dest_flags == dest_flags	&&
-	    info->func)
-	{
-	    goto found;
-	}
+    /* If the clip is within the source samples, and the samples are
+     * opaque, then the source is effectively opaque.
+     */
+#define NEAREST_OPAQUE	(FAST_PATH_SAMPLES_OPAQUE |			\
+			 FAST_PATH_NEAREST_FILTER |			\
+			 FAST_PATH_SAMPLES_COVER_CLIP_NEAREST)
+#define BILINEAR_OPAQUE	(FAST_PATH_SAMPLES_OPAQUE |			\
+			 FAST_PATH_BILINEAR_FILTER |			\
+			 FAST_PATH_SAMPLES_COVER_CLIP_BILINEAR)
+
+    if ((info.src_flags & NEAREST_OPAQUE) == NEAREST_OPAQUE ||
+	(info.src_flags & BILINEAR_OPAQUE) == BILINEAR_OPAQUE)
+    {
+	info.src_flags |= FAST_PATH_IS_OPAQUE;
     }
 
-    while (imp)
+    if ((info.mask_flags & NEAREST_OPAQUE) == NEAREST_OPAQUE ||
+	(info.mask_flags & BILINEAR_OPAQUE) == BILINEAR_OPAQUE)
     {
-	info = imp->fast_paths;
+	info.mask_flags |= FAST_PATH_IS_OPAQUE;
+    }
 
-	while (info->op != PIXMAN_OP_NONE)
-	{
-	    if ((info->op == op || info->op == PIXMAN_OP_any)		&&
-		/* Formats */
-		((info->src_format == src_format) ||
-		 (info->src_format == PIXMAN_any))			&&
-		((info->mask_format == mask_format) ||
-		 (info->mask_format == PIXMAN_any))			&&
-		((info->dest_format == dest_format) ||
-		 (info->dest_format == PIXMAN_any))			&&
-		/* Flags */
-		(info->src_flags & src_flags) == info->src_flags	&&
-		(info->mask_flags & mask_flags) == info->mask_flags	&&
-		(info->dest_flags & dest_flags) == info->dest_flags)
-	    {
-		/* Set i to the last spot in the cache so that the
-		 * move-to-front code below will work
-		 */
-		i = N_CACHED_FAST_PATHS - 1;
+    /*
+     * Check if we can replace our operator by a simpler one
+     * if the src or dest are opaque. The output operator should be
+     * mathematically equivalent to the source.
+     */
+    info.op = optimize_operator (op, info.src_flags, info.mask_flags, info.dest_flags);
 
-		goto found;
-	    }
+    _pixman_implementation_lookup_composite (
+	get_implementation (), info.op,
+	src_format, info.src_flags,
+	mask_format, info.mask_flags,
+	dest_format, info.dest_flags,
+	&imp, &func);
 
-	    ++info;
-	}
+    info.src_image = src;
+    info.mask_image = mask;
+    info.dest_image = dest;
 
-	imp = imp->delegate;
-    }
+    pbox = pixman_region32_rectangles (&region, &n);
 
-    /* We didn't find a compositing routine. This should not happen, but if
-     * it somehow does, just exit rather than crash.
-     */
-    goto out;
-
-found:
-    walk_region_internal (imp, op,
-			  src, mask, dest,
-			  src_x, src_y, mask_x, mask_y,
-			  dest_x, dest_y,
-			  width, height,
-			  (src_flags & FAST_PATH_SIMPLE_REPEAT),
-			  (mask_flags & FAST_PATH_SIMPLE_REPEAT),
-			  &region, info->func);
-
-    if (i)
+    while (n--)
     {
-	/* Make a copy of info->func, because info->func may change when
-	 * we update the cache.
-	 */
-	pixman_composite_func_t func = info->func;
-	
-	while (i--)
-	    cache->cache[i + 1] = cache->cache[i];
-
-	cache->cache[0].op = op;
-	cache->cache[0].src_format = src_format;
-	cache->cache[0].src_flags = src_flags;
-	cache->cache[0].mask_format = mask_format;
-	cache->cache[0].mask_flags = mask_flags;
-	cache->cache[0].dest_format = dest_format;
-	cache->cache[0].dest_flags = dest_flags;
-	cache->cache[0].func = func;
-    }
+	info.src_x = pbox->x1 + src_x - dest_x;
+	info.src_y = pbox->y1 + src_y - dest_y;
+	info.mask_x = pbox->x1 + mask_x - dest_x;
+	info.mask_y = pbox->y1 + mask_y - dest_y;
+	info.dest_x = pbox->x1;
+	info.dest_y = pbox->y1;
+	info.width = pbox->x2 - pbox->x1;
+	info.height = pbox->y2 - pbox->y1;
 
-out:
-    if (need_workaround)
-    {
-	unapply_workaround (src, src_bits, src_dx, src_dy);
-	unapply_workaround (mask, mask_bits, mask_dx, mask_dy);
-	unapply_workaround (dest, dest_bits, dest_dx, dest_dy);
+	func (imp, &info);
+
+	pbox++;
     }
 
+out:
     pixman_region32_fini (&region);
 }
 
@@ -775,55 +731,6 @@ pixman_image_composite (pixman_op_t      op,
                               mask_x, mask_y, dest_x, dest_y, width, height);
 }
 
-/*
- * Work around GCC bug causing crashes in Mozilla with SSE2
- *
- * When using -msse, gcc generates movdqa instructions assuming that
- * the stack is 16 byte aligned. Unfortunately some applications, such
- * as Mozilla and Mono, end up aligning the stack to 4 bytes, which
- * causes the movdqa instructions to fail.
- *
- * The __force_align_arg_pointer__ makes gcc generate a prologue that
- * realigns the stack pointer to 16 bytes.
- *
- * On x86-64 this is not necessary because the standard ABI already
- * calls for a 16 byte aligned stack.
- *
- * See https://bugs.freedesktop.org/show_bug.cgi?id=15693
- */
-#if defined (USE_SSE2) && defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
-__attribute__((__force_align_arg_pointer__))
-#endif
-PIXMAN_EXPORT void
-pixman_image_composite32 (pixman_op_t      op,
-                          pixman_image_t * src,
-                          pixman_image_t * mask,
-                          pixman_image_t * dest,
-                          int32_t          src_x,
-                          int32_t          src_y,
-                          int32_t          mask_x,
-                          int32_t          mask_y,
-                          int32_t          dest_x,
-                          int32_t          dest_y,
-                          int32_t          width,
-                          int32_t          height)
-{
-    _pixman_image_validate (src);
-    if (mask)
-	_pixman_image_validate (mask);
-    _pixman_image_validate (dest);
-
-    if (!imp)
-	imp = _pixman_choose_implementation ();
-
-    do_composite (imp, op,
-		  src, mask, dest,
-		  src_x, src_y,
-		  mask_x, mask_y,
-		  dest_x, dest_y,
-		  width, height);
-}
-
 PIXMAN_EXPORT pixman_bool_t
 pixman_blt (uint32_t *src_bits,
             uint32_t *dst_bits,
@@ -833,18 +740,16 @@ pixman_blt (uint32_t *src_bits,
             int       dst_bpp,
             int       src_x,
             int       src_y,
-            int       dst_x,
-            int       dst_y,
+            int       dest_x,
+            int       dest_y,
             int       width,
             int       height)
 {
-    if (!imp)
-	imp = _pixman_choose_implementation ();
-
-    return _pixman_implementation_blt (imp, src_bits, dst_bits, src_stride, dst_stride,
+    return _pixman_implementation_blt (get_implementation(),
+				       src_bits, dst_bits, src_stride, dst_stride,
                                        src_bpp, dst_bpp,
                                        src_x, src_y,
-                                       dst_x, dst_y,
+                                       dest_x, dest_y,
                                        width, height);
 }
 
@@ -856,12 +761,10 @@ pixman_fill (uint32_t *bits,
              int       y,
              int       width,
              int       height,
-             uint32_t xor)
+             uint32_t  filler)
 {
-    if (!imp)
-	imp = _pixman_choose_implementation ();
-
-    return _pixman_implementation_fill (imp, bits, stride, bpp, x, y, width, height, xor);
+    return _pixman_implementation_fill (
+	get_implementation(), bits, stride, bpp, x, y, width, height, filler);
 }
 
 static uint32_t
@@ -875,9 +778,9 @@ color_to_uint32 (const pixman_color_t *color)
 }
 
 static pixman_bool_t
-color_to_pixel (pixman_color_t *     color,
-                uint32_t *           pixel,
-                pixman_format_code_t format)
+color_to_pixel (const pixman_color_t *color,
+                uint32_t *            pixel,
+                pixman_format_code_t  format)
 {
     uint32_t c = color_to_uint32 (color);
 
@@ -887,9 +790,12 @@ color_to_pixel (pixman_color_t *     color,
           format == PIXMAN_x8b8g8r8     ||
           format == PIXMAN_b8g8r8a8     ||
           format == PIXMAN_b8g8r8x8     ||
+          format == PIXMAN_r8g8b8a8     ||
+          format == PIXMAN_r8g8b8x8     ||
           format == PIXMAN_r5g6b5       ||
           format == PIXMAN_b5g6r5       ||
-          format == PIXMAN_a8))
+          format == PIXMAN_a8           ||
+          format == PIXMAN_a1))
     {
 	return FALSE;
     }
@@ -908,12 +814,16 @@ color_to_pixel (pixman_color_t *     color,
 	    ((c & 0x0000ff00) <<  8) |
 	    ((c & 0x000000ff) << 24);
     }
+    if (PIXMAN_FORMAT_TYPE (format) == PIXMAN_TYPE_RGBA)
+	c = ((c & 0xff000000) >> 24) | (c << 8);
 
-    if (format == PIXMAN_a8)
+    if (format == PIXMAN_a1)
+	c = c >> 31;
+    else if (format == PIXMAN_a8)
 	c = c >> 24;
     else if (format == PIXMAN_r5g6b5 ||
              format == PIXMAN_b5g6r5)
-	c = CONVERT_8888_TO_0565 (c);
+	c = convert_8888_to_0565 (c);
 
 #if 0
     printf ("color: %x %x %x %x\n", color->alpha, color->red, color->green, color->blue);
@@ -927,7 +837,7 @@ color_to_pixel (pixman_color_t *     color,
 PIXMAN_EXPORT pixman_bool_t
 pixman_image_fill_rectangles (pixman_op_t                 op,
                               pixman_image_t *            dest,
-                              pixman_color_t *            color,
+			      const pixman_color_t *      color,
                               int                         n_rects,
                               const pixman_rectangle16_t *rects)
 {
@@ -966,7 +876,7 @@ pixman_image_fill_rectangles (pixman_op_t                 op,
 PIXMAN_EXPORT pixman_bool_t
 pixman_image_fill_boxes (pixman_op_t           op,
                          pixman_image_t *      dest,
-                         pixman_color_t *      color,
+                         const pixman_color_t *color,
                          int                   n_boxes,
                          const pixman_box32_t *boxes)
 {
@@ -1111,15 +1021,19 @@ pixman_format_supported_source (pixman_format_code_t format)
     case PIXMAN_a2r10g10b10:
     case PIXMAN_x2r10g10b10:
     case PIXMAN_a8r8g8b8:
+    case PIXMAN_a8r8g8b8_sRGB:
     case PIXMAN_x8r8g8b8:
     case PIXMAN_a8b8g8r8:
     case PIXMAN_x8b8g8r8:
     case PIXMAN_b8g8r8a8:
     case PIXMAN_b8g8r8x8:
+    case PIXMAN_r8g8b8a8:
+    case PIXMAN_r8g8b8x8:
     case PIXMAN_r8g8b8:
     case PIXMAN_b8g8r8:
     case PIXMAN_r5g6b5:
     case PIXMAN_b5g6r5:
+    case PIXMAN_x14r6g6b6:
     /* 16 bpp formats */
     case PIXMAN_a1r5g5b5:
     case PIXMAN_x1r5g5b5:
@@ -1190,7 +1104,7 @@ PIXMAN_EXPORT pixman_bool_t
 pixman_compute_composite_region (pixman_region16_t * region,
                                  pixman_image_t *    src_image,
                                  pixman_image_t *    mask_image,
-                                 pixman_image_t *    dst_image,
+                                 pixman_image_t *    dest_image,
                                  int16_t             src_x,
                                  int16_t             src_y,
                                  int16_t             mask_x,
@@ -1205,8 +1119,8 @@ pixman_compute_composite_region (pixman_region16_t * region,
 
     pixman_region32_init (&r32);
 
-    retval = pixman_compute_composite_region32 (
-	&r32, src_image, mask_image, dst_image,
+    retval = _pixman_compute_composite_region32 (
+	&r32, src_image, mask_image, dest_image,
 	src_x, src_y, mask_x, mask_y, dest_x, dest_y,
 	width, height);
 
diff --git a/pixman/pixman/pixman.h b/pixman/pixman/pixman.h
index 964d04ab9..509ba5e53 100644
--- a/pixman/pixman/pixman.h
+++ b/pixman/pixman/pixman.h
@@ -226,6 +226,9 @@ pixman_bool_t pixman_transform_is_inverse       (const struct pixman_transform *
 /*
  * Floating point matrices
  */
+typedef struct pixman_f_transform pixman_f_transform_t;
+typedef struct pixman_f_vector pixman_f_vector_t;
+
 struct pixman_f_vector
 {
     double  v[3];
@@ -289,7 +292,28 @@ typedef enum
     PIXMAN_FILTER_BEST,
     PIXMAN_FILTER_NEAREST,
     PIXMAN_FILTER_BILINEAR,
-    PIXMAN_FILTER_CONVOLUTION
+    PIXMAN_FILTER_CONVOLUTION,
+
+    /* The SEPARABLE_CONVOLUTION filter takes the following parameters:
+     *
+     *         width:           integer given as 16.16 fixpoint number
+     *         height:          integer given as 16.16 fixpoint number
+     *         x_phase_bits:	integer given as 16.16 fixpoint
+     *         y_phase_bits:	integer given as 16.16 fixpoint
+     *         xtables:         (1 << x_phase_bits) tables of size width
+     *         ytables:         (1 << y_phase_bits) tables of size height
+     *
+     * When sampling at (x, y), the location is first rounded to one of
+     * n_x_phases * n_y_phases subpixel positions. These subpixel positions
+     * determine an xtable and a ytable to use.
+     *
+     * Conceptually a width x height matrix is then formed in which each entry
+     * is the product of the corresponding entries in the x and y tables.
+     * This matrix is then aligned with the image pixels such that its center
+     * is as close as possible to the subpixel location chosen earlier. Then
+     * the image is convolved with the matrix and the resulting pixel returned.
+     */
+    PIXMAN_FILTER_SEPARABLE_CONVOLUTION
 } pixman_filter_t;
 
 typedef enum
@@ -420,7 +444,6 @@ void                    pixman_region_init_from_image    (pixman_region16_t *reg
 void                    pixman_region_fini               (pixman_region16_t *region);
 
 
-
 /* manipulation */
 void                    pixman_region_translate          (pixman_region16_t *region,
 							  int                x,
@@ -439,6 +462,12 @@ pixman_bool_t           pixman_region_union_rect         (pixman_region16_t *des
 							  int                y,
 							  unsigned int       width,
 							  unsigned int       height);
+pixman_bool_t		pixman_region_intersect_rect     (pixman_region16_t *dest,
+							  pixman_region16_t *source,
+							  int                x,
+							  int                y,
+							  unsigned int       width,
+							  unsigned int       height);
 pixman_bool_t           pixman_region_subtract           (pixman_region16_t *reg_d,
 							  pixman_region16_t *reg_m,
 							  pixman_region16_t *reg_s);
@@ -461,6 +490,7 @@ pixman_bool_t           pixman_region_equal              (pixman_region16_t *reg
 pixman_bool_t           pixman_region_selfcheck          (pixman_region16_t *region);
 void                    pixman_region_reset              (pixman_region16_t *region,
 							  pixman_box16_t    *box);
+void			pixman_region_clear		 (pixman_region16_t *region);
 /*
  * 32 bit regions
  */
@@ -521,6 +551,12 @@ pixman_bool_t           pixman_region32_intersect          (pixman_region32_t *n
 pixman_bool_t           pixman_region32_union              (pixman_region32_t *new_reg,
 							    pixman_region32_t *reg1,
 							    pixman_region32_t *reg2);
+pixman_bool_t		pixman_region32_intersect_rect     (pixman_region32_t *dest,
+							    pixman_region32_t *source,
+							    int                x,
+							    int                y,
+							    unsigned int       width,
+							    unsigned int       height);
 pixman_bool_t           pixman_region32_union_rect         (pixman_region32_t *dest,
 							    pixman_region32_t *source,
 							    int                x,
@@ -549,6 +585,7 @@ pixman_bool_t           pixman_region32_equal              (pixman_region32_t *r
 pixman_bool_t           pixman_region32_selfcheck          (pixman_region32_t *region);
 void                    pixman_region32_reset              (pixman_region32_t *region,
 							    pixman_box32_t    *box);
+void			pixman_region32_clear		   (pixman_region32_t *region);
 
 
 /* Copy / Fill / Misc */
@@ -560,8 +597,8 @@ pixman_bool_t pixman_blt                (uint32_t           *src_bits,
 					 int                 dst_bpp,
 					 int                 src_x,
 					 int                 src_y,
-					 int                 dst_x,
-					 int                 dst_y,
+					 int                 dest_x,
+					 int                 dest_y,
 					 int                 width,
 					 int                 height);
 pixman_bool_t pixman_fill               (uint32_t           *bits,
@@ -639,11 +676,14 @@ struct pixman_indexed
 #define PIXMAN_TYPE_YUY2	6
 #define PIXMAN_TYPE_YV12	7
 #define PIXMAN_TYPE_BGRA	8
+#define PIXMAN_TYPE_RGBA	9
+#define PIXMAN_TYPE_ARGB_SRGB	10
 
 #define PIXMAN_FORMAT_COLOR(f)				\
 	(PIXMAN_FORMAT_TYPE(f) == PIXMAN_TYPE_ARGB ||	\
 	 PIXMAN_FORMAT_TYPE(f) == PIXMAN_TYPE_ABGR ||	\
-	 PIXMAN_FORMAT_TYPE(f) == PIXMAN_TYPE_BGRA)
+	 PIXMAN_FORMAT_TYPE(f) == PIXMAN_TYPE_BGRA ||	\
+	 PIXMAN_FORMAT_TYPE(f) == PIXMAN_TYPE_RGBA)
 
 /* 32bpp formats */
 typedef enum {
@@ -653,11 +693,17 @@ typedef enum {
     PIXMAN_x8b8g8r8 =	 PIXMAN_FORMAT(32,PIXMAN_TYPE_ABGR,0,8,8,8),
     PIXMAN_b8g8r8a8 =	 PIXMAN_FORMAT(32,PIXMAN_TYPE_BGRA,8,8,8,8),
     PIXMAN_b8g8r8x8 =	 PIXMAN_FORMAT(32,PIXMAN_TYPE_BGRA,0,8,8,8),
+    PIXMAN_r8g8b8a8 =	 PIXMAN_FORMAT(32,PIXMAN_TYPE_RGBA,8,8,8,8),
+    PIXMAN_r8g8b8x8 =	 PIXMAN_FORMAT(32,PIXMAN_TYPE_RGBA,0,8,8,8),
+    PIXMAN_x14r6g6b6 =	 PIXMAN_FORMAT(32,PIXMAN_TYPE_ARGB,0,6,6,6),
     PIXMAN_x2r10g10b10 = PIXMAN_FORMAT(32,PIXMAN_TYPE_ARGB,0,10,10,10),
     PIXMAN_a2r10g10b10 = PIXMAN_FORMAT(32,PIXMAN_TYPE_ARGB,2,10,10,10),
     PIXMAN_x2b10g10r10 = PIXMAN_FORMAT(32,PIXMAN_TYPE_ABGR,0,10,10,10),
     PIXMAN_a2b10g10r10 = PIXMAN_FORMAT(32,PIXMAN_TYPE_ABGR,2,10,10,10),
 
+/* sRGB formats */
+    PIXMAN_a8r8g8b8_sRGB = PIXMAN_FORMAT(32,PIXMAN_TYPE_ARGB_SRGB,8,8,8,8),
+
 /* 24bpp formats */
     PIXMAN_r8g8b8 =	 PIXMAN_FORMAT(24,PIXMAN_TYPE_ARGB,0,8,8,8),
     PIXMAN_b8g8r8 =	 PIXMAN_FORMAT(24,PIXMAN_TYPE_ABGR,0,8,8,8),
@@ -715,18 +761,18 @@ pixman_bool_t pixman_format_supported_destination (pixman_format_code_t format);
 pixman_bool_t pixman_format_supported_source      (pixman_format_code_t format);
 
 /* Constructors */
-pixman_image_t *pixman_image_create_solid_fill       (pixman_color_t               *color);
-pixman_image_t *pixman_image_create_linear_gradient  (pixman_point_fixed_t         *p1,
-						      pixman_point_fixed_t         *p2,
+pixman_image_t *pixman_image_create_solid_fill       (const pixman_color_t         *color);
+pixman_image_t *pixman_image_create_linear_gradient  (const pixman_point_fixed_t   *p1,
+						      const pixman_point_fixed_t   *p2,
 						      const pixman_gradient_stop_t *stops,
 						      int                           n_stops);
-pixman_image_t *pixman_image_create_radial_gradient  (pixman_point_fixed_t         *inner,
-						      pixman_point_fixed_t         *outer,
+pixman_image_t *pixman_image_create_radial_gradient  (const pixman_point_fixed_t   *inner,
+						      const pixman_point_fixed_t   *outer,
 						      pixman_fixed_t                inner_radius,
 						      pixman_fixed_t                outer_radius,
 						      const pixman_gradient_stop_t *stops,
 						      int                           n_stops);
-pixman_image_t *pixman_image_create_conical_gradient (pixman_point_fixed_t         *center,
+pixman_image_t *pixman_image_create_conical_gradient (const pixman_point_fixed_t   *center,
 						      pixman_fixed_t                angle,
 						      const pixman_gradient_stop_t *stops,
 						      int                           n_stops);
@@ -735,6 +781,11 @@ pixman_image_t *pixman_image_create_bits             (pixman_format_code_t
 						      int                           height,
 						      uint32_t                     *bits,
 						      int                           rowstride_bytes);
+pixman_image_t *pixman_image_create_bits_no_clear    (pixman_format_code_t format,
+						      int                  width,
+						      int                  height,
+						      uint32_t *           bits,
+						      int                  rowstride_bytes);
 
 /* Destructor */
 pixman_image_t *pixman_image_ref                     (pixman_image_t               *image);
@@ -768,6 +819,7 @@ void            pixman_image_set_alpha_map           (pixman_image_t
 						      int16_t                       y);
 void            pixman_image_set_component_alpha     (pixman_image_t               *image,
 						      pixman_bool_t                 component_alpha);
+pixman_bool_t   pixman_image_get_component_alpha     (pixman_image_t               *image);
 void		pixman_image_set_accessors	     (pixman_image_t		   *image,
 						      pixman_read_memory_func_t	    read_func,
 						      pixman_write_memory_func_t    write_func);
@@ -778,14 +830,42 @@ int		pixman_image_get_width               (pixman_image_t               *image);
 int             pixman_image_get_height              (pixman_image_t               *image);
 int		pixman_image_get_stride              (pixman_image_t               *image); /* in bytes */
 int		pixman_image_get_depth               (pixman_image_t		   *image);
+pixman_format_code_t pixman_image_get_format	     (pixman_image_t		   *image);
+
+typedef enum
+{
+    PIXMAN_KERNEL_IMPULSE,
+    PIXMAN_KERNEL_BOX,
+    PIXMAN_KERNEL_LINEAR,
+    PIXMAN_KERNEL_CUBIC,
+    PIXMAN_KERNEL_GAUSSIAN,
+    PIXMAN_KERNEL_LANCZOS2,
+    PIXMAN_KERNEL_LANCZOS3,
+    PIXMAN_KERNEL_LANCZOS3_STRETCHED       /* Jim Blinn's 'nice' filter */
+} pixman_kernel_t;
+
+/* Create the parameter list for a SEPARABLE_CONVOLUTION filter
+ * with the given kernels and scale parameters.
+ */
+pixman_fixed_t *
+pixman_filter_create_separable_convolution (int             *n_values,
+					    pixman_fixed_t   scale_x,
+					    pixman_fixed_t   scale_y,
+					    pixman_kernel_t  reconstruct_x,
+					    pixman_kernel_t  reconstruct_y,
+					    pixman_kernel_t  sample_x,
+					    pixman_kernel_t  sample_y,
+					    int              subsample_bits_x,
+					    int              subsample_bits_y);
+
 pixman_bool_t	pixman_image_fill_rectangles	     (pixman_op_t		    op,
 						      pixman_image_t		   *image,
-						      pixman_color_t		   *color,
+						      const pixman_color_t	   *color,
 						      int			    n_rects,
 						      const pixman_rectangle16_t   *rects);
 pixman_bool_t   pixman_image_fill_boxes              (pixman_op_t                   op,
                                                       pixman_image_t               *dest,
-                                                      pixman_color_t               *color,
+                                                      const pixman_color_t         *color,
                                                       int                           n_boxes,
                                                       const pixman_box32_t         *boxes);
 
@@ -793,7 +873,7 @@ pixman_bool_t   pixman_image_fill_boxes              (pixman_op_t
 pixman_bool_t pixman_compute_composite_region (pixman_region16_t *region,
 					       pixman_image_t    *src_image,
 					       pixman_image_t    *mask_image,
-					       pixman_image_t    *dst_image,
+					       pixman_image_t    *dest_image,
 					       int16_t            src_x,
 					       int16_t            src_y,
 					       int16_t            mask_x,
@@ -827,19 +907,84 @@ void          pixman_image_composite32        (pixman_op_t        op,
 					       int32_t            width,
 					       int32_t            height);
 
-/* Old X servers rely on out-of-bounds accesses when they are asked
- * to composite with a window as the source. They create a pixman image
- * pointing to some bogus position in memory, but then they set a clip
- * region to the position where the actual bits are.
+/* Executive Summary: This function is a no-op that only exists
+ * for historical reasons.
+ *
+ * There used to be a bug in the X server where it would rely on
+ * out-of-bounds accesses when it was asked to composite with a
+ * window as the source. It would create a pixman image pointing
+ * to some bogus position in memory, but then set a clip region
+ * to the position where the actual bits were.
  *
  * Due to a bug in old versions of pixman, where it would not clip
  * against the image bounds when a clip region was set, this would
- * actually work. So by default we allow certain out-of-bound access
- * to happen unless explicitly disabled.
+ * actually work. So when the pixman bug was fixed, a workaround was
+ * added to allow certain out-of-bound accesses. This function disabled
+ * those workarounds.
  *
- * Fixed X servers should call this function to disable the workaround.
+ * Since 0.21.2, pixman doesn't do these workarounds anymore, so now this
+ * function is a no-op.
  */
-void          pixman_disable_out_of_bounds_workaround (void);
+void pixman_disable_out_of_bounds_workaround (void);
+
+/*
+ * Glyphs
+ */
+typedef struct pixman_glyph_cache_t pixman_glyph_cache_t;
+typedef struct
+{
+    int		x, y;
+    const void *glyph;
+} pixman_glyph_t;
+
+pixman_glyph_cache_t *pixman_glyph_cache_create       (void);
+void                  pixman_glyph_cache_destroy      (pixman_glyph_cache_t *cache);
+void                  pixman_glyph_cache_freeze       (pixman_glyph_cache_t *cache);
+void                  pixman_glyph_cache_thaw         (pixman_glyph_cache_t *cache);
+const void *          pixman_glyph_cache_lookup       (pixman_glyph_cache_t *cache,
+						       void                 *font_key,
+						       void                 *glyph_key);
+const void *          pixman_glyph_cache_insert       (pixman_glyph_cache_t *cache,
+						       void                 *font_key,
+						       void                 *glyph_key,
+						       int		     origin_x,
+						       int                   origin_y,
+						       pixman_image_t       *glyph_image);
+void                  pixman_glyph_cache_remove       (pixman_glyph_cache_t *cache,
+						       void                 *font_key,
+						       void                 *glyph_key);
+void                  pixman_glyph_get_extents        (pixman_glyph_cache_t *cache,
+						       int                   n_glyphs,
+						       pixman_glyph_t       *glyphs,
+						       pixman_box32_t       *extents);
+pixman_format_code_t  pixman_glyph_get_mask_format    (pixman_glyph_cache_t *cache,
+						       int		     n_glyphs,
+						       const pixman_glyph_t *glyphs);
+void                  pixman_composite_glyphs         (pixman_op_t           op,
+						       pixman_image_t       *src,
+						       pixman_image_t       *dest,
+						       pixman_format_code_t  mask_format,
+						       int32_t               src_x,
+						       int32_t               src_y,
+						       int32_t		     mask_x,
+						       int32_t		     mask_y,
+						       int32_t               dest_x,
+						       int32_t               dest_y,
+						       int32_t		     width,
+						       int32_t		     height,
+						       pixman_glyph_cache_t *cache,
+						       int		     n_glyphs,
+						       const pixman_glyph_t *glyphs);
+void                  pixman_composite_glyphs_no_mask (pixman_op_t           op,
+						       pixman_image_t       *src,
+						       pixman_image_t       *dest,
+						       int32_t               src_x,
+						       int32_t               src_y,
+						       int32_t               dest_x,
+						       int32_t               dest_y,
+						       pixman_glyph_cache_t *cache,
+						       int		     n_glyphs,
+						       const pixman_glyph_t *glyphs);
 
 /*
  * Trapezoids
@@ -848,6 +993,7 @@ typedef struct pixman_edge pixman_edge_t;
 typedef struct pixman_trapezoid pixman_trapezoid_t;
 typedef struct pixman_trap pixman_trap_t;
 typedef struct pixman_span_fix pixman_span_fix_t;
+typedef struct pixman_triangle pixman_triangle_t;
 
 /*
  * An edge structure.  This represents a single polygon edge
@@ -875,12 +1021,16 @@ struct pixman_trapezoid
     pixman_line_fixed_t	left, right;
 };
 
+struct pixman_triangle
+{
+    pixman_point_fixed_t p1, p2, p3;
+};
 
 /* whether 't' is a well defined not obviously empty trapezoid */
 #define pixman_trapezoid_valid(t)				   \
     ((t)->left.p1.y != (t)->left.p2.y &&			   \
      (t)->right.p1.y != (t)->right.p2.y &&			   \
-     (int) ((t)->bottom - (t)->top) > 0)
+     ((t)->bottom > (t)->top))
 
 struct pixman_span_fix
 {
@@ -920,7 +1070,7 @@ void           pixman_add_traps            (pixman_image_t            *image,
 					    int16_t                    x_off,
 					    int16_t                    y_off,
 					    int                        ntrap,
-					    pixman_trap_t             *traps);
+					    const pixman_trap_t       *traps);
 void           pixman_add_trapezoids       (pixman_image_t            *image,
 					    int16_t                    x_off,
 					    int                        y_off,
@@ -930,6 +1080,31 @@ void           pixman_rasterize_trapezoid  (pixman_image_t            *image,
 					    const pixman_trapezoid_t  *trap,
 					    int                        x_off,
 					    int                        y_off);
+void          pixman_composite_trapezoids (pixman_op_t		       op,
+					   pixman_image_t *	       src,
+					   pixman_image_t *	       dst,
+					   pixman_format_code_t	       mask_format,
+					   int			       x_src,
+					   int			       y_src,
+					   int			       x_dst,
+					   int			       y_dst,
+					   int			       n_traps,
+					   const pixman_trapezoid_t *  traps);
+void          pixman_composite_triangles (pixman_op_t		       op,
+					  pixman_image_t *	       src,
+					  pixman_image_t *	       dst,
+					  pixman_format_code_t	       mask_format,
+					  int			       x_src,
+					  int			       y_src,
+					  int			       x_dst,
+					  int			       y_dst,
+					  int			       n_tris,
+					  const pixman_triangle_t *    tris);
+void	      pixman_add_triangles       (pixman_image_t              *image,
+					  int32_t	               x_off,
+					  int32_t	               y_off,
+					  int	                       n_tris,
+					  const pixman_triangle_t     *tris);
 
 PIXMAN_END_DECLS
 
diff --git a/pixman/pixman/refactor b/pixman/pixman/refactor
deleted file mode 100644
index 52fceab17..000000000
--- a/pixman/pixman/refactor
+++ /dev/null
@@ -1,478 +0,0 @@
-Roadmap
-
-- Move all the fetchers etc. into pixman-image to make pixman-compose.c
-  less intimidating.
-
-  DONE
-
-- Make combiners for unified alpha take a mask argument. That way
-  we won't need two separate paths for unified vs component in the
-  general compositing code.
-
-  DONE, except that the Altivec code needs to be updated. Luca is
-  looking into that.
-
-- Delete separate 'unified alpha' path
- 
-  DONE
-
-- Split images into their own files
-
-  DONE
-
-- Split the gradient walker code out into its own file
-
-  DONE
-
-- Add scanline getters per image
-
-  DONE
-
-- Generic 64 bit fetcher 
-
-  DONE
-
-- Split fast path tables into their respective architecture dependent
-  files.
-
-See "Render Algorithm" below for rationale
-
-Images will eventually have these virtual functions:
-
-       get_scanline()
-       get_scanline_wide()
-       get_pixel()
-       get_pixel_wide()
-       get_untransformed_pixel()
-       get_untransformed_pixel_wide()
-       get_unfiltered_pixel()
-       get_unfiltered_pixel_wide()
-
-       store_scanline()
-       store_scanline_wide()
-
-1.
-
-Initially we will just have get_scanline() and get_scanline_wide();
-these will be based on the ones in pixman-compose. Hopefully this will
-reduce the complexity in pixman_composite_rect_general().
-
-Note that there is access considerations - the compose function is
-being compiled twice.
-
-
-2.
-
-Split image types into their own source files. Export noop virtual
-reinit() call.  Call this whenever a property of the image changes.
-
-
-3. 
-
-Split the get_scanline() call into smaller functions that are
-initialized by the reinit() call.
-
-The Render Algorithm:
-	(first repeat, then filter, then transform, then clip)
-
-Starting from a destination pixel (x, y), do
-
-	1 x = x - xDst + xSrc
-	  y = y - yDst + ySrc
-
-	2 reject pixel that is outside the clip
-
-	This treats clipping as something that happens after
-	transformation, which I think is correct for client clips. For
-	hierarchy clips it is wrong, but who really cares? Without
-	GraphicsExposes hierarchy clips are basically irrelevant. Yes,
-	you could imagine cases where the pixels of a subwindow of a
-	redirected, transformed window should be treated as
-	transparent. I don't really care
-
-	Basically, I think the render spec should say that pixels that
-	are unavailable due to the hierarcy have undefined content,
-	and that GraphicsExposes are not generated. Ie., basically
-	that using non-redirected windows as sources is fail. This is
-	at least consistent with the current implementation and we can
-	update the spec later if someone makes it work.
-
-	The implication for render is that it should stop passing the
-	hierarchy clip to pixman. In pixman, if a souce image has a
-	clip it should be used in computing the composite region and
-	nowhere else, regardless of what "has_client_clip" says. The
-	default should be for there to not be any clip.
-
-	I would really like to get rid of the client clip as well for
-	source images, but unfortunately there is at least one
-	application in the wild that uses them.
-
-	3 Transform pixel: (x, y) = T(x, y)
-
-	4 Call p = GetUntransformedPixel (x, y)
-
-	5 If the image has an alpha map, then
-
-		Call GetUntransformedPixel (x, y) on the alpha map
-		
-		add resulting alpha channel to p
-
-	   return p
-
-	Where GetUnTransformedPixel is:
-
-	6 switch (filter)
-	  {
-	  case NEAREST:
-		return GetUnfilteredPixel (x, y);
-		break;
-
-	  case BILINEAR:
-		return GetUnfilteredPixel (...) // 4 times 
-		break;
-
-	  case CONVOLUTION:
-		return GetUnfilteredPixel (...) // as many times as necessary.
-		break;
-	  }
-
-	Where GetUnfilteredPixel (x, y) is
-
-	7 switch (repeat)
-	   {
-	   case REPEAT_NORMAL:
-	   case REPEAT_PAD:
-	   case REPEAT_REFLECT:
-		// adjust x, y as appropriate
-		break;
-
-	   case REPEAT_NONE:
-	        if (x, y) is outside image bounds
-		     return 0;
-		break;
-	   }
-
-	   return GetRawPixel(x, y)
-
-	Where GetRawPixel (x, y) is
-
-	8 Compute the pixel in question, depending on image type.
-
-For gradients, repeat has a totally different meaning, so
-UnfilteredPixel() and RawPixel() must be the same function so that
-gradients can do their own repeat algorithm.
-
-So, the GetRawPixel
-
-	for bits must deal with repeats
-	for gradients must deal with repeats (differently)
-	for solids, should ignore repeats.
-
-	for polygons, when we add them, either ignore repeats or do
-	something similar to bits (in which case, we may want an extra
-	layer of indirection to modify the coordinates).
-
-It is then possible to build things like "get scanline" or "get tile" on
-top of this. In the simplest case, just repeatedly calling GetPixel()
-would work, but specialized get_scanline()s or get_tile()s could be
-plugged in for common cases. 
-
-By not plugging anything in for images with access functions, we only
-have to compile the pixel functions twice, not the scanline functions.
-
-And we can get rid of fetchers for the bizarre formats that no one
-uses. Such as b2g3r3 etc. r1g2b1? Seriously? It is also worth
-considering a generic format based pixel fetcher for these edge cases.
-
-Since the actual routines depend on the image attributes, the images
-must be notified when those change and update their function pointers
-appropriately. So there should probably be a virtual function called
-(* reinit) or something like that.
-
-There will also be wide fetchers for both pixels and lines. The line
-fetcher will just call the wide pixel fetcher. The wide pixel fetcher
-will just call expand, except for 10 bit formats.
-
-Rendering pipeline:
-
-Drawable:
-	0. if (picture has alpha map)
-		0.1. Position alpha map according to the alpha_x/alpha_y
-	        0.2. Where the two drawables intersect, the alpha channel
-		     Replace the alpha channel of source with the one
-		     from the alpha map. Replacement only takes place
-		     in the intersection of the two drawables' geometries.
-	1. Repeat the drawable according to the repeat attribute
-	2. Reconstruct a continuous image according to the filter
-	3. Transform according to the transform attribute
-	4. Position image such that src_x, src_y is over dst_x, dst_y
-	5. Sample once per destination pixel 
-	6. Clip. If a pixel is not within the source clip, then no
-	   compositing takes place at that pixel. (Ie., it's *not*
-	   treated as 0).
-
-	Sampling a drawable: 
-
-	- If the channel does not have an alpha channel, the pixels in it
-	  are treated as opaque.
-
-	Note on reconstruction:
-
-	- The top left pixel has coordinates (0.5, 0.5) and pixels are
-	  spaced 1 apart.
-
-Gradient:
-	1. Unless gradient type is conical, repeat the underlying (0, 1)
-		gradient according to the repeat attribute
-	2. Integrate the gradient across the plane according to type.
-	3. Transform according to transform attribute
-	4. Position gradient 
-	5. Sample once per destination pixel.
- 	6. Clip
-
-Solid Fill:
-	1. Repeat has no effect
-	2. Image is already continuous and defined for the entire plane
-	3. Transform has no effect
-	4. Positioning has no effect
-	5. Sample once per destination pixel.
-	6. Clip
-
-Polygon:
-	1. Repeat has no effect
-	2. Image is already continuous and defined on the whole plane
-	3. Transform according to transform attribute
-	4. Position image
-	5. Supersample 15x17 per destination pixel.
-	6. Clip
-
-Possibly interesting additions:
-	- More general transformations, such as warping, or general
-	  shading.
-
-	- Shader image where a function is called to generate the
-          pixel (ie., uploading assembly code).
-
-	- Resampling kernels
-
-	  In principle the polygon image uses a 15x17 box filter for
-	  resampling. If we allow general resampling filters, then we
-	  get all the various antialiasing types for free. 
-
-	  Bilinear downsampling looks terrible and could be much 
-	  improved by a resampling filter. NEAREST reconstruction
-	  combined with a box resampling filter is what GdkPixbuf
-	  does, I believe.
-
-	  Useful for high frequency gradients as well.
-
-	  (Note that the difference between a reconstruction and a
-	  resampling filter is mainly where in the pipeline they
-	  occur. High quality resampling should use a correctly
-	  oriented kernel so it should happen after transformation.
-
-	  An implementation can transform the resampling kernel and
-	  convolve it with the reconstruction if it so desires, but it
-	  will need to deal with the fact that the resampling kernel
-	  will not necessarily be pixel aligned.
-
-	  "Output kernels"
-
-	  One could imagine doing the resampling after compositing,
-	  ie., for each destination pixel sample each source image 16
-	  times, then composite those subpixels individually, then
-	  finally apply a kernel.
-
-	  However, this is effectively the same as full screen
-	  antialiasing, which is a simpler way to think about it. So
-	  resampling kernels may make sense for individual images, but
-	  not as a post-compositing step.
-	  
-	  Fullscreen AA is inefficient without chained compositing
-	  though. Consider an (image scaled up to oversample size IN
-	  some polygon) scaled down to screen size. With the current
-	  implementation, there will be a huge temporary. With chained
-	  compositing, the whole thing ends up being equivalent to the
-	  output kernel from above.
-
-	- Color space conversion
-
-	  The complete model here is that each surface has a color
-	  space associated with it and that the compositing operation
-	  also has one associated with it. Note also that gradients
-	  should have associcated colorspaces.
-
-	- Dithering
-
-	  If people dither something that is already dithered, it will
-	  look terrible, but don't do that, then. (Dithering happens
-	  after resampling if at all - what is the relationship
-	  with color spaces? Presumably dithering should happen in linear
-	  intensity space).
-
-	- Floating point surfaces, 16, 32 and possibly 64 bit per
-	  channel.
-
-	Maybe crack:
-
-	- Glyph polygons
-
-	  If glyphs could be given as polygons, they could be
-	  positioned and rasterized more accurately. The glyph
-	  structure would need subpixel positioning though.
-
-	- Luminance vs. coverage for the alpha channel
-
-	  Whether the alpha channel should be interpreted as luminance
-          modulation or as coverage (intensity modulation). This is a
-          bit of a departure from the rendering model though. It could
-	  also be considered whether it should be possible to have 
-	  both channels in the same drawable.
-
-	- Alternative for component alpha
-
-	  - Set component-alpha on the output image.
-
-	    - This means each of the components are sampled
-	      independently and composited in the corresponding
-	      channel only.
-
-	  - Have 3 x oversampled mask
-
-	  - Scale it down by 3 horizontally, with [ 1/3, 1/3, 1/3 ]
-            resampling filter. 
-
-	    Is this equivalent to just using a component alpha mask?
-
-	Incompatible changes:
-
-	- Gradients could be specified with premultiplied colors. (You
-	  can use a mask to get things like gradients from solid red to
-	  transparent red.
-
-Refactoring pixman
-
-The pixman code is not particularly nice to put it mildly. Among the
-issues are
-
-- inconsistent naming style (fb vs Fb, camelCase vs
-  underscore_naming). Sometimes there is even inconsistency *within*
-  one name.
-
-      fetchProc32 ACCESS(pixman_fetchProcForPicture32)
-
-  may be one of the uglies names ever created.
-
-  coding style: 
-  	 use the one from cairo except that pixman uses this brace style:
-	 
-		while (blah)
-		{
-		}
-
-	Format do while like this:
-
-	       do 
-	       {
-
-	       } 
-	       while (...);
-
-- PIXMAN_COMPOSITE_RECT_GENERAL() is horribly complex
-
-- switch case logic in pixman-access.c
-
-  Instead it would be better to just store function pointers in the
-  image objects themselves,
-
-  	get_pixel()
-	get_scanline()
-
-- Much of the scanline fetching code is for formats that no one 
-  ever uses. a2r2g2b2 anyone?
-
-  It would probably be worthwhile having a generic fetcher for any
-  pixman format whatsoever.
-
-- Code related to particular image types should be split into individual
-  files.
-
-	pixman-bits-image.c
-	pixman-linear-gradient-image.c
-	pixman-radial-gradient-image.c
-	pixman-solid-image.c
-
-- Fast path code should be split into files based on architecture:
-
-       pixman-mmx-fastpath.c
-       pixman-sse2-fastpath.c
-       pixman-c-fastpath.c
-
-       etc.
-
-  Each of these files should then export a fastpath table, which would
-  be declared in pixman-private.h. This should allow us to get rid
-  of the pixman-mmx.h files.
-
-  The fast path table should describe each fast path. Ie there should
-  be bitfields indicating what things the fast path can handle, rather than
-  like now where it is only allowed to take one format per src/mask/dest. Ie., 
-
-  { 
-    FAST_a8r8g8b8 | FAST_x8r8g8b8,
-    FAST_null,
-    FAST_x8r8g8b8,
-    FAST_repeat_normal | FAST_repeat_none,
-    the_fast_path
-  }
-
-There should then be *one* file that implements pixman_image_composite(). 
-This should do this:
-
-     optimize_operator();
-
-     convert 1x1 repeat to solid (actually this should be done at
-     image creation time).
-     
-     is there a useful fastpath?
-
-There should be a file called pixman-cpu.c that contains all the
-architecture specific stuff to detect what CPU features we have.
-
-Issues that must be kept in mind:
-
-       - we need accessor code to be preserved
-
-       - maybe there should be a "store_scanline" too?
-
-         Is this sufficient?
-
-	 We should preserve the optimization where the
-	 compositing happens directly in the destination
-	 whenever possible.
-
-	- It should be possible to create GPU samplers from the
-	  images.
-
-The "horizontal" classification should be a bit in the image, the
-"vertical" classification should just happen inside the gradient
-file. Note though that
-
-      (a) these will change if the tranformation/repeat changes.
-
-      (b) at the moment the optimization for linear gradients
-          takes the source rectangle into account. Presumably
-	  this is to also optimize the case where the gradient
-	  is close enough to horizontal?
-
-Who is responsible for repeats? In principle it should be the scanline
-fetch. Right now NORMAL repeats are handled by walk_composite_region()
-while other repeats are handled by the scanline code.
-
-
-(Random note on filtering: do you filter before or after
-transformation?  Hardware is going to filter after transformation;
-this is also what pixman does currently). It's not completely clear
-what filtering *after* transformation means. One thing that might look
-good would be to do *supersampling*, ie., compute multiple subpixels
-per destination pixel, then average them together.
diff --git a/pixman/pixman/rounding.txt b/pixman/pixman/rounding.txt
new file mode 100644
index 000000000..b52b08439
--- /dev/null
+++ b/pixman/pixman/rounding.txt
@@ -0,0 +1,167 @@
+*** General notes about rounding
+
+Suppose a function is sampled at positions [k + o] where k is an
+integer and o is a fractional offset 0 <= o < 1.
+
+To round a value to the nearest sample, breaking ties by rounding up,
+we can do this:
+
+   round(x) = floor(x - o + 0.5) + o
+
+That is, first subtract o to let us pretend that the samples are at
+integer coordinates, then add 0.5 and floor to round to nearest
+integer, then add the offset back in.
+
+To break ties by rounding down:
+
+    round(x) = ceil(x - o - 0.5) + o
+
+or if we have an epsilon value:
+
+    round(x) = floor(x - o + 0.5 - e) + o
+
+To always round *up* to the next sample:
+
+    round_up(x) = ceil(x - o) + o
+
+To always round *down* to the previous sample:
+
+    round_down(x) = floor(x - o) + o
+
+If a set of samples is stored in an array, you get from the sample
+position to an index by subtracting the position of the first sample
+in the array:
+
+    index(s) = s - first_sample
+
+
+*** Application to pixman
+
+In pixman, images are sampled with o = 0.5, that is, pixels are
+located midways between integers. We usually break ties by rounding
+down (i.e., "round towards north-west").
+
+
+-- NEAREST filtering:
+
+The NEAREST filter simply picks the closest pixel to the given
+position:
+
+    round(x) = floor(x - 0.5 + 0.5 - e) + 0.5 = floor (x - e) + 0.5
+
+The first sample of a pixman image has position 0.5, so to find the
+index in the pixel array, we have to subtract 0.5:
+
+    floor (x - e) + 0.5 - 0.5 = floor (x - e).
+
+Therefore a 16.16 fixed-point image location is turned into a pixel
+value with NEAREST filtering by doing this:
+
+    pixels[((y - e) >> 16) * stride + ((x - e) >> 16)]
+
+where stride is the number of pixels allocated per scanline and e =
+0x0001.
+
+
+-- CONVOLUTION filtering:
+
+A convolution matrix is considered a sampling of a function f at
+values surrounding 0. For example, this convolution matrix:
+
+	[a, b, c, d]
+
+is interpreted as the values of a function f:
+
+   	a = f(-1.5)
+        b = f(-0.5)
+        c = f(0.5)
+        d = f(1.5)
+
+The sample offset in this case is o = 0.5 and the first sample has
+position s0 = -1.5. If the matrix is:
+
+        [a, b, c, d, e]
+
+the sample offset is o = 0 and the first sample has position s0 =
+-2.0. In general we have 
+
+      s0 = (- width / 2.0 + 0.5).
+
+and
+
+      o = frac (s0)
+
+To evaluate f at a position between the samples, we round to the
+closest sample, and then we subtract the position of the first sample
+to get the index in the matrix:
+
+	f(t) = matrix[floor(t - o + 0.5) + o - s0]
+
+Note that in this case we break ties by rounding up.
+
+If we write s0 = m + o, where m is an integer, this is equivalent to
+
+        f(t) = matrix[floor(t - o + 0.5) + o - (m + o)]
+	     = matrix[floor(t - o + 0.5 - m) + o - o]
+	     = matrix[floor(t - s0 + 0.5)]
+
+The convolution filter in pixman positions f such that 0 aligns with
+the given position x. For a given pixel x0 in the image, the closest
+sample of f is then computed by taking (x - x0) and rounding that to
+the closest index:
+
+	i = floor ((x0 - x) - s0 + 0.5)
+
+To perform the convolution, we have to find the first pixel x0 whose
+corresponding sample has index 0. We can write x0 = k + 0.5, where k
+is an integer:
+
+         0 = floor(k + 0.5 - x - s0 + 0.5)
+
+	   = k + floor(1 - x - s0)
+
+	   = k - ceil(x + s0 - 1)
+
+	   = k - floor(x + s0 - e)
+
+	   = k - floor(x - (width - 1) / 2.0 - e)
+
+And so the final formula for the index k of x0 in the image is:
+
+    	    k = floor(x - (width - 1) / 2.0 - e)
+
+Computing the result is then simply a matter of convolving all the
+pixels starting at k with all the samples in the matrix.
+
+
+--- SEPARABLE_CONVOLUTION
+
+For this filter, x is first rounded to one of n regularly spaced
+subpixel positions. This subpixel position determines which of n
+convolution matrices is being used.
+
+Then, as in a regular convolution filter, the first pixel to be used
+is determined:
+
+    	k = floor (x - (width - 1) / 2.0 - e)
+
+and then the image pixels starting there are convolved with the chosen
+matrix. If we write x = xi + frac, where xi is an integer, we get
+
+	k = xi + floor (frac - (width - 1) / 2.0 - e)
+
+so the location of k relative to x is given by:
+
+    (k + 0.5 - x) = xi + floor (frac - (width - 1) / 2.0 - e) + 0.5 - x
+
+                  = floor (frac - (width - 1) / 2.0 - e) + 0.5 - frac
+
+which means the contents of the matrix corresponding to (frac) should
+contain width samplings of the function, with the first sample at:
+
+       floor (frac - (width - 1) / 2.0 - e) + 0.5 - frac
+
+This filter is called separable because each of the k x k convolution
+matrices is specified with two k-wide vectors, one for each dimension,
+where each entry in the matrix is computed as the product of the
+corresponding entries in the vectors.
diff --git a/pixman/pixman/solaris-hwcap.mapfile b/pixman/pixman/solaris-hwcap.mapfile
index 3605ca79f..87efce1e3 100644
--- a/pixman/pixman/solaris-hwcap.mapfile
+++ b/pixman/pixman/solaris-hwcap.mapfile
@@ -1,6 +1,6 @@
 ###############################################################################
 #
-# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2009, Oracle and/or its affiliates. All rights reserved.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
diff --git a/pixman/test/Makefile.am b/pixman/test/Makefile.am
index 841ff8d7d..88dc36d2a 100644
--- a/pixman/test/Makefile.am
+++ b/pixman/test/Makefile.am
@@ -1,84 +1,13 @@
-TEST_LDADD = $(top_builddir)/pixman/libpixman-1.la
-INCLUDES = -I$(top_srcdir)/pixman -I$(top_builddir)/pixman
+include $(top_srcdir)/test/Makefile.sources
 
-TESTPROGRAMS =			\
-	a1-trap-test		\
-	region-test		\
-	fetch-test		\
-	oob-test		\
-	window-test		\
-	trap-crasher		\
-	alphamap		\
-	blitters-test		\
-	scaling-test		\
-	composite
+AM_CFLAGS = $(OPENMP_CFLAGS) $(PTHREAD_CFLAGS)
+AM_LDFLAGS = $(OPENMP_CFLAGS) $(TESTPROGS_EXTRA_LDFLAGS) $(PTHREAD_LDFLAGS)
+LDADD = libutils.la $(top_builddir)/pixman/libpixman-1.la -lm  $(PNG_LIBS) $(PTHREAD_LIBS)
+AM_CPPFLAGS = -I$(top_srcdir)/pixman -I$(top_builddir)/pixman $(PNG_CFLAGS)
 
-a1_trap_test_LDADD = $(TEST_LDADD)
-fetch_test_LDADD = $(TEST_LDADD)
-composite_LDADD = $(TEST_LDADD)
-trap_crasher_LDADD = $(TEST_LDADD)
-oob_test_LDADD = $(TEST_LDADD)
-window_test_LDADD = $(TEST_LDADD)
+libutils_la_SOURCES = $(libutils_sources) $(libutils_headers)
 
-region_test_LDADD = $(TEST_LDADD)
-region_test_SOURCES = region-test.c utils.c utils.h
-
-blitters_test_LDADD = $(TEST_LDADD)
-blitters_test_SOURCES = blitters-test.c utils.c utils.h
-
-scaling_test_LDADD = $(TEST_LDADD)
-scaling_test_SOURCES = scaling-test.c utils.c utils.h
-
-alphamap_LDADD = $(TEST_LDADD)
-alphamap_SOURCES = alphamap.c utils.c utils.h
-
-# GTK using test programs
-
-if HAVE_GTK
-
-GTK_LDADD = $(TEST_LDADD) $(GTK_LIBS)
-GTK_UTILS = gtk-utils.c gtk-utils.h
-
-TESTPROGRAMS_GTK =		\
-	clip-test		\
-	clip-in			\
-	composite-test		\
-	gradient-test		\
-	alpha-test		\
-	screen-test		\
-	convolution-test	\
-	trap-test		\
-	alphamap
-
-INCLUDES += $(GTK_CFLAGS)
-
-gradient_test_LDADD = $(GTK_LDADD)
-gradient_test_SOURCES = gradient-test.c $(GTK_UTILS)
-
-alpha_test_LDADD = $(GTK_LDADD)
-alpha_test_SOURCES = alpha-test.c $(GTK_UTILS)
-
-composite_test_LDADD = $(GTK_LDADD)
-composite_test_SOURCES = composite-test.c $(GTK_UTILS)
-
-clip_test_LDADD = $(GTK_LDADD)
-clip_test_SOURCES = clip-test.c $(GTK_UTILS)
-
-clip_in_LDADD = $(GTK_LDADD)
-clip_in_SOURCES = clip-in.c $(GTK_UTILS)
-
-trap_test_LDADD = $(GTK_LDADD)
-trap_test_SOURCES = trap-test.c $(GTK_UTILS)
-
-screen_test_LDADD = $(GTK_LDADD)
-screen_test_SOURCES = screen-test.c $(GTK_UTILS)
-
-convolution_test_LDADD = $(GTK_LDADD)
-convolution_test_SOURCES = convolution-test.c $(GTK_UTILS)
-
-endif
-
-noinst_PROGRAMS = $(TESTPROGRAMS) $(TESTPROGRAMS_GTK)
+noinst_LTLIBRARIES = libutils.la
+noinst_PROGRAMS = $(TESTPROGRAMS) $(OTHERPROGRAMS)
 
 TESTS = $(TESTPROGRAMS)
-
diff --git a/pixman/test/Makefile.sources b/pixman/test/Makefile.sources
new file mode 100644
index 000000000..2ae5d9f8d
--- /dev/null
+++ b/pixman/test/Makefile.sources
@@ -0,0 +1,49 @@
+# Tests (sorted by expected completion time)
+TESTPROGRAMS =			\
+	prng-test		\
+	a1-trap-test		\
+	pdf-op-test		\
+	region-test		\
+	region-translate-test	\
+	combiner-test		\
+	pixel-test		\
+	fetch-test		\
+	rotate-test		\
+	oob-test		\
+	infinite-loop		\
+	trap-crasher		\
+	alpha-loop		\
+	thread-test		\
+	scaling-crash-test	\
+	scaling-helpers-test	\
+	gradient-crash-test	\
+	region-contains-test	\
+	alphamap		\
+	matrix-test		\
+	stress-test		\
+	composite-traps-test	\
+	blitters-test		\
+	glyph-test		\
+	scaling-test		\
+	affine-test		\
+	composite		\
+	$(NULL)
+
+# Other programs
+OTHERPROGRAMS =                 \
+	lowlevel-blt-bench	\
+	radial-perf-test	\
+        check-formats           \
+	scaling-bench		\
+	$(NULL)
+
+# Utility functions
+libutils_sources =		\
+	utils.c			\
+	utils-prng.c		\
+	$(NULL)
+
+libutils_headers =		\
+	utils.h			\
+	utils-prng.h		\
+	$(NULL)
diff --git a/pixman/test/Makefile.win32 b/pixman/test/Makefile.win32
new file mode 100644
index 000000000..6cfb4a780
--- /dev/null
+++ b/pixman/test/Makefile.win32
@@ -0,0 +1,54 @@
+default: all
+
+top_srcdir = ..
+include $(top_srcdir)/test/Makefile.sources
+include $(top_srcdir)/Makefile.win32.common
+
+TEST_LDADD = \
+	$(top_builddir)/pixman/$(CFG_VAR)/$(LIBRARY).lib \
+	$(CFG_VAR)/libutils.lib \
+	$(NULL)
+
+libutils_OBJECTS = $(patsubst %.c, $(CFG_VAR)/%.obj, $(libutils_sources))
+
+SOURCES = $(patsubst %,   %.c,              $(TESTPROGRAMS) $(OTHERPROGRAMS))
+OBJECTS = $(patsubst %.c, $(CFG_VAR)/%.obj, $(SOURCES))
+TESTS   = $(patsubst %,   $(CFG_VAR)/%.exe, $(TESTPROGRAMS))
+OTHERS  = $(patsubst %,   $(CFG_VAR)/%.exe, $(OTHERPROGRAMS))
+
+all: pixman inform $(TESTS) $(OTHERS)
+
+check: pixman inform $(TESTS)
+	@failures=0 ; \
+	total=0 ; \
+	for test in $(TESTS) ; \
+	do \
+		total=`expr $$total + 1` ; \
+		if ./$$test ; \
+		then echo "PASS: $$test" ; \
+		else echo "FAIL: $$test" ; \
+		     failures=`expr $$failures + 1` ; \
+		fi ; \
+	done ; \
+	if test $$failures -eq 0 ; \
+	then banner="All $$total tests passed" ; \
+	else banner="$$failures of $$total tests failed" ; \
+	fi ; \
+	dashes=`echo "$$banner" | sed s/./=/g`; \
+	echo "$$dashes" ; \
+	echo "$$banner" ; \
+	echo "$$dashes" ; \
+	test $$failures -eq 0
+
+$(CFG_VAR)/libutils.lib: $(libutils_OBJECTS)
+	@$(AR) $(PIXMAN_ARFLAGS) -OUT:$@ $^
+
+$(CFG_VAR)/%.exe: $(CFG_VAR)/%.obj $(TEST_LDADD)
+	@$(LD) $(PIXMAN_LDFLAGS) -OUT:$@ $^
+
+$(top_builddir)/pixman/$(CFG_VAR)/$(LIBRARY).lib: pixman
+
+pixman:
+	@$(MAKE) -C $(top_builddir)/pixman -f Makefile.win32
+
+.PHONY: all check pixman
diff --git a/pixman/test/a1-trap-test.c b/pixman/test/a1-trap-test.c
index 6163e7c61..c2b488316 100644
--- a/pixman/test/a1-trap-test.c
+++ b/pixman/test/a1-trap-test.c
@@ -2,7 +2,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include "pixman.h"
+#include "utils.h"
 
 int
 main (int argc, char **argv)
@@ -45,6 +45,14 @@ main (int argc, char **argv)
     assert (bits[1] == 0xffffffff);
     assert (bits[1 * WIDTH + 0] == 0xffffffff);
     assert (bits[1 * WIDTH + 1] == 0xffffffff);
+
+    /* The check-formats test depends on operator_name() and format_name() returning
+     * these precise formats, so if those change, check-formats.c must be updated too.
+     */
+    assert (
+        strcmp (operator_name (PIXMAN_OP_DISJOINT_OVER), "PIXMAN_OP_DISJOINT_OVER") == 0);
+    assert (
+        strcmp (format_name (PIXMAN_r5g6b5), "r5g6b5") == 0);
     
     return 0;
 }
diff --git a/pixman/test/affine-test.c b/pixman/test/affine-test.c
new file mode 100644
index 000000000..8e19023a3
--- /dev/null
+++ b/pixman/test/affine-test.c
@@ -0,0 +1,324 @@
+/*
+ * Test program, which can detect some problems with affine transformations
+ * in pixman. Testing is done by running lots of random SRC and OVER
+ * compositing operations a8r8g8b8, x8a8r8g8b8, r5g6b5 and a8 color formats
+ * with random scaled, rotated and translated transforms.
+ *
+ * Script 'fuzzer-find-diff.pl' can be used to narrow down the problem in
+ * the case of test failure.
+ */
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include "utils.h"
+
+#define MAX_SRC_WIDTH  16
+#define MAX_SRC_HEIGHT 16
+#define MAX_DST_WIDTH  16
+#define MAX_DST_HEIGHT 16
+#define MAX_STRIDE     4
+
+/*
+ * Composite operation with pseudorandom images
+ */
+uint32_t
+test_composite (int      testnum,
+		int      verbose)
+{
+    int                i;
+    pixman_image_t *   src_img;
+    pixman_image_t *   dst_img;
+    pixman_transform_t transform;
+    pixman_region16_t  clip;
+    int                src_width, src_height;
+    int                dst_width, dst_height;
+    int                src_stride, dst_stride;
+    int                src_x, src_y;
+    int                dst_x, dst_y;
+    int                src_bpp;
+    int                dst_bpp;
+    int                w, h;
+    pixman_fixed_t     scale_x = 65536, scale_y = 65536;
+    pixman_fixed_t     translate_x = 0, translate_y = 0;
+    pixman_op_t        op;
+    pixman_repeat_t    repeat = PIXMAN_REPEAT_NONE;
+    pixman_format_code_t src_fmt, dst_fmt;
+    uint32_t *         srcbuf;
+    uint32_t *         dstbuf;
+    uint32_t           crc32;
+    FLOAT_REGS_CORRUPTION_DETECTOR_START ();
+
+    prng_srand (testnum);
+
+    src_bpp = (prng_rand_n (2) == 0) ? 2 : 4;
+    dst_bpp = (prng_rand_n (2) == 0) ? 2 : 4;
+    op = (prng_rand_n (2) == 0) ? PIXMAN_OP_SRC : PIXMAN_OP_OVER;
+
+    src_width = prng_rand_n (MAX_SRC_WIDTH) + 1;
+    src_height = prng_rand_n (MAX_SRC_HEIGHT) + 1;
+    dst_width = prng_rand_n (MAX_DST_WIDTH) + 1;
+    dst_height = prng_rand_n (MAX_DST_HEIGHT) + 1;
+    src_stride = src_width * src_bpp + prng_rand_n (MAX_STRIDE) * src_bpp;
+    dst_stride = dst_width * dst_bpp + prng_rand_n (MAX_STRIDE) * dst_bpp;
+
+    if (src_stride & 3)
+	src_stride += 2;
+
+    if (dst_stride & 3)
+	dst_stride += 2;
+
+    src_x = -(src_width / 4) + prng_rand_n (src_width * 3 / 2);
+    src_y = -(src_height / 4) + prng_rand_n (src_height * 3 / 2);
+    dst_x = -(dst_width / 4) + prng_rand_n (dst_width * 3 / 2);
+    dst_y = -(dst_height / 4) + prng_rand_n (dst_height * 3 / 2);
+    w = prng_rand_n (dst_width * 3 / 2 - dst_x);
+    h = prng_rand_n (dst_height * 3 / 2 - dst_y);
+
+    srcbuf = (uint32_t *)malloc (src_stride * src_height);
+    dstbuf = (uint32_t *)malloc (dst_stride * dst_height);
+
+    prng_randmemset (srcbuf, src_stride * src_height, 0);
+    prng_randmemset (dstbuf, dst_stride * dst_height, 0);
+
+    if (prng_rand_n (2) == 0)
+    {
+	srcbuf += (src_stride / 4) * (src_height - 1);
+	src_stride = - src_stride;
+    }
+
+    if (prng_rand_n (2) == 0)
+    {
+	dstbuf += (dst_stride / 4) * (dst_height - 1);
+	dst_stride = - dst_stride;
+    }
+    
+    src_fmt = src_bpp == 4 ? (prng_rand_n (2) == 0 ?
+                              PIXMAN_a8r8g8b8 : PIXMAN_x8r8g8b8) : PIXMAN_r5g6b5;
+
+    dst_fmt = dst_bpp == 4 ? (prng_rand_n (2) == 0 ?
+                              PIXMAN_a8r8g8b8 : PIXMAN_x8r8g8b8) : PIXMAN_r5g6b5;
+
+    src_img = pixman_image_create_bits (
+        src_fmt, src_width, src_height, srcbuf, src_stride);
+
+    dst_img = pixman_image_create_bits (
+        dst_fmt, dst_width, dst_height, dstbuf, dst_stride);
+
+    image_endian_swap (src_img);
+    image_endian_swap (dst_img);
+
+    pixman_transform_init_identity (&transform);
+
+    if (prng_rand_n (3) > 0)
+    {
+	scale_x = -65536 * 3 + prng_rand_n (65536 * 6);
+	if (prng_rand_n (2))
+	    scale_y = -65536 * 3 + prng_rand_n (65536 * 6);
+	else
+	    scale_y = scale_x;
+	pixman_transform_init_scale (&transform, scale_x, scale_y);
+    }
+    if (prng_rand_n (3) > 0)
+    {
+	translate_x = -65536 * 3 + prng_rand_n (6 * 65536);
+	if (prng_rand_n (2))
+	    translate_y = -65536 * 3 + prng_rand_n (6 * 65536);
+	else
+	    translate_y = translate_x;
+	pixman_transform_translate (&transform, NULL, translate_x, translate_y);
+    }
+
+    if (prng_rand_n (4) > 0)
+    {
+	int c, s, tx = 0, ty = 0;
+	switch (prng_rand_n (4))
+	{
+	case 0:
+	    /* 90 degrees */
+	    c = 0;
+	    s = pixman_fixed_1;
+	    tx = pixman_int_to_fixed (MAX_SRC_HEIGHT);
+	    break;
+	case 1:
+	    /* 180 degrees */
+	    c = -pixman_fixed_1;
+	    s = 0;
+	    tx = pixman_int_to_fixed (MAX_SRC_WIDTH);
+	    ty = pixman_int_to_fixed (MAX_SRC_HEIGHT);
+	    break;
+	case 2:
+	    /* 270 degrees */
+	    c = 0;
+	    s = -pixman_fixed_1;
+	    ty = pixman_int_to_fixed (MAX_SRC_WIDTH);
+	    break;
+	default:
+	    /* arbitrary rotation */
+	    c = prng_rand_n (2 * 65536) - 65536;
+	    s = prng_rand_n (2 * 65536) - 65536;
+	    break;
+	}
+	pixman_transform_rotate (&transform, NULL, c, s);
+	pixman_transform_translate (&transform, NULL, tx, ty);
+    }
+
+    if (prng_rand_n (8) == 0)
+    {
+	/* Flip random bits */
+	int maxflipcount = 8;
+	while (maxflipcount--)
+	{
+	    int i = prng_rand_n (2);
+	    int j = prng_rand_n (3);
+	    int bitnum = prng_rand_n (32);
+	    transform.matrix[i][j] ^= 1 << bitnum;
+	    if (prng_rand_n (2))
+		break;
+	}
+    }
+
+    pixman_image_set_transform (src_img, &transform);
+
+    switch (prng_rand_n (4))
+    {
+    case 0:
+	repeat = PIXMAN_REPEAT_NONE;
+	break;
+
+    case 1:
+	repeat = PIXMAN_REPEAT_NORMAL;
+	break;
+
+    case 2:
+	repeat = PIXMAN_REPEAT_PAD;
+	break;
+
+    case 3:
+	repeat = PIXMAN_REPEAT_REFLECT;
+	break;
+
+    default:
+        break;
+    }
+    pixman_image_set_repeat (src_img, repeat);
+
+    if (prng_rand_n (2))
+	pixman_image_set_filter (src_img, PIXMAN_FILTER_NEAREST, NULL, 0);
+    else
+	pixman_image_set_filter (src_img, PIXMAN_FILTER_BILINEAR, NULL, 0);
+
+    if (verbose)
+    {
+#define M(r,c)								\
+	transform.matrix[r][c]
+
+	printf ("src_fmt=%s, dst_fmt=%s\n", format_name (src_fmt), format_name (dst_fmt));
+	printf ("op=%s, repeat=%d, transform=\n",
+	        operator_name (op), repeat);
+	printf (" { { { 0x%08x, 0x%08x, 0x%08x },\n"
+		"     { 0x%08x, 0x%08x, 0x%08x },\n"
+		"     { 0x%08x, 0x%08x, 0x%08x },\n"
+		" } };\n",
+		M(0,0), M(0,1), M(0,2),
+		M(1,0), M(1,1), M(1,2),
+		M(2,0), M(2,1), M(2,2));
+	printf ("src_width=%d, src_height=%d, dst_width=%d, dst_height=%d\n",
+	        src_width, src_height, dst_width, dst_height);
+	printf ("src_x=%d, src_y=%d, dst_x=%d, dst_y=%d\n",
+	        src_x, src_y, dst_x, dst_y);
+	printf ("w=%d, h=%d\n", w, h);
+    }
+
+    if (prng_rand_n (8) == 0)
+    {
+	pixman_box16_t clip_boxes[2];
+	int            n = prng_rand_n (2) + 1;
+
+	for (i = 0; i < n; i++)
+	{
+	    clip_boxes[i].x1 = prng_rand_n (src_width);
+	    clip_boxes[i].y1 = prng_rand_n (src_height);
+	    clip_boxes[i].x2 =
+		clip_boxes[i].x1 + prng_rand_n (src_width - clip_boxes[i].x1);
+	    clip_boxes[i].y2 =
+		clip_boxes[i].y1 + prng_rand_n (src_height - clip_boxes[i].y1);
+
+	    if (verbose)
+	    {
+		printf ("source clip box: [%d,%d-%d,%d]\n",
+		        clip_boxes[i].x1, clip_boxes[i].y1,
+		        clip_boxes[i].x2, clip_boxes[i].y2);
+	    }
+	}
+
+	pixman_region_init_rects (&clip, clip_boxes, n);
+	pixman_image_set_clip_region (src_img, &clip);
+	pixman_image_set_source_clipping (src_img, 1);
+	pixman_region_fini (&clip);
+    }
+
+    if (prng_rand_n (8) == 0)
+    {
+	pixman_box16_t clip_boxes[2];
+	int            n = prng_rand_n (2) + 1;
+	for (i = 0; i < n; i++)
+	{
+	    clip_boxes[i].x1 = prng_rand_n (dst_width);
+	    clip_boxes[i].y1 = prng_rand_n (dst_height);
+	    clip_boxes[i].x2 =
+		clip_boxes[i].x1 + prng_rand_n (dst_width - clip_boxes[i].x1);
+	    clip_boxes[i].y2 =
+		clip_boxes[i].y1 + prng_rand_n (dst_height - clip_boxes[i].y1);
+
+	    if (verbose)
+	    {
+		printf ("destination clip box: [%d,%d-%d,%d]\n",
+		        clip_boxes[i].x1, clip_boxes[i].y1,
+		        clip_boxes[i].x2, clip_boxes[i].y2);
+	    }
+	}
+	pixman_region_init_rects (&clip, clip_boxes, n);
+	pixman_image_set_clip_region (dst_img, &clip);
+	pixman_region_fini (&clip);
+    }
+
+    pixman_image_composite (op, src_img, NULL, dst_img,
+                            src_x, src_y, 0, 0, dst_x, dst_y, w, h);
+
+    crc32 = compute_crc32_for_image (0, dst_img);
+    
+    if (verbose)
+	print_image (dst_img);
+
+    pixman_image_unref (src_img);
+    pixman_image_unref (dst_img);
+
+    if (src_stride < 0)
+	srcbuf += (src_stride / 4) * (src_height - 1);
+
+    if (dst_stride < 0)
+	dstbuf += (dst_stride / 4) * (dst_height - 1);
+    
+    free (srcbuf);
+    free (dstbuf);
+
+    FLOAT_REGS_CORRUPTION_DETECTOR_FINISH ();
+    return crc32;
+}
+
+#if BILINEAR_INTERPOLATION_BITS == 7
+#define CHECKSUM 0xBE724CFE
+#elif BILINEAR_INTERPOLATION_BITS == 4
+#define CHECKSUM 0x79BBE501
+#else
+#define CHECKSUM 0x00000000
+#endif
+
+int
+main (int argc, const char *argv[])
+{
+    pixman_disable_out_of_bounds_workaround ();
+
+    return fuzzer_test_main ("affine", 8000000, CHECKSUM,
+			     test_composite, argc, argv);
+}
diff --git a/pixman/test/alpha-loop.c b/pixman/test/alpha-loop.c
new file mode 100644
index 000000000..4d4384d00
--- /dev/null
+++ b/pixman/test/alpha-loop.c
@@ -0,0 +1,35 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include "utils.h"
+
+#define WIDTH 400
+#define HEIGHT 200
+
+int
+main (int argc, char **argv)
+{
+    pixman_image_t *a, *d, *s;
+    uint8_t *alpha;
+    uint32_t *src, *dest;
+
+    prng_srand (0);
+
+    alpha = make_random_bytes (WIDTH * HEIGHT);
+    src = (uint32_t *)make_random_bytes (WIDTH * HEIGHT * 4);
+    dest = (uint32_t *)make_random_bytes (WIDTH * HEIGHT * 4);
+
+    a = pixman_image_create_bits (PIXMAN_a8, WIDTH, HEIGHT, (uint32_t *)alpha, WIDTH);
+    d = pixman_image_create_bits (PIXMAN_a8r8g8b8, WIDTH, HEIGHT, dest, WIDTH * 4);
+    s = pixman_image_create_bits (PIXMAN_a2r10g10b10, WIDTH, HEIGHT, src, WIDTH * 4);
+
+    fail_after (5, "Infinite loop detected: 5 seconds without progress\n");
+
+    pixman_image_set_alpha_map (s, a, 0, 0);
+    pixman_image_set_alpha_map (a, s, 0, 0);
+
+    pixman_image_composite (PIXMAN_OP_SRC, s, NULL, d, 0, 0, 0, 0, 0, 0, WIDTH, HEIGHT);
+
+    pixman_image_unref (s);
+
+    return 0;
+}
diff --git a/pixman/test/alphamap.c b/pixman/test/alphamap.c
index e6a25efcb..4d09076fb 100644
--- a/pixman/test/alphamap.c
+++ b/pixman/test/alphamap.c
@@ -2,47 +2,313 @@
 #include <stdlib.h>
 #include "utils.h"
 
-#define WIDTH 400
-#define HEIGHT 200
+#define WIDTH 48
+#define HEIGHT 48
 
-int
-main (int argc, char **argv)
+static const pixman_format_code_t formats[] =
+{
+    PIXMAN_a8r8g8b8,
+    PIXMAN_a2r10g10b10,
+    PIXMAN_a4r4g4b4,
+    PIXMAN_a8
+};
+
+static const pixman_format_code_t alpha_formats[] =
+{
+    PIXMAN_null,
+    PIXMAN_a8,
+    PIXMAN_a2r10g10b10,
+    PIXMAN_a4r4g4b4
+};
+
+static const int origins[] =
+{
+    0, 10, -100
+};
+
+static void
+on_destroy (pixman_image_t *image, void *data)
+{
+    uint32_t *bits = pixman_image_get_data (image);
+
+    fence_free (bits);
+}
+
+static pixman_image_t *
+make_image (pixman_format_code_t format)
+{
+    uint32_t *bits;
+    uint8_t bpp = PIXMAN_FORMAT_BPP (format) / 8;
+    pixman_image_t *image;
+
+    bits = (uint32_t *)make_random_bytes (WIDTH * HEIGHT * bpp);
+
+    image = pixman_image_create_bits (format, WIDTH, HEIGHT, bits, WIDTH * bpp);
+
+    if (image && bits)
+	pixman_image_set_destroy_function (image, on_destroy, NULL);
+
+    return image;
+}
+
+static uint8_t
+get_alpha (pixman_image_t *image, int x, int y, int orig_x, int orig_y)
+{
+    uint8_t *bits;
+    uint8_t r;
+
+    if (image->common.alpha_map)
+    {
+	if (x - orig_x >= 0 && x - orig_x < WIDTH &&
+	    y - orig_y >= 0 && y - orig_y < HEIGHT)
+	{
+	    image = (pixman_image_t *)image->common.alpha_map;
+
+	    x -= orig_x;
+	    y -= orig_y;
+	}
+	else
+	{
+	    return 0;
+	}
+    }
+
+    bits = (uint8_t *)image->bits.bits;
+
+    if (image->bits.format == PIXMAN_a8)
+    {
+	r = bits[y * WIDTH + x];
+    }
+    else if (image->bits.format == PIXMAN_a2r10g10b10)
+    {
+	r = ((uint32_t *)bits)[y * WIDTH + x] >> 30;
+	r |= r << 2;
+	r |= r << 4;
+    }
+    else if (image->bits.format == PIXMAN_a8r8g8b8)
+    {
+	r = ((uint32_t *)bits)[y * WIDTH + x] >> 24;
+    }
+    else if (image->bits.format == PIXMAN_a4r4g4b4)
+    {
+	r = ((uint16_t *)bits)[y * WIDTH + x] >> 12;
+	r |= r << 4;
+    }
+    else
+    {
+	assert (0);
+    }
+
+    return r;
+}
+
+static uint16_t
+get_red (pixman_image_t *image, int x, int y, int orig_x, int orig_y)
+{
+    uint8_t *bits;
+    uint16_t r;
+
+    bits = (uint8_t *)image->bits.bits;
+
+    if (image->bits.format == PIXMAN_a8)
+    {
+	r = 0x00;
+    }
+    else if (image->bits.format == PIXMAN_a2r10g10b10)
+    {
+	r = ((uint32_t *)bits)[y * WIDTH + x] >> 14;
+	r &= 0xffc0;
+	r |= (r >> 10);
+    }
+    else if (image->bits.format == PIXMAN_a8r8g8b8)
+    {
+	r = ((uint32_t *)bits)[y * WIDTH + x] >> 16;
+	r &= 0xff;
+	r |= r << 8;
+    }
+    else if (image->bits.format == PIXMAN_a4r4g4b4)
+    {
+	r = ((uint16_t *)bits)[y * WIDTH + x] >> 8;
+	r &= 0xf;
+	r |= r << 4;
+	r |= r << 8;
+    }
+    else
+    {
+	assert (0);
+    }
+
+    return r;
+}
+
+static int
+run_test (int s, int d, int sa, int da, int soff, int doff)
 {
-    uint8_t *alpha = make_random_bytes (WIDTH * HEIGHT);
-    uint32_t *src = (uint32_t *)make_random_bytes (WIDTH * HEIGHT * 4);
-    uint32_t *dest = (uint32_t *)make_random_bytes (WIDTH * HEIGHT * 4);
-    int i;
+    pixman_format_code_t sf = formats[s];
+    pixman_format_code_t df = formats[d];
+    pixman_format_code_t saf = alpha_formats[sa];
+    pixman_format_code_t daf = alpha_formats[da];
+    pixman_image_t *src, *dst, *orig_dst, *alpha, *orig_alpha;
+    pixman_transform_t t1;
+    int j, k;
+    int n_alpha_bits, n_red_bits;
 
-    pixman_image_t *a = pixman_image_create_bits (PIXMAN_a8, WIDTH, HEIGHT, (uint32_t *)alpha, WIDTH);
-    pixman_image_t *d = pixman_image_create_bits (PIXMAN_a8r8g8b8, WIDTH, HEIGHT, dest, WIDTH * 4);
+    soff = origins[soff];
+    doff = origins[doff];
 
-    for (i = 0; i < 2; ++i)
+    n_alpha_bits = PIXMAN_FORMAT_A (df);
+    if (daf != PIXMAN_null)
+	n_alpha_bits = PIXMAN_FORMAT_A (daf);
+
+    n_red_bits = PIXMAN_FORMAT_R (df);
+
+    /* Source */
+    src = make_image (sf);
+    if (saf != PIXMAN_null)
     {
-	pixman_format_code_t sformat = (i == 0)? PIXMAN_a8r8g8b8 : PIXMAN_a2r10g10b10;
-	pixman_image_t *s = pixman_image_create_bits (sformat, WIDTH, HEIGHT, src, WIDTH * 4);
-	int j, k;
+	alpha = make_image (saf);
+	pixman_image_set_alpha_map (src, alpha, soff, soff);
+	pixman_image_unref (alpha);
+    }
 
-	pixman_image_set_alpha_map (s, a, 0, 0);
+    /* Destination */
+    orig_dst = make_image (df);
+    dst = make_image (df);
+    pixman_image_composite (PIXMAN_OP_SRC, orig_dst, NULL, dst,
+			    0, 0, 0, 0, 0, 0, WIDTH, HEIGHT);
 
-	pixman_image_composite (PIXMAN_OP_SRC, s, NULL, d, 0, 0, 0, 0, 0, 0, WIDTH, HEIGHT);
+    if (daf != PIXMAN_null)
+    {
+	orig_alpha = make_image (daf);
+	alpha = make_image (daf);
+
+	pixman_image_composite (PIXMAN_OP_SRC, orig_alpha, NULL, alpha,
+				0, 0, 0, 0, 0, 0, WIDTH, HEIGHT);
+
+	pixman_image_set_alpha_map (orig_dst, orig_alpha, doff, doff);
+	pixman_image_set_alpha_map (dst, alpha, doff, doff);
+
+	pixman_image_unref (orig_alpha);
+	pixman_image_unref (alpha);
+    }
+
+    /* Transformations, repeats and filters on destinations should be ignored,
+     * so just set some random ones.
+     */
+    pixman_transform_init_identity (&t1);
+    pixman_transform_scale (&t1, NULL, pixman_int_to_fixed (100), pixman_int_to_fixed (11));
+    pixman_transform_rotate (&t1, NULL, pixman_double_to_fixed (0.5), pixman_double_to_fixed (0.11));
+    pixman_transform_translate (&t1, NULL, pixman_int_to_fixed (11), pixman_int_to_fixed (17));
 
-	for (j = 0; j < HEIGHT; ++j)
+    pixman_image_set_transform (dst, &t1);
+    pixman_image_set_filter (dst, PIXMAN_FILTER_BILINEAR, NULL, 0);
+    pixman_image_set_repeat (dst, PIXMAN_REPEAT_REFLECT);
+
+    pixman_image_composite (PIXMAN_OP_ADD, src, NULL, dst,
+			    0, 0, 0, 0, 0, 0, WIDTH, HEIGHT);
+
+    for (j = MAX (doff, 0); j < MIN (HEIGHT, HEIGHT + doff); ++j)
+    {
+	for (k = MAX (doff, 0); k < MIN (WIDTH, WIDTH + doff); ++k)
 	{
-	    for (k = 0; k < WIDTH; ++k)
+	    uint8_t sa, da, oda, refa;
+	    uint16_t sr, dr, odr, refr;
+
+	    sa = get_alpha (src, k, j, soff, soff);
+	    da = get_alpha (dst, k, j, doff, doff);
+	    oda = get_alpha (orig_dst, k, j, doff, doff);
+
+	    if (sa + oda > 255)
+		refa = 255;
+	    else
+		refa = sa + oda;
+
+	    if (da >> (8 - n_alpha_bits) != refa >> (8 - n_alpha_bits))
 	    {
-		uint8_t ap = ((uint8_t *)alpha)[j * WIDTH + k];
-		uint32_t dap = (dest[j * WIDTH + k] >> 24);
-		uint32_t sap = (src[j * WIDTH + k] >> 24);
+		printf ("\nWrong alpha value at (%d, %d). Should be 0x%x; got 0x%x. Source was 0x%x, original dest was 0x%x\n",
+			k, j, refa, da, sa, oda);
 
-		if (ap != dap)
+		printf ("src: %s, alpha: %s, origin %d %d\ndst: %s, alpha: %s, origin: %d %d\n\n",
+			format_name (sf),
+			format_name (saf),
+			soff, soff,
+			format_name (df),
+			format_name (daf),
+			doff, doff);
+		return 1;
+	    }
+
+	    /* There are cases where we go through the 8 bit compositing
+	     * path even with 10bpc formats. This results in incorrect
+	     * results here, so only do the red check for narrow formats
+	     */
+	    if (n_red_bits <= 8)
+	    {
+		sr = get_red (src, k, j, soff, soff);
+		dr = get_red (dst, k, j, doff, doff);
+		odr = get_red (orig_dst, k, j, doff, doff);
+
+		if (sr + odr > 0xffff)
+		    refr = 0xffff;
+		else
+		    refr = sr + odr;
+
+		if (abs ((dr >> (16 - n_red_bits)) - (refr >> (16 - n_red_bits))) > 1)
 		{
-		    printf ("Wrong alpha value at (%d, %d). Should be %d; got %d (src was %d)\n", k, j, ap, dap, sap);
+		    printf ("%d red bits\n", n_red_bits);
+		    printf ("\nWrong red value at (%d, %d). Should be 0x%x; got 0x%x. Source was 0x%x, original dest was 0x%x\n",
+			    k, j, refr, dr, sr, odr);
+
+		    printf ("src: %s, alpha: %s, origin %d %d\ndst: %s, alpha: %s, origin: %d %d\n\n",
+			    format_name (sf),
+			    format_name (saf),
+			    soff, soff,
+			    format_name (df),
+			    format_name (daf),
+			    doff, doff);
 		    return 1;
 		}
 	    }
 	}
+    }
+
+    pixman_image_set_alpha_map (src, NULL, 0, 0);
+    pixman_image_set_alpha_map (dst, NULL, 0, 0);
+    pixman_image_set_alpha_map (orig_dst, NULL, 0, 0);
+
+    pixman_image_unref (src);
+    pixman_image_unref (dst);
+    pixman_image_unref (orig_dst);
+
+    return 0;
+}
 
-	pixman_image_unref (s);
+int
+main (int argc, char **argv)
+{
+    int i, j, a, b, x, y;
+
+    prng_srand (0);
+
+    for (i = 0; i < ARRAY_LENGTH (formats); ++i)
+    {
+	for (j = 0; j < ARRAY_LENGTH (formats); ++j)
+	{
+	    for (a = 0; a < ARRAY_LENGTH (alpha_formats); ++a)
+	    {
+		for (b = 0; b < ARRAY_LENGTH (alpha_formats); ++b)
+		{
+		    for (x = 0; x < ARRAY_LENGTH (origins); ++x)
+		    {
+			for (y = 0; y < ARRAY_LENGTH (origins); ++y)
+			{
+			    if (run_test (i, j, a, b, x, y) != 0)
+				return 1;
+			}
+		    }
+		}
+	    }
+	}
     }
 
     return 0;
diff --git a/pixman/test/blitters-test-bisect.rb b/pixman/test/blitters-test-bisect.rb
deleted file mode 100644
index 62ff782e7..000000000
--- a/pixman/test/blitters-test-bisect.rb
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env ruby
-
-if not ARGV[0] or not ARGV[1] then
-    printf("Please provide two 'blitters-test' static binaries in the command line.\n\n")
-    printf("The first should be linked with a correct reference pixman library.\n")
-    printf("The second binary should be linked with a pixman library which needs to be tested.\n")
-    exit(0)
-end
-
-def test_range(min, max)
-    if `#{ARGV[0]} #{min} #{max} 2>/dev/null` == `#{ARGV[1]} #{min} #{max} 2>/dev/null` then
-        return
-    end
-    while max != min + 1 do
-        avg = ((min + max) / 2).to_i
-        res1 = `#{ARGV[0]} #{min} #{avg} 2>/dev/null`
-        res2 = `#{ARGV[1]} #{min} #{avg} 2>/dev/null`
-        if res1 != res2 then
-            max = avg
-        else
-            min = avg
-        end
-    end
-    return max
-end
-
-base = 1
-while true do
-    # run infinitely, processing 100000 test cases per iteration
-    printf("running tests %d-%d\n", base, base + 100000 - 1);
-    res = test_range(base, base + 100000 - 1)
-    if res then
-        printf("-- ref --\n")
-        printf("%s\n", `#{ARGV[0]} -#{res}`)
-        printf("-- new --\n")
-        printf("%s\n", `#{ARGV[1]} -#{res}`)
-
-        printf("\nFailed test %d, you can reproduce the problematic conditions by running\n", res)
-        printf("#{ARGV[1]} -%d\n", res)
-        exit(1)
-    end
-    base += 100000
-end
diff --git a/pixman/test/blitters-test.c b/pixman/test/blitters-test.c
index 1ebf6d9ca..ea03f475d 100644
--- a/pixman/test/blitters-test.c
+++ b/pixman/test/blitters-test.c
@@ -2,47 +2,20 @@
  * Test program, which stresses the use of different color formats and
  * compositing operations.
  *
- * Just run it without any command line arguments, and it will report either
- *   "blitters test passed" - everything is ok
- *   "blitters test failed!" - there is some problem
- *
- * In the case of failure, finding the problem involves the following steps:
- * 1. Get the reference 'blitters-test' binary. It makes sense to disable all
- *    the cpu specific optimizations in pixman and also configure it with
- *    '--disable-shared' option. Those who are paranoid can also tweak the
- *    sources to disable all fastpath functions. The resulting binary
- *    can be renamed to something like 'blitters-test.ref'.
- * 2. Compile the buggy binary (also with the '--disable-shared' option).
- * 3. Run 'ruby blitters-test-bisect.rb ./blitters-test.ref ./blitters-test'
- * 4. Look at the information about failed case (destination buffer content
- *    will be shown) and try to figure out what is wrong. Loading
- *    test program in gdb, specifying failed test number in the command
- *    line with '-' character prepended and setting breakpoint on
- *    'pixman_image_composite' function can provide detailed information
- *    about function arguments
+ * Script 'fuzzer-find-diff.pl' can be used to narrow down the problem in
+ * the case of test failure.
  */
-#include <assert.h>
 #include <stdlib.h>
 #include <stdio.h>
-#include <config.h>
 #include "utils.h"
 
-static pixman_indexed_t palette;
-
-static void *
-aligned_malloc (size_t align, size_t size)
-{
-    void *result;
-
-#ifdef HAVE_POSIX_MEMALIGN
-    if (posix_memalign (&result, align, size) != 0)
-      result = NULL;
-#else
-    result = malloc (size);
-#endif
+static pixman_indexed_t rgb_palette[9];
+static pixman_indexed_t y_palette[9];
 
-    return result;
-}
+/* The first eight format in the list are by far the most widely
+ * used formats, so we test those more than the others
+ */
+#define N_MOST_LIKELY_FORMATS 8
 
 /* Create random image for testing purposes */
 static pixman_image_t *
@@ -52,43 +25,60 @@ create_random_image (pixman_format_code_t *allowed_formats,
 		     int                   max_extra_stride,
 		     pixman_format_code_t *used_fmt)
 {
-    int n = 0, i, width, height, stride;
+    int n = 0, width, height, stride;
     pixman_format_code_t fmt;
     uint32_t *buf;
     pixman_image_t *img;
 
-    while (allowed_formats[n] != -1)
+    while (allowed_formats[n] != PIXMAN_null)
 	n++;
-    fmt = allowed_formats[lcg_rand_n (n)];
 
-    width = lcg_rand_n (max_width) + 1;
-    height = lcg_rand_n (max_height) + 1;
+    if (n > N_MOST_LIKELY_FORMATS && prng_rand_n (4) != 0)
+	n = N_MOST_LIKELY_FORMATS;
+    fmt = allowed_formats[prng_rand_n (n)];
+
+    width = prng_rand_n (max_width) + 1;
+    height = prng_rand_n (max_height) + 1;
     stride = (width * PIXMAN_FORMAT_BPP (fmt) + 7) / 8 +
-	lcg_rand_n (max_extra_stride + 1);
+	prng_rand_n (max_extra_stride + 1);
     stride = (stride + 3) & ~3;
 
     /* do the allocation */
     buf = aligned_malloc (64, stride * height);
 
-    /* initialize image with random data */
-    for (i = 0; i < stride * height; i++)
+    if (prng_rand_n (4) == 0)
     {
-	/* generation is biased to having more 0 or 255 bytes as
-	 * they are more likely to be special-cased in code
-	 */
-	*((uint8_t *)buf + i) = lcg_rand_n (4) ? lcg_rand_n (256) :
-	    (lcg_rand_n (2) ? 0 : 255);
+	/* uniform distribution */
+	prng_randmemset (buf, stride * height, 0);
+    }
+    else
+    {
+	/* significantly increased probability for 0x00 and 0xFF */
+	prng_randmemset (buf, stride * height, RANDMEMSET_MORE_00_AND_FF);
     }
 
+    /* test negative stride */
+    if (prng_rand_n (4) == 0)
+    {
+	buf += (stride / 4) * (height - 1);
+	stride = - stride;
+    }
+    
     img = pixman_image_create_bits (fmt, width, height, buf, stride);
 
-    if (PIXMAN_FORMAT_TYPE (fmt) == PIXMAN_TYPE_COLOR	||
-	PIXMAN_FORMAT_TYPE (fmt) == PIXMAN_TYPE_GRAY)
+    if (PIXMAN_FORMAT_TYPE (fmt) == PIXMAN_TYPE_COLOR)
     {
-	pixman_image_set_indexed (img, &palette);
+	pixman_image_set_indexed (img, &(rgb_palette[PIXMAN_FORMAT_BPP (fmt)]));
+    }
+    else if (PIXMAN_FORMAT_TYPE (fmt) == PIXMAN_TYPE_GRAY)
+    {
+	pixman_image_set_indexed (img, &(y_palette[PIXMAN_FORMAT_BPP (fmt)]));
     }
 
-    image_endian_swap (img, PIXMAN_FORMAT_BPP (fmt));
+    if (prng_rand_n (16) == 0)
+	pixman_image_set_filter (img, PIXMAN_FILTER_BILINEAR, NULL, 0);
+
+    image_endian_swap (img);
 
     if (used_fmt) *used_fmt = fmt;
     return img;
@@ -101,36 +91,13 @@ free_random_image (uint32_t initcrc,
 		   pixman_format_code_t fmt)
 {
     uint32_t crc32 = 0;
-    int stride = pixman_image_get_stride (img);
     uint32_t *data = pixman_image_get_data (img);
-    int height = pixman_image_get_height (img);
 
-    if (fmt != -1)
-    {
-	/* mask unused 'x' part */
-	if (PIXMAN_FORMAT_BPP (fmt) - PIXMAN_FORMAT_DEPTH (fmt) &&
-	    PIXMAN_FORMAT_DEPTH (fmt) != 0)
-	{
-	    int i;
-	    uint32_t *data = pixman_image_get_data (img);
-	    uint32_t mask = (1 << PIXMAN_FORMAT_DEPTH (fmt)) - 1;
-
-	    if (PIXMAN_FORMAT_TYPE (fmt) == PIXMAN_TYPE_BGRA)
-		mask <<= (PIXMAN_FORMAT_BPP (fmt) - PIXMAN_FORMAT_DEPTH (fmt));
-
-	    for (i = 0; i < 32; i++)
-		mask |= mask << (i * PIXMAN_FORMAT_BPP (fmt));
-
-	    for (i = 0; i < stride * height / 4; i++)
-		data[i] &= mask;
-	}
+    if (fmt != PIXMAN_null)
+	crc32 = compute_crc32_for_image (initcrc, img);
 
-	/* swap endiannes in order to provide identical results on both big
-	 * and litte endian systems
-	 */
-	image_endian_swap (img, PIXMAN_FORMAT_BPP (fmt));
-	crc32 = compute_crc32 (initcrc, data, stride * height);
-    }
+    if (img->bits.rowstride < 0)
+	data += img->bits.rowstride * (img->bits.height - 1);
 
     pixman_image_unref (img);
     free (data);
@@ -201,22 +168,27 @@ static pixman_op_t op_list[] = {
 
 static pixman_format_code_t img_fmt_list[] = {
     PIXMAN_a8r8g8b8,
+    PIXMAN_a8b8g8r8,
     PIXMAN_x8r8g8b8,
+    PIXMAN_x8b8g8r8,
     PIXMAN_r5g6b5,
-    PIXMAN_r3g3b2,
+    PIXMAN_b5g6r5,
     PIXMAN_a8,
-    PIXMAN_a8b8g8r8,
-    PIXMAN_x8b8g8r8,
+    PIXMAN_a1,
+    PIXMAN_r3g3b2,
     PIXMAN_b8g8r8a8,
     PIXMAN_b8g8r8x8,
+    PIXMAN_r8g8b8a8,
+    PIXMAN_r8g8b8x8,
+    PIXMAN_x14r6g6b6,
     PIXMAN_r8g8b8,
     PIXMAN_b8g8r8,
-    PIXMAN_r5g6b5,
-    PIXMAN_b5g6r5,
+#if 0 /* These are going to use floating point in the near future */
     PIXMAN_x2r10g10b10,
     PIXMAN_a2r10g10b10,
     PIXMAN_x2b10g10r10,
     PIXMAN_a2b10g10r10,
+#endif
     PIXMAN_a1r5g5b5,
     PIXMAN_x1r5g5b5,
     PIXMAN_a1b5g5r5,
@@ -225,7 +197,6 @@ static pixman_format_code_t img_fmt_list[] = {
     PIXMAN_x4r4g4b4,
     PIXMAN_a4b4g4r4,
     PIXMAN_x4b4g4r4,
-    PIXMAN_a8,
     PIXMAN_r3g3b2,
     PIXMAN_b2g3r3,
     PIXMAN_a2r2g2b2,
@@ -243,8 +214,7 @@ static pixman_format_code_t img_fmt_list[] = {
     PIXMAN_b1g2r1,
     PIXMAN_a1r1g1b1,
     PIXMAN_a1b1g1r1,
-    PIXMAN_a1,
-    -1
+    PIXMAN_null
 };
 
 static pixman_format_code_t mask_fmt_list[] = {
@@ -252,7 +222,7 @@ static pixman_format_code_t mask_fmt_list[] = {
     PIXMAN_a8,
     PIXMAN_a4,
     PIXMAN_a1,
-    -1
+    PIXMAN_null
 };
 
 
@@ -260,9 +230,8 @@ static pixman_format_code_t mask_fmt_list[] = {
  * Composite operation with pseudorandom images
  */
 uint32_t
-test_composite (uint32_t initcrc, int testnum, int verbose)
+test_composite (int testnum, int verbose)
 {
-    int i;
     pixman_image_t *src_img = NULL;
     pixman_image_t *dst_img = NULL;
     pixman_image_t *mask_img = NULL;
@@ -273,11 +242,12 @@ test_composite (uint32_t initcrc, int testnum, int verbose)
     int dst_x, dst_y;
     int mask_x, mask_y;
     int w, h;
-    int op;
+    pixman_op_t op;
     pixman_format_code_t src_fmt, dst_fmt, mask_fmt;
-    uint32_t *dstbuf, *srcbuf, *maskbuf;
+    uint32_t *srcbuf, *maskbuf;
     uint32_t crc32;
     int max_width, max_height, max_extra_stride;
+    FLOAT_REGS_CORRUPTION_DETECTOR_START ();
 
     max_width = max_height = 24 + testnum / 10000;
     max_extra_stride = 4 + testnum / 1000000;
@@ -291,11 +261,11 @@ test_composite (uint32_t initcrc, int testnum, int verbose)
     if (max_extra_stride > 8)
 	max_extra_stride = 8;
 
-    lcg_srand (testnum);
+    prng_srand (testnum);
 
-    op = op_list[lcg_rand_n (sizeof (op_list) / sizeof (op_list[0]))];
+    op = op_list[prng_rand_n (ARRAY_LENGTH (op_list))];
 
-    if (lcg_rand_n (8))
+    if (prng_rand_n (8))
     {
 	/* normal image */
 	src_img = create_random_image (img_fmt_list, max_width, max_height,
@@ -321,25 +291,24 @@ test_composite (uint32_t initcrc, int testnum, int verbose)
     dst_height = pixman_image_get_height (dst_img);
     dst_stride = pixman_image_get_stride (dst_img);
 
-    dstbuf = pixman_image_get_data (dst_img);
     srcbuf = pixman_image_get_data (src_img);
 
-    src_x = lcg_rand_n (src_width);
-    src_y = lcg_rand_n (src_height);
-    dst_x = lcg_rand_n (dst_width);
-    dst_y = lcg_rand_n (dst_height);
+    src_x = prng_rand_n (src_width);
+    src_y = prng_rand_n (src_height);
+    dst_x = prng_rand_n (dst_width);
+    dst_y = prng_rand_n (dst_height);
 
     mask_img = NULL;
-    mask_fmt = -1;
+    mask_fmt = PIXMAN_null;
     mask_x = 0;
     mask_y = 0;
     maskbuf = NULL;
 
     if ((src_fmt == PIXMAN_x8r8g8b8 || src_fmt == PIXMAN_x8b8g8r8) &&
-	(lcg_rand_n (4) == 0))
+	(prng_rand_n (4) == 0))
     {
 	/* PIXBUF */
-	mask_fmt = lcg_rand_n (2) ? PIXMAN_a8r8g8b8 : PIXMAN_a8b8g8r8;
+	mask_fmt = prng_rand_n (2) ? PIXMAN_a8r8g8b8 : PIXMAN_a8b8g8r8;
 	mask_img = pixman_image_create_bits (mask_fmt,
 	                                     src_width,
 	                                     src_height,
@@ -349,9 +318,9 @@ test_composite (uint32_t initcrc, int testnum, int verbose)
 	mask_y = src_y;
 	maskbuf = srcbuf;
     }
-    else if (lcg_rand_n (2))
+    else if (prng_rand_n (2))
     {
-	if (lcg_rand_n (2))
+	if (prng_rand_n (2))
 	{
 	    mask_img = create_random_image (mask_fmt_list, max_width, max_height,
 					   max_extra_stride, &mask_fmt);
@@ -364,21 +333,23 @@ test_composite (uint32_t initcrc, int testnum, int verbose)
 	    pixman_image_set_repeat (mask_img, PIXMAN_REPEAT_NORMAL);
 	}
 
-	if (lcg_rand_n (2))
+	if (prng_rand_n (2))
 	    pixman_image_set_component_alpha (mask_img, 1);
 
-	mask_x = lcg_rand_n (pixman_image_get_width (mask_img));
-	mask_y = lcg_rand_n (pixman_image_get_height (mask_img));
+	mask_x = prng_rand_n (pixman_image_get_width (mask_img));
+	mask_y = prng_rand_n (pixman_image_get_height (mask_img));
     }
 
 
-    w = lcg_rand_n (dst_width - dst_x + 1);
-    h = lcg_rand_n (dst_height - dst_y + 1);
+    w = prng_rand_n (dst_width - dst_x + 1);
+    h = prng_rand_n (dst_height - dst_y + 1);
 
     if (verbose)
     {
-	printf ("op=%d, src_fmt=%08X, dst_fmt=%08X, mask_fmt=%08X\n",
-	    op, src_fmt, dst_fmt, mask_fmt);
+        printf ("op=%s\n", operator_name (op));
+	printf ("src_fmt=%s, dst_fmt=%s, mask_fmt=%s\n",
+	    format_name (src_fmt), format_name (dst_fmt),
+	    format_name (mask_fmt));
 	printf ("src_width=%d, src_height=%d, dst_width=%d, dst_height=%d\n",
 	    src_width, src_height, dst_width, dst_height);
 	printf ("src_x=%d, src_y=%d, dst_x=%d, dst_y=%d\n",
@@ -392,106 +363,37 @@ test_composite (uint32_t initcrc, int testnum, int verbose)
 			    src_x, src_y, mask_x, mask_y, dst_x, dst_y, w, h);
 
     if (verbose)
-    {
-	int j;
-
-	printf ("---\n");
-	for (i = 0; i < dst_height; i++)
-	{
-	    for (j = 0; j < dst_stride; j++)
-	    {
-		if (j == (dst_width * PIXMAN_FORMAT_BPP (dst_fmt) + 7) / 8)
-		    printf ("| ");
-
-		printf ("%02X ", *((uint8_t *)dstbuf + i * dst_stride + j));
-	    }
-	    printf ("\n");
-	}
-	printf ("---\n");
-    }
+	print_image (dst_img);
 
-    free_random_image (initcrc, src_img, -1);
-    crc32 = free_random_image (initcrc, dst_img, dst_fmt);
+    free_random_image (0, src_img, PIXMAN_null);
+    crc32 = free_random_image (0, dst_img, dst_fmt);
 
     if (mask_img)
     {
 	if (srcbuf == maskbuf)
 	    pixman_image_unref(mask_img);
 	else
-	    free_random_image (initcrc, mask_img, -1);
+	    free_random_image (0, mask_img, PIXMAN_null);
     }
 
-
+    FLOAT_REGS_CORRUPTION_DETECTOR_FINISH ();
     return crc32;
 }
 
-static void
-initialize_palette (void)
-{
-    int i;
-
-    for (i = 0; i < PIXMAN_MAX_INDEXED; ++i)
-	palette.rgba[i] = lcg_rand ();
-
-    for (i = 0; i < 32768; ++i)
-	palette.ent[i] = lcg_rand() & 0xff;
-}
-
 int
-main (int argc, char *argv[])
+main (int argc, const char *argv[])
 {
-    int i, n1 = 1, n2 = 0;
-    uint32_t crc = 0;
-    int verbose = getenv ("VERBOSE") != NULL;
-
-    initialize_palette();
+    int i;
 
-    if (argc >= 3)
-    {
-	n1 = atoi (argv[1]);
-	n2 = atoi (argv[2]);
-    }
-    else if (argc >= 2)
-    {
-	n2 = atoi (argv[1]);
-    }
-    else
-    {
-	n1 = 1;
-	n2 = 2000000;
-    }
+    prng_srand (0);
 
-    if (n2 < 0)
+    for (i = 1; i <= 8; i++)
     {
-	crc = test_composite (0, abs (n2), 1);
-	printf ("crc32=%08X\n", crc);
+	initialize_palette (&(rgb_palette[i]), i, TRUE);
+	initialize_palette (&(y_palette[i]), i, FALSE);
     }
-    else
-    {
-	for (i = n1; i <= n2; i++)
-	{
-	    crc = test_composite (crc, i, 0);
-
-	    if (verbose)
-		printf ("%d: %08X\n", i, crc);
-	}
-	printf ("crc32=%08X\n", crc);
 
-	if (n2 == 2000000)
-	{
-	    /* Predefined value for running with all the fastpath functions
-	       disabled. It needs to be updated every time when changes are
-	       introduced to this program or behavior of pixman changes! */
-	    if (crc == 0xBBACC28D)
-	    {
-		printf ("blitters test passed\n");
-	    }
-	    else
-	    {
-		printf ("blitters test failed!\n");
-		return 1;
-	    }
-	}
-    }
-    return 0;
+    return fuzzer_test_main("blitters", 2000000,
+			    0xE0A07495,
+			    test_composite, argc, argv);
 }
diff --git a/pixman/test/check-formats.c b/pixman/test/check-formats.c
new file mode 100644
index 000000000..7edc198c1
--- /dev/null
+++ b/pixman/test/check-formats.c
@@ -0,0 +1,352 @@
+#include <ctype.h>
+#include "utils.h"
+
+static int
+check_op (pixman_op_t          op,
+          pixman_format_code_t src_format,
+          pixman_format_code_t dest_format)
+{
+    uint32_t src_alpha_mask, src_green_mask;
+    uint32_t dest_alpha_mask, dest_green_mask;
+    pixel_checker_t src_checker, dest_checker;
+    pixman_image_t *si, *di;
+    uint32_t sa, sg, da, dg;
+    uint32_t s, d;
+    int retval = 0;
+
+    pixel_checker_init (&src_checker, src_format);
+    pixel_checker_init (&dest_checker, dest_format);
+
+    pixel_checker_get_masks (
+        &src_checker, &src_alpha_mask, NULL, &src_green_mask, NULL);
+    pixel_checker_get_masks (
+        &dest_checker, &dest_alpha_mask, NULL, &dest_green_mask, NULL);
+
+    /* printf ("masks: %x %x %x %x\n", */
+    /* 	    src_alpha_mask, src_green_mask, */
+    /* 	    dest_alpha_mask, dest_green_mask); */
+
+    si = pixman_image_create_bits (src_format, 1, 1, &s, 4);
+    di = pixman_image_create_bits (dest_format, 1, 1, &d, 4);
+
+    sa = 0;
+    do
+    {
+        sg = 0;
+        do
+        {
+            da = 0;
+            do
+            {
+                dg = 0;
+                do
+                {
+                    color_t src_color, dest_color, result_color;
+                    uint32_t orig_d;
+
+                    s = sa | sg;
+                    d = da | dg;
+
+                    orig_d = d;
+
+		    pixel_checker_convert_pixel_to_color (&src_checker, s, &src_color);
+		    pixel_checker_convert_pixel_to_color (&dest_checker, d, &dest_color);
+
+		    do_composite (op, &src_color, NULL, &dest_color, &result_color, FALSE);
+
+
+		    if (!is_little_endian())
+                    {
+			s <<= 32 - PIXMAN_FORMAT_BPP (src_format);
+			d <<= 32 - PIXMAN_FORMAT_BPP (dest_format);
+                    }
+
+		    pixman_image_composite32 (op, si, NULL, di,
+					      0, 0, 0, 0, 0, 0, 1, 1);
+
+		    if (!is_little_endian())
+                        d >>= (32 - PIXMAN_FORMAT_BPP (dest_format));
+
+                    if (!pixel_checker_check (&dest_checker, d, &result_color))
+                    {
+                        printf ("---- test failed ----\n");
+                        printf ("operator: %-32s\n", operator_name (op));
+                        printf ("source:   %-12s pixel: %08x\n", format_name (src_format), s);
+                        printf ("dest:     %-12s pixel: %08x\n", format_name (dest_format), orig_d);
+                        printf ("got:      %-12s pixel: %08x\n", format_name (dest_format), d);
+
+                        retval = 1;
+                    }
+
+                    dg -= dest_green_mask;
+                    dg &= dest_green_mask;
+                }
+                while (dg != 0);
+
+                da -= dest_alpha_mask;
+                da &= dest_alpha_mask;
+            }
+            while (da != 0);
+
+            sg -= src_green_mask;
+            sg &= src_green_mask;
+        }
+        while (sg != 0);
+
+        sa -= src_alpha_mask;
+        sa &= src_alpha_mask;
+    }
+    while (sa != 0);
+
+    pixman_image_unref (si);
+    pixman_image_unref (di);
+
+    return retval;
+}
+
+static const pixman_op_t op_list[] =
+{
+    PIXMAN_OP_CLEAR,
+    PIXMAN_OP_SRC,
+    PIXMAN_OP_DST,
+    PIXMAN_OP_OVER,
+    PIXMAN_OP_OVER_REVERSE,
+    PIXMAN_OP_IN,
+    PIXMAN_OP_IN_REVERSE,
+    PIXMAN_OP_OUT,
+    PIXMAN_OP_OUT_REVERSE,
+    PIXMAN_OP_ATOP,
+    PIXMAN_OP_ATOP_REVERSE,
+    PIXMAN_OP_XOR,
+    PIXMAN_OP_ADD,
+    PIXMAN_OP_SATURATE,
+
+    PIXMAN_OP_DISJOINT_CLEAR,
+    PIXMAN_OP_DISJOINT_SRC,
+    PIXMAN_OP_DISJOINT_DST,
+    PIXMAN_OP_DISJOINT_OVER,
+    PIXMAN_OP_DISJOINT_OVER_REVERSE,
+    PIXMAN_OP_DISJOINT_IN,
+    PIXMAN_OP_DISJOINT_IN_REVERSE,
+    PIXMAN_OP_DISJOINT_OUT,
+    PIXMAN_OP_DISJOINT_OUT_REVERSE,
+    PIXMAN_OP_DISJOINT_ATOP,
+    PIXMAN_OP_DISJOINT_ATOP_REVERSE,
+    PIXMAN_OP_DISJOINT_XOR,
+
+    PIXMAN_OP_CONJOINT_CLEAR,
+    PIXMAN_OP_CONJOINT_SRC,
+    PIXMAN_OP_CONJOINT_DST,
+    PIXMAN_OP_CONJOINT_OVER,
+    PIXMAN_OP_CONJOINT_OVER_REVERSE,
+    PIXMAN_OP_CONJOINT_IN,
+    PIXMAN_OP_CONJOINT_IN_REVERSE,
+    PIXMAN_OP_CONJOINT_OUT,
+    PIXMAN_OP_CONJOINT_OUT_REVERSE,
+    PIXMAN_OP_CONJOINT_ATOP,
+    PIXMAN_OP_CONJOINT_ATOP_REVERSE,
+    PIXMAN_OP_CONJOINT_XOR,
+};
+
+static const pixman_format_code_t format_list[] =
+{
+    PIXMAN_a8r8g8b8,
+    PIXMAN_x8r8g8b8,
+    PIXMAN_a8b8g8r8,
+    PIXMAN_x8b8g8r8,
+    PIXMAN_b8g8r8a8,
+    PIXMAN_b8g8r8x8,
+    PIXMAN_r8g8b8a8,
+    PIXMAN_r8g8b8x8,
+    PIXMAN_x14r6g6b6,
+    PIXMAN_x2r10g10b10,
+    PIXMAN_a2r10g10b10,
+    PIXMAN_x2b10g10r10,
+    PIXMAN_a2b10g10r10,
+    PIXMAN_a8r8g8b8_sRGB,
+    PIXMAN_r8g8b8,
+    PIXMAN_b8g8r8,
+    PIXMAN_r5g6b5,
+    PIXMAN_b5g6r5,
+    PIXMAN_a1r5g5b5,
+    PIXMAN_x1r5g5b5,
+    PIXMAN_a1b5g5r5,
+    PIXMAN_x1b5g5r5,
+    PIXMAN_a4r4g4b4,
+    PIXMAN_x4r4g4b4,
+    PIXMAN_a4b4g4r4,
+    PIXMAN_x4b4g4r4,
+    PIXMAN_a8,
+    PIXMAN_r3g3b2,
+    PIXMAN_b2g3r3,
+    PIXMAN_a2r2g2b2,
+    PIXMAN_a2b2g2r2,
+    PIXMAN_x4a4,
+    PIXMAN_a4,
+    PIXMAN_r1g2b1,
+    PIXMAN_b1g2r1,
+    PIXMAN_a1r1g1b1,
+    PIXMAN_a1b1g1r1,
+    PIXMAN_a1,
+};
+
+static pixman_format_code_t
+format_from_string (const char *s)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_LENGTH (format_list); ++i)
+    {
+        if (strcasecmp (format_name (format_list[i]), s) == 0)
+            return format_list[i];
+    }
+
+    return PIXMAN_null;
+}
+
+static void
+emit (const char *s, int *n_chars)
+{
+    *n_chars += printf ("%s,", s);
+    if (*n_chars > 60)
+    {
+        printf ("\n    ");
+        *n_chars = 0;
+    }
+    else
+    {
+        printf (" ");
+        (*n_chars)++;
+    }
+}
+
+static void
+list_formats (void)
+{
+    int n_chars;
+    int i;
+
+    printf ("Formats:\n    ");
+
+    n_chars = 0;
+    for (i = 0; i < ARRAY_LENGTH (format_list); ++i)
+        emit (format_name (format_list[i]), &n_chars);
+
+    printf ("\n\n");
+}
+
+static void
+list_operators (void)
+{
+    char short_name [128] = { 0 };
+    int i, n_chars;
+
+    printf ("Operators:\n    ");
+
+    n_chars = 0;
+    for (i = 0; i < ARRAY_LENGTH (op_list); ++i)
+    {
+        pixman_op_t op = op_list[i];
+        int j;
+
+        snprintf (short_name, sizeof (short_name) - 1, "%s",
+                  operator_name (op) + strlen ("PIXMAN_OP_"));
+
+        for (j = 0; short_name[j] != '\0'; ++j)
+            short_name[j] = tolower (short_name[j]);
+
+        emit (short_name, &n_chars);
+    }
+
+    printf ("\n\n");
+}
+
+static pixman_op_t
+operator_from_string (const char *s)
+{
+    char full_name[128] = { 0 };
+    int i;
+
+    snprintf (full_name, (sizeof full_name) - 1, "PIXMAN_OP_%s", s);
+
+    for (i = 0; i < ARRAY_LENGTH (op_list); ++i)
+    {
+        pixman_op_t op = op_list[i];
+
+        if (strcasecmp (operator_name (op), full_name) == 0)
+            return op;
+    }
+
+    return PIXMAN_OP_NONE;
+}
+
+int
+main (int argc, char **argv)
+{
+    enum { OPTION_OP, OPTION_SRC, OPTION_DEST, LAST_OPTION } option;
+    pixman_format_code_t src_fmt, dest_fmt;
+    pixman_op_t op;
+
+    op = PIXMAN_OP_NONE;
+    src_fmt = PIXMAN_null;
+    dest_fmt = PIXMAN_null;
+
+    argc--;
+    argv++;
+
+    for (option = OPTION_OP; option < LAST_OPTION; ++option)
+    {
+        char *arg = NULL;
+
+        if (argc)
+        {
+            argc--;
+            arg = *argv++;
+        }
+
+        switch (option)
+        {
+        case OPTION_OP:
+            if (!arg)
+                printf ("  - missing operator\n");
+            else if ((op = operator_from_string (arg)) == PIXMAN_OP_NONE)
+                printf ("  - unknown operator %s\n", arg);
+            break;
+
+        case OPTION_SRC:
+            if (!arg)
+                printf ("  - missing source format\n");
+            else if ((src_fmt = format_from_string (arg)) == PIXMAN_null)
+                printf ("  - unknown source format %s\n", arg);
+            break;
+
+        case OPTION_DEST:
+            if (!arg)
+                printf ("  - missing destination format\n");
+            else if ((dest_fmt = format_from_string (arg)) == PIXMAN_null)
+                printf ("  - unknown destination format %s\n", arg);
+            break;
+
+        default:
+            assert (0);
+            break;
+        }
+    }
+
+    while (argc--)
+    {
+        op = PIXMAN_OP_NONE;
+        printf ("  - unexpected argument: %s\n", *argv++);
+    }
+
+    if (op == PIXMAN_OP_NONE || src_fmt == PIXMAN_null || dest_fmt == PIXMAN_null)
+    {
+        printf ("\nUsage:\n    check-formats <operator> <src-format> <dest-format>\n\n");
+        list_operators();
+        list_formats();
+
+        return -1;
+    }
+
+    return check_op (op, src_fmt, dest_fmt);
+}
diff --git a/pixman/test/combiner-test.c b/pixman/test/combiner-test.c
new file mode 100644
index 000000000..01f63a56e
--- /dev/null
+++ b/pixman/test/combiner-test.c
@@ -0,0 +1,151 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include "utils.h"
+#include <sys/types.h>
+#include "pixman-private.h"
+
+static const pixman_op_t op_list[] =
+{
+    PIXMAN_OP_SRC,
+    PIXMAN_OP_OVER,
+    PIXMAN_OP_ADD,
+    PIXMAN_OP_CLEAR,
+    PIXMAN_OP_SRC,
+    PIXMAN_OP_DST,
+    PIXMAN_OP_OVER,
+    PIXMAN_OP_OVER_REVERSE,
+    PIXMAN_OP_IN,
+    PIXMAN_OP_IN_REVERSE,
+    PIXMAN_OP_OUT,
+    PIXMAN_OP_OUT_REVERSE,
+    PIXMAN_OP_ATOP,
+    PIXMAN_OP_ATOP_REVERSE,
+    PIXMAN_OP_XOR,
+    PIXMAN_OP_ADD,
+    PIXMAN_OP_SATURATE,
+    PIXMAN_OP_DISJOINT_CLEAR,
+    PIXMAN_OP_DISJOINT_SRC,
+    PIXMAN_OP_DISJOINT_DST,
+    PIXMAN_OP_DISJOINT_OVER,
+    PIXMAN_OP_DISJOINT_OVER_REVERSE,
+    PIXMAN_OP_DISJOINT_IN,
+    PIXMAN_OP_DISJOINT_IN_REVERSE,
+    PIXMAN_OP_DISJOINT_OUT,
+    PIXMAN_OP_DISJOINT_OUT_REVERSE,
+    PIXMAN_OP_DISJOINT_ATOP,
+    PIXMAN_OP_DISJOINT_ATOP_REVERSE,
+    PIXMAN_OP_DISJOINT_XOR,
+    PIXMAN_OP_CONJOINT_CLEAR,
+    PIXMAN_OP_CONJOINT_SRC,
+    PIXMAN_OP_CONJOINT_DST,
+    PIXMAN_OP_CONJOINT_OVER,
+    PIXMAN_OP_CONJOINT_OVER_REVERSE,
+    PIXMAN_OP_CONJOINT_IN,
+    PIXMAN_OP_CONJOINT_IN_REVERSE,
+    PIXMAN_OP_CONJOINT_OUT,
+    PIXMAN_OP_CONJOINT_OUT_REVERSE,
+    PIXMAN_OP_CONJOINT_ATOP,
+    PIXMAN_OP_CONJOINT_ATOP_REVERSE,
+    PIXMAN_OP_CONJOINT_XOR,
+    PIXMAN_OP_MULTIPLY,
+    PIXMAN_OP_SCREEN,
+    PIXMAN_OP_OVERLAY,
+    PIXMAN_OP_DARKEN,
+    PIXMAN_OP_LIGHTEN,
+    PIXMAN_OP_COLOR_DODGE,
+    PIXMAN_OP_COLOR_BURN,
+    PIXMAN_OP_HARD_LIGHT,
+    PIXMAN_OP_DIFFERENCE,
+    PIXMAN_OP_EXCLUSION,
+    PIXMAN_OP_SOFT_LIGHT,
+    PIXMAN_OP_HSL_HUE,
+    PIXMAN_OP_HSL_SATURATION,
+    PIXMAN_OP_HSL_COLOR,
+    PIXMAN_OP_HSL_LUMINOSITY,
+};
+
+static float
+rand_float (void)
+{
+    uint32_t u = prng_rand();
+
+    return *(float *)&u;
+}
+
+static void
+random_floats (argb_t *argb, int width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	argb_t *p = argb + i;
+
+	p->a = rand_float();
+	p->r = rand_float();
+	p->g = rand_float();
+	p->b = rand_float();
+    }
+}
+
+#define WIDTH	512
+
+static pixman_combine_float_func_t
+lookup_combiner (pixman_implementation_t *imp, pixman_op_t op,
+		 pixman_bool_t component_alpha)
+{
+    pixman_combine_float_func_t f;
+
+    do
+    {
+	if (component_alpha)
+	    f = imp->combine_float_ca[op];
+	else
+	    f = imp->combine_float[op];
+	
+	imp = imp->fallback;
+    }
+    while (!f);
+
+    return f;
+}
+
+int
+main ()
+{
+    pixman_implementation_t *impl;
+    argb_t *src_bytes = malloc (WIDTH * sizeof (argb_t));
+    argb_t *mask_bytes = malloc (WIDTH * sizeof (argb_t));
+    argb_t *dest_bytes = malloc (WIDTH * sizeof (argb_t));
+    int i;
+
+    enable_divbyzero_exceptions();
+    
+    impl = _pixman_internal_only_get_implementation();
+    
+    prng_srand (0);
+
+    for (i = 0; i < ARRAY_LENGTH (op_list); ++i)
+    {
+	pixman_op_t op = op_list[i];
+	pixman_combine_float_func_t combiner;
+	int ca;
+
+	for (ca = 0; ca < 2; ++ca)
+	{
+	    combiner = lookup_combiner (impl, op, ca);
+
+	    random_floats (src_bytes, WIDTH);
+	    random_floats (mask_bytes, WIDTH);
+	    random_floats (dest_bytes, WIDTH);
+
+	    combiner (impl, op,
+		      (float *)dest_bytes,
+		      (float *)mask_bytes,
+		      (float *)src_bytes,
+		      WIDTH);
+	}
+    }	
+
+    return 0;
+}
diff --git a/pixman/test/composite-traps-test.c b/pixman/test/composite-traps-test.c
new file mode 100644
index 000000000..86a035564
--- /dev/null
+++ b/pixman/test/composite-traps-test.c
@@ -0,0 +1,252 @@
+/* Based loosely on scaling-test */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include "utils.h"
+
+#define MAX_SRC_WIDTH  48
+#define MAX_SRC_HEIGHT 48
+#define MAX_DST_WIDTH  48
+#define MAX_DST_HEIGHT 48
+#define MAX_STRIDE     4
+
+static pixman_format_code_t formats[] =
+{
+    PIXMAN_a8r8g8b8, PIXMAN_a8, PIXMAN_r5g6b5, PIXMAN_a1, PIXMAN_a4
+};
+
+static pixman_format_code_t mask_formats[] =
+{
+    PIXMAN_a1, PIXMAN_a4, PIXMAN_a8,
+};
+
+static pixman_op_t operators[] =
+{
+    PIXMAN_OP_OVER, PIXMAN_OP_ADD, PIXMAN_OP_SRC, PIXMAN_OP_IN
+};
+
+#define RANDOM_ELT(array)						\
+    ((array)[prng_rand_n(ARRAY_LENGTH((array)))])
+
+static void
+destroy_bits (pixman_image_t *image, void *data)
+{
+    fence_free (data);
+}
+
+static pixman_fixed_t
+random_fixed (int n)
+{
+    return prng_rand_n (n << 16);
+}
+
+/*
+ * Composite operation with pseudorandom images
+ */
+uint32_t
+test_composite (int      testnum,
+		int      verbose)
+{
+    int                i;
+    pixman_image_t *   src_img;
+    pixman_image_t *   dst_img;
+    pixman_region16_t  clip;
+    int                dst_width, dst_height;
+    int                dst_stride;
+    int                dst_x, dst_y;
+    int                dst_bpp;
+    pixman_op_t        op;
+    uint32_t *         dst_bits;
+    uint32_t           crc32;
+    pixman_format_code_t mask_format, dst_format;
+    pixman_trapezoid_t *traps;
+    int src_x, src_y;
+    int n_traps;
+
+    static pixman_color_t colors[] =
+    {
+	{ 0xffff, 0xffff, 0xffff, 0xffff },
+	{ 0x0000, 0x0000, 0x0000, 0x0000 },
+	{ 0xabcd, 0xabcd, 0x0000, 0xabcd },
+	{ 0x0000, 0x0000, 0x0000, 0xffff },
+	{ 0x0101, 0x0101, 0x0101, 0x0101 },
+	{ 0x7777, 0x6666, 0x5555, 0x9999 },
+    };
+    
+    FLOAT_REGS_CORRUPTION_DETECTOR_START ();
+
+    prng_srand (testnum);
+
+    op = RANDOM_ELT (operators);
+    mask_format = RANDOM_ELT (mask_formats);
+
+    /* Create source image */
+    
+    if (prng_rand_n (4) == 0)
+    {
+	src_img = pixman_image_create_solid_fill (
+	    &(colors[prng_rand_n (ARRAY_LENGTH (colors))]));
+
+	src_x = 10;
+	src_y = 234;
+    }
+    else
+    {
+	pixman_format_code_t src_format = RANDOM_ELT(formats);
+	int src_bpp = (PIXMAN_FORMAT_BPP (src_format) + 7) / 8;
+	int src_width = prng_rand_n (MAX_SRC_WIDTH) + 1;
+	int src_height = prng_rand_n (MAX_SRC_HEIGHT) + 1;
+	int src_stride = src_width * src_bpp + prng_rand_n (MAX_STRIDE) * src_bpp;
+	uint32_t *bits, *orig;
+
+	src_x = -(src_width / 4) + prng_rand_n (src_width * 3 / 2);
+	src_y = -(src_height / 4) + prng_rand_n (src_height * 3 / 2);
+
+	src_stride = (src_stride + 3) & ~3;
+	
+	orig = bits = (uint32_t *)make_random_bytes (src_stride * src_height);
+
+	if (prng_rand_n (2) == 0)
+	{
+	    bits += (src_stride / 4) * (src_height - 1);
+	    src_stride = - src_stride;
+	}
+	
+	src_img = pixman_image_create_bits (
+	    src_format, src_width, src_height, bits, src_stride);
+
+	pixman_image_set_destroy_function (src_img, destroy_bits, orig);
+
+	if (prng_rand_n (8) == 0)
+	{
+	    pixman_box16_t clip_boxes[2];
+	    int            n = prng_rand_n (2) + 1;
+	    
+	    for (i = 0; i < n; i++)
+	    {
+		clip_boxes[i].x1 = prng_rand_n (src_width);
+		clip_boxes[i].y1 = prng_rand_n (src_height);
+		clip_boxes[i].x2 =
+		    clip_boxes[i].x1 + prng_rand_n (src_width - clip_boxes[i].x1);
+		clip_boxes[i].y2 =
+		    clip_boxes[i].y1 + prng_rand_n (src_height - clip_boxes[i].y1);
+		
+		if (verbose)
+		{
+		    printf ("source clip box: [%d,%d-%d,%d]\n",
+			    clip_boxes[i].x1, clip_boxes[i].y1,
+			    clip_boxes[i].x2, clip_boxes[i].y2);
+		}
+	    }
+	    
+	    pixman_region_init_rects (&clip, clip_boxes, n);
+	    pixman_image_set_clip_region (src_img, &clip);
+	    pixman_image_set_source_clipping (src_img, 1);
+	    pixman_region_fini (&clip);
+	}
+
+	image_endian_swap (src_img);
+    }
+
+    /* Create destination image */
+    {
+	dst_format = RANDOM_ELT(formats);
+	dst_bpp = (PIXMAN_FORMAT_BPP (dst_format) + 7) / 8;
+	dst_width = prng_rand_n (MAX_DST_WIDTH) + 1;
+	dst_height = prng_rand_n (MAX_DST_HEIGHT) + 1;
+	dst_stride = dst_width * dst_bpp + prng_rand_n (MAX_STRIDE) * dst_bpp;
+	dst_stride = (dst_stride + 3) & ~3;
+	
+	dst_bits = (uint32_t *)make_random_bytes (dst_stride * dst_height);
+
+	if (prng_rand_n (2) == 0)
+	{
+	    dst_bits += (dst_stride / 4) * (dst_height - 1);
+	    dst_stride = - dst_stride;
+	}
+	
+	dst_x = -(dst_width / 4) + prng_rand_n (dst_width * 3 / 2);
+	dst_y = -(dst_height / 4) + prng_rand_n (dst_height * 3 / 2);
+	
+	dst_img = pixman_image_create_bits (
+	    dst_format, dst_width, dst_height, dst_bits, dst_stride);
+
+	image_endian_swap (dst_img);
+    }
+
+    /* Create traps */
+    {
+	int i;
+
+	n_traps = prng_rand_n (25);
+	traps = fence_malloc (n_traps * sizeof (pixman_trapezoid_t));
+
+	for (i = 0; i < n_traps; ++i)
+	{
+	    pixman_trapezoid_t *t = &(traps[i]);
+	    
+	    t->top = random_fixed (MAX_DST_HEIGHT) - MAX_DST_HEIGHT / 2;
+	    t->bottom = t->top + random_fixed (MAX_DST_HEIGHT);
+	    t->left.p1.x = random_fixed (MAX_DST_WIDTH) - MAX_DST_WIDTH / 2;
+	    t->left.p1.y = t->top - random_fixed (50);
+	    t->left.p2.x = random_fixed (MAX_DST_WIDTH) - MAX_DST_WIDTH / 2;
+	    t->left.p2.y = t->bottom + random_fixed (50);
+	    t->right.p1.x = t->left.p1.x + random_fixed (MAX_DST_WIDTH);
+	    t->right.p1.y = t->top - random_fixed (50);
+	    t->right.p2.x = t->left.p2.x + random_fixed (MAX_DST_WIDTH);
+	    t->right.p2.y = t->bottom - random_fixed (50);
+	}
+    }
+    
+    if (prng_rand_n (8) == 0)
+    {
+	pixman_box16_t clip_boxes[2];
+	int            n = prng_rand_n (2) + 1;
+	for (i = 0; i < n; i++)
+	{
+	    clip_boxes[i].x1 = prng_rand_n (dst_width);
+	    clip_boxes[i].y1 = prng_rand_n (dst_height);
+	    clip_boxes[i].x2 =
+		clip_boxes[i].x1 + prng_rand_n (dst_width - clip_boxes[i].x1);
+	    clip_boxes[i].y2 =
+		clip_boxes[i].y1 + prng_rand_n (dst_height - clip_boxes[i].y1);
+
+	    if (verbose)
+	    {
+		printf ("destination clip box: [%d,%d-%d,%d]\n",
+		        clip_boxes[i].x1, clip_boxes[i].y1,
+		        clip_boxes[i].x2, clip_boxes[i].y2);
+	    }
+	}
+	pixman_region_init_rects (&clip, clip_boxes, n);
+	pixman_image_set_clip_region (dst_img, &clip);
+	pixman_region_fini (&clip);
+    }
+
+    pixman_composite_trapezoids (op, src_img, dst_img, mask_format,
+				 src_x, src_y, dst_x, dst_y, n_traps, traps);
+
+    crc32 = compute_crc32_for_image (0, dst_img);
+
+    if (verbose)
+	print_image (dst_img);
+
+    if (dst_stride < 0)
+	dst_bits += (dst_stride / 4) * (dst_height - 1);
+    
+    fence_free (dst_bits);
+    
+    pixman_image_unref (src_img);
+    pixman_image_unref (dst_img);
+    fence_free (traps);
+
+    FLOAT_REGS_CORRUPTION_DETECTOR_FINISH ();
+    return crc32;
+}
+
+int
+main (int argc, const char *argv[])
+{
+    return fuzzer_test_main("composite traps", 40000, 0xAF41D210,
+			    test_composite, argc, argv);
+}
diff --git a/pixman/test/composite.c b/pixman/test/composite.c
index 0624cd37d..9e51a8f65 100644
--- a/pixman/test/composite.c
+++ b/pixman/test/composite.c
@@ -1,6 +1,8 @@
 /*
  * Copyright © 2005 Eric Anholt
  * Copyright © 2009 Chris Wilson
+ * Copyright © 2010 Soeren Sandmann
+ * Copyright © 2010 Red Hat, Inc.
  *
  * Permission to use, copy, modify, distribute, and sell this software and its
  * documentation for any purpose is hereby granted without fee, provided that
@@ -20,44 +22,23 @@
  * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
  * PERFORMANCE OF THIS SOFTWARE.
  */
-
-#include <pixman.h>
 #include <stdio.h>
 #include <stdlib.h> /* abort() */
 #include <math.h>
-#include <config.h>
-
-#define FALSE 0
-#define TRUE !FALSE
+#include <time.h>
+#include "utils.h"
 
-#define ARRAY_LENGTH(A) ((int) (sizeof (A) / sizeof ((A) [0])))
-#define min(a,b) ((a) <= (b) ? (a) : (b))
-#define max(a,b) ((a) >= (b) ? (a) : (b))
-
-typedef struct color_t color_t;
-typedef struct format_t format_t;
 typedef struct image_t image_t;
-typedef struct operator_t operator_t;
-
-struct color_t
-{
-    double r, g, b, a;
-};
 
-struct format_t
+static const color_t colors[] =
 {
-    pixman_format_code_t format;
-    const char *name;
-};
-
-static color_t colors[] =
-{
-    /* these are premultiplied in main() */
     { 1.0, 1.0, 1.0, 1.0 },
+    { 1.0, 1.0, 1.0, 0.0 },
+    { 0.0, 0.0, 0.0, 1.0 },
+    { 0.0, 0.0, 0.0, 0.0 },
     { 1.0, 0.0, 0.0, 1.0 },
     { 0.0, 1.0, 0.0, 1.0 },
     { 0.0, 0.0, 1.0, 1.0 },
-    { 0.0, 0.0, 0.0, 1.0 },
     { 0.5, 0.0, 0.0, 0.5 },
 };
 
@@ -82,677 +63,322 @@ compute_pixman_color (const color_t *color,
     out->alpha = _color_double_to_short (color->a);
 }
 
-static const format_t formats[] =
+#define REPEAT 0x01000000
+#define FLAGS  0xff000000
+
+static const int sizes[] =
 {
-#define P(x) { PIXMAN_##x, #x }
-    P(a8),
-
-    /* 32bpp formats */
-    P(a8r8g8b8),
-    P(x8r8g8b8),
-    P(a8b8g8r8),
-    P(x8b8g8r8),
-    P(b8g8r8a8),
-    P(b8g8r8x8),
-
-    /* XXX: and here the errors begin! */
-#if 0
-    P(x2r10g10b10),
-    P(a2r10g10b10),
-    P(x2b10g10r10),
-    P(a2b10g10r10),
-
-    /* 24bpp formats */
-    P(r8g8b8),
-    P(b8g8r8),
-
-    /* 16bpp formats */
-    P(r5g6b5),
-    P(b5g6r5),
-
-    P(a1r5g5b5),
-    P(x1r5g5b5),
-    P(a1b5g5r5),
-    P(x1b5g5r5),
-    P(a4r4g4b4),
-    P(x4r4g4b4),
-    P(a4b4g4r4),
-    P(x4b4g4r4),
-
-    /* 8bpp formats */
-    P(a8),
-    P(r3g3b2),
-    P(b2g3r3),
-    P(a2r2g2b2),
-    P(a2b2g2r2),
-
-    P(x4a4),
-
-    /* 4bpp formats */
-    P(a4),
-    P(r1g2b1),
-    P(b1g2r1),
-    P(a1r1g1b1),
-    P(a1b1g1r1),
-
-    /* 1bpp formats */
-    P(a1)
-#endif
-#undef P
+    0,
+    1,
+    1 | REPEAT,
+    10
+};
+
+static const pixman_format_code_t formats[] =
+{
+    /* 32 bpp formats */
+    PIXMAN_a8r8g8b8,
+    PIXMAN_x8r8g8b8,
+    PIXMAN_a8b8g8r8,
+    PIXMAN_x8b8g8r8,
+    PIXMAN_b8g8r8a8,
+    PIXMAN_b8g8r8x8,
+    PIXMAN_r8g8b8a8,
+    PIXMAN_r8g8b8x8,
+    PIXMAN_x2r10g10b10,
+    PIXMAN_x2b10g10r10,
+    PIXMAN_a2r10g10b10,
+    PIXMAN_a2b10g10r10,
+    
+    /* sRGB formats */
+    PIXMAN_a8r8g8b8_sRGB,
+
+    /* 24 bpp formats */
+    PIXMAN_r8g8b8,
+    PIXMAN_b8g8r8,
+    PIXMAN_r5g6b5,
+    PIXMAN_b5g6r5,
+
+    /* 16 bpp formats */
+    PIXMAN_x1r5g5b5,
+    PIXMAN_x1b5g5r5,
+    PIXMAN_a1r5g5b5,
+    PIXMAN_a1b5g5r5,
+    PIXMAN_a4b4g4r4,
+    PIXMAN_x4b4g4r4,
+    PIXMAN_a4r4g4b4,
+    PIXMAN_x4r4g4b4,
+
+    /* 8 bpp formats */
+    PIXMAN_a8,
+    PIXMAN_r3g3b2,
+    PIXMAN_b2g3r3,
+    PIXMAN_a2r2g2b2,
+    PIXMAN_a2b2g2r2,
+    PIXMAN_x4a4,
+
+    /* 4 bpp formats */
+    PIXMAN_a4,
+    PIXMAN_r1g2b1,
+    PIXMAN_b1g2r1,
+    PIXMAN_a1r1g1b1,
+    PIXMAN_a1b1g1r1,
+
+    /* 1 bpp formats */
+    PIXMAN_a1,
 };
 
 struct image_t
 {
     pixman_image_t *image;
-    const format_t *format;
+    pixman_format_code_t format;
     const color_t *color;
     pixman_repeat_t repeat;
     int size;
 };
 
-struct operator_t
-{
-    pixman_op_t op;
-    const char *name;
-};
-
-static const operator_t operators[] =
+static const pixman_op_t operators[] =
 {
-#define P(x) { PIXMAN_OP_##x, #x }
-    P(CLEAR),
-    P(SRC),
-    P(DST),
-    P(OVER),
-    P(OVER_REVERSE),
-    P(IN),
-    P(IN_REVERSE),
-    P(OUT),
-    P(OUT_REVERSE),
-    P(ATOP),
-    P(ATOP_REVERSE),
-    P(XOR),
-    P(ADD),
-    P(SATURATE),
-
-    P(DISJOINT_CLEAR),
-    P(DISJOINT_SRC),
-    P(DISJOINT_DST),
-    P(DISJOINT_OVER),
-    P(DISJOINT_OVER_REVERSE),
-    P(DISJOINT_IN),
-    P(DISJOINT_IN_REVERSE),
-    P(DISJOINT_OUT),
-    P(DISJOINT_OUT_REVERSE),
-    P(DISJOINT_ATOP),
-    P(DISJOINT_ATOP_REVERSE),
-    P(DISJOINT_XOR),
-
-    P(CONJOINT_CLEAR),
-    P(CONJOINT_SRC),
-    P(CONJOINT_DST),
-    P(CONJOINT_OVER),
-    P(CONJOINT_OVER_REVERSE),
-    P(CONJOINT_IN),
-    P(CONJOINT_IN_REVERSE),
-    P(CONJOINT_OUT),
-    P(CONJOINT_OUT_REVERSE),
-    P(CONJOINT_ATOP),
-    P(CONJOINT_ATOP_REVERSE),
-    P(CONJOINT_XOR),
-#undef P
+    PIXMAN_OP_CLEAR,
+    PIXMAN_OP_SRC,
+    PIXMAN_OP_DST,
+    PIXMAN_OP_OVER,
+    PIXMAN_OP_OVER_REVERSE,
+    PIXMAN_OP_IN,
+    PIXMAN_OP_IN_REVERSE,
+    PIXMAN_OP_OUT,
+    PIXMAN_OP_OUT_REVERSE,
+    PIXMAN_OP_ATOP,
+    PIXMAN_OP_ATOP_REVERSE,
+    PIXMAN_OP_XOR,
+    PIXMAN_OP_ADD,
+    PIXMAN_OP_SATURATE,
+
+    PIXMAN_OP_DISJOINT_CLEAR,
+    PIXMAN_OP_DISJOINT_SRC,
+    PIXMAN_OP_DISJOINT_DST,
+    PIXMAN_OP_DISJOINT_OVER,
+    PIXMAN_OP_DISJOINT_OVER_REVERSE,
+    PIXMAN_OP_DISJOINT_IN,
+    PIXMAN_OP_DISJOINT_IN_REVERSE,
+    PIXMAN_OP_DISJOINT_OUT,
+    PIXMAN_OP_DISJOINT_OUT_REVERSE,
+    PIXMAN_OP_DISJOINT_ATOP,
+    PIXMAN_OP_DISJOINT_ATOP_REVERSE,
+    PIXMAN_OP_DISJOINT_XOR,
+
+    PIXMAN_OP_CONJOINT_CLEAR,
+    PIXMAN_OP_CONJOINT_SRC,
+    PIXMAN_OP_CONJOINT_DST,
+    PIXMAN_OP_CONJOINT_OVER,
+    PIXMAN_OP_CONJOINT_OVER_REVERSE,
+    PIXMAN_OP_CONJOINT_IN,
+    PIXMAN_OP_CONJOINT_IN_REVERSE,
+    PIXMAN_OP_CONJOINT_OUT,
+    PIXMAN_OP_CONJOINT_OUT_REVERSE,
+    PIXMAN_OP_CONJOINT_ATOP,
+    PIXMAN_OP_CONJOINT_ATOP_REVERSE,
+    PIXMAN_OP_CONJOINT_XOR,
 };
 
-static double
-calc_op (pixman_op_t op, double src, double dst, double srca, double dsta)
-{
-#define mult_chan(src, dst, Fa, Fb) min ((src) * (Fa) + (dst) * (Fb), 1.0)
-
-    double Fa, Fb;
-
-    switch (op)
-    {
-    case PIXMAN_OP_CLEAR:
-    case PIXMAN_OP_DISJOINT_CLEAR:
-    case PIXMAN_OP_CONJOINT_CLEAR:
-	return mult_chan (src, dst, 0.0, 0.0);
-
-    case PIXMAN_OP_SRC:
-    case PIXMAN_OP_DISJOINT_SRC:
-    case PIXMAN_OP_CONJOINT_SRC:
-	return mult_chan (src, dst, 1.0, 0.0);
-
-    case PIXMAN_OP_DST:
-    case PIXMAN_OP_DISJOINT_DST:
-    case PIXMAN_OP_CONJOINT_DST:
-	return mult_chan (src, dst, 0.0, 1.0);
-
-    case PIXMAN_OP_OVER:
-	return mult_chan (src, dst, 1.0, 1.0 - srca);
-
-    case PIXMAN_OP_OVER_REVERSE:
-	return mult_chan (src, dst, 1.0 - dsta, 1.0);
-
-    case PIXMAN_OP_IN:
-	return mult_chan (src, dst, dsta, 0.0);
-
-    case PIXMAN_OP_IN_REVERSE:
-	return mult_chan (src, dst, 0.0, srca);
-
-    case PIXMAN_OP_OUT:
-	return mult_chan (src, dst, 1.0 - dsta, 0.0);
-
-    case PIXMAN_OP_OUT_REVERSE:
-	return mult_chan (src, dst, 0.0, 1.0 - srca);
-
-    case PIXMAN_OP_ATOP:
-	return mult_chan (src, dst, dsta, 1.0 - srca);
-
-    case PIXMAN_OP_ATOP_REVERSE:
-	return mult_chan (src, dst, 1.0 - dsta,  srca);
-
-    case PIXMAN_OP_XOR:
-	return mult_chan (src, dst, 1.0 - dsta, 1.0 - srca);
-
-    case PIXMAN_OP_ADD:
-	return mult_chan (src, dst, 1.0, 1.0);
-
-    case PIXMAN_OP_SATURATE:
-    case PIXMAN_OP_DISJOINT_OVER_REVERSE:
-	if (srca == 0.0)
-	    Fa = 1.0;
-	else
-	    Fa = min (1.0, (1.0 - dsta) / srca);
-	return mult_chan (src, dst, Fa, 1.0);
-
-    case PIXMAN_OP_DISJOINT_OVER:
-	if (dsta == 0.0)
-	    Fb = 1.0;
-	else
-	    Fb = min (1.0, (1.0 - srca) / dsta);
-	return mult_chan (src, dst, 1.0, Fb);
-
-    case PIXMAN_OP_DISJOINT_IN:
-	if (srca == 0.0)
-	    Fa = 0.0;
-	else
-	    Fa = max (0.0, 1.0 - (1.0 - dsta) / srca);
-	return mult_chan (src, dst, Fa, 0.0);
-
-    case PIXMAN_OP_DISJOINT_IN_REVERSE:
-	if (dsta == 0.0)
-	    Fb = 0.0;
-	else
-	    Fb = max (0.0, 1.0 - (1.0 - srca) / dsta);
-	return mult_chan (src, dst, 0.0, Fb);
-
-    case PIXMAN_OP_DISJOINT_OUT:
-	if (srca == 0.0)
-	    Fa = 1.0;
-	else
-	    Fa = min (1.0, (1.0 - dsta) / srca);
-	return mult_chan (src, dst, Fa, 0.0);
-
-    case PIXMAN_OP_DISJOINT_OUT_REVERSE:
-	if (dsta == 0.0)
-	    Fb = 1.0;
-	else
-	    Fb = min (1.0, (1.0 - srca) / dsta);
-	return mult_chan (src, dst, 0.0, Fb);
-
-    case PIXMAN_OP_DISJOINT_ATOP:
-	if (srca == 0.0)
-	    Fa = 0.0;
-	else
-	    Fa = max (0.0, 1.0 - (1.0 - dsta) / srca);
-	if (dsta == 0.0)
-	    Fb = 1.0;
-	else
-	    Fb = min (1.0, (1.0 - srca) / dsta);
-	return mult_chan (src, dst, Fa, Fb);
-
-    case PIXMAN_OP_DISJOINT_ATOP_REVERSE:
-	if (srca == 0.0)
-	    Fa = 1.0;
-	else
-	    Fa = min (1.0, (1.0 - dsta) / srca);
-	if (dsta == 0.0)
-	    Fb = 0.0;
-	else
-	    Fb = max (0.0, 1.0 - (1.0 - srca) / dsta);
-	return mult_chan (src, dst, Fa, Fb);
-
-    case PIXMAN_OP_DISJOINT_XOR:
-	if (srca == 0.0)
-	    Fa = 1.0;
-	else
-	    Fa = min (1.0, (1.0 - dsta) / srca);
-	if (dsta == 0.0)
-	    Fb = 1.0;
-	else
-	    Fb = min (1.0, (1.0 - srca) / dsta);
-	return mult_chan (src, dst, Fa, Fb);
-
-    case PIXMAN_OP_CONJOINT_OVER:
-	if (dsta == 0.0)
-	    Fb = 0.0;
-	else
-	    Fb = max (0.0, 1.0 - srca / dsta);
-	return mult_chan (src, dst, 1.0, Fb);
-
-    case PIXMAN_OP_CONJOINT_OVER_REVERSE:
-	if (srca == 0.0)
-	    Fa = 0.0;
-	else
-	    Fa = max (0.0, 1.0 - dsta / srca);
-	return mult_chan (src, dst, Fa, 1.0);
-
-    case PIXMAN_OP_CONJOINT_IN:
-	if (srca == 0.0)
-	    Fa = 1.0;
-	else
-	    Fa = min (1.0, dsta / srca);
-	return mult_chan (src, dst, Fa, 0.0);
-
-    case PIXMAN_OP_CONJOINT_IN_REVERSE:
-	if (dsta == 0.0)
-	    Fb = 1.0;
-	else
-	    Fb = min (1.0, srca / dsta);
-	return mult_chan (src, dst, 0.0, Fb);
-
-    case PIXMAN_OP_CONJOINT_OUT:
-	if (srca == 0.0)
-	    Fa = 0.0;
-	else
-	    Fa = max (0.0, 1.0 - dsta / srca);
-	return mult_chan (src, dst, Fa, 0.0);
-
-    case PIXMAN_OP_CONJOINT_OUT_REVERSE:
-	if (dsta == 0.0)
-	    Fb = 0.0;
-	else
-	    Fb = max (0.0, 1.0 - srca / dsta);
-	return mult_chan (src, dst, 0.0, Fb);
-
-    case PIXMAN_OP_CONJOINT_ATOP:
-	if (srca == 0.0)
-	    Fa = 1.0;
-	else
-	    Fa = min (1.0, dsta / srca);
-	if (dsta == 0.0)
-	    Fb = 0.0;
-	else
-	    Fb = max (0.0, 1.0 - srca / dsta);
-	return mult_chan (src, dst, Fa, Fb);
-
-    case PIXMAN_OP_CONJOINT_ATOP_REVERSE:
-	if (srca == 0.0)
-	    Fa = 0.0;
-	else
-	    Fa = max (0.0, 1.0 - dsta / srca);
-	if (dsta == 0.0)
-	    Fb = 1.0;
-	else
-	    Fb = min (1.0, srca / dsta);
-	return mult_chan (src, dst, Fa, Fb);
-
-    case PIXMAN_OP_CONJOINT_XOR:
-	if (srca == 0.0)
-	    Fa = 0.0;
-	else
-	    Fa = max (0.0, 1.0 - dsta / srca);
-	if (dsta == 0.0)
-	    Fb = 0.0;
-	else
-	    Fb = max (0.0, 1.0 - srca / dsta);
-	return mult_chan (src, dst, Fa, Fb);
-
-    case PIXMAN_OP_MULTIPLY:
-    case PIXMAN_OP_SCREEN:
-    case PIXMAN_OP_OVERLAY:
-    case PIXMAN_OP_DARKEN:
-    case PIXMAN_OP_LIGHTEN:
-    case PIXMAN_OP_COLOR_DODGE:
-    case PIXMAN_OP_COLOR_BURN:
-    case PIXMAN_OP_HARD_LIGHT:
-    case PIXMAN_OP_SOFT_LIGHT:
-    case PIXMAN_OP_DIFFERENCE:
-    case PIXMAN_OP_EXCLUSION:
-    case PIXMAN_OP_HSL_HUE:
-    case PIXMAN_OP_HSL_SATURATION:
-    case PIXMAN_OP_HSL_COLOR:
-    case PIXMAN_OP_HSL_LUMINOSITY:
-    default:
-	abort();
-    }
-#undef mult_chan
-}
-
-static void
-do_composite (pixman_op_t op,
-	      const color_t *src,
-	      const color_t *mask,
-	      const color_t *dst,
-	      color_t *result,
-	      pixman_bool_t component_alpha)
-{
-    color_t srcval, srcalpha;
-
-    if (mask == NULL)
-    {
-	srcval = *src;
-
-	srcalpha.r = src->a;
-	srcalpha.g = src->a;
-	srcalpha.b = src->a;
-	srcalpha.a = src->a;
-    }
-    else if (component_alpha)
-    {
-	srcval.r = src->r * mask->r;
-	srcval.g = src->g * mask->g;
-	srcval.b = src->b * mask->b;
-	srcval.a = src->a * mask->a;
-
-	srcalpha.r = src->a * mask->r;
-	srcalpha.g = src->a * mask->g;
-	srcalpha.b = src->a * mask->b;
-	srcalpha.a = src->a * mask->a;
-    }
-    else
-    {
-	srcval.r = src->r * mask->a;
-	srcval.g = src->g * mask->a;
-	srcval.b = src->b * mask->a;
-	srcval.a = src->a * mask->a;
-
-	srcalpha.r = src->a * mask->a;
-	srcalpha.g = src->a * mask->a;
-	srcalpha.b = src->a * mask->a;
-	srcalpha.a = src->a * mask->a;
-    }
-
-    result->r = calc_op (op, srcval.r, dst->r, srcalpha.r, dst->a);
-    result->g = calc_op (op, srcval.g, dst->g, srcalpha.g, dst->a);
-    result->b = calc_op (op, srcval.b, dst->b, srcalpha.b, dst->a);
-    result->a = calc_op (op, srcval.a, dst->a, srcalpha.a, dst->a);
-}
-
-static void
-color_correct (pixman_format_code_t format,
-	       color_t *color)
+static uint32_t
+get_value (pixman_image_t *image)
 {
-#define round_pix(pix, mask) \
-    ((int)((pix) * (mask) + .5) / (double) (mask))
+    uint32_t value = *(uint32_t *)pixman_image_get_data (image);
 
-    if (PIXMAN_FORMAT_R (format) == 0)
-    {
-	color->r = 0.0;
-	color->g = 0.0;
-	color->b = 0.0;
-    }
-    else
+#ifdef WORDS_BIGENDIAN
     {
-	color->r = round_pix (color->r, PIXMAN_FORMAT_R (format));
-	color->g = round_pix (color->g, PIXMAN_FORMAT_G (format));
-	color->b = round_pix (color->b, PIXMAN_FORMAT_B (format));
+	pixman_format_code_t format = pixman_image_get_format (image);
+	value >>= 8 * sizeof(value) - PIXMAN_FORMAT_BPP (format);
     }
+#endif
 
-    if (PIXMAN_FORMAT_A (format) == 0)
-	color->a = 1.0;
-    else
-	color->a = round_pix (color->a, PIXMAN_FORMAT_A (format));
-
-#undef round_pix
+    return value;
 }
 
-static void
-get_pixel (pixman_image_t *image,
-	   pixman_format_code_t format,
-	   color_t *color)
+static char *
+describe_image (image_t *info, char *buf)
 {
-#define MASK(N) ((1UL << (N))-1)
-
-    unsigned long rs, gs, bs, as;
-    int a, r, g, b;
-    unsigned long val;
-
-    val = *(unsigned long *) pixman_image_get_data (image);
-#ifdef WORDS_BIGENDIAN
-    val >>= 8 * sizeof(val) - PIXMAN_FORMAT_BPP (format);
-#endif
-
-    /* Number of bits in each channel */
-    a = PIXMAN_FORMAT_A (format);
-    r = PIXMAN_FORMAT_R (format);
-    g = PIXMAN_FORMAT_G (format);
-    b = PIXMAN_FORMAT_B (format);
-
-    switch (PIXMAN_FORMAT_TYPE (format))
-    {
-    case PIXMAN_TYPE_ARGB:
-        bs = 0;
-        gs = b + bs;
-        rs = g + gs;
-        as = r + rs;
-	break;
-
-    case PIXMAN_TYPE_ABGR:
-        rs = 0;
-        gs = r + rs;
-        bs = g + gs;
-        as = b + bs;
-	break;
-
-    case PIXMAN_TYPE_BGRA:
-        as = 0;
-	rs = PIXMAN_FORMAT_BPP (format) - (b + g + r);
-        gs = r + rs;
-        bs = g + gs;
-	break;
-
-    case PIXMAN_TYPE_A:
-        as = 0;
-        rs = 0;
-        gs = 0;
-        bs = 0;
-	break;
-
-    case PIXMAN_TYPE_OTHER:
-    case PIXMAN_TYPE_COLOR:
-    case PIXMAN_TYPE_GRAY:
-    case PIXMAN_TYPE_YUY2:
-    case PIXMAN_TYPE_YV12:
-    default:
-	abort ();
-        as = 0;
-        rs = 0;
-        gs = 0;
-        bs = 0;
-	break;
-    }
-
-    if (MASK (a) != 0)
-	color->a = ((val >> as) & MASK (a)) / (double) MASK (a);
-    else
-	color->a = 1.0;
-
-    if (MASK (r) != 0)
+    if (info->size)
     {
-	color->r = ((val >> rs) & MASK (r)) / (double) MASK (r);
-	color->g = ((val >> gs) & MASK (g)) / (double) MASK (g);
-	color->b = ((val >> bs) & MASK (b)) / (double) MASK (b);
+	sprintf (buf, "%s, %dx%d%s",
+		 format_name (info->format),
+		 info->size, info->size,
+		 info->repeat ? " R" :"");
     }
     else
     {
-	color->r = 0.0;
-	color->g = 0.0;
-	color->b = 0.0;
+	sprintf (buf, "solid");
     }
 
-#undef MASK
-}
-
-static double
-eval_diff (color_t *expected, color_t *test)
-{
-    double rscale, gscale, bscale, ascale;
-    double rdiff, gdiff, bdiff, adiff;
-
-    /* XXX: Need to be provided mask shifts so we can produce useful error
-     * values.
-     */
-    rscale = 1.0 * (1 << 5);
-    gscale = 1.0 * (1 << 6);
-    bscale = 1.0 * (1 << 5);
-    ascale = 1.0 * 32;
-
-    rdiff = fabs (test->r - expected->r) * rscale;
-    bdiff = fabs (test->g - expected->g) * gscale;
-    gdiff = fabs (test->b - expected->b) * bscale;
-    adiff = fabs (test->a - expected->a) * ascale;
-
-    return max (max (max (rdiff, gdiff), bdiff), adiff);
+    return buf;
 }
 
 static char *
-describe_image (image_t *info, char *buf, int buflen)
+describe_color (const color_t *color, char *buf)
 {
-    if (info->size)
-    {
-	snprintf (buf, buflen, "%s %dx%d%s",
-		  info->format->name,
-		  info->size, info->size,
-		  info->repeat ? "R" :"");
-    }
-    else
-    {
-	snprintf (buf, buflen, "solid");
-    }
+    sprintf (buf, "%.3f %.3f %.3f %.3f",
+	     color->r, color->g, color->b, color->a);
 
     return buf;
 }
 
-/* Test a composite of a given operation, source, mask, and destination
- * picture.
- * Fills the window, and samples from the 0,0 pixel corner.
- */
 static pixman_bool_t
 composite_test (image_t *dst,
-		const operator_t *op,
+		pixman_op_t op,
 		image_t *src,
 		image_t *mask,
-		pixman_bool_t component_alpha)
+		pixman_bool_t component_alpha,
+		int testno)
 {
-    pixman_color_t fill;
-    pixman_rectangle16_t rect;
-    color_t expected, result, tdst, tsrc, tmsk;
-    double diff;
-    pixman_bool_t success = TRUE;
-
-    compute_pixman_color (dst->color, &fill);
-    rect.x = rect.y = 0;
-    rect.width = rect.height = dst->size;
-    pixman_image_fill_rectangles (PIXMAN_OP_SRC, dst->image,
-				  &fill, 1, &rect);
-
-    if (mask != NULL)
+    color_t expected, tdst, tsrc, tmsk;
+    pixel_checker_t checker;
+
+    if (mask)
     {
 	pixman_image_set_component_alpha (mask->image, component_alpha);
-	pixman_image_composite (op->op, src->image, mask->image, dst->image,
-				0, 0,
-				0, 0,
-				0, 0,
-				dst->size, dst->size);
 
-	tmsk = *mask->color;
-	if (mask->size)
-	{
-	    color_correct (mask->format->format, &tmsk);
-
-	    if (component_alpha &&
-		PIXMAN_FORMAT_R (mask->format->format) == 0)
-	    {
-		/* Ax component-alpha masks expand alpha into
-		 * all color channels.
-		 */
-		tmsk.r = tmsk.g = tmsk.b = tmsk.a;
-	    }
-	}
+	pixman_image_composite (op, src->image, mask->image, dst->image,
+				0, 0, 0, 0, 0, 0, dst->size, dst->size);
     }
     else
     {
-	pixman_image_composite (op->op, src->image, NULL, dst->image,
+	pixman_image_composite (op, src->image, NULL, dst->image,
 				0, 0,
 				0, 0,
 				0, 0,
 				dst->size, dst->size);
     }
-    get_pixel (dst->image, dst->format->format, &result);
 
     tdst = *dst->color;
-    color_correct (dst->format->format, &tdst);
     tsrc = *src->color;
+
+    if (mask)
+    {
+	tmsk = *mask->color;
+    }
+
+    /* It turns out that by construction all source, mask etc. colors are
+     * linear because they are made from fills, and fills are always in linear
+     * color space.  However, if they have been converted to bitmaps, we need
+     * to simulate the sRGB approximation to pass the test cases.
+     */
     if (src->size)
-	color_correct (src->format->format, &tsrc);
-    do_composite (op->op, &tsrc, mask ? &tmsk : NULL, &tdst,
-		  &expected, component_alpha);
-    color_correct (dst->format->format, &expected);
+    {
+	if (PIXMAN_FORMAT_TYPE (src->format) == PIXMAN_TYPE_ARGB_SRGB)
+        {
+	    tsrc.r = convert_linear_to_srgb (tsrc.r);
+	    tsrc.g = convert_linear_to_srgb (tsrc.g);
+	    tsrc.b = convert_linear_to_srgb (tsrc.b);
+	    round_color (src->format, &tsrc);
+	    tsrc.r = convert_srgb_to_linear (tsrc.r);
+	    tsrc.g = convert_srgb_to_linear (tsrc.g);
+	    tsrc.b = convert_srgb_to_linear (tsrc.b);
+	}
+        else
+        {
+	    round_color (src->format, &tsrc);
+	}
+    }
 
-    diff = eval_diff (&expected, &result);
-    if (diff > 3.0)
+    if (mask && mask->size)
     {
-	char buf[40];
-
-	snprintf (buf, sizeof (buf),
-		  "%s %scomposite",
-		  op->name,
-		  component_alpha ? "CA " : "");
-
-	printf ("%s test error of %.4f --\n"
-		"           R    G    B    A\n"
-		"got:       %.2f %.2f %.2f %.2f [%08lx]\n"
-		"expected:  %.2f %.2f %.2f %.2f\n",
-		buf, diff,
-		result.r, result.g, result.b, result.a,
-		*(unsigned long *) pixman_image_get_data (dst->image),
-		expected.r, expected.g, expected.b, expected.a);
-	
-	if (mask != NULL)
+	if (PIXMAN_FORMAT_TYPE (mask->format) == PIXMAN_TYPE_ARGB_SRGB)
 	{
-	    printf ("src color: %.2f %.2f %.2f %.2f\n"
-		    "msk color: %.2f %.2f %.2f %.2f\n"
-		    "dst color: %.2f %.2f %.2f %.2f\n",
-		    src->color->r, src->color->g,
-		    src->color->b, src->color->a,
-		    mask->color->r, mask->color->g,
-		    mask->color->b, mask->color->a,
-		    dst->color->r, dst->color->g,
-		    dst->color->b, dst->color->a);
-	    printf ("src: %s, ", describe_image (src, buf, sizeof (buf)));
-	    printf ("mask: %s, ", describe_image (mask, buf, sizeof (buf)));
-	    printf ("dst: %s\n\n", describe_image (dst, buf, sizeof (buf)));
+	    tmsk.r = convert_linear_to_srgb (tmsk.r);
+	    tmsk.g = convert_linear_to_srgb (tmsk.g);
+	    tmsk.b = convert_linear_to_srgb (tmsk.b);
+	    round_color (mask->format, &tmsk);
+	    tmsk.r = convert_srgb_to_linear (tmsk.r);
+	    tmsk.g = convert_srgb_to_linear (tmsk.g);
+	    tmsk.b = convert_srgb_to_linear (tmsk.b);
 	}
 	else
 	{
-	    printf ("src color: %.2f %.2f %.2f %.2f\n"
-		    "dst color: %.2f %.2f %.2f %.2f\n",
-		    src->color->r, src->color->g,
-		    src->color->b, src->color->a,
-		    dst->color->r, dst->color->g,
-		    dst->color->b, dst->color->a);
-	    printf ("src: %s, ", describe_image (src, buf, sizeof (buf)));
-	    printf ("dst: %s\n\n", describe_image (dst, buf, sizeof (buf)));
+	    round_color (mask->format, &tmsk);
 	}
+    }
 
-	success = FALSE;
+    if (mask)
+    {
+	if (component_alpha && PIXMAN_FORMAT_R (mask->format) == 0)
+	{
+	    /* Ax component-alpha masks expand alpha into
+	     * all color channels.
+	     */
+	    tmsk.r = tmsk.g = tmsk.b = tmsk.a;
+	}
     }
 
-    return success;
-}
+    if (PIXMAN_FORMAT_TYPE (dst->format) == PIXMAN_TYPE_ARGB_SRGB)
+    {
+	tdst.r = convert_linear_to_srgb (tdst.r);
+	tdst.g = convert_linear_to_srgb (tdst.g);
+	tdst.b = convert_linear_to_srgb (tdst.b);
+    	round_color (dst->format, &tdst);
+	tdst.r = convert_srgb_to_linear (tdst.r);
+	tdst.g = convert_srgb_to_linear (tdst.g);
+	tdst.b = convert_srgb_to_linear (tdst.b);
+    }
+    else
+    {
+    	round_color (dst->format, &tdst);
+    }
 
-#define REPEAT 0x01000000
-#define FLAGS  0xff000000
+    do_composite (op,
+		  &tsrc,
+		  mask? &tmsk : NULL,
+		  &tdst,
+		  &expected,
+		  component_alpha);
+
+    pixel_checker_init (&checker, dst->format);
+
+    if (!pixel_checker_check (&checker, get_value (dst->image), &expected))
+    {
+	char buf[40], buf2[40];
+	int a, r, g, b;
+	uint32_t pixel;
+
+	printf ("---- Test %d failed ----\n", testno);
+	printf ("Operator:      %s %s\n",
+                operator_name (op), component_alpha ? "CA" : "");
+
+	printf ("Source:        %s\n", describe_image (src, buf));
+	if (mask != NULL)
+	    printf ("Mask:          %s\n", describe_image (mask, buf));
+
+	printf ("Destination:   %s\n\n", describe_image (dst, buf));
+	printf ("               R     G     B     A         Rounded\n");
+	printf ("Source color:  %s     %s\n",
+		describe_color (src->color, buf),
+		describe_color (&tsrc, buf2));
+	if (mask)
+	{
+	    printf ("Mask color:    %s     %s\n",
+		    describe_color (mask->color, buf),
+		    describe_color (&tmsk, buf2));
+	}
+	printf ("Dest. color:   %s     %s\n",
+		describe_color (dst->color, buf),
+		describe_color (&tdst, buf2));
+
+	pixel = get_value (dst->image);
+
+	printf ("Expected:      %s\n", describe_color (&expected, buf));
+
+	pixel_checker_split_pixel (&checker, pixel, &a, &r, &g, &b);
+
+	printf ("Got:           %5d %5d %5d %5d  [pixel: 0x%08x]\n", r, g, b, a, pixel);
+	pixel_checker_get_min (&checker, &expected, &a, &r, &g, &b);
+	printf ("Min accepted:  %5d %5d %5d %5d\n", r, g, b, a);
+	pixel_checker_get_max (&checker, &expected, &a, &r, &g, &b);
+	printf ("Max accepted:  %5d %5d %5d %5d\n", r, g, b, a);
+
+	return FALSE;
+    }
+    return TRUE;
+}
 
 static void
 image_init (image_t *info,
@@ -765,24 +391,24 @@ image_init (image_t *info,
     info->color = &colors[color];
     compute_pixman_color (info->color, &fill);
 
-    info->format = &formats[format];
-    info->size = size & ~FLAGS;
+    info->format = formats[format];
+    info->size = sizes[size] & ~FLAGS;
     info->repeat = PIXMAN_REPEAT_NONE;
 
     if (info->size)
     {
-	pixman_rectangle16_t rect;
+	pixman_image_t *solid;
 
-	info->image = pixman_image_create_bits (info->format->format,
+	info->image = pixman_image_create_bits (info->format,
 						info->size, info->size,
 						NULL, 0);
 
-	rect.x = rect.y = 0;
-	rect.width = rect.height = info->size;
-	pixman_image_fill_rectangles (PIXMAN_OP_SRC, info->image, &fill,
-				      1, &rect);
+	solid = pixman_image_create_solid_fill (&fill);
+	pixman_image_composite32 (PIXMAN_OP_SRC, solid, NULL, info->image,
+				  0, 0, 0, 0, 0, 0, info->size, info->size);
+	pixman_image_unref (solid);
 
-	if (size & REPEAT)
+	if (sizes[size] & REPEAT)
 	{
 	    pixman_image_set_repeat (info->image, PIXMAN_REPEAT_NORMAL);
 	    info->repeat = PIXMAN_REPEAT_NORMAL;
@@ -800,102 +426,111 @@ image_fini (image_t *info)
     pixman_image_unref (info->image);
 }
 
-int
-main (void)
+static int
+random_size (void)
+{
+    return prng_rand_n (ARRAY_LENGTH (sizes));
+}
+
+static int
+random_color (void)
+{
+    return prng_rand_n (ARRAY_LENGTH (colors));
+}
+
+static int
+random_format (void)
+{
+    return prng_rand_n (ARRAY_LENGTH (formats));
+}
+
+static pixman_bool_t
+run_test (uint32_t seed)
 {
-    pixman_bool_t ok, group_ok = TRUE, ca;
-    int i, d, m, s;
-    int tests_passed = 0, tests_total = 0;
-    int sizes[] = { 1, 1 | REPEAT, 10 };
-    int num_tests;
+    image_t src, mask, dst;
+    pixman_op_t op;
+    int ca;
+    int ok;
+
+    prng_srand (seed);
 
-    for (i = 0; i < ARRAY_LENGTH (colors); i++)
+    image_init (&dst, random_color(), random_format(), 1);
+    image_init (&src, random_color(), random_format(), random_size());
+    image_init (&mask, random_color(), random_format(), random_size());
+
+    op = operators [prng_rand_n (ARRAY_LENGTH (operators))];
+
+    ca = prng_rand_n (3);
+
+    switch (ca)
     {
-	colors[i].r *= colors[i].a;
-	colors[i].g *= colors[i].a;
-	colors[i].b *= colors[i].a;
+    case 0:
+	ok = composite_test (&dst, op, &src, NULL, FALSE, seed);
+	break;
+    case 1:
+	ok = composite_test (&dst, op, &src, &mask, FALSE, seed);
+	break;
+    case 2:
+	ok = composite_test (&dst, op, &src, &mask,
+			     mask.size? TRUE : FALSE, seed);
+	break;
+    default:
+	ok = FALSE;
+	break;
     }
 
-    num_tests = ARRAY_LENGTH (colors) * ARRAY_LENGTH (formats);
+    image_fini (&src);
+    image_fini (&mask);
+    image_fini (&dst);
 
-    for (d = 0; d < num_tests; d++)
+    return ok;
+}
+
+int
+main (int argc, char **argv)
+{
+#define N_TESTS (8 * 1024 * 1024)
+    int result = 0;
+    uint32_t seed;
+    int32_t i;
+
+    if (argc > 1)
     {
-	image_t dst;
+	char *end;
 
-	image_init (
-	    &dst, d / ARRAY_LENGTH (formats), d % ARRAY_LENGTH (formats), 1);
+	i = strtol (argv[1], &end, 0);
+
+	if (end != argv[1])
+	{
+	    if (!run_test (i))
+		return 1;
+	    else
+		return 0;
+	}
+	else
+	{
+	    printf ("Usage:\n\n   %s <number>\n\n", argv[0]);
+	    return -1;
+	}
+    }
 
+    if (getenv ("PIXMAN_RANDOMIZE_TESTS"))
+	seed = get_random_seed();
+    else
+	seed = 1;
 
-	for (s = -ARRAY_LENGTH (colors);
-	     s < ARRAY_LENGTH (sizes) * num_tests;
-	     s++)
+#ifdef USE_OPENMP
+#   pragma omp parallel for default(none) shared(result, argv, seed)
+#endif
+    for (i = 0; i <= N_TESTS; ++i)
+    {
+	if (!result && !run_test (i + seed))
 	{
-	    image_t src;
+	    printf ("Test 0x%08X failed.\n", seed + i);
 
-	    if (s < 0)
-	    {
-		image_init (&src, -s - 1, 0, 0);
-	    }
-	    else
-	    {
-		image_init (&src,
-			    s / ARRAY_LENGTH (sizes) / ARRAY_LENGTH (formats),
-			    s / ARRAY_LENGTH (sizes) % ARRAY_LENGTH (formats),
-			    sizes[s % ARRAY_LENGTH (sizes)]);
-	    }
-
-	    for (m = -ARRAY_LENGTH (colors);
-		 m < ARRAY_LENGTH (sizes) * num_tests;
-		 m++)
-	    {
-		image_t mask;
-
-		if (m < 0)
-		{
-		    image_init (&mask, -m - 1, 0, 0);
-		}
-		else
-		{
-		    image_init (
-			&mask,
-			m / ARRAY_LENGTH (sizes) / ARRAY_LENGTH (formats),
-			m / ARRAY_LENGTH (sizes) % ARRAY_LENGTH (formats),
-			sizes[m % ARRAY_LENGTH (sizes)]);
-		}
-
-		for (ca = -1; ca <= 1; ca++)
-		{
-		    for (i = 0; i < ARRAY_LENGTH (operators); i++)
-		    {
-			const operator_t *op = &operators[i];
-
-			switch (ca)
-			{
-			case -1:
-			    ok = composite_test (&dst, op, &src, NULL, FALSE);
-			    break;
-			case 0:
-			    ok = composite_test (&dst, op, &src, &mask, FALSE);
-			    break;
-			case 1:
-			    ok = composite_test (&dst, op, &src, &mask,
-						 mask.size? TRUE : FALSE);
-			    break;
-                        default:
-                            break;
-			}
-			group_ok = group_ok && ok;
-			tests_passed += ok;
-			tests_total++;
-		    }
-		}
-
-		image_fini (&mask);
-	    }
-	    image_fini (&src);
+	    result = seed + i;
 	}
-	image_fini (&dst);
     }
 
-    return group_ok == FALSE;
+    return result;
 }
diff --git a/pixman/test/fetch-test.c b/pixman/test/fetch-test.c
index 2ca16ddbf..04e8cc583 100644
--- a/pixman/test/fetch-test.c
+++ b/pixman/test/fetch-test.c
@@ -1,14 +1,13 @@
 #include <assert.h>
 #include <stdlib.h>
 #include <stdio.h>
-#include "pixman.h"
-#include <config.h>
+#include "utils.h"
 
 #define SIZE 1024
 
 static pixman_indexed_t mono_palette =
 {
-    .rgba = { 0x00000000, 0x00ffffff },
+    0, { 0x00000000, 0x00ffffff },
 };
 
 
@@ -24,57 +23,63 @@ typedef struct {
 static testcase_t testcases[] =
 {
     {
-	.format = PIXMAN_a8r8g8b8,
-	.width = 2, .height = 2,
-	.stride = 8,
-	.src = { 0x00112233, 0x44556677,
-	         0x8899aabb, 0xccddeeff },
-	.dst = { 0x00112233, 0x44556677,
-	         0x8899aabb, 0xccddeeff },
-	.indexed = NULL,
+	PIXMAN_a8r8g8b8,
+	2, 2,
+	8,
+	{ 0x00112233, 0x44556677,
+	  0x8899aabb, 0xccddeeff },
+	{ 0x00112233, 0x44556677,
+	  0x8899aabb, 0xccddeeff },
+	NULL,
     },
     {
-	.format = PIXMAN_g1,
-	.width = 8, .height = 2,
-	.stride = 4,
+	PIXMAN_r8g8b8a8,
+	2, 2,
+	8,
+	{ 0x11223300, 0x55667744,
+	  0x99aabb88, 0xddeeffcc },
+	{ 0x00112233, 0x44556677,
+	  0x8899aabb, 0xccddeeff },
+	NULL,
+    },
+    {
+	PIXMAN_g1,
+	8, 2,
+	4,
 #ifdef WORDS_BIGENDIAN
-	.src =
 	{
 	    0xaa000000,
 	    0x55000000
 	},
 #else
-	.src =
 	{
 	    0x00000055,
 	    0x000000aa
 	},
 #endif
-	.dst =
 	{
 	    0x00ffffff, 0x00000000, 0x00ffffff, 0x00000000, 0x00ffffff, 0x00000000, 0x00ffffff, 0x00000000,
 	    0x00000000, 0x00ffffff, 0x00000000, 0x00ffffff, 0x00000000, 0x00ffffff, 0x00000000, 0x00ffffff
 	},
-	.indexed = &mono_palette,
+	&mono_palette,
     },
 #if 0
     {
-	.format = PIXMAN_g8,
-	.width = 4, .height = 2,
-	.stride = 4,
-	.src = { 0x01234567,
-	         0x89abcdef },
-	.dst = { 0x00010101, 0x00232323, 0x00454545, 0x00676767,
-	         0x00898989, 0x00ababab, 0x00cdcdcd, 0x00efefef, },
+	PIXMAN_g8,
+	4, 2,
+	4,
+	{ 0x01234567,
+	  0x89abcdef },
+	{ 0x00010101, 0x00232323, 0x00454545, 0x00676767,
+	  0x00898989, 0x00ababab, 0x00cdcdcd, 0x00efefef, },
     },
 #endif
     /* FIXME: make this work on big endian */
     {
-	.format = PIXMAN_yv12,
-	.width = 8, .height = 2,
-	.stride = 8,
+	PIXMAN_yv12,
+	8, 2,
+	8,
 #ifdef WORDS_BIGENDIAN
-	.src =
 	{
 	    0x00ff00ff, 0x00ff00ff,
 	    0xff00ff00, 0xff00ff00,
@@ -82,7 +87,6 @@ static testcase_t testcases[] =
 	    0x800080ff
 	},
 #else
-	.src =
 	{
 	    0xff00ff00, 0xff00ff00,
 	    0x00ff00ff, 0x00ff00ff,
@@ -90,7 +94,6 @@ static testcase_t testcases[] =
 	    0xff800080
 	},
 #endif
-	.dst =
 	{
 	    0xff000000, 0xffffffff, 0xffb80000, 0xffffe113,
 	    0xff000000, 0xffffffff, 0xff0023ee, 0xff4affff,
@@ -100,7 +103,7 @@ static testcase_t testcases[] =
     },
 };
 
-int n_test_cases = sizeof(testcases)/sizeof(testcases[0]);
+int n_test_cases = ARRAY_LENGTH (testcases);
 
 
 static uint32_t
@@ -116,6 +119,7 @@ reader (const void *src, int size)
 	return *(uint32_t *)src;
     default:
 	assert(0);
+	return 0; /* silence MSVC */
     }
 }
 
diff --git a/pixman/test/fuzzer-find-diff.pl b/pixman/test/fuzzer-find-diff.pl
new file mode 100755
index 000000000..e1d67fbf7
--- /dev/null
+++ b/pixman/test/fuzzer-find-diff.pl
@@ -0,0 +1,75 @@
+#!/usr/bin/env perl
+
+$usage = "Usage:
+  fuzzer-find-diff.pl reference_binary new_binary [number_of_tests_to_run]
+
+The first two input arguments are the commands to run the test programs
+based on fuzzer_test_main() function from 'util.c' (preferably they should
+be statically compiled, this can be achieved via '--disable-shared' pixman
+configure option). The third optional argument is the number of test rounds
+to run (if not specified, then testing runs infinitely or until some problem
+is detected).
+
+Usage examples:
+  fuzzer-find-diff.pl ./blitters-test-with-sse-disabled ./blitters-test 9000000
+  fuzzer-find-diff.pl ./blitters-test \"ssh ppc64_host /path/to/blitters-test\"
+";
+
+$#ARGV >= 1 or die $usage;
+
+$batch_size = 10000;
+
+if ($#ARGV >= 2) {
+    $number_of_tests = int($ARGV[2]);
+} else {
+    $number_of_tests = -1
+}
+
+sub test_range {
+    my $min = shift;
+    my $max = shift;
+
+    # check that [$min, $max] range is "bad", otherwise return
+    if (`$ARGV[0] $min $max 2>/dev/null` eq `$ARGV[1] $min $max 2>/dev/null`) {
+        return;
+    }
+
+    # check that $min itself is "good", otherwise return
+    if (`$ARGV[0] $min 2>/dev/null` ne `$ARGV[1] $min 2>/dev/null`) {
+        return $min;
+    }
+
+    # start bisecting
+    while ($max != $min + 1) {
+        my $avg = int(($min + $max) / 2);
+        my $res1 = `$ARGV[0] $min $avg 2>/dev/null`;
+        my $res2 = `$ARGV[1] $min $avg 2>/dev/null`;
+        if ($res1 ne $res2) {
+            $max = $avg;
+        } else {
+            $min = $avg;
+        }
+    }
+    return $max;
+}
+
+$base = 1;
+while ($number_of_tests <= 0 || $base <= $number_of_tests) {
+    printf("testing %-12d\r", $base + $batch_size - 1);
+    my $res = test_range($base, $base + $batch_size - 1);
+    if ($res) {
+        printf("Failure: results are different for test %d:\n", $res);
+
+        printf("\n-- ref --\n");
+        print `$ARGV[0] $res`;
+        printf("-- new --\n");
+        print `$ARGV[1] $res`;
+
+        printf("The problematic conditions can be reproduced by running:\n");
+        printf("$ARGV[1] %d\n", $res);
+
+        exit(1);
+    }
+    $base += $batch_size;
+}
+printf("Success: %d tests finished\n", $base - 1);
diff --git a/pixman/test/glyph-test.c b/pixman/test/glyph-test.c
new file mode 100644
index 000000000..1811add73
--- /dev/null
+++ b/pixman/test/glyph-test.c
@@ -0,0 +1,332 @@
+#include <stdlib.h>
+#include "utils.h"
+
+static const pixman_format_code_t glyph_formats[] =
+{
+    PIXMAN_a8r8g8b8,
+    PIXMAN_a8,
+    PIXMAN_a4,
+    PIXMAN_a1,
+    PIXMAN_x8r8g8b8,
+    PIXMAN_r3g3b2,
+    PIXMAN_null,
+};
+
+static const pixman_format_code_t formats[] =
+{
+    PIXMAN_a8r8g8b8,
+    PIXMAN_a8b8g8r8,
+    PIXMAN_x8r8g8b8,
+    PIXMAN_x8b8g8r8,
+    PIXMAN_r5g6b5,
+    PIXMAN_b5g6r5,
+    PIXMAN_a8,
+    PIXMAN_a1,
+    PIXMAN_r3g3b2,
+    PIXMAN_b8g8r8a8,
+    PIXMAN_b8g8r8x8,
+    PIXMAN_r8g8b8a8,
+    PIXMAN_r8g8b8x8,
+    PIXMAN_x14r6g6b6,
+    PIXMAN_r8g8b8,
+    PIXMAN_b8g8r8,
+#if 0
+    /* These use floating point */
+    PIXMAN_x2r10g10b10,
+    PIXMAN_a2r10g10b10,
+    PIXMAN_x2b10g10r10,
+    PIXMAN_a2b10g10r10,
+#endif
+    PIXMAN_a1r5g5b5,
+    PIXMAN_x1r5g5b5,
+    PIXMAN_a1b5g5r5,
+    PIXMAN_x1b5g5r5,
+    PIXMAN_a4r4g4b4,
+    PIXMAN_x4r4g4b4,
+    PIXMAN_a4b4g4r4,
+    PIXMAN_x4b4g4r4,
+    PIXMAN_r3g3b2,
+    PIXMAN_b2g3r3,
+    PIXMAN_a2r2g2b2,
+    PIXMAN_a2b2g2r2,
+    PIXMAN_x4a4,
+    PIXMAN_a4,
+    PIXMAN_r1g2b1,
+    PIXMAN_b1g2r1,
+    PIXMAN_a1r1g1b1,
+    PIXMAN_a1b1g1r1,
+    PIXMAN_null,
+};
+
+static const pixman_op_t operators[] =
+{
+    PIXMAN_OP_SRC,
+    PIXMAN_OP_OVER,
+    PIXMAN_OP_ADD,
+    PIXMAN_OP_CLEAR,
+    PIXMAN_OP_SRC,
+    PIXMAN_OP_DST,
+    PIXMAN_OP_OVER,
+    PIXMAN_OP_OVER_REVERSE,
+    PIXMAN_OP_IN,
+    PIXMAN_OP_IN_REVERSE,
+    PIXMAN_OP_OUT,
+    PIXMAN_OP_OUT_REVERSE,
+    PIXMAN_OP_ATOP,
+    PIXMAN_OP_ATOP_REVERSE,
+    PIXMAN_OP_XOR,
+    PIXMAN_OP_ADD
+};
+
+enum
+{
+    ALLOW_CLIPPED		= (1 << 0),
+    ALLOW_ALPHA_MAP		= (1 << 1),
+    ALLOW_SOURCE_CLIPPING	= (1 << 2),
+    ALLOW_REPEAT		= (1 << 3),
+    ALLOW_SOLID			= (1 << 4),
+    ALLOW_FENCED_MEMORY		= (1 << 5),
+};
+
+static void
+destroy_fenced (pixman_image_t *image, void *data)
+{
+    fence_free (data);
+}
+
+static void
+destroy_malloced (pixman_image_t *image, void *data)
+{
+    free (data);
+}
+
+static pixman_format_code_t
+random_format (const pixman_format_code_t *formats)
+{
+    int i;
+    i = 0;
+    while (formats[i] != PIXMAN_null)
+	++i;
+    return formats[prng_rand_n (i)];
+}
+
+static pixman_image_t *
+create_image (int max_size, const pixman_format_code_t *formats, uint32_t flags)
+{
+    int width, height;
+    pixman_image_t *image;
+    pixman_format_code_t format;
+    uint32_t *data;
+    int bpp;
+    int stride;
+    int i;
+    pixman_image_destroy_func_t destroy;
+
+    if ((flags & ALLOW_SOLID) && prng_rand_n (4) == 0)
+    {
+	pixman_color_t color;
+
+	color.alpha = prng_rand();
+	color.red = prng_rand();
+	color.green = prng_rand();
+	color.blue = prng_rand();
+
+	return pixman_image_create_solid_fill (&color);
+    }
+
+    width = prng_rand_n (max_size) + 1;
+    height = prng_rand_n (max_size) + 1;
+    format = random_format (formats);
+
+    bpp = PIXMAN_FORMAT_BPP (format);
+    stride = (width * bpp + 7) / 8 + prng_rand_n (17);
+    stride = (stride + 3) & ~3;
+
+    if (prng_rand_n (64) == 0)
+    {
+	if (!(data = (uint32_t *)make_random_bytes (stride * height)))
+	{
+	    fprintf (stderr, "Out of memory\n");
+	    abort ();
+	}
+	destroy = destroy_fenced;
+    }
+    else
+    {
+	data = malloc (stride * height);
+	prng_randmemset (data, height * stride, 0);
+	destroy = destroy_malloced;
+    }
+
+    image = pixman_image_create_bits (format, width, height, data, stride);
+    pixman_image_set_destroy_function (image, destroy, data);
+
+    if ((flags & ALLOW_CLIPPED) && prng_rand_n (8) == 0)
+    {
+	pixman_box16_t clip_boxes[8];
+	pixman_region16_t clip;
+	int n = prng_rand_n (8) + 1;
+
+	for (i = 0; i < n; i++)
+	{
+	    clip_boxes[i].x1 = prng_rand_n (width);
+	    clip_boxes[i].y1 = prng_rand_n (height);
+	    clip_boxes[i].x2 =
+		clip_boxes[i].x1 + prng_rand_n (width - clip_boxes[i].x1);
+	    clip_boxes[i].y2 =
+		clip_boxes[i].y1 + prng_rand_n (height - clip_boxes[i].y1);
+	}
+
+	pixman_region_init_rects (&clip, clip_boxes, n);
+	pixman_image_set_clip_region (image, &clip);
+	pixman_region_fini (&clip);
+    }
+
+    if ((flags & ALLOW_SOURCE_CLIPPING) && prng_rand_n (4) == 0)
+    {
+	pixman_image_set_source_clipping (image, TRUE);
+	pixman_image_set_has_client_clip (image, TRUE);
+    }
+
+    if ((flags & ALLOW_ALPHA_MAP) && prng_rand_n (16) == 0)
+    {
+	pixman_image_t *alpha_map;
+	int alpha_x, alpha_y;
+
+	alpha_x = prng_rand_n (width);
+	alpha_y = prng_rand_n (height);
+	alpha_map =
+	    create_image (max_size, formats, (flags & ~(ALLOW_ALPHA_MAP | ALLOW_SOLID)));
+	pixman_image_set_alpha_map (image, alpha_map, alpha_x, alpha_y);
+	pixman_image_unref (alpha_map);
+    }
+
+    if ((flags & ALLOW_REPEAT) && prng_rand_n (2) == 0)
+	pixman_image_set_repeat (image, prng_rand_n (4));
+
+    image_endian_swap (image);
+
+    return image;
+}
+
+#define KEY1(p) ((void *)(((uintptr_t)p) ^ (0xa7e23dfaUL)))
+#define KEY2(p) ((void *)(((uintptr_t)p) ^ (0xabcd9876UL)))
+
+#define MAX_GLYPHS 32
+
+uint32_t
+test_glyphs (int testnum, int verbose)
+{
+    pixman_image_t *glyph_images[MAX_GLYPHS];
+    pixman_glyph_t glyphs[4 * MAX_GLYPHS];
+    uint32_t crc32 = 0;
+    pixman_image_t *source, *dest;
+    int n_glyphs, i;
+    pixman_glyph_cache_t *cache;
+
+    prng_srand (testnum);
+
+    cache = pixman_glyph_cache_create ();
+
+    source = create_image (300, formats,
+			   ALLOW_CLIPPED | ALLOW_ALPHA_MAP |
+			   ALLOW_SOURCE_CLIPPING |
+			   ALLOW_REPEAT | ALLOW_SOLID);
+
+    dest = create_image (128, formats,
+			 ALLOW_CLIPPED | ALLOW_ALPHA_MAP |
+			 ALLOW_SOURCE_CLIPPING);
+
+    pixman_glyph_cache_freeze (cache);
+
+    n_glyphs = prng_rand_n (MAX_GLYPHS);
+    for (i = 0; i < n_glyphs; ++i)
+	glyph_images[i] = create_image (32, glyph_formats, 0);
+
+    for (i = 0; i < 4 * n_glyphs; ++i)
+    {
+	int g = prng_rand_n (n_glyphs);
+	pixman_image_t *glyph_img = glyph_images[g];
+	void *key1 = KEY1 (glyph_img);
+	void *key2 = KEY2 (glyph_img);
+	const void *glyph;
+
+	if (!(glyph = pixman_glyph_cache_lookup (cache, key1, key2)))
+	{
+	    glyph =
+		pixman_glyph_cache_insert (cache, key1, key2, 5, 8, glyph_img);
+	}
+
+	glyphs[i].glyph = glyph;
+	glyphs[i].x = prng_rand_n (128);
+	glyphs[i].y = prng_rand_n (128);
+    }
+
+    if (prng_rand_n (2) == 0)
+    {
+	int src_x = prng_rand_n (300) - 150;
+	int src_y = prng_rand_n (300) - 150;
+	int mask_x = prng_rand_n (64) - 32;
+	int mask_y = prng_rand_n (64) - 32;
+	int dest_x = prng_rand_n (64) - 32;
+	int dest_y = prng_rand_n (64) - 32;
+	int width = prng_rand_n (64);
+	int height = prng_rand_n (64);
+	pixman_op_t op = operators[prng_rand_n (ARRAY_LENGTH (operators))];
+	pixman_format_code_t format = random_format (glyph_formats);
+
+	pixman_composite_glyphs (
+	    op,
+	    source, dest, format,
+	    src_x, src_y,
+	    mask_x, mask_y,
+	    dest_x, dest_y,
+	    width, height,
+	    cache, 4 * n_glyphs, glyphs);
+    }
+    else
+    {
+	pixman_op_t op = operators[prng_rand_n (ARRAY_LENGTH (operators))];
+	int src_x = prng_rand_n (300) - 150;
+	int src_y = prng_rand_n (300) - 150;
+	int dest_x = prng_rand_n (64) - 32;
+	int dest_y = prng_rand_n (64) - 32;
+
+	pixman_composite_glyphs_no_mask (
+	    op, source, dest,
+	    src_x, src_y,
+	    dest_x, dest_y,
+	    cache, 4 * n_glyphs, glyphs);
+    }
+
+    pixman_glyph_cache_thaw (cache);
+
+    for (i = 0; i < n_glyphs; ++i)
+    {
+	pixman_image_t *img = glyph_images[i];
+	void *key1, *key2;
+
+	key1 = KEY1 (img);
+	key2 = KEY2 (img);
+
+	pixman_glyph_cache_remove (cache, key1, key2);
+	pixman_image_unref (glyph_images[i]);
+    }
+
+    crc32 = compute_crc32_for_image (0, dest);
+
+    pixman_image_unref (source);
+    pixman_image_unref (dest);
+
+    pixman_glyph_cache_destroy (cache);
+
+    return crc32;
+}
+
+int
+main (int argc, const char *argv[])
+{
+    return fuzzer_test_main ("glyph", 30000,	
+			     0xFA478A79,
+			     test_glyphs, argc, argv);
+}
diff --git a/pixman/test/gradient-crash-test.c b/pixman/test/gradient-crash-test.c
new file mode 100644
index 000000000..962d1cbe8
--- /dev/null
+++ b/pixman/test/gradient-crash-test.c
@@ -0,0 +1,158 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include "utils.h"
+
+int
+main (int argc, char **argv)
+{
+#define WIDTH 400
+#define HEIGHT 200
+    
+    uint32_t *dest = malloc (WIDTH * HEIGHT * 4);
+    pixman_image_t *src_img;
+    pixman_image_t *dest_img;
+    int i, j, k, p;
+
+    typedef struct
+    {
+	pixman_point_fixed_t p0;
+	pixman_point_fixed_t p1;
+    } point_pair_t;
+    
+    pixman_gradient_stop_t onestop[1] =
+	{
+	    { pixman_int_to_fixed (1), { 0xffff, 0xeeee, 0xeeee, 0xeeee } },
+	};
+
+    pixman_gradient_stop_t subsetstops[2] =
+	{
+	    { pixman_int_to_fixed (1), { 0xffff, 0xeeee, 0xeeee, 0xeeee } },
+	    { pixman_int_to_fixed (1), { 0xffff, 0xeeee, 0xeeee, 0xeeee } },
+	};
+
+    pixman_gradient_stop_t stops01[2] =
+	{
+	    { pixman_int_to_fixed (0), { 0xffff, 0xeeee, 0xeeee, 0xeeee } },
+	    { pixman_int_to_fixed (1), { 0xffff, 0x1111, 0x1111, 0x1111 } }
+	};
+
+    point_pair_t point_pairs [] =
+	{ { { pixman_double_to_fixed (0), 0 },
+	    { pixman_double_to_fixed (WIDTH / 8.), pixman_int_to_fixed (0) } },
+	  { { pixman_double_to_fixed (WIDTH / 2.0), pixman_double_to_fixed (HEIGHT / 2.0) },
+	    { pixman_double_to_fixed (WIDTH / 2.0), pixman_double_to_fixed (HEIGHT / 2.0) } }
+	};
+    
+    pixman_transform_t transformations[] = {
+	{
+	    { { pixman_double_to_fixed (2), pixman_double_to_fixed (0.5), pixman_double_to_fixed (-100), },
+	      { pixman_double_to_fixed (0), pixman_double_to_fixed (3), pixman_double_to_fixed (0), },
+	      { pixman_double_to_fixed (0), pixman_double_to_fixed (0.000), pixman_double_to_fixed (1.0) } 
+	    }
+	},
+	{
+	    { { pixman_double_to_fixed (1), pixman_double_to_fixed (0), pixman_double_to_fixed (0), },
+	      { pixman_double_to_fixed (0), pixman_double_to_fixed (1), pixman_double_to_fixed (0), },
+	      { pixman_double_to_fixed (0), pixman_double_to_fixed (0.000), pixman_double_to_fixed (1.0) } 
+	    }
+	},
+	{
+	    { { pixman_double_to_fixed (2), pixman_double_to_fixed (1), pixman_double_to_fixed (0), },
+	      { pixman_double_to_fixed (1), pixman_double_to_fixed (1), pixman_double_to_fixed (0), },
+	      { pixman_double_to_fixed (2), pixman_double_to_fixed (1.000), pixman_double_to_fixed (1.0) } 
+	    }
+	},
+	{
+	    { { pixman_double_to_fixed (2), pixman_double_to_fixed (1), pixman_double_to_fixed (0), },
+	      { pixman_double_to_fixed (1), pixman_double_to_fixed (1), pixman_double_to_fixed (0), },
+	      { pixman_double_to_fixed (0), pixman_double_to_fixed (0), pixman_double_to_fixed (0) } 
+	    }
+	},
+	{
+	    { { pixman_double_to_fixed (2), pixman_double_to_fixed (1), pixman_double_to_fixed (0), },
+	      { pixman_double_to_fixed (1), pixman_double_to_fixed (1), pixman_double_to_fixed (0), },
+	      { pixman_double_to_fixed (2), pixman_double_to_fixed (-1), pixman_double_to_fixed (0) } 
+	    }
+	},
+	{
+	    { { pixman_double_to_fixed (2), pixman_double_to_fixed (1), pixman_double_to_fixed (3), },
+	      { pixman_double_to_fixed (1), pixman_double_to_fixed (1), pixman_double_to_fixed (0), },
+	      { pixman_double_to_fixed (2), pixman_double_to_fixed (-1), pixman_double_to_fixed (0) } 
+	    }
+	},
+    };
+    
+    pixman_fixed_t r_inner;
+    pixman_fixed_t r_outer;
+
+    enable_divbyzero_exceptions();
+    
+    for (i = 0; i < WIDTH * HEIGHT; ++i)
+	dest[i] = 0x4f00004f; /* pale blue */
+    
+    dest_img = pixman_image_create_bits (PIXMAN_a8r8g8b8,
+					 WIDTH, HEIGHT, 
+					 dest,
+					 WIDTH * 4);
+
+    r_inner = 0;
+    r_outer = pixman_double_to_fixed (50.0);
+    
+    for (i = 0; i < 3; ++i)
+    {
+	pixman_gradient_stop_t *stops;
+        int num_stops;
+
+	if (i == 0)
+	{
+	    stops = onestop;
+	    num_stops = ARRAY_LENGTH (onestop);
+	}
+	else if (i == 1)
+	{
+	    stops = subsetstops;
+	    num_stops = ARRAY_LENGTH (subsetstops);
+	}
+	else
+	{
+	    stops = stops01;
+	    num_stops = ARRAY_LENGTH (stops01);
+	}
+	
+	for (j = 0; j < 3; ++j)
+	{
+	    for (p = 0; p < ARRAY_LENGTH (point_pairs); ++p)
+	    {
+		point_pair_t *pair = &(point_pairs[p]);
+
+		if (j == 0)
+		    src_img = pixman_image_create_conical_gradient (&(pair->p0), r_inner,
+								    stops, num_stops);
+		else if (j == 1)
+		    src_img = pixman_image_create_radial_gradient  (&(pair->p0), &(pair->p1),
+								    r_inner, r_outer,
+								    stops, num_stops);
+		else
+		    src_img = pixman_image_create_linear_gradient  (&(pair->p0), &(pair->p1),
+								    stops, num_stops);
+		
+		for (k = 0; k < ARRAY_LENGTH (transformations); ++k)
+		{
+		    pixman_image_set_transform (src_img, &transformations[k]);
+		    
+		    pixman_image_set_repeat (src_img, PIXMAN_REPEAT_NONE);
+		    pixman_image_composite (PIXMAN_OP_OVER, src_img, NULL, dest_img,
+					    0, 0, 0, 0, 0, 0, 10 * WIDTH, HEIGHT);
+		}
+
+		pixman_image_unref (src_img);
+	    }
+
+	}
+    }
+
+    pixman_image_unref (dest_img);
+    free (dest);
+    
+    return 0;
+}
diff --git a/pixman/test/gtk-utils.c b/pixman/test/gtk-utils.c
deleted file mode 100644
index 751a164c0..000000000
--- a/pixman/test/gtk-utils.c
+++ /dev/null
@@ -1,113 +0,0 @@
-#include <gtk/gtk.h>
-#include <config.h>
-#include "pixman-private.h"	/* For image->bits.format
-				 * FIXME: there should probably be public API for this
-				 */
-#include "gtk-utils.h"
-
-GdkPixbuf *
-pixbuf_from_argb32 (uint32_t *bits,
-		    gboolean has_alpha,
-		    int width,
-		    int height,
-		    int stride)
-{
-    GdkPixbuf *pixbuf = gdk_pixbuf_new (GDK_COLORSPACE_RGB, TRUE,
-					8, width, height);
-    int p_stride = gdk_pixbuf_get_rowstride (pixbuf);
-    guint32 *p_bits = (guint32 *)gdk_pixbuf_get_pixels (pixbuf);
-    int w, h;
-    
-    for (h = 0; h < height; ++h)
-    {
-	for (w = 0; w < width; ++w)
-	{
-	    uint32_t argb = bits[h * (stride / 4) + w];
-	    guint r, g, b, a;
-	    char *pb = (char *)p_bits;
-
-	    pb += h * p_stride + w * 4;
-
-	    r = (argb & 0x00ff0000) >> 16;
-	    g = (argb & 0x0000ff00) >> 8;
-	    b = (argb & 0x000000ff) >> 0;
-	    a = has_alpha? (argb & 0xff000000) >> 24 : 0xff;
-
-	    if (a)
-	    {
-		r = (r * 255) / a;
-		g = (g * 255) / a;
-		b = (b * 255) / a;
-	    }
-
-	    if (r > 255) r = 255;
-	    if (g > 255) g = 255;
-	    if (b > 255) b = 255;
-	    
-	    pb[0] = r;
-	    pb[1] = g;
-	    pb[2] = b;
-	    pb[3] = a;
-	}
-    }
-    
-    return pixbuf;
-}
-
-
-static gboolean
-on_expose (GtkWidget *widget, GdkEventExpose *expose, gpointer data)
-{
-    GdkPixbuf *pixbuf = data;
-    
-    gdk_draw_pixbuf (widget->window, NULL,
-		     pixbuf, 0, 0, 0, 0,
-		     gdk_pixbuf_get_width (pixbuf),
-		     gdk_pixbuf_get_height (pixbuf),
-		     GDK_RGB_DITHER_NONE,
-		     0, 0);
-    
-    return TRUE;
-}
-
-void
-show_image (pixman_image_t *image)
-{
-    GtkWidget *window;
-    GdkPixbuf *pixbuf;
-    int width, height, stride;
-    int argc;
-    char **argv;
-    char *arg0 = g_strdup ("pixman-test-program");
-    gboolean has_alpha;
-    pixman_format_code_t format;
-
-    argc = 1;
-    argv = (char **)&arg0;
-
-    gtk_init (&argc, &argv);
-    
-    window = gtk_window_new (GTK_WINDOW_TOPLEVEL);
-    width = pixman_image_get_width (image);
-    height = pixman_image_get_height (image);
-    stride = pixman_image_get_stride (image);
-
-    format = image->bits.format;
-    
-    if (format == PIXMAN_a8r8g8b8)
-	has_alpha = TRUE;
-    else if (format == PIXMAN_x8r8g8b8)
-	has_alpha = FALSE;
-    else
-	g_error ("Can't deal with this format: %x\n", format);
-    
-    pixbuf = pixbuf_from_argb32 (pixman_image_get_data (image), has_alpha,
-				 width, height, stride);
-    
-    g_signal_connect (window, "expose_event", G_CALLBACK (on_expose), pixbuf);
-    g_signal_connect (window, "delete_event", G_CALLBACK (gtk_main_quit), NULL);
-    
-    gtk_widget_show (window);
-    
-    gtk_main ();
-}
diff --git a/pixman/test/infinite-loop.c b/pixman/test/infinite-loop.c
new file mode 100644
index 000000000..02addaab2
--- /dev/null
+++ b/pixman/test/infinite-loop.c
@@ -0,0 +1,39 @@
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "utils.h"
+
+int
+main (int argc, char **argv)
+{
+#define SRC_WIDTH 16
+#define SRC_HEIGHT 12
+#define DST_WIDTH 7
+#define DST_HEIGHT 2
+
+    static const pixman_transform_t transform = {
+	{ { 0x200017bd, 0x00000000, 0x000e6465 },
+	  { 0x00000000, 0x000a42fd, 0x000e6465 },
+	  { 0x00000000, 0x00000000, 0x00010000 },
+	}
+    };
+    pixman_image_t *src, *dest;
+
+    src = pixman_image_create_bits (
+	PIXMAN_a8r8g8b8, SRC_WIDTH, SRC_HEIGHT, NULL, -1);
+    dest = pixman_image_create_bits (
+	PIXMAN_a8r8g8b8, DST_WIDTH, DST_HEIGHT, NULL, -1);
+
+    pixman_image_set_transform (src, &transform);
+    pixman_image_set_repeat (src, PIXMAN_REPEAT_NORMAL);
+    pixman_image_set_filter (src, PIXMAN_FILTER_BILINEAR, NULL, 0);
+
+    if (argc == 1 || strcmp (argv[1], "-nf") != 0)
+	fail_after (1, "infinite loop detected");
+
+    pixman_image_composite (
+	PIXMAN_OP_OVER, src, NULL, dest, -3, -3, 0, 0, 0, 0, 6, 2);
+
+    return 0;
+}
diff --git a/pixman/test/lowlevel-blt-bench.c b/pixman/test/lowlevel-blt-bench.c
new file mode 100644
index 000000000..1049e21e7
--- /dev/null
+++ b/pixman/test/lowlevel-blt-bench.c
@@ -0,0 +1,820 @@
+/*
+ * Copyright © 2009 Nokia Corporation
+ * Copyright © 2010 Movial Creative Technologies Oy
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "utils.h"
+
+#define SOLID_FLAG 1
+#define CA_FLAG    2
+
+#define L1CACHE_SIZE (8 * 1024)
+#define L2CACHE_SIZE (128 * 1024)
+
+/* This is applied to both L1 and L2 tests - alternatively, you could
+ * parameterise bench_L or split it into two functions. It could be
+ * read at runtime on some architectures, but it only really matters
+ * that it's a number that's an integer divisor of both cacheline
+ * lengths, and further, it only really matters for caches that don't
+ * do allocate0on-write. */
+#define CACHELINE_LENGTH (32) /* bytes */
+
+#define WIDTH  1920
+#define HEIGHT 1080
+#define BUFSIZE (WIDTH * HEIGHT * 4)
+#define XWIDTH 256
+#define XHEIGHT 256
+#define TILEWIDTH 32
+#define TINYWIDTH 8
+
+#define EXCLUDE_OVERHEAD 1
+
+uint32_t *dst;
+uint32_t *src;
+uint32_t *mask;
+
+double bandwidth = 0;
+
+double
+bench_memcpy ()
+{
+    int64_t n = 0, total;
+    double  t1, t2;
+    int     x = 0;
+
+    t1 = gettime ();
+    while (1)
+    {
+	memcpy (dst, src, BUFSIZE - 64);
+	memcpy (src, dst, BUFSIZE - 64);
+	n += 4 * (BUFSIZE - 64);
+	t2 = gettime ();
+	if (t2 - t1 > 0.5)
+	    break;
+    }
+    n = total = n * 5;
+    t1 = gettime ();
+    while (n > 0)
+    {
+	if (++x >= 64)
+	    x = 0;
+	memcpy ((char *)dst + 1, (char *)src + x, BUFSIZE - 64);
+	memcpy ((char *)src + 1, (char *)dst + x, BUFSIZE - 64);
+	n -= 4 * (BUFSIZE - 64);
+    }
+    t2 = gettime ();
+    return (double)total / (t2 - t1);
+}
+
+static pixman_bool_t use_scaling = FALSE;
+static pixman_filter_t filter = PIXMAN_FILTER_NEAREST;
+
+/* nearly 1x scale factor */
+static pixman_transform_t m =
+{
+    {
+        { pixman_fixed_1 + 1, 0,              0              },
+        { 0,                  pixman_fixed_1, 0              },
+        { 0,                  0,              pixman_fixed_1 }
+    }
+};
+
+static void
+pixman_image_composite_wrapper (pixman_implementation_t *impl,
+				pixman_composite_info_t *info)
+{
+    if (use_scaling)
+    {
+        pixman_image_set_filter (info->src_image, filter, NULL, 0);
+        pixman_image_set_transform(info->src_image, &m);
+    }
+    pixman_image_composite (info->op,
+			    info->src_image, info->mask_image, info->dest_image,
+			    info->src_x, info->src_y,
+			    info->mask_x, info->mask_y,
+			    info->dest_x, info->dest_y,
+			    info->width, info->height);
+}
+
+static void
+pixman_image_composite_empty (pixman_implementation_t *impl,
+			      pixman_composite_info_t *info)
+{
+    if (use_scaling)
+    {
+        pixman_image_set_filter (info->src_image, filter, NULL, 0);
+        pixman_image_set_transform(info->src_image, &m);
+    }
+    pixman_image_composite (info->op,
+			    info->src_image, info->mask_image, info->dest_image,
+			    0, 0, 0, 0, 0, 0, 1, 1);
+}
+
+static inline void
+call_func (pixman_composite_func_t func,
+	   pixman_op_t             op,
+	   pixman_image_t *        src_image,
+	   pixman_image_t *        mask_image,
+	   pixman_image_t *        dest_image,
+	   int32_t		   src_x,
+	   int32_t		   src_y,
+	   int32_t                 mask_x,
+	   int32_t                 mask_y,
+	   int32_t                 dest_x,
+	   int32_t                 dest_y,
+	   int32_t                 width,
+	   int32_t                 height)
+{
+    pixman_composite_info_t info;
+
+    info.op = op;
+    info.src_image = src_image;
+    info.mask_image = mask_image;
+    info.dest_image = dest_image;
+    info.src_x = src_x;
+    info.src_y = src_y;
+    info.mask_x = mask_x;
+    info.mask_y = mask_y;
+    info.dest_x = dest_x;
+    info.dest_y = dest_y;
+    info.width = width;
+    info.height = height;
+
+    func (0, &info);
+}
+
+void
+noinline
+bench_L  (pixman_op_t              op,
+          pixman_image_t *         src_img,
+          pixman_image_t *         mask_img,
+          pixman_image_t *         dst_img,
+          int64_t                  n,
+          pixman_composite_func_t  func,
+          int                      width,
+          int                      lines_count)
+{
+    int64_t      i, j, k;
+    int          x = 0;
+    int          q = 0;
+    volatile int qx;
+
+    for (i = 0; i < n; i++)
+    {
+        /* For caches without allocate-on-write, we need to force the
+         * destination buffer back into the cache on each iteration,
+         * otherwise if they are evicted during the test, they remain
+         * uncached. This doesn't matter for tests which read the
+         * destination buffer, or for caches that do allocate-on-write,
+         * but in those cases this loop just adds constant time, which
+         * should be successfully cancelled out.
+         */
+        for (j = 0; j < lines_count; j++)
+        {
+            for (k = 0; k < width + 62; k += CACHELINE_LENGTH / sizeof *dst)
+            {
+                q += dst[j * WIDTH + k];
+            }
+            q += dst[j * WIDTH + width + 62];
+        }
+	if (++x >= 64)
+	    x = 0;
+	call_func (func, op, src_img, mask_img, dst_img, x, 0, x, 0, 63 - x, 0, width, lines_count);
+    }
+    qx = q;
+}
+
+void
+noinline
+bench_M (pixman_op_t              op,
+         pixman_image_t *         src_img,
+         pixman_image_t *         mask_img,
+         pixman_image_t *         dst_img,
+         int64_t                  n,
+         pixman_composite_func_t  func)
+{
+    int64_t i;
+    int     x = 0;
+
+    for (i = 0; i < n; i++)
+    {
+	if (++x >= 64)
+	    x = 0;
+	call_func (func, op, src_img, mask_img, dst_img, x, 0, x, 0, 1, 0, WIDTH - 64, HEIGHT);
+    }
+}
+
+double
+noinline
+bench_HT (pixman_op_t              op,
+          pixman_image_t *         src_img,
+          pixman_image_t *         mask_img,
+          pixman_image_t *         dst_img,
+          int64_t                  n,
+          pixman_composite_func_t  func)
+{
+    double  pix_cnt = 0;
+    int     x = 0;
+    int     y = 0;
+    int64_t i;
+
+    srand (0);
+    for (i = 0; i < n; i++)
+    {
+	int w = (rand () % (TILEWIDTH * 2)) + 1;
+	int h = (rand () % (TILEWIDTH * 2)) + 1;
+	if (x + w > WIDTH)
+	{
+	    x = 0;
+	    y += TILEWIDTH * 2;
+	}
+	if (y + h > HEIGHT)
+	{
+	    y = 0;
+	}
+	call_func (func, op, src_img, mask_img, dst_img, x, y, x, y, x, y, w, h);
+	x += w;
+	pix_cnt += w * h;
+    }
+    return pix_cnt;
+}
+
+double
+noinline
+bench_VT (pixman_op_t              op,
+          pixman_image_t *         src_img,
+          pixman_image_t *         mask_img,
+          pixman_image_t *         dst_img,
+          int64_t                  n,
+          pixman_composite_func_t  func)
+{
+    double  pix_cnt = 0;
+    int     x = 0;
+    int     y = 0;
+    int64_t i;
+
+    srand (0);
+    for (i = 0; i < n; i++)
+    {
+	int w = (rand () % (TILEWIDTH * 2)) + 1;
+	int h = (rand () % (TILEWIDTH * 2)) + 1;
+	if (y + h > HEIGHT)
+	{
+	    y = 0;
+	    x += TILEWIDTH * 2;
+	}
+	if (x + w > WIDTH)
+	{
+	    x = 0;
+	}
+	call_func (func, op, src_img, mask_img, dst_img, x, y, x, y, x, y, w, h);
+	y += h;
+	pix_cnt += w * h;
+    }
+    return pix_cnt;
+}
+
+double
+noinline
+bench_R (pixman_op_t              op,
+         pixman_image_t *         src_img,
+         pixman_image_t *         mask_img,
+         pixman_image_t *         dst_img,
+         int64_t                  n,
+         pixman_composite_func_t  func,
+         int                      maxw,
+         int                      maxh)
+{
+    double  pix_cnt = 0;
+    int64_t i;
+
+    if (maxw <= TILEWIDTH * 2 || maxh <= TILEWIDTH * 2)
+    {
+	printf("error: maxw <= TILEWIDTH * 2 || maxh <= TILEWIDTH * 2\n");
+        return 0;
+    }
+
+    srand (0);
+    for (i = 0; i < n; i++)
+    {
+	int w = (rand () % (TILEWIDTH * 2)) + 1;
+	int h = (rand () % (TILEWIDTH * 2)) + 1;
+	int sx = rand () % (maxw - TILEWIDTH * 2);
+	int sy = rand () % (maxh - TILEWIDTH * 2);
+	int dx = rand () % (maxw - TILEWIDTH * 2);
+	int dy = rand () % (maxh - TILEWIDTH * 2);
+	call_func (func, op, src_img, mask_img, dst_img, sx, sy, sx, sy, dx, dy, w, h);
+	pix_cnt += w * h;
+    }
+    return pix_cnt;
+}
+
+double
+noinline
+bench_RT (pixman_op_t              op,
+          pixman_image_t *         src_img,
+          pixman_image_t *         mask_img,
+          pixman_image_t *         dst_img,
+          int64_t                  n,
+          pixman_composite_func_t  func,
+          int                      maxw,
+          int                      maxh)
+{
+    double  pix_cnt = 0;
+    int64_t i;
+
+    if (maxw <= TINYWIDTH * 2 || maxh <= TINYWIDTH * 2)
+    {
+	printf("error: maxw <= TINYWIDTH * 2 || maxh <= TINYWIDTH * 2\n");
+        return 0;
+    }
+
+    srand (0);
+    for (i = 0; i < n; i++)
+    {
+	int w = (rand () % (TINYWIDTH * 2)) + 1;
+	int h = (rand () % (TINYWIDTH * 2)) + 1;
+	int sx = rand () % (maxw - TINYWIDTH * 2);
+	int sy = rand () % (maxh - TINYWIDTH * 2);
+	int dx = rand () % (maxw - TINYWIDTH * 2);
+	int dy = rand () % (maxh - TINYWIDTH * 2);
+	call_func (func, op, src_img, mask_img, dst_img, sx, sy, sx, sy, dx, dy, w, h);
+	pix_cnt += w * h;
+    }
+    return pix_cnt;
+}
+
+void
+bench_composite (char * testname,
+                 int    src_fmt,
+                 int    src_flags,
+                 int    op,
+                 int    mask_fmt,
+                 int    mask_flags,
+                 int    dst_fmt,
+                 double npix)
+{
+    pixman_image_t *                src_img;
+    pixman_image_t *                dst_img;
+    pixman_image_t *                mask_img;
+    pixman_image_t *                xsrc_img;
+    pixman_image_t *                xdst_img;
+    pixman_image_t *                xmask_img;
+    double                          t1, t2, t3, pix_cnt;
+    int64_t                         n, l1test_width, nlines;
+    double                             bytes_per_pix = 0;
+    pixman_bool_t                   bench_pixbuf = FALSE;
+
+    pixman_composite_func_t func = pixman_image_composite_wrapper;
+
+    if (!(src_flags & SOLID_FLAG))
+    {
+        bytes_per_pix += (src_fmt >> 24) / 8.0;
+        src_img = pixman_image_create_bits (src_fmt,
+                                            WIDTH, HEIGHT,
+                                            src,
+                                            WIDTH * 4);
+        xsrc_img = pixman_image_create_bits (src_fmt,
+                                             XWIDTH, XHEIGHT,
+                                             src,
+                                             XWIDTH * 4);
+    }
+    else
+    {
+        src_img = pixman_image_create_bits (src_fmt,
+                                            1, 1,
+                                            src,
+                                            4);
+        xsrc_img = pixman_image_create_bits (src_fmt,
+                                             1, 1,
+                                             src,
+                                             4);
+        pixman_image_set_repeat (src_img, PIXMAN_REPEAT_NORMAL);
+        pixman_image_set_repeat (xsrc_img, PIXMAN_REPEAT_NORMAL);
+    }
+
+    bytes_per_pix += (dst_fmt >> 24) / 8.0;
+    dst_img = pixman_image_create_bits (dst_fmt,
+                                        WIDTH, HEIGHT,
+                                        dst,
+                                        WIDTH * 4);
+
+    mask_img = NULL;
+    xmask_img = NULL;
+    if (strcmp (testname, "pixbuf") == 0 || strcmp (testname, "rpixbuf") == 0)
+    {
+        bench_pixbuf = TRUE;
+    }
+    if (!(mask_flags & SOLID_FLAG) && mask_fmt != PIXMAN_null)
+    {
+        bytes_per_pix += (mask_fmt >> 24) / ((op == PIXMAN_OP_SRC) ? 8.0 : 4.0);
+        mask_img = pixman_image_create_bits (mask_fmt,
+                                             WIDTH, HEIGHT,
+                                             bench_pixbuf ? src : mask,
+                                             WIDTH * 4);
+        xmask_img = pixman_image_create_bits (mask_fmt,
+                                             XWIDTH, XHEIGHT,
+                                             bench_pixbuf ? src : mask,
+                                             XWIDTH * 4);
+    }
+    else if (mask_fmt != PIXMAN_null)
+    {
+        mask_img = pixman_image_create_bits (mask_fmt,
+                                             1, 1,
+                                             mask,
+                                             4);
+        xmask_img = pixman_image_create_bits (mask_fmt,
+                                             1, 1,
+                                             mask,
+                                             4 * 4);
+       pixman_image_set_repeat (mask_img, PIXMAN_REPEAT_NORMAL);
+       pixman_image_set_repeat (xmask_img, PIXMAN_REPEAT_NORMAL);
+    }
+    if ((mask_flags & CA_FLAG) && mask_fmt != PIXMAN_null)
+    {
+       pixman_image_set_component_alpha (mask_img, 1);
+    }
+    xdst_img = pixman_image_create_bits (dst_fmt,
+                                         XWIDTH, XHEIGHT,
+                                         dst,
+                                         XWIDTH * 4);
+
+
+    printf ("%24s %c", testname, func != pixman_image_composite_wrapper ?
+            '-' : '=');
+
+    memcpy (dst, src, BUFSIZE);
+    memcpy (src, dst, BUFSIZE);
+
+    l1test_width = L1CACHE_SIZE / 8 - 64;
+    if (l1test_width < 1)
+	l1test_width = 1;
+    if (l1test_width > WIDTH - 64)
+	l1test_width = WIDTH - 64;
+    n = 1 + npix / (l1test_width * 8);
+    t1 = gettime ();
+#if EXCLUDE_OVERHEAD
+    bench_L (op, src_img, mask_img, dst_img, n, pixman_image_composite_empty, l1test_width, 1);
+#endif
+    t2 = gettime ();
+    bench_L (op, src_img, mask_img, dst_img, n, func, l1test_width, 1);
+    t3 = gettime ();
+    printf ("  L1:%7.2f", (double)n * l1test_width * 1 /
+            ((t3 - t2) - (t2 - t1)) / 1000000.);
+    fflush (stdout);
+
+    memcpy (dst, src, BUFSIZE);
+    memcpy (src, dst, BUFSIZE);
+
+    nlines = (L2CACHE_SIZE / l1test_width) /
+	((PIXMAN_FORMAT_BPP(src_fmt) + PIXMAN_FORMAT_BPP(dst_fmt)) / 8);
+    if (nlines < 1)
+	nlines = 1;
+    n = 1 + npix / (l1test_width * nlines);
+    t1 = gettime ();
+#if EXCLUDE_OVERHEAD
+    bench_L (op, src_img, mask_img, dst_img, n, pixman_image_composite_empty, l1test_width, nlines);
+#endif
+    t2 = gettime ();
+    bench_L (op, src_img, mask_img, dst_img, n, func, l1test_width, nlines);
+    t3 = gettime ();
+    printf ("  L2:%7.2f", (double)n * l1test_width * nlines /
+            ((t3 - t2) - (t2 - t1)) / 1000000.);
+    fflush (stdout);
+
+    memcpy (dst, src, BUFSIZE);
+    memcpy (src, dst, BUFSIZE);
+
+    n = 1 + npix / (WIDTH * HEIGHT);
+    t1 = gettime ();
+#if EXCLUDE_OVERHEAD
+    bench_M (op, src_img, mask_img, dst_img, n, pixman_image_composite_empty);
+#endif
+    t2 = gettime ();
+    bench_M (op, src_img, mask_img, dst_img, n, func);
+    t3 = gettime ();
+    printf ("  M:%6.2f (%6.2f%%)",
+        ((double)n * (WIDTH - 64) * HEIGHT / ((t3 - t2) - (t2 - t1))) / 1000000.,
+        ((double)n * (WIDTH - 64) * HEIGHT / ((t3 - t2) - (t2 - t1)) * bytes_per_pix) * (100.0 / bandwidth) );
+    fflush (stdout);
+
+    memcpy (dst, src, BUFSIZE);
+    memcpy (src, dst, BUFSIZE);
+
+    n = 1 + npix / (8 * TILEWIDTH * TILEWIDTH);
+    t1 = gettime ();
+#if EXCLUDE_OVERHEAD
+    pix_cnt = bench_HT (op, src_img, mask_img, dst_img, n, pixman_image_composite_empty);
+#endif
+    t2 = gettime ();
+    pix_cnt = bench_HT (op, src_img, mask_img, dst_img, n, func);
+    t3 = gettime ();
+    printf ("  HT:%6.2f", (double)pix_cnt / ((t3 - t2) - (t2 - t1)) / 1000000.);
+    fflush (stdout);
+
+    memcpy (dst, src, BUFSIZE);
+    memcpy (src, dst, BUFSIZE);
+
+    n = 1 + npix / (8 * TILEWIDTH * TILEWIDTH);
+    t1 = gettime ();
+#if EXCLUDE_OVERHEAD
+    pix_cnt = bench_VT (op, src_img, mask_img, dst_img, n, pixman_image_composite_empty);
+#endif
+    t2 = gettime ();
+    pix_cnt = bench_VT (op, src_img, mask_img, dst_img, n, func);
+    t3 = gettime ();
+    printf ("  VT:%6.2f", (double)pix_cnt / ((t3 - t2) - (t2 - t1)) / 1000000.);
+    fflush (stdout);
+
+    memcpy (dst, src, BUFSIZE);
+    memcpy (src, dst, BUFSIZE);
+
+    n = 1 + npix / (8 * TILEWIDTH * TILEWIDTH);
+    t1 = gettime ();
+#if EXCLUDE_OVERHEAD
+    pix_cnt = bench_R (op, src_img, mask_img, dst_img, n, pixman_image_composite_empty, WIDTH, HEIGHT);
+#endif
+    t2 = gettime ();
+    pix_cnt = bench_R (op, src_img, mask_img, dst_img, n, func, WIDTH, HEIGHT);
+    t3 = gettime ();
+    printf ("  R:%6.2f", (double)pix_cnt / ((t3 - t2) - (t2 - t1)) / 1000000.);
+    fflush (stdout);
+
+    memcpy (dst, src, BUFSIZE);
+    memcpy (src, dst, BUFSIZE);
+
+    n = 1 + npix / (16 * TINYWIDTH * TINYWIDTH);
+    t1 = gettime ();
+#if EXCLUDE_OVERHEAD
+    pix_cnt = bench_RT (op, src_img, mask_img, dst_img, n, pixman_image_composite_empty, WIDTH, HEIGHT);
+#endif
+    t2 = gettime ();
+    pix_cnt = bench_RT (op, src_img, mask_img, dst_img, n, func, WIDTH, HEIGHT);
+    t3 = gettime ();
+    printf ("  RT:%6.2f (%4.0fKops/s)\n", (double)pix_cnt / ((t3 - t2) - (t2 - t1)) / 1000000., (double) n / ((t3 - t2) * 1000));
+
+    if (mask_img) {
+	pixman_image_unref (mask_img);
+	pixman_image_unref (xmask_img);
+    }
+    pixman_image_unref (src_img);
+    pixman_image_unref (dst_img);
+    pixman_image_unref (xsrc_img);
+    pixman_image_unref (xdst_img);
+}
+
+#define PIXMAN_OP_OUT_REV (PIXMAN_OP_OUT_REVERSE)
+
+struct
+{
+    char *testname;
+    int   src_fmt;
+    int   src_flags;
+    int   op;
+    int   mask_fmt;
+    int   mask_flags;
+    int   dst_fmt;
+}
+tests_tbl[] =
+{
+    { "add_8_8_8",             PIXMAN_a8,          0, PIXMAN_OP_ADD,     PIXMAN_a8,       0, PIXMAN_a8 },
+    { "add_n_8_8",             PIXMAN_a8r8g8b8,    1, PIXMAN_OP_ADD,     PIXMAN_a8,       0, PIXMAN_a8 },
+    { "add_n_8_8888",          PIXMAN_a8r8g8b8,    1, PIXMAN_OP_ADD,     PIXMAN_a8,       0, PIXMAN_a8r8g8b8 },
+    { "add_n_8_x888",          PIXMAN_a8r8g8b8,    1, PIXMAN_OP_ADD,     PIXMAN_a8,       0, PIXMAN_x8r8g8b8 },
+    { "add_n_8_0565",          PIXMAN_a8r8g8b8,    1, PIXMAN_OP_ADD,     PIXMAN_a8,       0, PIXMAN_r5g6b5 },
+    { "add_n_8_1555",          PIXMAN_a8r8g8b8,    1, PIXMAN_OP_ADD,     PIXMAN_a8,       0, PIXMAN_a1r5g5b5 },
+    { "add_n_8_4444",          PIXMAN_a8r8g8b8,    1, PIXMAN_OP_ADD,     PIXMAN_a8,       0, PIXMAN_a4r4g4b4 },
+    { "add_n_8_2222",          PIXMAN_a8r8g8b8,    1, PIXMAN_OP_ADD,     PIXMAN_a8,       0, PIXMAN_a2r2g2b2 },
+    { "add_n_8_2x10",          PIXMAN_a8r8g8b8,    1, PIXMAN_OP_ADD,     PIXMAN_a8,       0, PIXMAN_x2r10g10b10 },
+    { "add_n_8_2a10",          PIXMAN_a8r8g8b8,    1, PIXMAN_OP_ADD,     PIXMAN_a8,       0, PIXMAN_a2r10g10b10 },
+    { "add_n_8",               PIXMAN_a8r8g8b8,    1, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_a8 },
+    { "add_n_8888",            PIXMAN_a8r8g8b8,    1, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_a8r8g8b8 },
+    { "add_n_x888",            PIXMAN_a8r8g8b8,    1, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_x8r8g8b8 },
+    { "add_n_0565",            PIXMAN_a8r8g8b8,    1, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_r5g6b5 },
+    { "add_n_1555",            PIXMAN_a8r8g8b8,    1, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_a1r5g5b5 },
+    { "add_n_4444",            PIXMAN_a8r8g8b8,    1, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_a4r4g4b4 },
+    { "add_n_2222",            PIXMAN_a8r8g8b8,    1, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_a2r2g2b2 },
+    { "add_n_2x10",            PIXMAN_a2r10g10b10, 1, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_x2r10g10b10 },
+    { "add_n_2a10",            PIXMAN_a2r10g10b10, 1, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_a2r10g10b10 },
+    { "add_8_8",               PIXMAN_a8,          0, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_a8 },
+    { "add_x888_x888",         PIXMAN_x8r8g8b8,    0, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_x8r8g8b8 },
+    { "add_8888_8888",         PIXMAN_a8r8g8b8,    0, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_a8r8g8b8 },
+    { "add_8888_0565",         PIXMAN_a8r8g8b8,    0, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_r5g6b5 },
+    { "add_8888_1555",         PIXMAN_a8r8g8b8,    0, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_a1r5g5b5 },
+    { "add_8888_4444",         PIXMAN_a8r8g8b8,    0, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_a4r4g4b4 },
+    { "add_8888_2222",         PIXMAN_a8r8g8b8,    0, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_a2r2g2b2 },
+    { "add_0565_0565",         PIXMAN_r5g6b5,      0, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_r5g6b5 },
+    { "add_1555_1555",         PIXMAN_a1r5g5b5,    0, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_a1r5g5b5 },
+    { "add_0565_2x10",         PIXMAN_r5g6b5,      0, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_x2r10g10b10 },
+    { "add_2a10_2a10",         PIXMAN_a2r10g10b10, 0, PIXMAN_OP_ADD,     PIXMAN_null,     0, PIXMAN_a2r10g10b10 },
+    { "in_n_8_8",              PIXMAN_a8r8g8b8,    1, PIXMAN_OP_IN,      PIXMAN_a8,       0, PIXMAN_a8 },
+    { "in_8_8",                PIXMAN_a8,          0, PIXMAN_OP_IN,      PIXMAN_null,     0, PIXMAN_a8 },
+    { "src_n_2222",            PIXMAN_a8r8g8b8,    1, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_a2r2g2b2 },
+    { "src_n_0565",            PIXMAN_a8r8g8b8,    1, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_r5g6b5 },
+    { "src_n_1555",            PIXMAN_a8r8g8b8,    1, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_a1r5g5b5 },
+    { "src_n_4444",            PIXMAN_a8r8g8b8,    1, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_a4r4g4b4 },
+    { "src_n_x888",            PIXMAN_a8r8g8b8,    1, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_x8r8g8b8 },
+    { "src_n_8888",            PIXMAN_a8r8g8b8,    1, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_a8r8g8b8 },
+    { "src_n_2x10",            PIXMAN_a2r10g10b10, 1, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_x2r10g10b10 },
+    { "src_n_2a10",            PIXMAN_a2r10g10b10, 1, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_a2r10g10b10 },
+    { "src_8888_0565",         PIXMAN_a8r8g8b8,    0, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_r5g6b5 },
+    { "src_0565_8888",         PIXMAN_r5g6b5,      0, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_a8r8g8b8 },
+    { "src_8888_4444",         PIXMAN_a8r8g8b8,    0, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_a4r4g4b4 },
+    { "src_8888_2222",         PIXMAN_a8r8g8b8,    0, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_a2r2g2b2 },
+    { "src_8888_2x10",         PIXMAN_a8r8g8b8,    0, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_x2r10g10b10 },
+    { "src_8888_2a10",         PIXMAN_a8r8g8b8,    0, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_a2r10g10b10 },
+    { "src_0888_0565",         PIXMAN_r8g8b8,      0, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_r5g6b5 },
+    { "src_0888_8888",         PIXMAN_r8g8b8,      0, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_a8r8g8b8 },
+    { "src_0888_x888",         PIXMAN_r8g8b8,      0, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_x8r8g8b8 },
+    { "src_0888_8888_rev",     PIXMAN_b8g8r8,      0, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_x8r8g8b8 },
+    { "src_0888_0565_rev",     PIXMAN_b8g8r8,      0, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_r5g6b5 },
+    { "src_x888_x888",         PIXMAN_x8r8g8b8,    0, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_x8r8g8b8 },
+    { "src_x888_8888",         PIXMAN_x8r8g8b8,    0, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_a8r8g8b8 },
+    { "src_8888_8888",         PIXMAN_a8r8g8b8,    0, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_a8r8g8b8 },
+    { "src_0565_0565",         PIXMAN_r5g6b5,      0, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_r5g6b5 },
+    { "src_1555_0565",         PIXMAN_a1r5g5b5,    0, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_r5g6b5 },
+    { "src_0565_1555",         PIXMAN_r5g6b5,      0, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_a1r5g5b5 },
+    { "src_8_8",               PIXMAN_a8,          0, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_a8 },
+    { "src_n_8",               PIXMAN_a8,          1, PIXMAN_OP_SRC,     PIXMAN_null,     0, PIXMAN_a8 },
+    { "src_n_8_0565",          PIXMAN_a8r8g8b8,    1, PIXMAN_OP_SRC,     PIXMAN_a8,       0, PIXMAN_r5g6b5 },
+    { "src_n_8_1555",          PIXMAN_a8r8g8b8,    1, PIXMAN_OP_SRC,     PIXMAN_a8,       0, PIXMAN_a1r5g5b5 },
+    { "src_n_8_4444",          PIXMAN_a8r8g8b8,    1, PIXMAN_OP_SRC,     PIXMAN_a8,       0, PIXMAN_a4r4g4b4 },
+    { "src_n_8_2222",          PIXMAN_a8r8g8b8,    1, PIXMAN_OP_SRC,     PIXMAN_a8,       0, PIXMAN_a2r2g2b2 },
+    { "src_n_8_x888",          PIXMAN_a8r8g8b8,    1, PIXMAN_OP_SRC,     PIXMAN_a8,       0, PIXMAN_x8r8g8b8 },
+    { "src_n_8_8888",          PIXMAN_a8r8g8b8,    1, PIXMAN_OP_SRC,     PIXMAN_a8,       0, PIXMAN_a8r8g8b8 },
+    { "src_n_8_2x10",          PIXMAN_a8r8g8b8,    1, PIXMAN_OP_SRC,     PIXMAN_a8,       0, PIXMAN_x2r10g10b10 },
+    { "src_n_8_2a10",          PIXMAN_a8r8g8b8,    1, PIXMAN_OP_SRC,     PIXMAN_a8,       0, PIXMAN_a2r10g10b10 },
+    { "src_8888_8_0565",       PIXMAN_a8r8g8b8,    0, PIXMAN_OP_SRC,     PIXMAN_a8,       0, PIXMAN_r5g6b5 },
+    { "src_0888_8_0565",       PIXMAN_r8g8b8,      0, PIXMAN_OP_SRC,     PIXMAN_a8,       0, PIXMAN_r5g6b5 },
+    { "src_0888_8_8888",       PIXMAN_r8g8b8,      0, PIXMAN_OP_SRC,     PIXMAN_a8,       0, PIXMAN_a8r8g8b8 },
+    { "src_0888_8_x888",       PIXMAN_r8g8b8,      0, PIXMAN_OP_SRC,     PIXMAN_a8,       0, PIXMAN_x8r8g8b8 },
+    { "src_x888_8_x888",       PIXMAN_x8r8g8b8,    0, PIXMAN_OP_SRC,     PIXMAN_a8,       0, PIXMAN_x8r8g8b8 },
+    { "src_x888_8_8888",       PIXMAN_x8r8g8b8,    0, PIXMAN_OP_SRC,     PIXMAN_a8,       0, PIXMAN_a8r8g8b8 },
+    { "src_0565_8_0565",       PIXMAN_r5g6b5,      0, PIXMAN_OP_SRC,     PIXMAN_a8,       0, PIXMAN_r5g6b5 },
+    { "src_1555_8_0565",       PIXMAN_a1r5g5b5,    0, PIXMAN_OP_SRC,     PIXMAN_a8,       0, PIXMAN_r5g6b5 },
+    { "src_0565_8_1555",       PIXMAN_r5g6b5,      0, PIXMAN_OP_SRC,     PIXMAN_a8,       0, PIXMAN_a1r5g5b5 },
+    { "over_n_x888",           PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_null,     0, PIXMAN_x8r8g8b8 },
+    { "over_n_8888",           PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_null,     0, PIXMAN_a8r8g8b8 },
+    { "over_n_0565",           PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_null,     0, PIXMAN_r5g6b5 },
+    { "over_n_1555",           PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_null,     0, PIXMAN_a1r5g5b5 },
+    { "over_8888_0565",        PIXMAN_a8r8g8b8,    0, PIXMAN_OP_OVER,    PIXMAN_null,     0, PIXMAN_r5g6b5 },
+    { "over_8888_8888",        PIXMAN_a8r8g8b8,    0, PIXMAN_OP_OVER,    PIXMAN_null,     0, PIXMAN_a8r8g8b8 },
+    { "over_8888_x888",        PIXMAN_a8r8g8b8,    0, PIXMAN_OP_OVER,    PIXMAN_null,     0, PIXMAN_x8r8g8b8 },
+    { "over_x888_8_0565",      PIXMAN_x8r8g8b8,    0, PIXMAN_OP_OVER,    PIXMAN_a8,       0, PIXMAN_r5g6b5 },
+    { "over_x888_8_8888",      PIXMAN_x8r8g8b8,    0, PIXMAN_OP_OVER,    PIXMAN_a8,       0, PIXMAN_a8r8g8b8 },
+    { "over_n_8_0565",         PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_a8,       0, PIXMAN_r5g6b5 },
+    { "over_n_8_1555",         PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_a8,       0, PIXMAN_a1r5g5b5 },
+    { "over_n_8_4444",         PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_a8,       0, PIXMAN_a4r4g4b4 },
+    { "over_n_8_2222",         PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_a8,       0, PIXMAN_a2r2g2b2 },
+    { "over_n_8_x888",         PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_a8,       0, PIXMAN_x8r8g8b8 },
+    { "over_n_8_8888",         PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_a8,       0, PIXMAN_a8r8g8b8 },
+    { "over_n_8_2x10",         PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_a8,       0, PIXMAN_x2r10g10b10 },
+    { "over_n_8_2a10",         PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_a8,       0, PIXMAN_a2r10g10b10 },
+    { "over_n_8888_8888_ca",   PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_a8r8g8b8, 2, PIXMAN_a8r8g8b8 },
+    { "over_n_8888_x888_ca",   PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_a8r8g8b8, 2, PIXMAN_x8r8g8b8 },
+    { "over_n_8888_0565_ca",   PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_a8r8g8b8, 2, PIXMAN_r5g6b5 },
+    { "over_n_8888_1555_ca",   PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_a8r8g8b8, 2, PIXMAN_a1r5g5b5 },
+    { "over_n_8888_4444_ca",   PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_a8r8g8b8, 2, PIXMAN_a4r4g4b4 },
+    { "over_n_8888_2222_ca",   PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_a8r8g8b8, 2, PIXMAN_a2r2g2b2 },
+    { "over_n_8888_2x10_ca",   PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_a8r8g8b8, 2, PIXMAN_x2r10g10b10 },
+    { "over_n_8888_2a10_ca",   PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OVER,    PIXMAN_a8r8g8b8, 2, PIXMAN_a2r10g10b10 },
+    { "over_8888_n_8888",      PIXMAN_a8r8g8b8,    0, PIXMAN_OP_OVER,    PIXMAN_a8,       1, PIXMAN_a8r8g8b8 },
+    { "over_8888_n_x888",      PIXMAN_a8r8g8b8,    0, PIXMAN_OP_OVER,    PIXMAN_a8,       1, PIXMAN_x8r8g8b8 },
+    { "over_8888_n_0565",      PIXMAN_a8r8g8b8,    0, PIXMAN_OP_OVER,    PIXMAN_a8,       1, PIXMAN_r5g6b5 },
+    { "over_8888_n_1555",      PIXMAN_a8r8g8b8,    0, PIXMAN_OP_OVER,    PIXMAN_a8,       1, PIXMAN_a1r5g5b5 },
+    { "over_x888_n_8888",      PIXMAN_x8r8g8b8,    0, PIXMAN_OP_OVER,    PIXMAN_a8,       1, PIXMAN_a8r8g8b8 },
+    { "outrev_n_8_0565",       PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OUT_REV, PIXMAN_a8,       0, PIXMAN_r5g6b5 },
+    { "outrev_n_8_1555",       PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OUT_REV, PIXMAN_a8,       0, PIXMAN_a1r5g5b5 },
+    { "outrev_n_8_x888",       PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OUT_REV, PIXMAN_a8,       0, PIXMAN_x8r8g8b8 },
+    { "outrev_n_8_8888",       PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OUT_REV, PIXMAN_a8,       0, PIXMAN_a8r8g8b8 },
+    { "outrev_n_8888_0565_ca", PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OUT_REV, PIXMAN_a8r8g8b8, 2, PIXMAN_r5g6b5 },
+    { "outrev_n_8888_1555_ca", PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OUT_REV, PIXMAN_a8r8g8b8, 2, PIXMAN_a1r5g5b5 },
+    { "outrev_n_8888_x888_ca", PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OUT_REV, PIXMAN_a8r8g8b8, 2, PIXMAN_x8r8g8b8 },
+    { "outrev_n_8888_8888_ca", PIXMAN_a8r8g8b8,    1, PIXMAN_OP_OUT_REV, PIXMAN_a8r8g8b8, 2, PIXMAN_a8r8g8b8 },
+    { "over_reverse_n_8888",   PIXMAN_a8r8g8b8,    0, PIXMAN_OP_OVER_REVERSE, PIXMAN_null, 0, PIXMAN_a8r8g8b8 },
+    { "pixbuf",                PIXMAN_x8b8g8r8,    0, PIXMAN_OP_SRC,     PIXMAN_a8b8g8r8, 0, PIXMAN_a8r8g8b8 },
+    { "rpixbuf",               PIXMAN_x8b8g8r8,    0, PIXMAN_OP_SRC,     PIXMAN_a8b8g8r8, 0, PIXMAN_a8b8g8r8 },
+};
+
+int
+main (int argc, char *argv[])
+{
+    double x;
+    int i;
+    const char *pattern = NULL;
+    for (i = 1; i < argc; i++)
+    {
+	if (argv[i][0] == '-')
+	{
+	    if (strchr (argv[i] + 1, 'b'))
+	    {
+		use_scaling = TRUE;
+		filter = PIXMAN_FILTER_BILINEAR;
+	    }
+	    else if (strchr (argv[i] + 1, 'n'))
+	    {
+		use_scaling = TRUE;
+		filter = PIXMAN_FILTER_NEAREST;
+	    }
+	}
+	else
+	{
+	    pattern = argv[i];
+	}
+    }
+
+    if (!pattern)
+    {
+	printf ("Usage: lowlevel-blt-bench [-b] [-n] pattern\n");
+	printf ("  -n : benchmark nearest scaling\n");
+	printf ("  -b : benchmark bilinear scaling\n");
+	return 1;
+    }
+
+    src = aligned_malloc (4096, BUFSIZE * 3);
+    memset (src, 0xCC, BUFSIZE * 3);
+    dst = src + (BUFSIZE / 4);
+    mask = dst + (BUFSIZE / 4);
+
+    printf ("Benchmark for a set of most commonly used functions\n");
+    printf ("---\n");
+    printf ("All results are presented in millions of pixels per second\n");
+    printf ("L1  - small Xx1 rectangle (fitting L1 cache), always blitted at the same\n");
+    printf ("      memory location with small drift in horizontal direction\n");
+    printf ("L2  - small XxY rectangle (fitting L2 cache), always blitted at the same\n");
+    printf ("      memory location with small drift in horizontal direction\n");
+    printf ("M   - large %dx%d rectangle, always blitted at the same\n",
+            WIDTH - 64, HEIGHT);
+    printf ("      memory location with small drift in horizontal direction\n");
+    printf ("HT  - random rectangles with %dx%d average size are copied from\n",
+            TILEWIDTH, TILEWIDTH);
+    printf ("      one %dx%d buffer to another, traversing from left to right\n",
+            WIDTH, HEIGHT);
+    printf ("      and from top to bottom\n");
+    printf ("VT  - random rectangles with %dx%d average size are copied from\n",
+            TILEWIDTH, TILEWIDTH);
+    printf ("      one %dx%d buffer to another, traversing from top to bottom\n",
+            WIDTH, HEIGHT);
+    printf ("      and from left to right\n");
+    printf ("R   - random rectangles with %dx%d average size are copied from\n",
+            TILEWIDTH, TILEWIDTH);
+    printf ("      random locations of one %dx%d buffer to another\n",
+            WIDTH, HEIGHT);
+    printf ("RT  - as R, but %dx%d average sized rectangles are copied\n",
+            TINYWIDTH, TINYWIDTH);
+    printf ("---\n");
+    bandwidth = x = bench_memcpy ();
+    printf ("reference memcpy speed = %.1fMB/s (%.1fMP/s for 32bpp fills)\n",
+            x / 1000000., x / 4000000);
+    if (use_scaling)
+    {
+	printf ("---\n");
+	if (filter == PIXMAN_FILTER_BILINEAR)
+	    printf ("BILINEAR scaling\n");
+	else if (filter == PIXMAN_FILTER_NEAREST)
+	    printf ("NEAREST scaling\n");
+	else
+	    printf ("UNKNOWN scaling\n");
+    }
+    printf ("---\n");
+
+    for (i = 0; i < ARRAY_LENGTH (tests_tbl); i++)
+    {
+	if (strcmp (pattern, "all") == 0 || strcmp (tests_tbl[i].testname, pattern) == 0)
+	{
+	    bench_composite (tests_tbl[i].testname,
+			     tests_tbl[i].src_fmt,
+			     tests_tbl[i].src_flags,
+			     tests_tbl[i].op,
+			     tests_tbl[i].mask_fmt,
+			     tests_tbl[i].mask_flags,
+			     tests_tbl[i].dst_fmt,
+			     bandwidth/8);
+	}
+    }
+
+    free (src);
+    return 0;
+}
diff --git a/pixman/test/matrix-test.c b/pixman/test/matrix-test.c
new file mode 100644
index 000000000..0a5f203f5
--- /dev/null
+++ b/pixman/test/matrix-test.c
@@ -0,0 +1,235 @@
+/*
+ * Copyright © 2012 Siarhei Siamashka <siarhei.siamashka@gmail.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include "utils.h"
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <math.h>
+
+#ifdef HAVE_FLOAT128
+
+#define pixman_fixed_to_float128(x) (((__float128)(x)) / 65536.0Q)
+
+typedef struct { __float128 v[3]; } pixman_vector_f128_t;
+typedef struct { __float128 m[3][3]; } pixman_transform_f128_t;
+
+pixman_bool_t
+pixman_transform_point_f128 (const pixman_transform_f128_t *t,
+                             const pixman_vector_f128_t    *v,
+                             pixman_vector_f128_t          *result)
+{
+    int i;
+    for (i = 0; i < 3; i++)
+    {
+        result->v[i] = t->m[i][0] * v->v[0] +
+                       t->m[i][1] * v->v[1] +
+                       t->m[i][2] * v->v[2];
+    }
+    if (result->v[2] != 0)
+    {
+        result->v[0] /= result->v[2];
+        result->v[1] /= result->v[2];
+        result->v[2] = 1;
+        return TRUE;
+    }
+    else
+    {
+        return FALSE;
+    }
+}
+
+pixman_bool_t does_it_fit_fixed_48_16 (__float128 x)
+{
+    if (x >= 65536.0Q * 65536.0Q * 32768.0Q)
+        return FALSE;
+    if (x <= -65536.0Q * 65536.0Q * 32768.0Q)
+        return FALSE;
+    return TRUE;
+}
+
+#endif
+
+static inline uint32_t
+byteswap32 (uint32_t x)
+{
+    return ((x & ((uint32_t)0xFF << 24)) >> 24) |
+           ((x & ((uint32_t)0xFF << 16)) >>  8) |
+           ((x & ((uint32_t)0xFF <<  8)) <<  8) |
+           ((x & ((uint32_t)0xFF <<  0)) << 24);
+}
+
+static inline uint64_t
+byteswap64 (uint64_t x)
+{
+    return ((x & ((uint64_t)0xFF << 56)) >> 56) |
+           ((x & ((uint64_t)0xFF << 48)) >> 40) |
+           ((x & ((uint64_t)0xFF << 40)) >> 24) |
+           ((x & ((uint64_t)0xFF << 32)) >>  8) |
+           ((x & ((uint64_t)0xFF << 24)) <<  8) |
+           ((x & ((uint64_t)0xFF << 16)) << 24) |
+           ((x & ((uint64_t)0xFF <<  8)) << 40) |
+           ((x & ((uint64_t)0xFF <<  0)) << 56);
+}
+
+static void
+byteswap_transform (pixman_transform_t *t)
+{
+    int i, j;
+
+    if (is_little_endian ())
+        return;
+
+    for (i = 0; i < 3; i++)
+        for (j = 0; j < 3; j++)
+            t->matrix[i][j] = byteswap32 (t->matrix[i][j]);
+}
+
+static void
+byteswap_vector_48_16 (pixman_vector_48_16_t *v)
+{
+    int i;
+
+    if (is_little_endian ())
+        return;
+
+    for (i = 0; i < 3; i++)
+        v->v[i] = byteswap64 (v->v[i]);
+}
+
+uint32_t
+test_matrix (int testnum, int verbose)
+{
+    uint32_t crc32 = 0;
+    int i, j, k;
+    pixman_bool_t is_affine;
+
+    prng_srand (testnum);
+
+    for (i = 0; i < 100; i++)
+    {
+        pixman_bool_t           transform_ok;
+        pixman_transform_t      ti;
+        pixman_vector_48_16_t   vi, result_i;
+#ifdef HAVE_FLOAT128
+        pixman_transform_f128_t tf;
+        pixman_vector_f128_t    vf, result_f;
+#endif
+        prng_randmemset (&ti, sizeof(ti), 0);
+        prng_randmemset (&vi, sizeof(vi), 0);
+        byteswap_transform (&ti);
+        byteswap_vector_48_16 (&vi);
+
+        for (j = 0; j < 3; j++)
+        {
+            /* make sure that "vi" contains 31.16 fixed point data */
+            vi.v[j] >>= 17;
+            /* and apply random shift */
+            if (prng_rand_n (3) == 0)
+                vi.v[j] >>= prng_rand_n (46);
+        }
+
+        if (prng_rand_n (2))
+        {
+            /* random shift for the matrix */
+            for (j = 0; j < 3; j++)
+                for (k = 0; k < 3; k++)
+                    ti.matrix[j][k] >>= prng_rand_n (30);
+        }
+
+        if (prng_rand_n (2))
+        {
+            /* affine matrix */
+            ti.matrix[2][0] = 0;
+            ti.matrix[2][1] = 0;
+            ti.matrix[2][2] = pixman_fixed_1;
+        }
+
+        if (prng_rand_n (2))
+        {
+            /* cartesian coordinates */
+            vi.v[2] = pixman_fixed_1;
+        }
+
+        is_affine = (ti.matrix[2][0] == 0 && ti.matrix[2][1] == 0 &&
+                     ti.matrix[2][2] == pixman_fixed_1 &&
+                     vi.v[2] == pixman_fixed_1);
+
+        transform_ok = TRUE;
+        if (is_affine && prng_rand_n (2))
+            pixman_transform_point_31_16_affine (&ti, &vi, &result_i);
+        else
+            transform_ok = pixman_transform_point_31_16 (&ti, &vi, &result_i);
+
+#ifdef HAVE_FLOAT128
+        /* compare with a reference 128-bit floating point implementation */
+        for (j = 0; j < 3; j++)
+        {
+            vf.v[j] = pixman_fixed_to_float128 (vi.v[j]);
+            for (k = 0; k < 3; k++)
+            {
+                tf.m[j][k] = pixman_fixed_to_float128 (ti.matrix[j][k]);
+            }
+        }
+
+        if (pixman_transform_point_f128 (&tf, &vf, &result_f))
+        {
+            if (transform_ok ||
+                (does_it_fit_fixed_48_16 (result_f.v[0]) &&
+                 does_it_fit_fixed_48_16 (result_f.v[1]) &&
+                 does_it_fit_fixed_48_16 (result_f.v[2])))
+            {
+                for (j = 0; j < 3; j++)
+                {
+                    double diff = fabs (result_f.v[j] -
+                                        pixman_fixed_to_float128 (result_i.v[j]));
+
+                    if (is_affine && diff > (0.51 / 65536.0))
+                    {
+                        printf ("%d:%d: bad precision for affine (%.12f)\n",
+                               testnum, i, diff);
+                        abort ();
+                    }
+                    else if (diff > (0.71 / 65536.0))
+                    {
+                        printf ("%d:%d: bad precision for projective (%.12f)\n",
+                               testnum, i, diff);
+                        abort ();
+                    }
+                }
+            }
+        }
+#endif
+        byteswap_vector_48_16 (&result_i);
+        crc32 = compute_crc32 (crc32, &result_i, sizeof (result_i));
+    }
+    return crc32;
+}
+
+int
+main (int argc, const char *argv[])
+{
+    return fuzzer_test_main ("matrix", 20000,
+			     0xBEBF98C3,
+			     test_matrix, argc, argv);
+}
diff --git a/pixman/test/oob-test.c b/pixman/test/oob-test.c
index 4f9e5a244..0d19b504a 100644
--- a/pixman/test/oob-test.c
+++ b/pixman/test/oob-test.c
@@ -1,6 +1,6 @@
 #include <stdio.h>
 #include <stdlib.h>
-#include "pixman.h"
+#include "utils.h"
 
 typedef struct
 {
@@ -94,7 +94,7 @@ main (int argc, char **argv)
 {
     int i;
 
-    for (i = 0; i < sizeof (info) / sizeof (info[0]); ++i)
+    for (i = 0; i < ARRAY_LENGTH (info); ++i)
 	test_composite (&info[i]);
     
     return 0;
diff --git a/pixman/test/pdf-op-test.c b/pixman/test/pdf-op-test.c
new file mode 100644
index 000000000..dcb3a603a
--- /dev/null
+++ b/pixman/test/pdf-op-test.c
@@ -0,0 +1,83 @@
+#include <stdlib.h>
+#include "utils.h"
+
+static const pixman_op_t pdf_ops[] =
+{
+    PIXMAN_OP_MULTIPLY,
+    PIXMAN_OP_SCREEN,
+    PIXMAN_OP_OVERLAY,
+    PIXMAN_OP_DARKEN,
+    PIXMAN_OP_LIGHTEN,
+    PIXMAN_OP_COLOR_DODGE,
+    PIXMAN_OP_COLOR_BURN,
+    PIXMAN_OP_HARD_LIGHT,
+    PIXMAN_OP_SOFT_LIGHT,
+    PIXMAN_OP_DIFFERENCE,
+    PIXMAN_OP_EXCLUSION,
+    PIXMAN_OP_HSL_HUE,
+    PIXMAN_OP_HSL_SATURATION,
+    PIXMAN_OP_HSL_COLOR,
+    PIXMAN_OP_HSL_LUMINOSITY
+};
+
+static const uint32_t pixels[] =
+{
+    0x00808080,
+    0x80123456,
+    0x00000000,
+    0xffffffff,
+    0x00ffffff,
+    0x80808080,
+    0x00123456,
+};
+
+int
+main ()
+{
+    int o, s, m, d;
+
+    enable_divbyzero_exceptions();
+
+    for (o = 0; o < ARRAY_LENGTH (pdf_ops); ++o)
+    {
+	pixman_op_t op = pdf_ops[o];
+
+	for (s = 0; s < ARRAY_LENGTH (pixels); ++s)
+	{
+	    pixman_image_t *src;
+
+	    src = pixman_image_create_bits (
+		PIXMAN_a8r8g8b8, 1, 1, (uint32_t *)&(pixels[s]), 4);
+
+	    for (m = -1; m < ARRAY_LENGTH (pixels); ++m)
+	    {
+		pixman_image_t *msk = NULL;
+		if (m >= 0)
+		{
+		    msk = pixman_image_create_bits (
+			PIXMAN_a8r8g8b8, 1, 1, (uint32_t *)&(pixels[m]), 4);
+		}
+
+		for (d = 0; d < ARRAY_LENGTH (pixels); ++d)
+		{
+		    pixman_image_t *dst;
+		    uint32_t dp = pixels[d];
+
+		    dst = pixman_image_create_bits (
+			PIXMAN_a8r8g8b8, 1, 1, &dp, 4);
+
+		    pixman_image_composite (op, src, msk, dst,
+					    0, 0, 0, 0, 0, 0, 1, 1);
+
+		    pixman_image_unref (dst);
+		}
+		if (msk)
+		    pixman_image_unref (msk);
+	    }
+
+	    pixman_image_unref (src);
+	}
+    }
+
+    return 0;
+}
diff --git a/pixman/test/pixel-test.c b/pixman/test/pixel-test.c
new file mode 100644
index 000000000..8c525d202
--- /dev/null
+++ b/pixman/test/pixel-test.c
@@ -0,0 +1,267 @@
+/*
+ * Copyright © 2013 Soeren Sandmann
+ * Copyright © 2013 Red Hat, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#include <stdio.h>
+#include <stdlib.h> /* abort() */
+#include <math.h>
+#include <time.h>
+#include "utils.h"
+
+typedef struct pixel_combination_t pixel_combination_t;
+struct pixel_combination_t
+{
+    pixman_op_t			op;
+    pixman_format_code_t	src_format;
+    uint32_t			src_pixel;
+    pixman_format_code_t	dest_format;
+    uint32_t			dest_pixel;
+};
+
+static const pixel_combination_t regressions[] =
+{
+    { PIXMAN_OP_OVER,
+      PIXMAN_a8r8g8b8,	0x0f00c300,
+      PIXMAN_x14r6g6b6,	0x003c0,
+    },
+    { PIXMAN_OP_DISJOINT_XOR,
+      PIXMAN_a4r4g4b4,	0xd0c0,
+      PIXMAN_a8r8g8b8,	0x5300ea00,
+    },
+    { PIXMAN_OP_OVER,
+      PIXMAN_a8r8g8b8,	0x20c6bf00,
+      PIXMAN_r5g6b5,	0xb9ff
+    },
+    { PIXMAN_OP_OVER,
+      PIXMAN_a8r8g8b8,	0x204ac7ff,
+      PIXMAN_r5g6b5,	0xc1ff
+    },
+    { PIXMAN_OP_OVER_REVERSE,
+      PIXMAN_r5g6b5,	0xffc3,
+      PIXMAN_a8r8g8b8,	0x102d00dd
+    },
+    { PIXMAN_OP_OVER_REVERSE,
+      PIXMAN_r5g6b5,	0x1f00,
+      PIXMAN_a8r8g8b8,	0x1bdf0c89
+    },
+    { PIXMAN_OP_OVER_REVERSE,
+      PIXMAN_r5g6b5,	0xf9d2,
+      PIXMAN_a8r8g8b8,	0x1076bcf7
+    },
+    { PIXMAN_OP_OVER_REVERSE,
+      PIXMAN_r5g6b5,	0x00c3,
+      PIXMAN_a8r8g8b8,	0x1bfe9ae5
+    },
+    { PIXMAN_OP_OVER_REVERSE,
+      PIXMAN_r5g6b5,	0x09ff,
+      PIXMAN_a8r8g8b8,	0x0b00c16c
+    },
+    { PIXMAN_OP_DISJOINT_ATOP,
+      PIXMAN_a2r2g2b2,	0xbc,
+      PIXMAN_a8r8g8b8,	0x9efff1ff
+    },
+    { PIXMAN_OP_DISJOINT_ATOP,
+      PIXMAN_a4r4g4b4,	0xae5f,
+      PIXMAN_a8r8g8b8,	0xf215b675
+    },
+    { PIXMAN_OP_DISJOINT_ATOP_REVERSE,
+      PIXMAN_a8r8g8b8,	0xce007980,
+      PIXMAN_a8r8g8b8,	0x80ffe4ad
+    },
+    { PIXMAN_OP_DISJOINT_XOR,
+      PIXMAN_a8r8g8b8,	0xb8b07bea,
+      PIXMAN_a4r4g4b4,	0x939c
+    },
+    { PIXMAN_OP_CONJOINT_ATOP_REVERSE,
+      PIXMAN_r5g6b5,	0x0063,
+      PIXMAN_a8r8g8b8,	0x10bb1ed7,
+    },
+};
+
+static void
+fill (pixman_image_t *image, uint32_t pixel)
+{
+    uint8_t *data = (uint8_t *)pixman_image_get_data (image);
+    int bytes_per_pixel = PIXMAN_FORMAT_BPP (pixman_image_get_format (image)) / 8;
+    int n_bytes = pixman_image_get_stride (image) * pixman_image_get_height (image);
+    int i;
+
+    switch (bytes_per_pixel)
+    {
+    case 4:
+	for (i = 0; i < n_bytes / 4; ++i)
+	    ((uint32_t *)data)[i] = pixel;
+	break;
+
+    case 2:
+	pixel &= 0xffff;
+	for (i = 0; i < n_bytes / 2; ++i)
+	    ((uint16_t *)data)[i] = pixel;
+	break;
+
+    case 1:
+	pixel &= 0xff;
+	for (i = 0; i < n_bytes; ++i)
+	    ((uint8_t *)data)[i] = pixel;
+	break;
+
+    default:
+	assert (0);
+	break;
+    }
+}
+
+static uint32_t
+access (pixman_image_t *image, int x, int y)
+{
+    int bytes_per_pixel;
+    int stride;
+    uint32_t result;
+    uint8_t *location;
+
+    if (x < 0 || x >= image->bits.width || y < 0 || y >= image->bits.height)
+        return 0;
+
+    bytes_per_pixel = PIXMAN_FORMAT_BPP (image->bits.format) / 8;
+    stride = image->bits.rowstride * 4;
+
+    location = (uint8_t *)image->bits.bits + y * stride + x * bytes_per_pixel;
+
+    if (bytes_per_pixel == 4)
+        result = *(uint32_t *)location;
+    else if (bytes_per_pixel == 2)
+        result = *(uint16_t *)location;
+    else if (bytes_per_pixel == 1)
+        result = *(uint8_t *)location;
+    else
+	assert (0);
+
+    return result;
+}
+
+static pixman_bool_t
+verify (int test_no, const pixel_combination_t *combination, int size)
+{
+    pixman_image_t *src, *dest;
+    pixel_checker_t src_checker, dest_checker;
+    color_t source_color, dest_color, reference_color;
+    pixman_bool_t result = TRUE;
+    int i, j;
+
+    /* Compute reference color */
+    pixel_checker_init (&src_checker, combination->src_format);
+    pixel_checker_init (&dest_checker, combination->dest_format);
+    pixel_checker_convert_pixel_to_color (
+	&src_checker, combination->src_pixel, &source_color);
+    pixel_checker_convert_pixel_to_color (
+	&dest_checker, combination->dest_pixel, &dest_color);
+    do_composite (combination->op,
+		  &source_color, NULL, &dest_color,
+		  &reference_color, FALSE);
+
+    src = pixman_image_create_bits (
+	combination->src_format, size, size, NULL, -1);
+    dest = pixman_image_create_bits (
+	combination->dest_format, size, size, NULL, -1);
+
+    fill (src, combination->src_pixel);
+    fill (dest, combination->dest_pixel);
+
+    pixman_image_composite32 (
+	combination->op, src, NULL, dest, 0, 0, 0, 0, 0, 0, size, size);
+
+    for (j = 0; j < size; ++j)
+    {
+	for (i = 0; i < size; ++i)
+	{
+	    uint32_t computed = access (dest, i, j);
+	    int32_t a, r, g, b;
+
+	    if (!pixel_checker_check (&dest_checker, computed, &reference_color))
+	    {
+		printf ("----------- Test %d failed ----------\n", test_no);
+
+		printf ("   operator:         %s\n", operator_name (combination->op));
+		printf ("   src format:       %s\n", format_name (combination->src_format));
+		printf ("   dest format:      %s\n", format_name (combination->dest_format));
+                printf (" - source ARGB:      %f  %f  %f  %f   (pixel: %8x)\n",
+                        source_color.a, source_color.r, source_color.g, source_color.b,
+                        combination->src_pixel);
+		pixel_checker_split_pixel (&src_checker, combination->src_pixel,
+					   &a, &r, &g, &b);
+                printf ("                     %8d  %8d  %8d  %8d\n", a, r, g, b);
+
+                printf (" - dest ARGB:        %f  %f  %f  %f   (pixel: %8x)\n",
+                        dest_color.a, dest_color.r, dest_color.g, dest_color.b,
+                        combination->dest_pixel);
+		pixel_checker_split_pixel (&dest_checker, combination->dest_pixel,
+					   &a, &r, &g, &b);
+                printf ("                     %8d  %8d  %8d  %8d\n", a, r, g, b);
+
+                pixel_checker_split_pixel (&dest_checker, computed, &a, &r, &g, &b);
+                printf (" - expected ARGB:    %f  %f  %f  %f\n",
+                        reference_color.a, reference_color.r, reference_color.g, reference_color.b);
+
+                pixel_checker_get_min (&dest_checker, &reference_color, &a, &r, &g, &b);
+                printf ("   min acceptable:   %8d  %8d  %8d  %8d\n", a, r, g, b);
+
+                pixel_checker_split_pixel (&dest_checker, computed, &a, &r, &g, &b);
+                printf ("   got:              %8d  %8d  %8d  %8d   (pixel: %8x)\n", a, r, g, b, computed);
+
+                pixel_checker_get_max (&dest_checker, &reference_color, &a, &r, &g, &b);
+                printf ("   max acceptable:   %8d  %8d  %8d  %8d\n", a, r, g, b);
+
+		result = FALSE;
+		goto done;
+	    }
+	}
+    }
+
+done:
+    pixman_image_unref (src);
+    pixman_image_unref (dest);
+
+    return result;
+}
+
+int
+main (int argc, char **argv)
+{
+    int result = 0;
+    int i, j;
+
+    for (i = 0; i < ARRAY_LENGTH (regressions); ++i)
+    {
+	const pixel_combination_t *combination = &(regressions[i]);
+
+	for (j = 1; j < 34; ++j)
+	{
+	    if (!verify (i, combination, j))
+	    {
+		result = 1;
+		break;
+	    }
+	}
+    }
+
+    return result;
+}
diff --git a/pixman/test/prng-test.c b/pixman/test/prng-test.c
new file mode 100644
index 000000000..c1d9320cc
--- /dev/null
+++ b/pixman/test/prng-test.c
@@ -0,0 +1,175 @@
+/*
+ * Copyright © 2012 Siarhei Siamashka <siarhei.siamashka@gmail.com>
+ *
+ * Based on the public domain implementation of small noncryptographic PRNG
+ * authored by Bob Jenkins: http://burtleburtle.net/bob/rand/smallprng.html
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <assert.h>
+#include <stdlib.h>
+#include "utils-prng.h"
+#include "utils.h"
+
+/* The original code from http://www.burtleburtle.net/bob/rand/smallprng.html */
+
+typedef uint32_t u4;
+typedef struct ranctx { u4 a; u4 b; u4 c; u4 d; } ranctx;
+
+#define rot(x,k) (((x)<<(k))|((x)>>(32-(k))))
+u4 ranval( ranctx *x ) {
+    u4 e = x->a - rot(x->b, 27);
+    x->a = x->b ^ rot(x->c, 17);
+    x->b = x->c + x->d;
+    x->c = x->d + e;
+    x->d = e + x->a;
+    return x->d;
+}
+
+void raninit( ranctx *x, u4 seed ) {
+    u4 i;
+    x->a = 0xf1ea5eed, x->b = x->c = x->d = seed;
+    for (i=0; i<20; ++i) {
+        (void)ranval(x);
+    }
+}
+
+/*****************************************************************************/
+
+#define BUFSIZE (8 * 1024 * 1024)
+#define N 50
+
+void bench (void)
+{
+    double t1, t2;
+    int i;
+    prng_t prng;
+    uint8_t *buf = aligned_malloc (16, BUFSIZE + 1);
+
+    prng_srand_r (&prng, 1234);
+    t1 = gettime();
+    for (i = 0; i < N; i++)
+        prng_randmemset_r (&prng, buf, BUFSIZE, 0);
+    t2 = gettime();
+    printf ("aligned randmemset                    : %.2f MB/s\n",
+            (double)BUFSIZE * N / 1000000. / (t2 - t1));
+
+    t1 = gettime();
+    for (i = 0; i < N; i++)
+        prng_randmemset_r (&prng, buf + 1, BUFSIZE, 0);
+    t2 = gettime();
+    printf ("unaligned randmemset                  : %.2f MB/s\n",
+            (double)BUFSIZE * N / 1000000. / (t2 - t1));
+
+    t1 = gettime();
+    for (i = 0; i < N; i++)
+    {
+        prng_randmemset_r (&prng, buf, BUFSIZE, RANDMEMSET_MORE_00_AND_FF);
+    }
+    t2 = gettime ();
+    printf ("aligned randmemset (more 00 and FF)   : %.2f MB/s\n",
+            (double)BUFSIZE * N / 1000000. / (t2 - t1));
+
+    t1 = gettime();
+    for (i = 0; i < N; i++)
+    {
+        prng_randmemset_r (&prng, buf + 1, BUFSIZE, RANDMEMSET_MORE_00_AND_FF);
+    }
+    t2 = gettime ();
+    printf ("unaligned randmemset (more 00 and FF) : %.2f MB/s\n",
+            (double)BUFSIZE * N / 1000000. / (t2 - t1));
+
+    free (buf);
+}
+
+#define SMALLBUFSIZE 100
+
+int main (int argc, char *argv[])
+{
+    const uint32_t ref_crc[RANDMEMSET_MORE_00_AND_FF + 1] =
+    {
+        0xBA06763D, 0x103FC550, 0x8B59ABA5, 0xD82A0F39,
+        0xD2321099, 0xFD8C5420, 0xD3B7C42A, 0xFC098093,
+        0x85E01DE0, 0x6680F8F7, 0x4D32DD3C, 0xAE52382B,
+        0x149E6CB5, 0x8B336987, 0x15DCB2B3, 0x8A71B781
+    };
+    uint32_t crc1, crc2;
+    uint32_t ref, seed, seed0, seed1, seed2, seed3;
+    prng_rand_128_data_t buf;
+    uint8_t *bytebuf = aligned_malloc(16, SMALLBUFSIZE + 1);
+    ranctx x;
+    prng_t prng;
+    prng_randmemset_flags_t flags;
+
+    if (argc > 1 && strcmp(argv[1], "-bench") == 0)
+    {
+        bench ();
+        return 0;
+    }
+
+    /* basic test */
+    raninit (&x, 0);
+    prng_srand_r (&prng, 0);
+    assert (ranval (&x) == prng_rand_r (&prng));
+
+    /* test for simd code */
+    seed = 0;
+    prng_srand_r (&prng, seed);
+    seed0 = (seed = seed * 1103515245 + 12345);
+    seed1 = (seed = seed * 1103515245 + 12345);
+    seed2 = (seed = seed * 1103515245 + 12345);
+    seed3 = (seed = seed * 1103515245 + 12345);
+    prng_rand_128_r (&prng, &buf);
+
+    raninit (&x, seed0);
+    ref = ranval (&x);
+    assert (ref == buf.w[0]);
+
+    raninit (&x, seed1);
+    ref = ranval (&x);
+    assert (ref == buf.w[1]);
+
+    raninit (&x, seed2);
+    ref = ranval (&x);
+    assert (ref == buf.w[2]);
+
+    raninit (&x, seed3);
+    ref = ranval (&x);
+    assert (ref == buf.w[3]);
+
+    /* test for randmemset */
+    for (flags = 0; flags <= RANDMEMSET_MORE_00_AND_FF; flags++)
+    {
+        prng_srand_r (&prng, 1234);
+        prng_randmemset_r (&prng, bytebuf, 16, flags);
+        prng_randmemset_r (&prng, bytebuf + 16, SMALLBUFSIZE - 17, flags);
+        crc1 = compute_crc32 (0, bytebuf, SMALLBUFSIZE - 1);
+        prng_srand_r (&prng, 1234);
+        prng_randmemset_r (&prng, bytebuf + 1, SMALLBUFSIZE - 1, flags);
+        crc2 = compute_crc32 (0, bytebuf + 1, SMALLBUFSIZE - 1);
+        assert (ref_crc[flags] == crc1);
+        assert (ref_crc[flags] == crc2);
+    }
+
+    free (bytebuf);
+
+    return 0;
+}
diff --git a/pixman/test/radial-perf-test.c b/pixman/test/radial-perf-test.c
new file mode 100644
index 000000000..71092e27b
--- /dev/null
+++ b/pixman/test/radial-perf-test.c
@@ -0,0 +1,58 @@
+#include "utils.h"
+#include <stdio.h>
+
+int
+main ()
+{
+    static const pixman_point_fixed_t inner = { 0x0000, 0x0000 };
+    static const pixman_point_fixed_t outer = { 0x0000, 0x0000 };
+    static const pixman_fixed_t r_inner = 0;
+    static const pixman_fixed_t r_outer = 64 << 16;
+    static const pixman_gradient_stop_t stops[] = {
+	{ 0x00000, { 0x6666, 0x6666, 0x6666, 0xffff } },
+	{ 0x10000, { 0x0000, 0x0000, 0x0000, 0xffff } }
+    };
+    static const pixman_transform_t transform = {
+	{ { 0x0,        0x26ee, 0x0}, 
+	  { 0xffffeeef, 0x0,    0x0}, 
+	  { 0x0,        0x0,    0x10000}
+	}
+    };
+    static const pixman_color_t z = { 0x0000, 0x0000, 0x0000, 0x0000 };
+    pixman_image_t *dest, *radial, *zero;
+    int i;
+    double before, after;
+
+    dest = pixman_image_create_bits (
+	PIXMAN_x8r8g8b8, 640, 429, NULL, -1);
+    zero = pixman_image_create_solid_fill (&z);
+    radial = pixman_image_create_radial_gradient (
+	&inner, &outer, r_inner, r_outer, stops, ARRAY_LENGTH (stops));
+    pixman_image_set_transform (radial, &transform);
+    pixman_image_set_repeat (radial, PIXMAN_REPEAT_PAD);
+
+#define N_COMPOSITE	500
+
+    before = gettime();
+    for (i = 0; i < N_COMPOSITE; ++i)
+    {
+	before -= gettime();
+
+	pixman_image_composite (
+	    PIXMAN_OP_SRC, zero, NULL, dest,
+	    0, 0, 0, 0, 0, 0, 640, 429);
+
+	before += gettime();
+
+	pixman_image_composite32 (
+	    PIXMAN_OP_OVER, radial, NULL, dest,
+	    - 150, -158, 0, 0, 0, 0, 640, 361);
+    }
+
+    after = gettime();
+
+    write_png (dest, "radial.png");
+
+    printf ("Average time to composite: %f\n", (after - before) / N_COMPOSITE);
+    return 0;
+}
diff --git a/pixman/test/region-contains-test.c b/pixman/test/region-contains-test.c
new file mode 100644
index 000000000..096e65179
--- /dev/null
+++ b/pixman/test/region-contains-test.c
@@ -0,0 +1,169 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include "utils.h"
+
+static void
+make_random_region (pixman_region32_t *region)
+{
+    int n_boxes;
+
+    pixman_region32_init (region);
+
+    n_boxes = prng_rand_n (64);
+    while (n_boxes--)
+    {
+	int32_t x, y;
+	uint32_t w, h;
+
+	x = (int32_t)prng_rand() >> 2;
+	y = (int32_t)prng_rand() >> 2;
+	w = prng_rand() >> 2;
+	h = prng_rand() >> 2;
+
+	pixman_region32_union_rect (region, region, x, y, w, h);
+    }
+}
+
+static void
+print_box (pixman_box32_t *box)
+{
+    printf ("    %d %d %d %d\n", box->x1, box->y1, box->x2, box->y2);
+}
+
+static int32_t
+random_coord (pixman_region32_t *region, pixman_bool_t x)
+{
+    pixman_box32_t *b, *bb;
+    int n_boxes;
+    int begin, end;
+
+    if (prng_rand_n (14))
+    {
+	bb = pixman_region32_rectangles (region, &n_boxes);
+	if (n_boxes == 0)
+	    goto use_extent;
+	b = bb + prng_rand_n (n_boxes);
+    }
+    else
+    {
+    use_extent:
+	b = pixman_region32_extents (region);
+	n_boxes = 1;
+    }
+
+    if (x)
+    {
+	begin = b->x1;
+	end = b->x2;
+    }
+    else
+    {
+	begin = b->y1;
+	end = b->y2;
+    }
+
+    switch (prng_rand_n (5))
+    {
+    case 0:
+	return begin - prng_rand();
+    case 1:
+	return end + prng_rand ();
+    case 2:
+	return end;
+    case 3:
+	return begin;
+    default:
+	return (end - begin) / 2 + begin;
+    }
+    return 0;
+}
+
+static uint32_t
+compute_crc32_u32 (uint32_t crc32, uint32_t v)
+{
+    if (!is_little_endian())
+    {
+	v = ((v & 0xff000000) >> 24)	|
+	    ((v & 0x00ff0000) >> 8)	|
+	    ((v & 0x0000ff00) << 8)	|
+	    ((v & 0x000000ff) << 24);
+    }
+
+    return compute_crc32 (crc32, &v, sizeof (int32_t));
+}
+
+static uint32_t
+crc32_box32 (uint32_t crc32, pixman_box32_t *box)
+{
+    crc32 = compute_crc32_u32 (crc32, box->x1);
+    crc32 = compute_crc32_u32 (crc32, box->y1);
+    crc32 = compute_crc32_u32 (crc32, box->x2);
+    crc32 = compute_crc32_u32 (crc32, box->y2);
+
+    return crc32;
+}
+
+static uint32_t
+test_region_contains_rectangle (int i, int verbose)
+{
+    pixman_box32_t box;
+    pixman_box32_t rbox = { 0, 0, 0, 0 };
+    pixman_region32_t region;
+    uint32_t r, r1, r2, r3, r4, crc32;
+
+    prng_srand (i);
+
+    make_random_region (&region);
+
+    box.x1 = random_coord (&region, TRUE);
+    box.x2 = box.x1 + prng_rand ();
+    box.y1 = random_coord (&region, FALSE);
+    box.y2 = box.y1 + prng_rand ();
+
+    if (verbose)
+    {
+	int n_rects;
+	pixman_box32_t *boxes;
+
+	boxes = pixman_region32_rectangles (&region, &n_rects);
+
+	printf ("region:\n");
+	while (n_rects--)
+	    print_box (boxes++);
+	printf ("box:\n");
+	print_box (&box);
+    }
+
+    crc32 = 0;
+
+    r1 = pixman_region32_contains_point (&region, box.x1, box.y1, &rbox);
+    crc32 = crc32_box32 (crc32, &rbox);
+    r2 = pixman_region32_contains_point (&region, box.x1, box.y2, &rbox);
+    crc32 = crc32_box32 (crc32, &rbox);
+    r3 = pixman_region32_contains_point (&region, box.x2, box.y1, &rbox);
+    crc32 = crc32_box32 (crc32, &rbox);
+    r4 = pixman_region32_contains_point (&region, box.x2, box.y2, &rbox);
+    crc32 = crc32_box32 (crc32, &rbox);
+
+    r = pixman_region32_contains_rectangle (&region, &box);
+    r = (i << 8) | (r << 4) | (r1 << 3) | (r2 << 2) | (r3 << 1) | (r4 << 0);
+
+    crc32 = compute_crc32_u32 (crc32, r);
+
+    if (verbose)
+	printf ("results: %d %d %d %d %d\n", (r & 0xf0) >> 4, r1, r2, r3, r4);
+
+    pixman_region32_fini (&region);
+
+    return crc32;
+}
+
+int
+main (int argc, const char *argv[])
+{
+    return fuzzer_test_main ("region_contains",
+			     1000000,
+			     0x548E0F3F,
+			     test_region_contains_rectangle,
+			     argc, argv);
+}
diff --git a/pixman/test/region-test.c b/pixman/test/region-test.c
index 9d5a41eb9..bfc219bc7 100644
--- a/pixman/test/region-test.c
+++ b/pixman/test/region-test.c
@@ -32,6 +32,8 @@ main ()
 	0xffff
     };
 
+    prng_srand (0);
+
     /* This used to go into an infinite loop before pixman-region.c
      * was fixed to not use explict "short" variables
      */
@@ -91,10 +93,10 @@ main ()
 	/* Add some random rectangles */
 	for (j = 0; j < 64; j++)
 	    pixman_region32_union_rect (&r1, &r1,
-					lcg_rand_n (image_size),
-					lcg_rand_n (image_size),
-					lcg_rand_n (25),
-					lcg_rand_n (25));
+					prng_rand_n (image_size),
+					prng_rand_n (image_size),
+					prng_rand_n (25),
+					prng_rand_n (25));
 
 	/* Clip to image size */
 	pixman_region32_init_rect (&r2, 0, 0, image_size, image_size);
diff --git a/pixman/test/region-translate-test.c b/pixman/test/region-translate-test.c
new file mode 100644
index 000000000..5a03027e8
--- /dev/null
+++ b/pixman/test/region-translate-test.c
@@ -0,0 +1,30 @@
+#include <assert.h>
+#include "utils.h"
+
+/* Pixman had a bug where 32bit regions where clipped to 16bit sizes when
+ * pixman_region32_translate() was called. This test exercises that bug.
+ */
+
+#define LARGE 32000
+
+int
+main (int argc, char **argv)
+{
+  pixman_box32_t rect = { -LARGE, -LARGE, LARGE, LARGE };
+  pixman_region32_t r1, r2;
+
+  pixman_region32_init_rects (&r1, &rect, 1);
+  pixman_region32_init_rect (&r2, rect.x1, rect.y1, rect.x2 - rect.x1, rect.y2 - rect.y1);
+
+  assert (pixman_region32_equal (&r1,  &r2));
+
+  pixman_region32_translate (&r1, -LARGE, LARGE);
+  pixman_region32_translate (&r1, LARGE, -LARGE);
+
+  assert (pixman_region32_equal (&r1,  &r2));
+
+  pixman_region32_fini (&r1);
+  pixman_region32_fini (&r2);
+
+  return 0;
+}
diff --git a/pixman/test/rotate-test.c b/pixman/test/rotate-test.c
new file mode 100644
index 000000000..18ca60d9b
--- /dev/null
+++ b/pixman/test/rotate-test.c
@@ -0,0 +1,120 @@
+#include <stdlib.h>
+#include "utils.h"
+
+#define WIDTH	32
+#define HEIGHT	32
+
+static const pixman_format_code_t formats[] =
+{
+    PIXMAN_a8r8g8b8,
+    PIXMAN_a8b8g8r8,
+    PIXMAN_x8r8g8b8,
+    PIXMAN_x8b8g8r8,
+    PIXMAN_r5g6b5,
+    PIXMAN_b5g6r5,
+    PIXMAN_a8,
+    PIXMAN_a1,
+};
+
+static const pixman_op_t ops[] =
+{
+    PIXMAN_OP_OVER,
+    PIXMAN_OP_SRC,
+    PIXMAN_OP_ADD,
+};
+
+#define TRANSFORM(v00, v01, v10, v11)					\
+    { { { v00, v01, WIDTH * pixman_fixed_1 / 2 },			\
+        { v10, v11, HEIGHT * pixman_fixed_1 / 2 },			\
+	{ 0, 0, pixman_fixed_1 } } }
+
+#define F1 pixman_fixed_1
+
+static const pixman_transform_t transforms[] =
+{
+    TRANSFORM (0, -1, 1, 0),		/* wrong 90 degree rotation */
+    TRANSFORM (0, 1, -1, 0),		/* wrong 270 degree rotation */
+    TRANSFORM (1, 0, 0, 1),		/* wrong identity */
+    TRANSFORM (-1, 0, 0, -1),		/* wrong 180 degree rotation */
+    TRANSFORM (0, -F1, F1, 0),		/* correct 90 degree rotation */
+    TRANSFORM (0, F1, -F1, 0),		/* correct 270 degree rotation */
+    TRANSFORM (F1, 0, 0, F1),		/* correct identity */
+    TRANSFORM (-F1, 0, 0, -F1),		/* correct 180 degree rotation */
+};
+
+#define RANDOM_FORMAT()							\
+    (formats[prng_rand_n (ARRAY_LENGTH (formats))])
+
+#define RANDOM_OP()							\
+    (ops[prng_rand_n (ARRAY_LENGTH (ops))])
+
+#define RANDOM_TRANSFORM()						\
+    (&(transforms[prng_rand_n (ARRAY_LENGTH (transforms))]))
+
+static void
+on_destroy (pixman_image_t *image, void *data)
+{
+    free (data);
+}
+
+static pixman_image_t *
+make_image (void)
+{
+    pixman_format_code_t format = RANDOM_FORMAT();
+    uint32_t *bytes, *orig;
+    pixman_image_t *image;
+    int stride;
+
+    orig = bytes = malloc (WIDTH * HEIGHT * 4);
+    prng_randmemset (bytes, WIDTH * HEIGHT * 4, 0);
+
+    stride = WIDTH * 4;
+    if (prng_rand_n (2) == 0)
+    {
+	bytes += (stride / 4) * (HEIGHT - 1);
+	stride = - stride;
+    }
+
+    image = pixman_image_create_bits (
+	format, WIDTH, HEIGHT, bytes, stride);
+
+    pixman_image_set_transform (image, RANDOM_TRANSFORM());
+    pixman_image_set_destroy_function (image, on_destroy, orig);
+    pixman_image_set_repeat (image, PIXMAN_REPEAT_NORMAL);
+
+    image_endian_swap (image);
+
+    return image;
+}
+
+static uint32_t
+test_transform (int testnum, int verbose)
+{
+    pixman_image_t *src, *dest;
+    uint32_t crc;
+
+    prng_srand (testnum);
+    
+    src = make_image ();
+    dest = make_image ();
+
+    pixman_image_composite (RANDOM_OP(),
+			    src, NULL, dest,
+			    0, 0, 0, 0, WIDTH / 2, HEIGHT / 2,
+			    WIDTH, HEIGHT);
+
+    crc = compute_crc32_for_image (0, dest);
+
+    pixman_image_unref (src);
+    pixman_image_unref (dest);
+
+    return crc;
+}
+
+int
+main (int argc, const char *argv[])
+{
+    return fuzzer_test_main ("rotate", 15000,
+			     0x81E9EC2F,
+			     test_transform, argc, argv);
+}
diff --git a/pixman/test/scaling-bench.c b/pixman/test/scaling-bench.c
new file mode 100644
index 000000000..365e79850
--- /dev/null
+++ b/pixman/test/scaling-bench.c
@@ -0,0 +1,80 @@
+#include <stdlib.h>
+#include "utils.h"
+
+#define SOURCE_WIDTH 320
+#define SOURCE_HEIGHT 240
+#define TEST_REPEATS 3
+
+static pixman_image_t *
+make_source (void)
+{
+    size_t n_bytes = (SOURCE_WIDTH + 2) * (SOURCE_HEIGHT + 2) * 4;
+    uint32_t *data = malloc (n_bytes);
+    pixman_image_t *source;
+
+    prng_randmemset (data, n_bytes, 0);
+    
+    source = pixman_image_create_bits (
+	PIXMAN_a8r8g8b8, SOURCE_WIDTH + 2, SOURCE_HEIGHT + 2,
+	data,
+	(SOURCE_WIDTH + 2) * 4);
+
+    pixman_image_set_filter (source, PIXMAN_FILTER_BILINEAR, NULL, 0);
+
+    return source;
+}
+
+int
+main ()
+{
+    double scale;
+    pixman_image_t *src;
+
+    prng_srand (23874);
+    
+    src = make_source ();
+    printf ("# %-6s %-22s   %-14s %-12s\n",
+	    "ratio",
+	    "resolutions",
+	    "time / ms",
+	    "time per pixel / ns");
+    for (scale = 0.1; scale < 10.005; scale += 0.01)
+    {
+	int i;
+	int dest_width = SOURCE_WIDTH * scale + 0.5;
+	int dest_height = SOURCE_HEIGHT * scale + 0.5;
+	int dest_byte_stride = (dest_width * 4 + 15) & ~15;
+	pixman_fixed_t s = (1 / scale) * 65536.0 + 0.5;
+	pixman_transform_t transform;
+	pixman_image_t *dest;
+	double t1, t2, t = -1;
+	uint32_t *dest_buf = aligned_malloc (16, dest_byte_stride * dest_height);
+	memset (dest_buf, 0, dest_byte_stride * dest_height);
+
+	pixman_transform_init_scale (&transform, s, s);
+	pixman_image_set_transform (src, &transform);
+	
+	dest = pixman_image_create_bits (
+	    PIXMAN_a8r8g8b8, dest_width, dest_height, dest_buf, dest_byte_stride);
+
+	for (i = 0; i < TEST_REPEATS; i++)
+	{
+	    t1 = gettime();
+	    pixman_image_composite (
+		PIXMAN_OP_OVER, src, NULL, dest,
+		scale, scale, 0, 0, 0, 0, dest_width, dest_height);
+	    t2 = gettime();
+	    if (t < 0 || t2 - t1 < t)
+		t = t2 - t1;
+	}
+
+	printf ("%6.2f : %4dx%-4d => %4dx%-4d : %12.4f : %12.4f\n",
+		scale, SOURCE_WIDTH, SOURCE_HEIGHT, dest_width, dest_height,
+		t * 1000, (t / (dest_width * dest_height)) * 1000000000);
+
+	pixman_image_unref (dest);
+	free (dest_buf);
+    }
+
+    return 0;
+}
diff --git a/pixman/test/scaling-crash-test.c b/pixman/test/scaling-crash-test.c
new file mode 100644
index 000000000..0dac892b5
--- /dev/null
+++ b/pixman/test/scaling-crash-test.c
@@ -0,0 +1,219 @@
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include "utils.h"
+
+/*
+ * We have a source image filled with solid color, set NORMAL or PAD repeat,
+ * and some transform which results in nearest neighbour scaling.
+ *
+ * The expected result is either that the destination image filled with this solid
+ * color or, if the transformation is such that we can't composite anything at
+ * all, that nothing has changed in the destination.
+ *
+ * The surrounding memory of the source image is a different solid color so that
+ * we are sure to get failures if we access it.
+ */
+static int
+run_test (int32_t		dst_width,
+	  int32_t		dst_height,
+	  int32_t		src_width,
+	  int32_t		src_height,
+	  int32_t		src_x,
+	  int32_t		src_y,
+	  int32_t		scale_x,
+	  int32_t		scale_y,
+	  pixman_filter_t	filter,
+	  pixman_repeat_t	repeat)
+{
+    pixman_image_t *   src_img;
+    pixman_image_t *   dst_img;
+    pixman_transform_t transform;
+    uint32_t *         srcbuf;
+    uint32_t *         dstbuf;
+    pixman_color_t     color_cc = { 0xcccc, 0xcccc, 0xcccc, 0xcccc };
+    pixman_image_t *   solid;
+    int result;
+    int i;
+
+    static const pixman_fixed_t kernel[] =
+    {
+#define D(f)	(pixman_double_to_fixed (f) + 0x0001)
+
+	pixman_int_to_fixed (5),
+	pixman_int_to_fixed (5),
+	D(1/25.0), D(1/25.0), D(1/25.0), D(1/25.0), D(1/25.0),
+	D(1/25.0), D(1/25.0), D(1/25.0), D(1/25.0), D(1/25.0),
+	D(1/25.0), D(1/25.0), D(1/25.0), D(1/25.0), D(1/25.0),
+	D(1/25.0), D(1/25.0), D(1/25.0), D(1/25.0), D(1/25.0),
+	D(1/25.0), D(1/25.0), D(1/25.0), D(1/25.0), D(1/25.0)
+    };
+
+    result = 0;
+
+    srcbuf = (uint32_t *)malloc ((src_width + 10) * (src_height + 10) * 4);
+    dstbuf = (uint32_t *)malloc (dst_width * dst_height * 4);
+
+    memset (srcbuf, 0x88, src_width * src_height * 4);
+    memset (dstbuf, 0x33, dst_width * dst_height * 4);
+
+    src_img = pixman_image_create_bits (
+        PIXMAN_a8r8g8b8, src_width, src_height,
+	srcbuf + (src_width + 10) * 5 + 5, (src_width + 10) * 4);
+
+    solid = pixman_image_create_solid_fill (&color_cc);
+    pixman_image_composite32 (PIXMAN_OP_SRC, solid, NULL, src_img,
+			      0, 0, 0, 0, 0, 0, src_width, src_height);
+    pixman_image_unref (solid);
+
+    dst_img = pixman_image_create_bits (
+        PIXMAN_a8r8g8b8, dst_width, dst_height, dstbuf, dst_width * 4);
+
+    pixman_transform_init_scale (&transform, scale_x, scale_y);
+    pixman_image_set_transform (src_img, &transform);
+    pixman_image_set_repeat (src_img, repeat);
+    if (filter == PIXMAN_FILTER_CONVOLUTION)
+	pixman_image_set_filter (src_img, filter, kernel, 27);
+    else
+	pixman_image_set_filter (src_img, filter, NULL, 0);
+
+    pixman_image_composite (PIXMAN_OP_SRC, src_img, NULL, dst_img,
+                            src_x, src_y, 0, 0, 0, 0, dst_width, dst_height);
+
+    pixman_image_unref (src_img);
+    pixman_image_unref (dst_img);
+
+    for (i = 0; i < dst_width * dst_height; i++)
+    {
+	if (dstbuf[i] != 0xCCCCCCCC && dstbuf[i] != 0x33333333)
+	{
+	    result = 1;
+	    break;
+	}
+    }
+
+    free (srcbuf);
+    free (dstbuf);
+    return result;
+}
+
+typedef struct filter_info_t filter_info_t;
+struct filter_info_t
+{
+    pixman_filter_t value;
+    char name[28];
+};
+
+static const filter_info_t filters[] =
+{
+    { PIXMAN_FILTER_NEAREST, "NEAREST" },
+    { PIXMAN_FILTER_BILINEAR, "BILINEAR" },
+    { PIXMAN_FILTER_CONVOLUTION, "CONVOLUTION" },
+};
+
+typedef struct repeat_info_t repeat_info_t;
+struct repeat_info_t
+{
+    pixman_repeat_t value;
+    char name[28];
+};
+
+
+static const repeat_info_t repeats[] =
+{
+    { PIXMAN_REPEAT_PAD, "PAD" },
+    { PIXMAN_REPEAT_REFLECT, "REFLECT" },
+    { PIXMAN_REPEAT_NORMAL, "NORMAL" }
+};
+
+static int
+do_test (int32_t		dst_size,
+	 int32_t		src_size,
+	 int32_t		src_offs,
+	 int32_t		scale_factor)
+{
+    int i, j;
+
+    for (i = 0; i < ARRAY_LENGTH (filters); ++i)
+    {
+	for (j = 0; j < ARRAY_LENGTH (repeats); ++j)
+	{
+	    /* horizontal test */
+	    if (run_test (dst_size, 1,
+			  src_size, 1,
+			  src_offs, 0,
+			  scale_factor, 65536,
+			  filters[i].value,
+			  repeats[j].value) != 0)
+	    {
+		printf ("Vertical test failed with %s filter and repeat mode %s\n",
+			filters[i].name, repeats[j].name);
+
+		return 1;
+	    }
+
+	    /* vertical test */
+	    if (run_test (1, dst_size,
+			  1, src_size,
+			  0, src_offs,
+			  65536, scale_factor,
+			  filters[i].value,
+			  repeats[j].value) != 0)
+	    {
+		printf ("Vertical test failed with %s filter and repeat mode %s\n",
+			filters[i].name, repeats[j].name);
+
+		return 1;
+	    }
+	}
+    }
+
+    return 0;
+}
+
+int
+main (int argc, char *argv[])
+{
+    int i;
+
+    pixman_disable_out_of_bounds_workaround ();
+
+    /* can potentially crash */
+    assert (do_test (
+		48000, 32767, 1, 65536 * 128) == 0);
+
+    /* can potentially get into a deadloop */
+    assert (do_test (
+		16384, 65536, 32, 32768) == 0);
+
+    /* can potentially access memory outside source image buffer */
+    assert (do_test (
+		10, 10, 0, 1) == 0);
+    assert (do_test (
+		10, 10, 0, 0) == 0);
+
+    for (i = 0; i < 100; ++i)
+    {
+	pixman_fixed_t one_seventh =
+	    (((pixman_fixed_48_16_t)pixman_fixed_1) << 16) / (7 << 16);
+
+	assert (do_test (
+		    1, 7, 3, one_seventh + i - 50) == 0);
+    }
+
+    for (i = 0; i < 100; ++i)
+    {
+	pixman_fixed_t scale =
+	    (((pixman_fixed_48_16_t)pixman_fixed_1) << 16) / (32767 << 16);
+
+	assert (do_test (
+		    1, 32767, 16383, scale + i - 50) == 0);
+    }
+
+    /* can potentially provide invalid results (out of range matrix stuff) */
+    assert (do_test (
+	48000, 32767, 16384, 65536 * 128) == 0);
+
+    return 0;
+}
diff --git a/pixman/test/scaling-helpers-test.c b/pixman/test/scaling-helpers-test.c
new file mode 100644
index 000000000..cd5ace0b2
--- /dev/null
+++ b/pixman/test/scaling-helpers-test.c
@@ -0,0 +1,92 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include "utils.h"
+#include "pixman-inlines.h"
+
+/* A trivial reference implementation for
+ * 'bilinear_pad_repeat_get_scanline_bounds'
+ */
+static void
+bilinear_pad_repeat_get_scanline_bounds_ref (int32_t        source_image_width,
+					     pixman_fixed_t vx_,
+					     pixman_fixed_t unit_x,
+					     int32_t *      left_pad,
+					     int32_t *      left_tz,
+					     int32_t *      width,
+					     int32_t *      right_tz,
+					     int32_t *      right_pad)
+{
+    int w = *width;
+    int64_t vx = vx_;
+    *left_pad = 0;
+    *left_tz = 0;
+    *width = 0;
+    *right_tz = 0;
+    *right_pad = 0;
+    while (--w >= 0)
+    {
+	if (vx < 0)
+	{
+	    if (vx + pixman_fixed_1 < 0)
+		*left_pad += 1;
+	    else
+		*left_tz += 1;
+	}
+	else if (vx + pixman_fixed_1 >= pixman_int_to_fixed (source_image_width))
+	{
+	    if (vx >= pixman_int_to_fixed (source_image_width))
+		*right_pad += 1;
+	    else
+		*right_tz += 1;
+	}
+	else
+	{
+	    *width += 1;
+	}
+	vx += unit_x;
+    }
+}
+
+int
+main (void)
+{
+    int i;
+    prng_srand (0);
+    for (i = 0; i < 10000; i++)
+    {
+	int32_t left_pad1, left_tz1, width1, right_tz1, right_pad1;
+	int32_t left_pad2, left_tz2, width2, right_tz2, right_pad2;
+	pixman_fixed_t vx = prng_rand_n(10000 << 16) - (3000 << 16);
+	int32_t width = prng_rand_n(10000);
+	int32_t source_image_width = prng_rand_n(10000) + 1;
+	pixman_fixed_t unit_x = prng_rand_n(10 << 16) + 1;
+	width1 = width2 = width;
+
+	bilinear_pad_repeat_get_scanline_bounds_ref (source_image_width,
+						     vx,
+						     unit_x,
+						     &left_pad1,
+						     &left_tz1,
+						     &width1,
+						     &right_tz1,
+						     &right_pad1);
+
+	bilinear_pad_repeat_get_scanline_bounds (source_image_width,
+						 vx,
+						 unit_x,
+						 &left_pad2,
+						 &left_tz2,
+						 &width2,
+						 &right_tz2,
+						 &right_pad2);
+
+	assert (left_pad1 == left_pad2);
+	assert (left_tz1 == left_tz2);
+	assert (width1 == width2);
+	assert (right_tz1 == right_tz2);
+	assert (right_pad1 == right_pad2);
+    }
+
+    return 0;
+}
diff --git a/pixman/test/scaling-test-bisect.rb b/pixman/test/scaling-test-bisect.rb
deleted file mode 100644
index ab3426244..000000000
--- a/pixman/test/scaling-test-bisect.rb
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env ruby
-
-if not ARGV[0] or not ARGV[1] then
-    printf("Please provide two 'scaling-test' static binaries in the command line.\n\n")
-    printf("The first should be linked with the correct reference pixman library.\n")
-    printf("The second binrary should be linked with the pixman library to be tested.\n")
-    exit(0)
-end
-
-$MAX = 3000000
-$MIN = 1
-$AVG = 0
-
-if `#{ARGV[0]} #{$MAX} 2>/dev/null` == `#{ARGV[1]} #{$MAX} 2>/dev/null` then
-    printf("test ok\n")
-    exit(0)
-end
-
-printf("test failed, bisecting...\n")
-
-while $MAX != $MIN + 1 do
-    $AVG = (($MIN + $MAX) / 2).to_i
-    res1 = `#{ARGV[0]} #{$AVG} 2>/dev/null`
-    res2 = `#{ARGV[1]} #{$AVG} 2>/dev/null`
-    if res1 != res2 then
-        $MAX = $AVG
-    else
-        $MIN = $AVG
-    end
-end
-
-printf("-- ref --\n")
-printf("%s\n", `#{ARGV[0]} -#{$MAX}`)
-printf("-- new --\n")
-printf("%s\n", `#{ARGV[1]} -#{$MAX}`)
-
-printf("\nFailed test number is %d, you can reproduce the problematic conditions\n", $MAX)
-printf("by running 'scaling-test -%d'\n", $MAX)
diff --git a/pixman/test/scaling-test.c b/pixman/test/scaling-test.c
index 29772906d..e2f7fa9f4 100644
--- a/pixman/test/scaling-test.c
+++ b/pixman/test/scaling-test.c
@@ -1,124 +1,228 @@
 /*
- * Test program, which can detect problems with nearest neighbout scaling
- * implementation. Also SRC and OVER opetations tested for 16bpp and 32bpp
- * images.
+ * Test program, which can detect some problems with nearest neighbour
+ * and bilinear scaling in pixman. Testing is done by running lots
+ * of random SRC and OVER compositing operations a8r8g8b8, x8a8r8g8b8
+ * and r5g6b5 color formats.
  *
- * Just run it without any command line arguments, and it will report either
- *   "scaling test passed" - everything is ok
- *   "scaling test failed!" - there is some problem
- *
- * In the case of failure, finding the problem involves the following steps:
- * 1. Get the reference 'scaling-test' binary. It makes sense to disable all
- *    the cpu specific optimizations in pixman and also configure it with
- *    '--disable-shared' option. Those who are paranoid can also tweak the
- *    sources to disable all fastpath functions. The resulting binary
- *    can be renamed to something like 'scaling-test.ref'.
- * 2. Compile the buggy binary (also with the '--disable-shared' option).
- * 3. Run 'ruby scaling-test-bisect.rb ./scaling-test.ref ./scaling-test'
- * 4. Look at the information about failed case (destination buffer content
- *    will be shown) and try to figure out what is wrong. It is possible
- *    to use debugging print to stderr in pixman to get more information,
- *    this does not interfere with the testing script.
+ * Script 'fuzzer-find-diff.pl' can be used to narrow down the problem in
+ * the case of test failure.
  */
-#include <assert.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include "utils.h"
 
-#define MAX_SRC_WIDTH  10
-#define MAX_SRC_HEIGHT 10
-#define MAX_DST_WIDTH  10
-#define MAX_DST_HEIGHT 10
+#define MAX_SRC_WIDTH  48
+#define MAX_SRC_HEIGHT 8
+#define MAX_DST_WIDTH  48
+#define MAX_DST_HEIGHT 8
 #define MAX_STRIDE     4
 
 /*
  * Composite operation with pseudorandom images
  */
+
+static pixman_format_code_t
+get_format (int bpp)
+{
+    if (bpp == 4)
+    {
+	switch (prng_rand_n (4))
+	{
+	default:
+	case 0:
+	    return PIXMAN_a8r8g8b8;
+	case 1:
+	    return PIXMAN_x8r8g8b8;
+	case 2:
+	    return PIXMAN_a8b8g8r8;
+	case 3:
+	    return PIXMAN_x8b8g8r8;
+	}
+    }
+    else
+    {
+	return PIXMAN_r5g6b5;
+    }
+}
+
 uint32_t
-test_composite (uint32_t initcrc,
-		int      testnum,
+test_composite (int      testnum,
 		int      verbose)
 {
     int                i;
     pixman_image_t *   src_img;
+    pixman_image_t *   mask_img;
     pixman_image_t *   dst_img;
     pixman_transform_t transform;
     pixman_region16_t  clip;
     int                src_width, src_height;
+    int                mask_width, mask_height;
     int                dst_width, dst_height;
-    int                src_stride, dst_stride;
+    int                src_stride, mask_stride, dst_stride;
     int                src_x, src_y;
+    int                mask_x, mask_y;
     int                dst_x, dst_y;
     int                src_bpp;
+    int                mask_bpp = 1;
     int                dst_bpp;
     int                w, h;
-    int                scale_x = 32768, scale_y = 32768;
-    int                op;
-    int                repeat = 0;
-    int                src_fmt, dst_fmt;
+    pixman_fixed_t     scale_x = 65536, scale_y = 65536;
+    pixman_fixed_t     translate_x = 0, translate_y = 0;
+    pixman_fixed_t     mask_scale_x = 65536, mask_scale_y = 65536;
+    pixman_fixed_t     mask_translate_x = 0, mask_translate_y = 0;
+    pixman_op_t        op;
+    pixman_repeat_t    repeat = PIXMAN_REPEAT_NONE;
+    pixman_repeat_t    mask_repeat = PIXMAN_REPEAT_NONE;
+    pixman_format_code_t src_fmt, dst_fmt;
     uint32_t *         srcbuf;
     uint32_t *         dstbuf;
+    uint32_t *         maskbuf;
     uint32_t           crc32;
+    FLOAT_REGS_CORRUPTION_DETECTOR_START ();
+
+    prng_srand (testnum);
+
+    src_bpp = (prng_rand_n (2) == 0) ? 2 : 4;
+    dst_bpp = (prng_rand_n (2) == 0) ? 2 : 4;
+    switch (prng_rand_n (3))
+    {
+    case 0:
+	op = PIXMAN_OP_SRC;
+	break;
+    case 1:
+	op = PIXMAN_OP_OVER;
+	break;
+    default:
+	op = PIXMAN_OP_ADD;
+	break;
+    }
 
-    lcg_srand (testnum);
+    src_width = prng_rand_n (MAX_SRC_WIDTH) + 1;
+    src_height = prng_rand_n (MAX_SRC_HEIGHT) + 1;
 
-    src_bpp = (lcg_rand_n (2) == 0) ? 2 : 4;
-    dst_bpp = (lcg_rand_n (2) == 0) ? 2 : 4;
-    op = (lcg_rand_n (2) == 0) ? PIXMAN_OP_SRC : PIXMAN_OP_OVER;
+    if (prng_rand_n (2))
+    {
+	mask_width = prng_rand_n (MAX_SRC_WIDTH) + 1;
+	mask_height = prng_rand_n (MAX_SRC_HEIGHT) + 1;
+    }
+    else
+    {
+	mask_width = mask_height = 1;
+    }
 
-    src_width = lcg_rand_n (MAX_SRC_WIDTH) + 1;
-    src_height = lcg_rand_n (MAX_SRC_HEIGHT) + 1;
-    dst_width = lcg_rand_n (MAX_DST_WIDTH) + 1;
-    dst_height = lcg_rand_n (MAX_DST_HEIGHT) + 1;
-    src_stride = src_width * src_bpp + lcg_rand_n (MAX_STRIDE) * src_bpp;
-    dst_stride = dst_width * dst_bpp + lcg_rand_n (MAX_STRIDE) * dst_bpp;
+    dst_width = prng_rand_n (MAX_DST_WIDTH) + 1;
+    dst_height = prng_rand_n (MAX_DST_HEIGHT) + 1;
+    src_stride = src_width * src_bpp + prng_rand_n (MAX_STRIDE) * src_bpp;
+    mask_stride = mask_width * mask_bpp + prng_rand_n (MAX_STRIDE) * mask_bpp;
+    dst_stride = dst_width * dst_bpp + prng_rand_n (MAX_STRIDE) * dst_bpp;
 
     if (src_stride & 3)
 	src_stride += 2;
 
+    if (mask_stride & 1)
+	mask_stride += 1;
+    if (mask_stride & 2)
+	mask_stride += 2;
+
     if (dst_stride & 3)
 	dst_stride += 2;
 
-    src_x = -(src_width / 4) + lcg_rand_n (src_width * 3 / 2);
-    src_y = -(src_height / 4) + lcg_rand_n (src_height * 3 / 2);
-    dst_x = -(dst_width / 4) + lcg_rand_n (dst_width * 3 / 2);
-    dst_y = -(dst_height / 4) + lcg_rand_n (dst_height * 3 / 2);
-    w = lcg_rand_n (dst_width * 3 / 2 - dst_x);
-    h = lcg_rand_n (dst_height * 3 / 2 - dst_y);
+    src_x = -(src_width / 4) + prng_rand_n (src_width * 3 / 2);
+    src_y = -(src_height / 4) + prng_rand_n (src_height * 3 / 2);
+    mask_x = -(mask_width / 4) + prng_rand_n (mask_width * 3 / 2);
+    mask_y = -(mask_height / 4) + prng_rand_n (mask_height * 3 / 2);
+    dst_x = -(dst_width / 4) + prng_rand_n (dst_width * 3 / 2);
+    dst_y = -(dst_height / 4) + prng_rand_n (dst_height * 3 / 2);
+    w = prng_rand_n (dst_width * 3 / 2 - dst_x);
+    h = prng_rand_n (dst_height * 3 / 2 - dst_y);
 
     srcbuf = (uint32_t *)malloc (src_stride * src_height);
+    maskbuf = (uint32_t *)malloc (mask_stride * mask_height);
     dstbuf = (uint32_t *)malloc (dst_stride * dst_height);
 
-    for (i = 0; i < src_stride * src_height; i++)
-	*((uint8_t *)srcbuf + i) = lcg_rand_n (256);
+    prng_randmemset (srcbuf, src_stride * src_height, 0);
+    prng_randmemset (maskbuf, mask_stride * mask_height, 0);
+    prng_randmemset (dstbuf, dst_stride * dst_height, 0);
 
-    for (i = 0; i < dst_stride * dst_height; i++)
-	*((uint8_t *)dstbuf + i) = lcg_rand_n (256);
+    src_fmt = get_format (src_bpp);
+    dst_fmt = get_format (dst_bpp);
 
-    src_fmt = src_bpp == 4 ? (lcg_rand_n (2) == 0 ?
-                              PIXMAN_a8r8g8b8 : PIXMAN_x8r8g8b8) : PIXMAN_r5g6b5;
+    if (prng_rand_n (2))
+    {
+	srcbuf += (src_stride / 4) * (src_height - 1);
+	src_stride = - src_stride;
+    }
 
-    dst_fmt = dst_bpp == 4 ? (lcg_rand_n (2) == 0 ?
-                              PIXMAN_a8r8g8b8 : PIXMAN_x8r8g8b8) : PIXMAN_r5g6b5;
+    if (prng_rand_n (2))
+    {
+	maskbuf += (mask_stride / 4) * (mask_height - 1);
+	mask_stride = - mask_stride;
+    }
+
+    if (prng_rand_n (2))
+    {
+	dstbuf += (dst_stride / 4) * (dst_height - 1);
+	dst_stride = - dst_stride;
+    }
 
     src_img = pixman_image_create_bits (
         src_fmt, src_width, src_height, srcbuf, src_stride);
 
+    mask_img = pixman_image_create_bits (
+        PIXMAN_a8, mask_width, mask_height, maskbuf, mask_stride);
+
     dst_img = pixman_image_create_bits (
         dst_fmt, dst_width, dst_height, dstbuf, dst_stride);
 
-    image_endian_swap (src_img, src_bpp * 8);
-    image_endian_swap (dst_img, dst_bpp * 8);
+    image_endian_swap (src_img);
+    image_endian_swap (dst_img);
 
-    if (lcg_rand_n (8) > 0)
+    if (prng_rand_n (4) > 0)
     {
-	scale_x = 32768 + lcg_rand_n (65536);
-	scale_y = 32768 + lcg_rand_n (65536);
+	scale_x = -32768 * 3 + prng_rand_n (65536 * 5);
+	scale_y = -32768 * 3 + prng_rand_n (65536 * 5);
+	translate_x = prng_rand_n (65536);
+	translate_y = prng_rand_n (65536);
 	pixman_transform_init_scale (&transform, scale_x, scale_y);
+	pixman_transform_translate (&transform, NULL, translate_x, translate_y);
 	pixman_image_set_transform (src_img, &transform);
     }
 
-    switch (lcg_rand_n (4))
+    if (prng_rand_n (2) > 0)
+    {
+	mask_scale_x = -32768 * 3 + prng_rand_n (65536 * 5);
+	mask_scale_y = -32768 * 3 + prng_rand_n (65536 * 5);
+	mask_translate_x = prng_rand_n (65536);
+	mask_translate_y = prng_rand_n (65536);
+	pixman_transform_init_scale (&transform, mask_scale_x, mask_scale_y);
+	pixman_transform_translate (&transform, NULL, mask_translate_x, mask_translate_y);
+	pixman_image_set_transform (mask_img, &transform);
+    }
+
+    switch (prng_rand_n (4))
+    {
+    case 0:
+	mask_repeat = PIXMAN_REPEAT_NONE;
+	break;
+
+    case 1:
+	mask_repeat = PIXMAN_REPEAT_NORMAL;
+	break;
+
+    case 2:
+	mask_repeat = PIXMAN_REPEAT_PAD;
+	break;
+
+    case 3:
+	mask_repeat = PIXMAN_REPEAT_REFLECT;
+	break;
+
+    default:
+        break;
+    }
+    pixman_image_set_repeat (mask_img, mask_repeat);
+
+    switch (prng_rand_n (4))
     {
     case 0:
 	repeat = PIXMAN_REPEAT_NONE;
@@ -141,16 +245,24 @@ test_composite (uint32_t initcrc,
     }
     pixman_image_set_repeat (src_img, repeat);
 
-    if (lcg_rand_n (2))
+    if (prng_rand_n (2))
 	pixman_image_set_filter (src_img, PIXMAN_FILTER_NEAREST, NULL, 0);
     else
 	pixman_image_set_filter (src_img, PIXMAN_FILTER_BILINEAR, NULL, 0);
 
+    if (prng_rand_n (2))
+	pixman_image_set_filter (mask_img, PIXMAN_FILTER_NEAREST, NULL, 0);
+    else
+	pixman_image_set_filter (mask_img, PIXMAN_FILTER_BILINEAR, NULL, 0);
+
     if (verbose)
     {
-	printf ("src_fmt=%08X, dst_fmt=%08X\n", src_fmt, dst_fmt);
-	printf ("op=%d, scale_x=%d, scale_y=%d, repeat=%d\n",
-	        op, scale_x, scale_y, repeat);
+	printf ("src_fmt=%s, dst_fmt=%s\n", 
+		format_name (src_fmt), format_name (dst_fmt));
+	printf ("op=%s, scale_x=%d, scale_y=%d, repeat=%d\n",
+	        operator_name (op), scale_x, scale_y, repeat);
+	printf ("translate_x=%d, translate_y=%d\n",
+	        translate_x, translate_y);
 	printf ("src_width=%d, src_height=%d, dst_width=%d, dst_height=%d\n",
 	        src_width, src_height, dst_width, dst_height);
 	printf ("src_x=%d, src_y=%d, dst_x=%d, dst_y=%d\n",
@@ -158,19 +270,19 @@ test_composite (uint32_t initcrc,
 	printf ("w=%d, h=%d\n", w, h);
     }
 
-    if (lcg_rand_n (8) == 0)
+    if (prng_rand_n (8) == 0)
     {
 	pixman_box16_t clip_boxes[2];
-	int            n = lcg_rand_n (2) + 1;
+	int            n = prng_rand_n (2) + 1;
 
 	for (i = 0; i < n; i++)
 	{
-	    clip_boxes[i].x1 = lcg_rand_n (src_width);
-	    clip_boxes[i].y1 = lcg_rand_n (src_height);
+	    clip_boxes[i].x1 = prng_rand_n (src_width);
+	    clip_boxes[i].y1 = prng_rand_n (src_height);
 	    clip_boxes[i].x2 =
-		clip_boxes[i].x1 + lcg_rand_n (src_width - clip_boxes[i].x1);
+		clip_boxes[i].x1 + prng_rand_n (src_width - clip_boxes[i].x1);
 	    clip_boxes[i].y2 =
-		clip_boxes[i].y1 + lcg_rand_n (src_height - clip_boxes[i].y1);
+		clip_boxes[i].y1 + prng_rand_n (src_height - clip_boxes[i].y1);
 
 	    if (verbose)
 	    {
@@ -186,106 +298,105 @@ test_composite (uint32_t initcrc,
 	pixman_region_fini (&clip);
     }
 
-    if (lcg_rand_n (8) == 0)
+    if (prng_rand_n (8) == 0)
     {
 	pixman_box16_t clip_boxes[2];
-	int            n = lcg_rand_n (2) + 1;
+	int            n = prng_rand_n (2) + 1;
+
 	for (i = 0; i < n; i++)
 	{
-	    clip_boxes[i].x1 = lcg_rand_n (dst_width);
-	    clip_boxes[i].y1 = lcg_rand_n (dst_height);
+	    clip_boxes[i].x1 = prng_rand_n (mask_width);
+	    clip_boxes[i].y1 = prng_rand_n (mask_height);
 	    clip_boxes[i].x2 =
-		clip_boxes[i].x1 + lcg_rand_n (dst_width - clip_boxes[i].x1);
+		clip_boxes[i].x1 + prng_rand_n (mask_width - clip_boxes[i].x1);
 	    clip_boxes[i].y2 =
-		clip_boxes[i].y1 + lcg_rand_n (dst_height - clip_boxes[i].y1);
+		clip_boxes[i].y1 + prng_rand_n (mask_height - clip_boxes[i].y1);
 
 	    if (verbose)
 	    {
-		printf ("destination clip box: [%d,%d-%d,%d]\n",
+		printf ("mask clip box: [%d,%d-%d,%d]\n",
 		        clip_boxes[i].x1, clip_boxes[i].y1,
 		        clip_boxes[i].x2, clip_boxes[i].y2);
 	    }
 	}
+
 	pixman_region_init_rects (&clip, clip_boxes, n);
-	pixman_image_set_clip_region (dst_img, &clip);
+	pixman_image_set_clip_region (mask_img, &clip);
+	pixman_image_set_source_clipping (mask_img, 1);
 	pixman_region_fini (&clip);
     }
 
-    pixman_image_composite (op, src_img, NULL, dst_img,
-                            src_x, src_y, 0, 0, dst_x, dst_y, w, h);
-
-    if (dst_fmt == PIXMAN_x8r8g8b8)
-    {
-	/* ignore unused part */
-	for (i = 0; i < dst_stride * dst_height / 4; i++)
-	    dstbuf[i] &= 0xFFFFFF;
-    }
-
-    image_endian_swap (dst_img, dst_bpp * 8);
-
-    if (verbose)
+    if (prng_rand_n (8) == 0)
     {
-	int j;
-	
-	for (i = 0; i < dst_height; i++)
+	pixman_box16_t clip_boxes[2];
+	int            n = prng_rand_n (2) + 1;
+	for (i = 0; i < n; i++)
 	{
-	    for (j = 0; j < dst_stride; j++)
-		printf ("%02X ", *((uint8_t *)dstbuf + i * dst_stride + j));
+	    clip_boxes[i].x1 = prng_rand_n (dst_width);
+	    clip_boxes[i].y1 = prng_rand_n (dst_height);
+	    clip_boxes[i].x2 =
+		clip_boxes[i].x1 + prng_rand_n (dst_width - clip_boxes[i].x1);
+	    clip_boxes[i].y2 =
+		clip_boxes[i].y1 + prng_rand_n (dst_height - clip_boxes[i].y1);
 
-	    printf ("\n");
+	    if (verbose)
+	    {
+		printf ("destination clip box: [%d,%d-%d,%d]\n",
+		        clip_boxes[i].x1, clip_boxes[i].y1,
+		        clip_boxes[i].x2, clip_boxes[i].y2);
+	    }
 	}
+	pixman_region_init_rects (&clip, clip_boxes, n);
+	pixman_image_set_clip_region (dst_img, &clip);
+	pixman_region_fini (&clip);
     }
 
+    if (prng_rand_n (2) == 0)
+	pixman_image_composite (op, src_img, NULL, dst_img,
+                            src_x, src_y, 0, 0, dst_x, dst_y, w, h);
+    else
+	pixman_image_composite (op, src_img, mask_img, dst_img,
+                            src_x, src_y, mask_x, mask_y, dst_x, dst_y, w, h);
+
+    crc32 = compute_crc32_for_image (0, dst_img);
+    
+    if (verbose)
+	print_image (dst_img);
+
     pixman_image_unref (src_img);
+    pixman_image_unref (mask_img);
     pixman_image_unref (dst_img);
 
-    crc32 = compute_crc32 (initcrc, dstbuf, dst_stride * dst_height);
+    if (src_stride < 0)
+	srcbuf += (src_stride / 4) * (src_height - 1);
+
+    if (mask_stride < 0)
+	maskbuf += (mask_stride / 4) * (mask_height - 1);
+
+    if (dst_stride < 0)
+	dstbuf += (dst_stride / 4) * (dst_height - 1);
+
     free (srcbuf);
+    free (maskbuf);
     free (dstbuf);
+
+    FLOAT_REGS_CORRUPTION_DETECTOR_FINISH ();
     return crc32;
 }
 
+#if BILINEAR_INTERPOLATION_BITS == 7
+#define CHECKSUM 0x92E0F068
+#elif BILINEAR_INTERPOLATION_BITS == 4
+#define CHECKSUM 0x8EFFA1E5
+#else
+#define CHECKSUM 0x00000000
+#endif
+
 int
-main (int   argc, char *argv[])
+main (int argc, const char *argv[])
 {
-    int      i, n = 0;
-    uint32_t crc = 0;
-
     pixman_disable_out_of_bounds_workaround ();
 
-    if (argc >= 2)
-	n = atoi (argv[1]);
-
-    if (n == 0) n = 3000000;
-
-    if (n < 0)
-    {
-	crc = test_composite (0, -n, 1);
-	printf ("crc32=%08X\n", crc);
-    }
-    else
-    {
-	for (i = 1; i <= n; i++)
-	    crc = test_composite (crc, i, 0);
-
-	printf ("crc32=%08X\n", crc);
-
-	if (n == 3000000)
-	{
-	    /* predefined value for running with all the fastpath functions disabled  */
-	    /* it needs to be updated every time changes are introduced to this program! */
-
-	    if (crc == 0x2168ACD1)
-	    {
-		printf ("scaling test passed\n");
-	    }
-	    else
-	    {
-		printf ("scaling test failed!\n");
-		return 1;
-	    }
-	}
-    }
-
-    return 0;
+    return fuzzer_test_main("scaling", 8000000, CHECKSUM,
+			    test_composite, argc, argv);
 }
diff --git a/pixman/test/stress-test.c b/pixman/test/stress-test.c
new file mode 100644
index 000000000..1f03c7543
--- /dev/null
+++ b/pixman/test/stress-test.c
@@ -0,0 +1,1040 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include "utils.h"
+#include <sys/types.h>
+
+#if 0
+#define fence_malloc malloc
+#define fence_free free
+#define make_random_bytes malloc
+#endif
+
+static const pixman_format_code_t image_formats[] =
+{
+    PIXMAN_a8r8g8b8,
+    PIXMAN_x8r8g8b8,
+    PIXMAN_r5g6b5,
+    PIXMAN_r3g3b2,
+    PIXMAN_a8,
+    PIXMAN_a8b8g8r8,
+    PIXMAN_x8b8g8r8,
+    PIXMAN_b8g8r8a8,
+    PIXMAN_b8g8r8x8,
+    PIXMAN_r8g8b8a8,
+    PIXMAN_r8g8b8x8,
+    PIXMAN_x14r6g6b6,
+    PIXMAN_r8g8b8,
+    PIXMAN_b8g8r8,
+    PIXMAN_a8r8g8b8_sRGB,
+    PIXMAN_r5g6b5,
+    PIXMAN_b5g6r5,
+    PIXMAN_x2r10g10b10,
+    PIXMAN_a2r10g10b10,
+    PIXMAN_x2b10g10r10,
+    PIXMAN_a2b10g10r10,
+    PIXMAN_a1r5g5b5,
+    PIXMAN_x1r5g5b5,
+    PIXMAN_a1b5g5r5,
+    PIXMAN_x1b5g5r5,
+    PIXMAN_a4r4g4b4,
+    PIXMAN_x4r4g4b4,
+    PIXMAN_a4b4g4r4,
+    PIXMAN_x4b4g4r4,
+    PIXMAN_a8,
+    PIXMAN_r3g3b2,
+    PIXMAN_b2g3r3,
+    PIXMAN_a2r2g2b2,
+    PIXMAN_a2b2g2r2,
+    PIXMAN_c8,
+    PIXMAN_g8,
+    PIXMAN_x4c4,
+    PIXMAN_x4g4,
+    PIXMAN_c4,
+    PIXMAN_g4,
+    PIXMAN_g1,
+    PIXMAN_x4a4,
+    PIXMAN_a4,
+    PIXMAN_r1g2b1,
+    PIXMAN_b1g2r1,
+    PIXMAN_a1r1g1b1,
+    PIXMAN_a1b1g1r1,
+    PIXMAN_a1
+};
+
+static pixman_filter_t filters[] =
+{
+    PIXMAN_FILTER_NEAREST,
+    PIXMAN_FILTER_BILINEAR,
+    PIXMAN_FILTER_FAST,
+    PIXMAN_FILTER_GOOD,
+    PIXMAN_FILTER_BEST,
+    PIXMAN_FILTER_CONVOLUTION
+};
+
+static int
+get_size (void)
+{
+    switch (prng_rand_n (28))
+    {
+    case 0:
+	return 1;
+
+    case 1:
+	return 2;
+
+    default:
+    case 2:
+	return prng_rand_n (100);
+
+    case 4:
+	return prng_rand_n (2000) + 1000;
+
+    case 5:
+	return 65535;
+
+    case 6:
+	return 65536;
+
+    case 7:
+	return prng_rand_n (64000) + 63000;
+    }
+}
+
+static void
+destroy (pixman_image_t *image, void *data)
+{
+    if (image->type == BITS && image->bits.free_me != image->bits.bits)
+    {
+	uint32_t *bits;
+
+	if (image->bits.bits != (void *)0x01)
+	{
+	    bits = image->bits.bits;
+
+	    if (image->bits.rowstride < 0)
+		bits -= (- image->bits.rowstride * (image->bits.height - 1));
+
+	    fence_free (bits);
+	}
+    }
+
+    free (data);
+}
+
+static uint32_t
+real_reader (const void *src, int size)
+{
+    switch (size)
+    {
+    case 1:
+	return *(uint8_t *)src;
+    case 2:
+	return *(uint16_t *)src;
+    case 4:
+	return *(uint32_t *)src;
+    default:
+	assert (0);
+	return 0; /* silence MSVC */
+    }
+}
+
+static void
+real_writer (void *src, uint32_t value, int size)
+{
+    switch (size)
+    {
+    case 1:
+	*(uint8_t *)src = value;
+	break;
+
+    case 2:
+	*(uint16_t *)src = value;
+	break;
+
+    case 4:
+	*(uint32_t *)src = value;
+	break;
+
+    default:
+	assert (0);
+	break;
+    }
+}
+
+static uint32_t
+fake_reader (const void *src, int size)
+{
+    uint32_t r = prng_rand ();
+
+    assert (size == 1 || size == 2 || size == 4);
+
+    return r >> (32 - (size * 8));
+}
+
+static void
+fake_writer (void *src, uint32_t value, int size)
+{
+    assert (size == 1 || size == 2 || size == 4);
+}
+
+static int32_t
+log_rand (void)
+{
+    uint32_t mask;
+
+    mask = (1 << prng_rand_n (10)) - 1;
+
+    return (prng_rand () & mask) - (mask >> 1);
+}
+
+static int32_t
+rand_x (pixman_image_t *image)
+{
+    if (image->type == BITS)
+	return prng_rand_n (image->bits.width);
+    else
+	return log_rand ();
+}
+
+static int32_t
+rand_y (pixman_image_t *image)
+{
+    if (image->type == BITS)
+	return prng_rand_n (image->bits.height);
+    else
+	return log_rand ();
+}
+
+typedef enum
+{
+    DONT_CARE,
+    PREFER_ALPHA,
+    REQUIRE_ALPHA
+} alpha_preference_t;
+
+static pixman_format_code_t
+random_format (alpha_preference_t alpha)
+{
+    pixman_format_code_t format;
+    int n = prng_rand_n (ARRAY_LENGTH (image_formats));
+
+    if (alpha >= PREFER_ALPHA &&
+	(alpha == REQUIRE_ALPHA || prng_rand_n (4) != 0))
+    {
+        do
+        {
+            format = image_formats[n++ % ARRAY_LENGTH (image_formats)];
+        } while (PIXMAN_FORMAT_TYPE (format) != PIXMAN_TYPE_A);
+    }
+    else
+    {
+        format = image_formats[n];
+    }
+
+    return format;
+}
+
+static pixman_image_t *
+create_random_bits_image (alpha_preference_t alpha_preference)
+{
+    pixman_format_code_t format;
+    pixman_indexed_t *indexed;
+    pixman_image_t *image;
+    int width, height, stride;
+    uint32_t *bits;
+    pixman_read_memory_func_t read_func = NULL;
+    pixman_write_memory_func_t write_func = NULL;
+    pixman_filter_t filter;
+    pixman_fixed_t *coefficients = NULL;
+    int n_coefficients = 0;
+
+    /* format */
+    format = random_format (alpha_preference);
+
+    indexed = NULL;
+    if (PIXMAN_FORMAT_TYPE (format) == PIXMAN_TYPE_COLOR)
+    {
+	indexed = malloc (sizeof (pixman_indexed_t));
+
+	initialize_palette (indexed, PIXMAN_FORMAT_BPP (format), TRUE);
+    }
+    else if (PIXMAN_FORMAT_TYPE (format) == PIXMAN_TYPE_GRAY)
+    {
+	indexed = malloc (sizeof (pixman_indexed_t));
+
+	initialize_palette (indexed, PIXMAN_FORMAT_BPP (format), FALSE);
+    }
+    else
+    {
+	indexed = NULL;
+    }
+
+    /* size */
+    width = get_size ();
+    height = get_size ();
+
+    while ((uint64_t)width * height > 200000)
+    {
+	if (prng_rand_n(2) == 0)
+	    height = 200000 / width;
+	else
+	    width = 200000 / height;
+    }
+
+    if (height == 0)
+	height = 1;
+    if (width == 0)
+	width = 1;
+
+    /* bits */
+    switch (prng_rand_n (7))
+    {
+    default:
+    case 0:
+	stride = width * PIXMAN_FORMAT_BPP (format) + prng_rand_n (17);
+	stride = (stride + 3) & (~3);
+	bits = (uint32_t *)make_random_bytes (height * stride);
+	break;
+
+    case 1:
+	stride = 0;
+	bits = NULL;
+	break;
+
+    case 2: /* Zero-filled */
+	stride = width * PIXMAN_FORMAT_BPP (format) + prng_rand_n (17);
+	stride = (stride + 3) & (~3);
+	bits = fence_malloc (height * stride);
+	if (!bits)
+	    return NULL;
+	memset (bits, 0, height * stride);
+	break;
+
+    case 3: /* Filled with 0xFF */
+	stride = width * PIXMAN_FORMAT_BPP (format) + prng_rand_n (17);
+	stride = (stride + 3) & (~3);
+	bits = fence_malloc (height * stride);
+	if (!bits)
+	    return NULL;
+	memset (bits, 0xff, height * stride);
+	break;
+
+    case 4: /* bits is a bad pointer, has read/write functions */
+	stride = 232;
+	bits = (void *)0x01;
+	read_func = fake_reader;
+	write_func = fake_writer;
+	break;
+
+    case 5: /* bits is a real pointer, has read/write functions */
+	stride = width * PIXMAN_FORMAT_BPP (format) + prng_rand_n (17);
+	stride = (stride + 3) & (~3);
+	bits = fence_malloc (height * stride);
+	if (!bits)
+	    return NULL;
+	memset (bits, 0xff, height * stride);
+	read_func = real_reader;
+	write_func = real_writer;
+	break;
+
+    case 6: /* bits is a real pointer, stride is negative */
+	stride = (width * PIXMAN_FORMAT_BPP (format) + prng_rand_n (17));
+	stride = (stride + 3) & (~3);
+	bits = (uint32_t *)make_random_bytes (height * stride);
+	if (!bits)
+	    return NULL;
+	bits += ((height - 1) * stride) / 4;
+	stride = - stride;
+	break;
+    }
+
+    /* Filter */
+    filter = filters[prng_rand_n (ARRAY_LENGTH (filters))];
+    if (filter == PIXMAN_FILTER_CONVOLUTION)
+    {
+	int width = prng_rand_n (3);
+	int height = prng_rand_n (4);
+
+	n_coefficients = width * height + 2;
+	coefficients = malloc (n_coefficients * sizeof (pixman_fixed_t));
+
+	if (coefficients)
+	{
+	    int i;
+
+	    for (i = 0; i < width * height; ++i)
+		coefficients[i + 2] = prng_rand();
+
+	    coefficients[0] = width << 16;
+	    coefficients[1] = height << 16;
+	}
+	else
+	{
+	    filter = PIXMAN_FILTER_BEST;
+	}
+    }
+
+    /* Finally create the image */
+    image = pixman_image_create_bits (format, width, height, bits, stride);
+    if (!image)
+	return NULL;
+
+    pixman_image_set_indexed (image, indexed);
+    pixman_image_set_destroy_function (image, destroy, indexed);
+    pixman_image_set_accessors (image, read_func, write_func);
+    pixman_image_set_filter (image, filter, coefficients, n_coefficients);
+
+    return image;
+}
+
+static pixman_repeat_t repeats[] =
+{
+    PIXMAN_REPEAT_NONE,
+    PIXMAN_REPEAT_NORMAL,
+    PIXMAN_REPEAT_REFLECT,
+    PIXMAN_REPEAT_PAD
+};
+
+static uint32_t
+absolute (int32_t i)
+{
+    return i < 0? -i : i;
+}
+
+static void
+set_general_properties (pixman_image_t *image, pixman_bool_t allow_alpha_map)
+{
+    pixman_repeat_t repeat;
+
+    /* Set properties that are generic to all images */
+
+    /* Repeat */
+    repeat = repeats[prng_rand_n (ARRAY_LENGTH (repeats))];
+    pixman_image_set_repeat (image, repeat);
+
+    /* Alpha map */
+    if (allow_alpha_map && prng_rand_n (4) == 0)
+    {
+	pixman_image_t *alpha_map;
+	int16_t x, y;
+
+	alpha_map = create_random_bits_image (DONT_CARE);
+
+	if (alpha_map)
+	{
+	    set_general_properties (alpha_map, FALSE);
+
+	    x = rand_x (image) - image->bits.width / 2;
+	    y = rand_y (image) - image->bits.height / 2;
+
+	    pixman_image_set_alpha_map (image, alpha_map, x, y);
+
+	    pixman_image_unref (alpha_map);
+	}
+    }
+
+    /* Component alpha */
+    pixman_image_set_component_alpha (image, prng_rand_n (3) == 0);
+
+    /* Clip region */
+    if (prng_rand_n (8) < 2)
+    {
+	pixman_region32_t region;
+	int i, n_rects;
+
+	pixman_region32_init (&region);
+
+	switch (prng_rand_n (12))
+	{
+	case 0:
+	    n_rects = 0;
+	    break;
+
+	case 1: case 2: case 3:
+	    n_rects = 1;
+	    break;
+
+	case 4: case 5:
+	    n_rects = 2;
+	    break;
+
+	case 6: case 7:
+	    n_rects = 3;
+	    break;
+
+	default:
+	    n_rects = prng_rand_n (100);
+	    break;
+	}
+
+	for (i = 0; i < n_rects; ++i)
+	{
+	    uint32_t width, height;
+	    int x, y;
+
+	    x = log_rand();
+	    y = log_rand();
+	    width = absolute (log_rand ()) + 1;
+	    height = absolute (log_rand ()) + 1;
+
+	    pixman_region32_union_rect (
+		&region, &region, x, y, width, height);
+	}
+
+	if (image->type == BITS && prng_rand_n (8) != 0)
+	{
+	    uint32_t width, height;
+	    int x, y;
+	    int i;
+
+	    /* Also add a couple of clip rectangles inside the image
+	     * so that compositing will actually take place.
+	     */
+	    for (i = 0; i < 5; ++i)
+	    {
+		x = prng_rand_n (2 * image->bits.width) - image->bits.width;
+		y = prng_rand_n (2 * image->bits.height) - image->bits.height;
+		width = prng_rand_n (image->bits.width) - x + 10;
+		height = prng_rand_n (image->bits.height) - y + 10;
+
+		if (width + x < x)
+		    width = INT32_MAX - x;
+		if (height + y < y)
+		    height = INT32_MAX - y;
+
+		pixman_region32_union_rect (
+		    &region, &region, x, y, width, height);
+	    }
+	}
+
+	pixman_image_set_clip_region32 (image, &region);
+
+	pixman_region32_fini (&region);
+    }
+
+    /* Whether source clipping is enabled */
+    pixman_image_set_source_clipping (image, !!prng_rand_n (2));
+
+    /* Client clip */
+    pixman_image_set_has_client_clip (image, !!prng_rand_n (2));
+
+    /* Transform */
+    if (prng_rand_n (5) < 2)
+    {
+	pixman_transform_t xform;
+	int i, j, k;
+	uint32_t tx, ty, sx, sy;
+	uint32_t c, s;
+
+	memset (&xform, 0, sizeof xform);
+	xform.matrix[0][0] = pixman_fixed_1;
+	xform.matrix[1][1] = pixman_fixed_1;
+	xform.matrix[2][2] = pixman_fixed_1;
+
+	for (k = 0; k < 3; ++k)
+	{
+	    switch (prng_rand_n (4))
+	    {
+	    case 0:
+		/* rotation */
+		c = prng_rand_n (2 * 65536) - 65536;
+		s = prng_rand_n (2 * 65536) - 65536;
+		pixman_transform_rotate (&xform, NULL, c, s);
+		break;
+
+	    case 1:
+		/* translation */
+		tx = prng_rand();
+		ty = prng_rand();
+		pixman_transform_translate (&xform, NULL, tx, ty);
+		break;
+
+	    case 2:
+		/* scale */
+		sx = prng_rand();
+		sy = prng_rand();
+		pixman_transform_scale (&xform, NULL, sx, sy);
+		break;
+
+	    case 3:
+		if (prng_rand_n (16) == 0)
+		{
+		    /* random */
+		    for (i = 0; i < 3; ++i)
+			for (j = 0; j < 3; ++j)
+			    xform.matrix[i][j] = prng_rand();
+		    break;
+		}
+		else if (prng_rand_n (16) == 0)
+		{
+		    /* zero */
+		    memset (&xform, 0, sizeof xform);
+		}
+		break;
+	    }
+	}
+
+	pixman_image_set_transform (image, &xform);
+    }
+}
+
+static pixman_color_t
+random_color (void)
+{
+    pixman_color_t color =
+    {
+	prng_rand() & 0xffff,
+	prng_rand() & 0xffff,
+	prng_rand() & 0xffff,
+	prng_rand() & 0xffff,
+    };
+
+    return color;
+}
+
+
+static pixman_image_t *
+create_random_solid_image (void)
+{
+    pixman_color_t color = random_color();
+    pixman_image_t *image = pixman_image_create_solid_fill (&color);
+
+    return image;
+}
+
+static pixman_gradient_stop_t *
+create_random_stops (int *n_stops)
+{
+    pixman_fixed_t step;
+    pixman_fixed_t s;
+    int i;
+    pixman_gradient_stop_t *stops;
+
+    *n_stops = prng_rand_n (50) + 1;
+
+    step = pixman_fixed_1 / *n_stops;
+
+    stops = malloc (*n_stops * sizeof (pixman_gradient_stop_t));
+
+    s = 0;
+    for (i = 0; i < (*n_stops) - 1; ++i)
+    {
+	stops[i].x = s;
+	stops[i].color = random_color();
+
+	s += step;
+    }
+
+    stops[*n_stops - 1].x = pixman_fixed_1;
+    stops[*n_stops - 1].color = random_color();
+
+    return stops;
+}
+
+static pixman_point_fixed_t
+create_random_point (void)
+{
+    pixman_point_fixed_t p;
+
+    p.x = log_rand ();
+    p.y = log_rand ();
+
+    return p;
+}
+
+static pixman_image_t *
+create_random_linear_image (void)
+{
+    int n_stops;
+    pixman_gradient_stop_t *stops;
+    pixman_point_fixed_t p1, p2;
+    pixman_image_t *result;
+
+    stops = create_random_stops (&n_stops);
+    if (!stops)
+	return NULL;
+
+    p1 = create_random_point ();
+    p2 = create_random_point ();
+
+    result = pixman_image_create_linear_gradient (&p1, &p2, stops, n_stops);
+
+    free (stops);
+
+    return result;
+}
+
+static pixman_image_t *
+create_random_radial_image (void)
+{
+    int n_stops;
+    pixman_gradient_stop_t *stops;
+    pixman_point_fixed_t inner_c, outer_c;
+    pixman_fixed_t inner_r, outer_r;
+    pixman_image_t *result;
+
+    inner_c = create_random_point();
+    outer_c = create_random_point();
+    inner_r = prng_rand();
+    outer_r = prng_rand();
+
+    stops = create_random_stops (&n_stops);
+
+    if (!stops)
+	return NULL;
+
+    result = pixman_image_create_radial_gradient (
+	&inner_c, &outer_c, inner_r, outer_r, stops, n_stops);
+
+    free (stops);
+
+    return result;
+}
+
+static pixman_image_t *
+create_random_conical_image (void)
+{
+    pixman_gradient_stop_t *stops;
+    int n_stops;
+    pixman_point_fixed_t c;
+    pixman_fixed_t angle;
+    pixman_image_t *result;
+
+    c = create_random_point();
+    angle = prng_rand();
+
+    stops = create_random_stops (&n_stops);
+
+    if (!stops)
+	return NULL;
+
+    result = pixman_image_create_conical_gradient (&c, angle, stops, n_stops);
+
+    free (stops);
+
+    return result;
+}
+
+static pixman_image_t *
+create_random_image (void)
+{
+    pixman_image_t *result;
+
+    switch (prng_rand_n (5))
+    {
+    default:
+    case 0:
+	result = create_random_bits_image (DONT_CARE);
+	break;
+
+    case 1:
+	result = create_random_solid_image ();
+	break;
+
+    case 2:
+	result = create_random_linear_image ();
+	break;
+
+    case 3:
+	result = create_random_radial_image ();
+	break;
+
+    case 4:
+	result = create_random_conical_image ();
+	break;
+    }
+
+    if (result)
+	set_general_properties (result, TRUE);
+
+    return result;
+}
+
+static void
+random_line (pixman_line_fixed_t *line, int width, int height)
+{
+    line->p1.x = prng_rand_n (width) << 16;
+    line->p1.y = prng_rand_n (height) << 16;
+    line->p2.x = prng_rand_n (width) << 16;
+    line->p2.y = prng_rand_n (height) << 16;
+}
+
+static pixman_trapezoid_t *
+create_random_trapezoids (int *n_traps, int height, int width)
+{
+    pixman_trapezoid_t *trapezoids;
+    int i;
+
+    *n_traps = prng_rand_n (16) + 1;
+
+    trapezoids = malloc (sizeof (pixman_trapezoid_t) * *n_traps);
+
+    for (i = 0; i < *n_traps; ++i)
+    {
+        pixman_trapezoid_t *t = &(trapezoids[i]);
+
+        t->top = prng_rand_n (height) << 16;
+        t->bottom = prng_rand_n (height) << 16;
+
+        random_line (&t->left, height, width);
+        random_line (&t->right, height, width);
+    }
+
+    return trapezoids;
+}
+
+static const pixman_op_t op_list[] =
+{
+    PIXMAN_OP_SRC,
+    PIXMAN_OP_OVER,
+    PIXMAN_OP_ADD,
+    PIXMAN_OP_CLEAR,
+    PIXMAN_OP_SRC,
+    PIXMAN_OP_DST,
+    PIXMAN_OP_OVER,
+    PIXMAN_OP_OVER_REVERSE,
+    PIXMAN_OP_IN,
+    PIXMAN_OP_IN_REVERSE,
+    PIXMAN_OP_OUT,
+    PIXMAN_OP_OUT_REVERSE,
+    PIXMAN_OP_ATOP,
+    PIXMAN_OP_ATOP_REVERSE,
+    PIXMAN_OP_XOR,
+    PIXMAN_OP_ADD,
+    PIXMAN_OP_SATURATE,
+    PIXMAN_OP_DISJOINT_CLEAR,
+    PIXMAN_OP_DISJOINT_SRC,
+    PIXMAN_OP_DISJOINT_DST,
+    PIXMAN_OP_DISJOINT_OVER,
+    PIXMAN_OP_DISJOINT_OVER_REVERSE,
+    PIXMAN_OP_DISJOINT_IN,
+    PIXMAN_OP_DISJOINT_IN_REVERSE,
+    PIXMAN_OP_DISJOINT_OUT,
+    PIXMAN_OP_DISJOINT_OUT_REVERSE,
+    PIXMAN_OP_DISJOINT_ATOP,
+    PIXMAN_OP_DISJOINT_ATOP_REVERSE,
+    PIXMAN_OP_DISJOINT_XOR,
+    PIXMAN_OP_CONJOINT_CLEAR,
+    PIXMAN_OP_CONJOINT_SRC,
+    PIXMAN_OP_CONJOINT_DST,
+    PIXMAN_OP_CONJOINT_OVER,
+    PIXMAN_OP_CONJOINT_OVER_REVERSE,
+    PIXMAN_OP_CONJOINT_IN,
+    PIXMAN_OP_CONJOINT_IN_REVERSE,
+    PIXMAN_OP_CONJOINT_OUT,
+    PIXMAN_OP_CONJOINT_OUT_REVERSE,
+    PIXMAN_OP_CONJOINT_ATOP,
+    PIXMAN_OP_CONJOINT_ATOP_REVERSE,
+    PIXMAN_OP_CONJOINT_XOR,
+    PIXMAN_OP_MULTIPLY,
+    PIXMAN_OP_SCREEN,
+    PIXMAN_OP_OVERLAY,
+    PIXMAN_OP_DARKEN,
+    PIXMAN_OP_LIGHTEN,
+    PIXMAN_OP_COLOR_DODGE,
+    PIXMAN_OP_COLOR_BURN,
+    PIXMAN_OP_HARD_LIGHT,
+    PIXMAN_OP_DIFFERENCE,
+    PIXMAN_OP_EXCLUSION,
+    PIXMAN_OP_SOFT_LIGHT,
+    PIXMAN_OP_HSL_HUE,
+    PIXMAN_OP_HSL_SATURATION,
+    PIXMAN_OP_HSL_COLOR,
+    PIXMAN_OP_HSL_LUMINOSITY,
+};
+
+static void
+run_test (uint32_t seed, pixman_bool_t verbose, uint32_t mod)
+{
+    pixman_image_t *source, *mask, *dest;
+    pixman_op_t op;
+
+    if (verbose)
+    {
+	if (mod == 0 || (seed % mod) == 0)
+	    printf ("Seed 0x%08x\n", seed);
+    }
+
+    source = mask = dest = NULL;
+
+    prng_srand (seed);
+
+    if (prng_rand_n (8) == 0)
+    {
+        int n_traps;
+        pixman_trapezoid_t *trapezoids;
+	int p = prng_rand_n (3);
+
+	if (p == 0)
+	    dest = create_random_bits_image (DONT_CARE);
+	else
+	    dest = create_random_bits_image (REQUIRE_ALPHA);
+
+	if (!dest)
+	    goto out;
+
+	set_general_properties (dest, TRUE);
+
+	if (!(trapezoids = create_random_trapezoids (
+		  &n_traps, dest->bits.width, dest->bits.height)))
+	{
+	    goto out;
+	}
+
+	switch (p)
+	{
+	case 0:
+	    source = create_random_image ();
+
+	    if (source)
+	    {
+		op = op_list [prng_rand_n (ARRAY_LENGTH (op_list))];
+
+		pixman_composite_trapezoids (
+		    op, source, dest,
+		    random_format (REQUIRE_ALPHA),
+		    rand_x (source), rand_y (source),
+		    rand_x (dest), rand_y (dest),
+		    n_traps, trapezoids);
+	    }
+	    break;
+
+	case 1:
+	    pixman_rasterize_trapezoid (
+		dest, &trapezoids[prng_rand_n (n_traps)],
+		rand_x (dest), rand_y (dest));
+	    break;
+
+	case 2:
+	    pixman_add_trapezoids (
+		dest, rand_x (dest), rand_y (dest), n_traps, trapezoids);
+	    break;
+        }
+
+	free (trapezoids);
+    }
+    else
+    {
+        dest = create_random_bits_image (DONT_CARE);
+        source = create_random_image ();
+        mask = create_random_image ();
+
+        if (source && mask && dest)
+        {
+            set_general_properties (dest, TRUE);
+
+            op = op_list [prng_rand_n (ARRAY_LENGTH (op_list))];
+
+            pixman_image_composite32 (op,
+                                      source, mask, dest,
+                                      rand_x (source), rand_y (source),
+                                      rand_x (mask), rand_y (mask),
+                                      0, 0,
+                                      dest->bits.width,
+                                      dest->bits.height);
+        }
+    }
+
+out:
+    if (source)
+	pixman_image_unref (source);
+    if (mask)
+	pixman_image_unref (mask);
+    if (dest)
+	pixman_image_unref (dest);
+}
+
+static pixman_bool_t
+get_int (char *s, uint32_t *i)
+{
+    char *end;
+    int p;
+
+    p = strtol (s, &end, 0);
+
+    if (end != s && *end == 0)
+    {
+	*i = p;
+	return TRUE;
+    }
+
+    return FALSE;
+}
+
+int
+main (int argc, char **argv)
+{
+    int verbose = FALSE;
+    uint32_t seed = 1;
+    uint32_t n_tests = 8000;
+    uint32_t mod = 0;
+    pixman_bool_t use_threads = TRUE;
+    int32_t i;
+
+    pixman_disable_out_of_bounds_workaround ();
+
+    enable_divbyzero_exceptions();
+
+    if (getenv ("VERBOSE") != NULL)
+	verbose = TRUE;
+
+    for (i = 1; i < argc; ++i)
+    {
+	if (strcmp (argv[i], "-v") == 0)
+	{
+	    verbose = TRUE;
+
+	    if (i + 1 < argc)
+	    {
+		get_int (argv[i + 1], &mod);
+		i++;
+	    }
+	}
+	else if (strcmp (argv[i], "-s") == 0 && i + 1 < argc)
+	{
+	    get_int (argv[i + 1], &seed);
+	    use_threads = FALSE;
+	    i++;
+	}
+	else if (strcmp (argv[i], "-n") == 0 && i + 1 < argc)
+	{
+	    get_int (argv[i + 1], &n_tests);
+	    i++;
+	}
+	else
+	{
+	    if (strcmp (argv[i], "-h") != 0)
+		printf ("Unknown option '%s'\n\n", argv[i]);
+
+	    printf ("Options:\n\n"
+		    "-n <number>        Number of tests to run\n"
+		    "-s <seed> 	        Seed of first test (ignored if PIXMAN_RANDOMIZE_TESTS is set)\n"
+		    "-v                 Print out seeds\n"
+		    "-v <n>             Print out every n'th seed\n\n");
+
+	    exit (-1);
+	}
+    }
+
+    if (getenv ("PIXMAN_RANDOMIZE_TESTS"))
+    {
+	seed = get_random_seed();
+	printf ("First seed: 0x%08x\n", seed);
+    }
+
+    if (use_threads)
+    {
+#ifdef USE_OPENMP
+#   pragma omp parallel for default(none) shared(verbose, n_tests, mod, seed)
+#endif
+	for (i = 0; i < (int32_t)n_tests; ++i)
+	    run_test (seed + i, verbose, mod);
+    }
+    else
+    {
+	for (i = 0; i < (int32_t)n_tests; ++i)
+	    run_test (seed + i, verbose, mod);
+    }
+
+    return 0;
+}
diff --git a/pixman/test/thread-test.c b/pixman/test/thread-test.c
new file mode 100644
index 000000000..0b07b269d
--- /dev/null
+++ b/pixman/test/thread-test.c
@@ -0,0 +1,199 @@
+#include "utils.h"
+
+#ifndef HAVE_PTHREADS
+
+int main ()
+{
+    printf ("Skipped thread-test - pthreads not supported\n");
+    return 0;
+}
+
+#else
+
+#include <stdlib.h>
+#include <pthread.h>
+
+typedef struct
+{
+    int       thread_no;
+    uint32_t *dst_buf;
+    prng_t    prng_state;
+} info_t;
+
+static const pixman_op_t operators[] = 
+{
+    PIXMAN_OP_SRC,
+    PIXMAN_OP_OVER,
+    PIXMAN_OP_ADD,
+    PIXMAN_OP_CLEAR,
+    PIXMAN_OP_SRC,
+    PIXMAN_OP_DST,
+    PIXMAN_OP_OVER,
+    PIXMAN_OP_OVER_REVERSE,
+    PIXMAN_OP_IN,
+    PIXMAN_OP_IN_REVERSE,
+    PIXMAN_OP_OUT,
+    PIXMAN_OP_OUT_REVERSE,
+    PIXMAN_OP_ATOP,
+    PIXMAN_OP_ATOP_REVERSE,
+    PIXMAN_OP_XOR,
+    PIXMAN_OP_ADD,
+    PIXMAN_OP_SATURATE,
+    PIXMAN_OP_DISJOINT_CLEAR,
+    PIXMAN_OP_DISJOINT_SRC,
+    PIXMAN_OP_DISJOINT_DST,
+    PIXMAN_OP_DISJOINT_OVER,
+    PIXMAN_OP_DISJOINT_OVER_REVERSE,
+    PIXMAN_OP_DISJOINT_IN,
+    PIXMAN_OP_DISJOINT_IN_REVERSE,
+    PIXMAN_OP_DISJOINT_OUT,
+    PIXMAN_OP_DISJOINT_OUT_REVERSE,
+    PIXMAN_OP_DISJOINT_ATOP,
+    PIXMAN_OP_DISJOINT_ATOP_REVERSE,
+    PIXMAN_OP_DISJOINT_XOR,
+    PIXMAN_OP_CONJOINT_CLEAR,
+    PIXMAN_OP_CONJOINT_SRC,
+    PIXMAN_OP_CONJOINT_DST,
+    PIXMAN_OP_CONJOINT_OVER,
+    PIXMAN_OP_CONJOINT_OVER_REVERSE,
+    PIXMAN_OP_CONJOINT_IN,
+    PIXMAN_OP_CONJOINT_IN_REVERSE,
+    PIXMAN_OP_CONJOINT_OUT,
+    PIXMAN_OP_CONJOINT_OUT_REVERSE,
+    PIXMAN_OP_CONJOINT_ATOP,
+    PIXMAN_OP_CONJOINT_ATOP_REVERSE,
+    PIXMAN_OP_CONJOINT_XOR,
+    PIXMAN_OP_MULTIPLY,
+    PIXMAN_OP_SCREEN,
+    PIXMAN_OP_OVERLAY,
+    PIXMAN_OP_DARKEN,
+    PIXMAN_OP_LIGHTEN,
+    PIXMAN_OP_COLOR_DODGE,
+    PIXMAN_OP_COLOR_BURN,
+    PIXMAN_OP_HARD_LIGHT,
+    PIXMAN_OP_DIFFERENCE,
+    PIXMAN_OP_EXCLUSION,
+};
+
+static const pixman_format_code_t formats[] =
+{
+    PIXMAN_a8r8g8b8,
+    PIXMAN_r5g6b5,
+    PIXMAN_a8,
+    PIXMAN_a4,
+    PIXMAN_a1,
+    PIXMAN_b5g6r5,
+    PIXMAN_r8g8b8a8,
+    PIXMAN_a4r4g4b4
+};
+
+#define N_ROUNDS 8192
+
+#define RAND_ELT(arr)							\
+    arr[prng_rand_r(&info->prng_state) % ARRAY_LENGTH (arr)]
+
+#define DEST_WIDTH (7)
+
+static void *
+thread (void *data)
+{
+    info_t *info = data;
+    uint32_t crc32 = 0x0;
+    uint32_t src_buf[64];
+    pixman_image_t *dst_img, *src_img;
+    int i;
+
+    prng_srand_r (&info->prng_state, info->thread_no);
+
+    for (i = 0; i < N_ROUNDS; ++i)
+    {
+	pixman_op_t op;
+	int rand1, rand2;
+
+	prng_randmemset_r (&info->prng_state, info->dst_buf,
+			   DEST_WIDTH * sizeof (uint32_t), 0);
+	prng_randmemset_r (&info->prng_state, src_buf,
+			   sizeof (src_buf), 0);
+
+	src_img = pixman_image_create_bits (
+	    RAND_ELT (formats), 4, 4, src_buf, 16);
+	dst_img = pixman_image_create_bits (
+	    RAND_ELT (formats), DEST_WIDTH, 1, info->dst_buf,
+	    DEST_WIDTH * sizeof (uint32_t));
+
+	image_endian_swap (src_img);
+	image_endian_swap (dst_img);
+	
+	rand2 = prng_rand_r (&info->prng_state) % 4;
+	rand1 = prng_rand_r (&info->prng_state) % 4;
+	op = RAND_ELT (operators);
+
+	pixman_image_composite32 (
+	    op,
+	    src_img, NULL, dst_img,
+	    rand1, rand2, 0, 0, 0, 0, DEST_WIDTH, 1);
+
+	crc32 = compute_crc32_for_image (crc32, dst_img);
+
+	pixman_image_unref (src_img);
+	pixman_image_unref (dst_img);
+    }
+
+    return (void *)(uintptr_t)crc32;
+}
+
+static inline uint32_t
+byteswap32 (uint32_t x)
+{
+    return ((x & ((uint32_t)0xFF << 24)) >> 24) |
+           ((x & ((uint32_t)0xFF << 16)) >>  8) |
+           ((x & ((uint32_t)0xFF <<  8)) <<  8) |
+           ((x & ((uint32_t)0xFF <<  0)) << 24);
+}
+
+int
+main (void)
+{
+    uint32_t dest[16 * DEST_WIDTH];
+    info_t info[16] = { { 0 } };
+    pthread_t threads[16];
+    void *retvals[16];
+    uint32_t crc32s[16], crc32;
+    int i;
+
+    for (i = 0; i < 16; ++i)
+    {
+	info[i].thread_no = i;
+	info[i].dst_buf = &dest[i * DEST_WIDTH];
+    }
+
+    for (i = 0; i < 16; ++i)
+	pthread_create (&threads[i], NULL, thread, &info[i]);
+
+    for (i = 0; i < 16; ++i)
+	pthread_join (threads[i], &retvals[i]);
+
+    for (i = 0; i < 16; ++i)
+    {
+	crc32s[i] = (uintptr_t)retvals[i];
+
+	if (is_little_endian())
+	    crc32s[i] = byteswap32 (crc32s[i]);
+    }
+
+    crc32 = compute_crc32 (0, crc32s, sizeof crc32s);
+
+#define EXPECTED 0xE299B18E
+
+    if (crc32 != EXPECTED)
+    {
+	printf ("thread-test failed. Got checksum 0x%08X, expected 0x%08X\n",
+		crc32, EXPECTED);
+	return 1;
+    }
+
+    return 0;
+}
+
+#endif
+
diff --git a/pixman/test/trap-crasher.c b/pixman/test/trap-crasher.c
index 42b82f674..77be1c98b 100644
--- a/pixman/test/trap-crasher.c
+++ b/pixman/test/trap-crasher.c
@@ -1,27 +1,39 @@
 #include <stdlib.h>
-#include <pixman.h>
+#include "utils.h"
 
 int
 main()
 {
     pixman_image_t *dst;
-    pixman_trapezoid_t traps[1] = {
+    pixman_trapezoid_t traps[] = {
 	{
-	    .top = 2147483646,
-	    .bottom = 2147483647,
-	    .left = {
-		.p1 = { .x = 0, .y = 0 },
-		.p2 = { .x = 0, .y = 2147483647 }
+	    2147483646,
+	    2147483647,
+	    {
+		{ 0, 0 },
+		{ 0, 2147483647 }
 	    },
-	    .right = {
-		.p1 = { .x = 65536, .y = 0 },
-		.p2 = { .x = 0, .y = 2147483647 }
+	    {
+		{ 65536, 0 },
+		{ 0, 2147483647 }
+	    }
+	},
+	{
+	    32768,
+	    - 2147483647,
+	    {
+		{ 0, 0 },
+		{ 0, 2147483647 }
+	    },
+	    {
+		{ 65536, 0 },
+		{ 0, 2147483647 }
 	    }
 	},
     };
-    
+
     dst = pixman_image_create_bits (PIXMAN_a8, 1, 1, NULL, -1);
-    
-    pixman_add_trapezoids (dst, 0, 0, sizeof (traps)/sizeof (traps[0]), traps);
+
+    pixman_add_trapezoids (dst, 0, 0, ARRAY_LENGTH (traps), traps);
     return (0);
 }
diff --git a/pixman/test/utils-prng.c b/pixman/test/utils-prng.c
new file mode 100644
index 000000000..c27b5be83
--- /dev/null
+++ b/pixman/test/utils-prng.c
@@ -0,0 +1,298 @@
+/*
+ * Copyright © 2012 Siarhei Siamashka <siarhei.siamashka@gmail.com>
+ *
+ * Based on the public domain implementation of small noncryptographic PRNG
+ * authored by Bob Jenkins: http://burtleburtle.net/bob/rand/smallprng.html
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include "utils.h"
+#include "utils-prng.h"
+
+#if defined(HAVE_GCC_VECTOR_EXTENSIONS) && defined(__SSE2__)
+#include <xmmintrin.h>
+#endif
+
+void smallprng_srand_r (smallprng_t *x, uint32_t seed)
+{
+    uint32_t i;
+    x->a = 0xf1ea5eed, x->b = x->c = x->d = seed;
+    for (i = 0; i < 20; ++i)
+        smallprng_rand_r (x);
+}
+
+/*
+ * Set a 32-bit seed for PRNG
+ *
+ * LCG is used here for generating independent seeds for different
+ * smallprng instances (in the case if smallprng is also used for
+ * generating these seeds, "Big Crush" test from TestU01 detects
+ * some problems in the glued 'prng_rand_128_r' output data).
+ * Actually we might be even better using some cryptographic
+ * hash for this purpose, but LCG seems to be also enough for
+ * passing "Big Crush".
+ */
+void prng_srand_r (prng_t *x, uint32_t seed)
+{
+#ifdef HAVE_GCC_VECTOR_EXTENSIONS
+    int i;
+    prng_rand_128_data_t dummy;
+    smallprng_srand_r (&x->p0, seed);
+    x->a[0] = x->a[1] = x->a[2] = x->a[3] = 0xf1ea5eed;
+    x->b[0] = x->c[0] = x->d[0] = (seed = seed * 1103515245 + 12345);
+    x->b[1] = x->c[1] = x->d[1] = (seed = seed * 1103515245 + 12345);
+    x->b[2] = x->c[2] = x->d[2] = (seed = seed * 1103515245 + 12345);
+    x->b[3] = x->c[3] = x->d[3] = (seed = seed * 1103515245 + 12345);
+    for (i = 0; i < 20; ++i)
+        prng_rand_128_r (x, &dummy);
+#else
+    smallprng_srand_r (&x->p0, seed);
+    smallprng_srand_r (&x->p1, (seed = seed * 1103515245 + 12345));
+    smallprng_srand_r (&x->p2, (seed = seed * 1103515245 + 12345));
+    smallprng_srand_r (&x->p3, (seed = seed * 1103515245 + 12345));
+    smallprng_srand_r (&x->p4, (seed = seed * 1103515245 + 12345));
+#endif
+}
+
+static force_inline void
+store_rand_128_data (void *addr, prng_rand_128_data_t *d, int aligned)
+{
+#ifdef HAVE_GCC_VECTOR_EXTENSIONS
+    if (aligned)
+    {
+        *(uint8x16 *)addr = d->vb;
+        return;
+    }
+    else
+    {
+#ifdef __SSE2__
+        /* workaround for http://gcc.gnu.org/PR55614 */
+        _mm_storeu_si128 (addr, _mm_loadu_si128 ((__m128i *)d));
+        return;
+#endif
+    }
+#endif
+    /* we could try something better for unaligned writes (packed attribute),
+     * but GCC is not very reliable: http://gcc.gnu.org/PR55454 */
+    memcpy (addr, d, 16);
+}
+
+/*
+ * Helper function and the actual code for "prng_randmemset_r" function
+ */
+static force_inline void
+randmemset_internal (prng_t                  *prng,
+                     uint8_t                 *buf,
+                     size_t                   size,
+                     prng_randmemset_flags_t  flags,
+                     int                      aligned)
+{
+    prng_t local_prng = *prng;
+    prng_rand_128_data_t randdata;
+    size_t i;
+
+    while (size >= 16)
+    {
+        prng_rand_128_data_t t;
+        if (flags == 0)
+        {
+            prng_rand_128_r (&local_prng, &randdata);
+        }
+        else
+        {
+            prng_rand_128_r (&local_prng, &t);
+            prng_rand_128_r (&local_prng, &randdata);
+#ifdef HAVE_GCC_VECTOR_EXTENSIONS
+            if (flags & RANDMEMSET_MORE_FF)
+            {
+                const uint8x16 const_C0 =
+                {
+                    0xC0, 0xC0, 0xC0, 0xC0, 0xC0, 0xC0, 0xC0, 0xC0,
+                    0xC0, 0xC0, 0xC0, 0xC0, 0xC0, 0xC0, 0xC0, 0xC0
+                };
+                randdata.vb |= (t.vb >= const_C0);
+            }
+            if (flags & RANDMEMSET_MORE_00)
+            {
+                const uint8x16 const_40 =
+                {
+                    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+                    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40
+                };
+                randdata.vb &= (t.vb >= const_40);
+            }
+            if (flags & RANDMEMSET_MORE_FFFFFFFF)
+            {
+                const uint32x4 const_C0000000 =
+                {
+                    0xC0000000, 0xC0000000, 0xC0000000, 0xC0000000
+                };
+                randdata.vw |= ((t.vw << 30) >= const_C0000000);
+            }
+            if (flags & RANDMEMSET_MORE_00000000)
+            {
+                const uint32x4 const_40000000 =
+                {
+                    0x40000000, 0x40000000, 0x40000000, 0x40000000
+                };
+                randdata.vw &= ((t.vw << 30) >= const_40000000);
+            }
+#else
+            #define PROCESS_ONE_LANE(i)                                       \
+                if (flags & RANDMEMSET_MORE_FF)                               \
+                {                                                             \
+                    uint32_t mask_ff = (t.w[i] & (t.w[i] << 1)) & 0x80808080; \
+                    mask_ff |= mask_ff >> 1;                                  \
+                    mask_ff |= mask_ff >> 2;                                  \
+                    mask_ff |= mask_ff >> 4;                                  \
+                    randdata.w[i] |= mask_ff;                                 \
+                }                                                             \
+                if (flags & RANDMEMSET_MORE_00)                               \
+                {                                                             \
+                    uint32_t mask_00 = (t.w[i] | (t.w[i] << 1)) & 0x80808080; \
+                    mask_00 |= mask_00 >> 1;                                  \
+                    mask_00 |= mask_00 >> 2;                                  \
+                    mask_00 |= mask_00 >> 4;                                  \
+                    randdata.w[i] &= mask_00;                                 \
+                }                                                             \
+                if (flags & RANDMEMSET_MORE_FFFFFFFF)                         \
+                {                                                             \
+                    int32_t mask_ff = ((t.w[i] << 30) & (t.w[i] << 31)) &     \
+                                       0x80000000;                            \
+                    randdata.w[i] |= mask_ff >> 31;                           \
+                }                                                             \
+                if (flags & RANDMEMSET_MORE_00000000)                         \
+                {                                                             \
+                    int32_t mask_00 = ((t.w[i] << 30) | (t.w[i] << 31)) &     \
+                                       0x80000000;                            \
+                    randdata.w[i] &= mask_00 >> 31;                           \
+                }
+
+            PROCESS_ONE_LANE (0)
+            PROCESS_ONE_LANE (1)
+            PROCESS_ONE_LANE (2)
+            PROCESS_ONE_LANE (3)
+#endif
+        }
+        if (is_little_endian ())
+        {
+            store_rand_128_data (buf, &randdata, aligned);
+            buf += 16;
+        }
+        else
+        {
+#ifdef HAVE_GCC_VECTOR_EXTENSIONS
+            const uint8x16 bswap_shufflemask =
+            {
+                3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
+            };
+            randdata.vb = __builtin_shuffle (randdata.vb, bswap_shufflemask);
+            store_rand_128_data (buf, &randdata, aligned);
+            buf += 16;
+#else
+            uint8_t t1, t2, t3, t4;
+            #define STORE_ONE_LANE(i)                                         \
+                t1 = randdata.b[i * 4 + 3];                                   \
+                t2 = randdata.b[i * 4 + 2];                                   \
+                t3 = randdata.b[i * 4 + 1];                                   \
+                t4 = randdata.b[i * 4 + 0];                                   \
+                *buf++ = t1;                                                  \
+                *buf++ = t2;                                                  \
+                *buf++ = t3;                                                  \
+                *buf++ = t4;
+
+            STORE_ONE_LANE (0)
+            STORE_ONE_LANE (1)
+            STORE_ONE_LANE (2)
+            STORE_ONE_LANE (3)
+#endif
+        }
+        size -= 16;
+    }
+    i = 0;
+    while (i < size)
+    {
+        uint8_t randbyte = prng_rand_r (&local_prng) & 0xFF;
+        if (flags != 0)
+        {
+            uint8_t t = prng_rand_r (&local_prng) & 0xFF;
+            if ((flags & RANDMEMSET_MORE_FF) && (t >= 0xC0))
+                randbyte = 0xFF;
+            if ((flags & RANDMEMSET_MORE_00) && (t < 0x40))
+                randbyte = 0x00;
+            if (i % 4 == 0 && i + 4 <= size)
+            {
+                t = prng_rand_r (&local_prng) & 0xFF;
+                if ((flags & RANDMEMSET_MORE_FFFFFFFF) && (t >= 0xC0))
+                {
+                    memset(&buf[i], 0xFF, 4);
+                    i += 4;
+                    continue;
+                }
+                if ((flags & RANDMEMSET_MORE_00000000) && (t < 0x40))
+                {
+                    memset(&buf[i], 0x00, 4);
+                    i += 4;
+                    continue;
+                }
+            }
+        }
+        buf[i] = randbyte;
+        i++;
+    }
+    *prng = local_prng;
+}
+
+/*
+ * Fill memory buffer with random data. Flags argument may be used
+ * to tweak some statistics properties:
+ *    RANDMEMSET_MORE_00        - set ~25% of bytes to 0x00
+ *    RANDMEMSET_MORE_FF        - set ~25% of bytes to 0xFF
+ *    RANDMEMSET_MORE_00000000  - ~25% chance for 00000000 4-byte clusters
+ *    RANDMEMSET_MORE_FFFFFFFF  - ~25% chance for FFFFFFFF 4-byte clusters
+ */
+void prng_randmemset_r (prng_t                  *prng,
+                        void                    *voidbuf,
+                        size_t                   size,
+                        prng_randmemset_flags_t  flags)
+{
+    uint8_t *buf = (uint8_t *)voidbuf;
+    if ((uintptr_t)buf & 15)
+    {
+        /* unaligned buffer */
+        if (flags == 0)
+            randmemset_internal (prng, buf, size, 0, 0);
+        else if (flags == RANDMEMSET_MORE_00_AND_FF)
+            randmemset_internal (prng, buf, size, RANDMEMSET_MORE_00_AND_FF, 0);
+        else
+            randmemset_internal (prng, buf, size, flags, 0);
+    }
+    else
+    {
+        /* aligned buffer */
+        if (flags == 0)
+            randmemset_internal (prng, buf, size, 0, 1);
+        else if (flags == RANDMEMSET_MORE_00_AND_FF)
+            randmemset_internal (prng, buf, size, RANDMEMSET_MORE_00_AND_FF, 1);
+        else
+            randmemset_internal (prng, buf, size, flags, 1);
+    }
+}
diff --git a/pixman/test/utils-prng.h b/pixman/test/utils-prng.h
new file mode 100644
index 000000000..f9ae8ddf7
--- /dev/null
+++ b/pixman/test/utils-prng.h
@@ -0,0 +1,170 @@
+/*
+ * Copyright © 2012 Siarhei Siamashka <siarhei.siamashka@gmail.com>
+ *
+ * Based on the public domain implementation of small noncryptographic PRNG
+ * authored by Bob Jenkins: http://burtleburtle.net/bob/rand/smallprng.html
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __UTILS_PRNG_H__
+#define __UTILS_PRNG_H__
+
+/*
+ * This file provides a fast SIMD-optimized noncryptographic PRNG (pseudorandom
+ * number generator), with the output good enough to pass "Big Crush" tests
+ * from TestU01 (http://en.wikipedia.org/wiki/TestU01).
+ *
+ * SIMD code uses http://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html
+ * which is a GCC specific extension. There is also a slower alternative
+ * code path, which should work with any C compiler.
+ *
+ * The "prng_t" structure keeps the internal state of the random number
+ * generator. It is possible to have multiple instances of the random number
+ * generator active at the same time, in this case each of them needs to have
+ * its own "prng_t". All the functions take a pointer to "prng_t"
+ * as the first argument.
+ *
+ * Functions:
+ *
+ * ----------------------------------------------------------------------------
+ * void prng_srand_r (prng_t *prng, uint32_t seed);
+ *
+ * Initialize the pseudorandom number generator. The sequence of preudorandom
+ * numbers is deterministic and only depends on "seed". Any two generators
+ * initialized with the same seed will produce exactly the same sequence.
+ *
+ * ----------------------------------------------------------------------------
+ * uint32_t prng_rand_r (prng_t *prng);
+ *
+ * Generate a single uniformly distributed 32-bit pseudorandom value.
+ *
+ * ----------------------------------------------------------------------------
+ * void prng_randmemset_r (prng_t                  *prng,
+ *                         void                    *buffer,
+ *                         size_t                   size,
+ *                         prng_randmemset_flags_t  flags);
+ *
+ * Fills the memory buffer "buffer" with "size" bytes of pseudorandom data.
+ * The "flags" argument may be used to tweak some statistics properties:
+ *    RANDMEMSET_MORE_00 - set ~25% of bytes to 0x00
+ *    RANDMEMSET_MORE_FF - set ~25% of bytes to 0xFF
+ * The flags can be combined. This allows a bit better simulation of typical
+ * pixel data, which normally contains a lot of fully transparent or fully
+ * opaque pixels.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "pixman-private.h"
+
+/*****************************************************************************/
+
+#ifdef HAVE_GCC_VECTOR_EXTENSIONS
+typedef uint32_t uint32x4 __attribute__ ((vector_size(16)));
+typedef uint8_t  uint8x16 __attribute__ ((vector_size(16)));
+#endif
+
+typedef struct
+{
+    uint32_t a, b, c, d;
+} smallprng_t;
+
+typedef struct
+{
+#ifdef HAVE_GCC_VECTOR_EXTENSIONS
+    uint32x4 a, b, c, d;
+#else
+    smallprng_t p1, p2, p3, p4;
+#endif
+    smallprng_t p0;
+} prng_t;
+
+typedef union
+{
+    uint8_t  b[16];
+    uint32_t w[4];
+#ifdef HAVE_GCC_VECTOR_EXTENSIONS
+    uint8x16 vb;
+    uint32x4 vw;
+#endif
+} prng_rand_128_data_t;
+
+/*****************************************************************************/
+
+static force_inline uint32_t
+smallprng_rand_r (smallprng_t *x)
+{
+    uint32_t e = x->a - ((x->b << 27) + (x->b >> (32 - 27)));
+    x->a = x->b ^ ((x->c << 17) ^ (x->c >> (32 - 17)));
+    x->b = x->c + x->d;
+    x->c = x->d + e;
+    x->d = e + x->a;
+    return x->d;
+}
+
+/* Generate 4 bytes (32-bits) of random data */
+static force_inline uint32_t
+prng_rand_r (prng_t *x)
+{
+    return smallprng_rand_r (&x->p0);
+}
+
+/* Generate 16 bytes (128-bits) of random data */
+static force_inline void
+prng_rand_128_r (prng_t *x, prng_rand_128_data_t *data)
+{
+#ifdef HAVE_GCC_VECTOR_EXTENSIONS
+    uint32x4 e = x->a - ((x->b << 27) + (x->b >> (32 - 27)));
+    x->a = x->b ^ ((x->c << 17) ^ (x->c >> (32 - 17)));
+    x->b = x->c + x->d;
+    x->c = x->d + e;
+    x->d = e + x->a;
+    data->vw = x->d;
+#else
+    data->w[0] = smallprng_rand_r (&x->p1);
+    data->w[1] = smallprng_rand_r (&x->p2);
+    data->w[2] = smallprng_rand_r (&x->p3);
+    data->w[3] = smallprng_rand_r (&x->p4);
+#endif
+}
+
+typedef enum
+{
+    RANDMEMSET_MORE_00        = 1, /* ~25% chance for 0x00 bytes */
+    RANDMEMSET_MORE_FF        = 2, /* ~25% chance for 0xFF bytes */
+    RANDMEMSET_MORE_00000000  = 4, /* ~25% chance for 0x00000000 clusters */
+    RANDMEMSET_MORE_FFFFFFFF  = 8, /* ~25% chance for 0xFFFFFFFF clusters */
+    RANDMEMSET_MORE_00_AND_FF = (RANDMEMSET_MORE_00 | RANDMEMSET_MORE_00000000 |
+                                 RANDMEMSET_MORE_FF | RANDMEMSET_MORE_FFFFFFFF)
+} prng_randmemset_flags_t;
+
+/* Set the 32-bit seed for PRNG */
+void prng_srand_r (prng_t *prng, uint32_t seed);
+
+/* Fill memory buffer with random data */
+void prng_randmemset_r (prng_t                  *prng,
+                        void                    *buffer,
+                        size_t                   size,
+                        prng_randmemset_flags_t  flags);
+
+#endif
diff --git a/pixman/test/utils.c b/pixman/test/utils.c
index 58cd100e2..ebe0ccc09 100644
--- a/pixman/test/utils.c
+++ b/pixman/test/utils.c
@@ -1,9 +1,37 @@
+#define _GNU_SOURCE
+
 #include "utils.h"
+#include <math.h>
+#include <signal.h>
+#include <stdlib.h>
+
+#ifdef HAVE_GETTIMEOFDAY
+#include <sys/time.h>
+#else
+#include <time.h>
+#endif
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef HAVE_SYS_MMAN_H
+#include <sys/mman.h>
+#endif
 
-/* Random number seed
+#ifdef HAVE_FENV_H
+#include <fenv.h>
+#endif
+
+#ifdef HAVE_LIBPNG
+#include <png.h>
+#endif
+
+/* Random number generator state
  */
 
-uint32_t lcg_seed;
+prng_t prng_state_data;
+prng_t *prng_state;
 
 /*----------------------------------------------------------------------------*\
  *  CRC-32 version 2.0.0 by Craig Bruce, 2006-04-29.
@@ -109,29 +137,172 @@ compute_crc32 (uint32_t    in_crc32,
     return (crc32 ^ 0xFFFFFFFF);
 }
 
+static uint32_t
+compute_crc32_for_image_internal (uint32_t        crc32,
+				  pixman_image_t *img,
+				  pixman_bool_t	  remove_alpha,
+				  pixman_bool_t	  remove_rgb)
+{
+    pixman_format_code_t fmt = pixman_image_get_format (img);
+    uint32_t *data = pixman_image_get_data (img);
+    int stride = pixman_image_get_stride (img);
+    int height = pixman_image_get_height (img);
+    uint32_t mask = 0xffffffff;
+    int i;
+
+    if (stride < 0)
+    {
+	data += (stride / 4) * (height - 1);
+	stride = - stride;
+    }
+
+    /* mask unused 'x' part */
+    if (PIXMAN_FORMAT_BPP (fmt) - PIXMAN_FORMAT_DEPTH (fmt) &&
+	PIXMAN_FORMAT_DEPTH (fmt) != 0)
+    {
+	uint32_t m = (1 << PIXMAN_FORMAT_DEPTH (fmt)) - 1;
+
+	if (PIXMAN_FORMAT_TYPE (fmt) == PIXMAN_TYPE_BGRA ||
+	    PIXMAN_FORMAT_TYPE (fmt) == PIXMAN_TYPE_RGBA)
+	{
+	    m <<= (PIXMAN_FORMAT_BPP (fmt) - PIXMAN_FORMAT_DEPTH (fmt));
+	}
+
+	mask &= m;
+    }
+
+    /* mask alpha channel */
+    if (remove_alpha && PIXMAN_FORMAT_A (fmt))
+    {
+	uint32_t m;
+
+	if (PIXMAN_FORMAT_BPP (fmt) == 32)
+	    m = 0xffffffff;
+	else
+	    m = (1 << PIXMAN_FORMAT_BPP (fmt)) - 1;
+
+	m >>= PIXMAN_FORMAT_A (fmt);
+
+	if (PIXMAN_FORMAT_TYPE (fmt) == PIXMAN_TYPE_BGRA ||
+	    PIXMAN_FORMAT_TYPE (fmt) == PIXMAN_TYPE_RGBA ||
+	    PIXMAN_FORMAT_TYPE (fmt) == PIXMAN_TYPE_A)
+	{
+	    /* Alpha is at the bottom of the pixel */
+	    m <<= PIXMAN_FORMAT_A (fmt);
+	}
+
+	mask &= m;
+    }
+
+    /* mask rgb channels */
+    if (remove_rgb && PIXMAN_FORMAT_RGB (fmt))
+    {
+	uint32_t m = ((uint32_t)~0) >> (32 - PIXMAN_FORMAT_BPP (fmt));
+	uint32_t size = PIXMAN_FORMAT_R (fmt) + PIXMAN_FORMAT_G (fmt) + PIXMAN_FORMAT_B (fmt);
+
+	m &= ~((1 << size) - 1);
+
+	if (PIXMAN_FORMAT_TYPE (fmt) == PIXMAN_TYPE_BGRA ||
+	    PIXMAN_FORMAT_TYPE (fmt) == PIXMAN_TYPE_RGBA)
+	{
+	    /* RGB channels are at the top of the pixel */
+	    m >>= size;
+	}
+
+	mask &= m;
+    }
+
+    for (i = 0; i * PIXMAN_FORMAT_BPP (fmt) < 32; i++)
+	mask |= mask << (i * PIXMAN_FORMAT_BPP (fmt));
+
+    for (i = 0; i < stride * height / 4; i++)
+	data[i] &= mask;
+
+    /* swap endiannes in order to provide identical results on both big
+     * and litte endian systems
+     */
+    image_endian_swap (img);
+
+    return compute_crc32 (crc32, data, stride * height);
+}
+
+uint32_t
+compute_crc32_for_image (uint32_t        crc32,
+			 pixman_image_t *img)
+{
+    if (img->common.alpha_map)
+    {
+	crc32 = compute_crc32_for_image_internal (crc32, img, TRUE, FALSE);
+	crc32 = compute_crc32_for_image_internal (
+	    crc32, (pixman_image_t *)img->common.alpha_map, FALSE, TRUE);
+    }
+    else
+    {
+	crc32 = compute_crc32_for_image_internal (crc32, img, FALSE, FALSE);
+    }
+
+    return crc32;
+}
+
+void
+print_image (pixman_image_t *image)
+{
+    int i, j;
+    int width, height, stride;
+    pixman_format_code_t format;
+    uint8_t *buffer;
+    int s;
+
+    width = pixman_image_get_width (image);
+    height = pixman_image_get_height (image);
+    stride = pixman_image_get_stride (image);
+    format = pixman_image_get_format (image);
+    buffer = (uint8_t *)pixman_image_get_data (image);
+
+    s = (stride >= 0)? stride : - stride;
+    
+    printf ("---\n");
+    for (i = 0; i < height; i++)
+    {
+	for (j = 0; j < s; j++)
+	{
+	    if (j == (width * PIXMAN_FORMAT_BPP (format) + 7) / 8)
+		printf ("| ");
+
+	    printf ("%02X ", *((uint8_t *)buffer + i * stride + j));
+	}
+	printf ("\n");
+    }
+    printf ("---\n");
+}
+
 /* perform endian conversion of pixel data
  */
 void
-image_endian_swap (pixman_image_t *img, int bpp)
+image_endian_swap (pixman_image_t *img)
 {
     int stride = pixman_image_get_stride (img);
     uint32_t *data = pixman_image_get_data (img);
     int height = pixman_image_get_height (img);
+    int bpp = PIXMAN_FORMAT_BPP (pixman_image_get_format (img));
     int i, j;
 
     /* swap bytes only on big endian systems */
-    volatile uint16_t endian_check_var = 0x1234;
-    if (*(volatile uint8_t *)&endian_check_var != 0x12)
+    if (is_little_endian())
+	return;
+
+    if (bpp == 8)
 	return;
 
     for (i = 0; i < height; i++)
     {
 	uint8_t *line_data = (uint8_t *)data + stride * i;
-	/* swap bytes only for 16, 24 and 32 bpp for now */
+	int s = (stride >= 0)? stride : - stride;
+    	
 	switch (bpp)
 	{
 	case 1:
-	    for (j = 0; j < stride; j++)
+	    for (j = 0; j < s; j++)
 	    {
 		line_data[j] =
 		    ((line_data[j] & 0x80) >> 7) |
@@ -145,13 +316,13 @@ image_endian_swap (pixman_image_t *img, int bpp)
 	    }
 	    break;
 	case 4:
-	    for (j = 0; j < stride; j++)
+	    for (j = 0; j < s; j++)
 	    {
 		line_data[j] = (line_data[j] >> 4) | (line_data[j] << 4);
 	    }
 	    break;
 	case 16:
-	    for (j = 0; j + 2 <= stride; j += 2)
+	    for (j = 0; j + 2 <= s; j += 2)
 	    {
 		char t1 = line_data[j + 0];
 		char t2 = line_data[j + 1];
@@ -161,7 +332,7 @@ image_endian_swap (pixman_image_t *img, int bpp)
 	    }
 	    break;
 	case 24:
-	    for (j = 0; j + 3 <= stride; j += 3)
+	    for (j = 0; j + 3 <= s; j += 3)
 	    {
 		char t1 = line_data[j + 0];
 		char t2 = line_data[j + 1];
@@ -173,7 +344,7 @@ image_endian_swap (pixman_image_t *img, int bpp)
 	    }
 	    break;
 	case 32:
-	    for (j = 0; j + 4 <= stride; j += 4)
+	    for (j = 0; j + 4 <= s; j += 4)
 	    {
 		char t1 = line_data[j + 0];
 		char t2 = line_data[j + 1];
@@ -187,22 +358,1261 @@ image_endian_swap (pixman_image_t *img, int bpp)
 	    }
 	    break;
 	default:
+	    assert (FALSE);
 	    break;
 	}
     }
 }
 
+#define N_LEADING_PROTECTED	10
+#define N_TRAILING_PROTECTED	10
+
+typedef struct
+{
+    void *addr;
+    uint32_t len;
+    uint8_t *trailing;
+    int n_bytes;
+} info_t;
+
+#if defined(HAVE_MPROTECT) && defined(HAVE_GETPAGESIZE) && defined(HAVE_SYS_MMAN_H) && defined(HAVE_MMAP)
+
+/* This is apparently necessary on at least OS X */
+#ifndef MAP_ANONYMOUS
+#define MAP_ANONYMOUS MAP_ANON
+#endif
+
+void *
+fence_malloc (int64_t len)
+{
+    unsigned long page_size = getpagesize();
+    unsigned long page_mask = page_size - 1;
+    uint32_t n_payload_bytes = (len + page_mask) & ~page_mask;
+    uint32_t n_bytes =
+	(page_size * (N_LEADING_PROTECTED + N_TRAILING_PROTECTED + 2) +
+	 n_payload_bytes) & ~page_mask;
+    uint8_t *initial_page;
+    uint8_t *leading_protected;
+    uint8_t *trailing_protected;
+    uint8_t *payload;
+    uint8_t *addr;
+
+    if (len < 0)
+	abort();
+    
+    addr = mmap (NULL, n_bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS,
+		 -1, 0);
+
+    if (addr == MAP_FAILED)
+    {
+	printf ("mmap failed on %lld %u\n", (long long int)len, n_bytes);
+	return NULL;
+    }
+
+    initial_page = (uint8_t *)(((uintptr_t)addr + page_mask) & ~page_mask);
+    leading_protected = initial_page + page_size;
+    payload = leading_protected + N_LEADING_PROTECTED * page_size;
+    trailing_protected = payload + n_payload_bytes;
+
+    ((info_t *)initial_page)->addr = addr;
+    ((info_t *)initial_page)->len = len;
+    ((info_t *)initial_page)->trailing = trailing_protected;
+    ((info_t *)initial_page)->n_bytes = n_bytes;
+
+    if ((mprotect (leading_protected, N_LEADING_PROTECTED * page_size,
+		  PROT_NONE) == -1) ||
+	(mprotect (trailing_protected, N_TRAILING_PROTECTED * page_size,
+		  PROT_NONE) == -1))
+    {
+	munmap (addr, n_bytes);
+	return NULL;
+    }
+
+    return payload;
+}
+
+void
+fence_free (void *data)
+{
+    uint32_t page_size = getpagesize();
+    uint8_t *payload = data;
+    uint8_t *leading_protected = payload - N_LEADING_PROTECTED * page_size;
+    uint8_t *initial_page = leading_protected - page_size;
+    info_t *info = (info_t *)initial_page;
+
+    munmap (info->addr, info->n_bytes);
+}
+
+#else
+
+void *
+fence_malloc (int64_t len)
+{
+    return malloc (len);
+}
+
+void
+fence_free (void *data)
+{
+    free (data);
+}
+
+#endif
+
 uint8_t *
 make_random_bytes (int n_bytes)
 {
-    uint8_t *bytes = malloc (n_bytes);
-    int i;
+    uint8_t *bytes = fence_malloc (n_bytes);
 
     if (!bytes)
 	return NULL;
 
-    for (i = 0; i < n_bytes; ++i)
-	bytes[i] = lcg_rand () & 0xff;
+    prng_randmemset (bytes, n_bytes, 0);
 
     return bytes;
 }
+
+void
+a8r8g8b8_to_rgba_np (uint32_t *dst, uint32_t *src, int n_pixels)
+{
+    uint8_t *dst8 = (uint8_t *)dst;
+    int i;
+
+    for (i = 0; i < n_pixels; ++i)
+    {
+	uint32_t p = src[i];
+	uint8_t a, r, g, b;
+
+	a = (p & 0xff000000) >> 24;
+	r = (p & 0x00ff0000) >> 16;
+	g = (p & 0x0000ff00) >> 8;
+	b = (p & 0x000000ff) >> 0;
+
+	if (a != 0)
+	{
+#define DIVIDE(c, a)							\
+	    do								\
+	    {								\
+		int t = ((c) * 255) / a;				\
+		(c) = t < 0? 0 : t > 255? 255 : t;			\
+	    } while (0)
+
+	    DIVIDE (r, a);
+	    DIVIDE (g, a);
+	    DIVIDE (b, a);
+	}
+
+	*dst8++ = r;
+	*dst8++ = g;
+	*dst8++ = b;
+	*dst8++ = a;
+    }
+}
+
+#ifdef HAVE_LIBPNG
+
+pixman_bool_t
+write_png (pixman_image_t *image, const char *filename)
+{
+    int width = pixman_image_get_width (image);
+    int height = pixman_image_get_height (image);
+    int stride = width * 4;
+    uint32_t *data = malloc (height * stride);
+    pixman_image_t *copy;
+    png_struct *write_struct;
+    png_info *info_struct;
+    pixman_bool_t result = FALSE;
+    FILE *f = fopen (filename, "wb");
+    png_bytep *row_pointers;
+    int i;
+
+    if (!f)
+	return FALSE;
+
+    row_pointers = malloc (height * sizeof (png_bytep));
+
+    copy = pixman_image_create_bits (
+	PIXMAN_a8r8g8b8, width, height, data, stride);
+
+    pixman_image_composite32 (
+	PIXMAN_OP_SRC, image, NULL, copy, 0, 0, 0, 0, 0, 0, width, height);
+
+    a8r8g8b8_to_rgba_np (data, data, height * width);
+
+    for (i = 0; i < height; ++i)
+	row_pointers[i] = (png_bytep)(data + i * width);
+
+    if (!(write_struct = png_create_write_struct (
+	      PNG_LIBPNG_VER_STRING, NULL, NULL, NULL)))
+	goto out1;
+
+    if (!(info_struct = png_create_info_struct (write_struct)))
+	goto out2;
+
+    png_init_io (write_struct, f);
+
+    png_set_IHDR (write_struct, info_struct, width, height,
+		  8, PNG_COLOR_TYPE_RGB_ALPHA,
+		  PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_BASE,
+		  PNG_FILTER_TYPE_BASE);
+
+    png_write_info (write_struct, info_struct);
+
+    png_write_image (write_struct, row_pointers);
+
+    png_write_end (write_struct, NULL);
+
+    result = TRUE;
+
+out2:
+    png_destroy_write_struct (&write_struct, &info_struct);
+
+out1:
+    if (fclose (f) != 0)
+	result = FALSE;
+
+    pixman_image_unref (copy);
+    free (row_pointers);
+    free (data);
+    return result;
+}
+
+#else /* no libpng */
+
+pixman_bool_t
+write_png (pixman_image_t *image, const char *filename)
+{
+    return FALSE;
+}
+
+#endif
+
+static void
+color8_to_color16 (uint32_t color8, pixman_color_t *color16)
+{
+    color16->alpha = ((color8 & 0xff000000) >> 24);
+    color16->red =   ((color8 & 0x00ff0000) >> 16);
+    color16->green = ((color8 & 0x0000ff00) >> 8);
+    color16->blue =  ((color8 & 0x000000ff) >> 0);
+
+    color16->alpha |= color16->alpha << 8;
+    color16->red   |= color16->red << 8;
+    color16->blue  |= color16->blue << 8;
+    color16->green |= color16->green << 8;
+}
+
+void
+draw_checkerboard (pixman_image_t *image,
+		   int check_size,
+		   uint32_t color1, uint32_t color2)
+{
+    pixman_color_t check1, check2;
+    pixman_image_t *c1, *c2;
+    int n_checks_x, n_checks_y;
+    int i, j;
+
+    color8_to_color16 (color1, &check1);
+    color8_to_color16 (color2, &check2);
+    
+    c1 = pixman_image_create_solid_fill (&check1);
+    c2 = pixman_image_create_solid_fill (&check2);
+
+    n_checks_x = (
+	pixman_image_get_width (image) + check_size - 1) / check_size;
+    n_checks_y = (
+	pixman_image_get_height (image) + check_size - 1) / check_size;
+
+    for (j = 0; j < n_checks_y; j++)
+    {
+	for (i = 0; i < n_checks_x; i++)
+	{
+	    pixman_image_t *src;
+
+	    if (((i ^ j) & 1))
+		src = c1;
+	    else
+		src = c2;
+
+	    pixman_image_composite32 (PIXMAN_OP_SRC, src, NULL, image,
+				      0, 0, 0, 0,
+				      i * check_size, j * check_size,
+				      check_size, check_size);
+	}
+    }
+}
+
+static uint32_t
+call_test_function (uint32_t    (*test_function)(int testnum, int verbose),
+		    int		testnum,
+		    int		verbose)
+{
+    uint32_t retval;
+
+#if defined (__GNUC__) && defined (_WIN32) && (defined (__i386) || defined (__i386__))
+    __asm__ (
+	/* Deliberately avoid aligning the stack to 16 bytes */
+	"pushl	%1\n\t"
+	"pushl	%2\n\t"
+	"call	*%3\n\t"
+	"addl	$8, %%esp\n\t"
+	: "=a" (retval)
+	: "r" (verbose),
+	  "r" (testnum),
+	  "r" (test_function)
+	: "edx", "ecx"); /* caller save registers */
+#else
+    retval = test_function (testnum, verbose);
+#endif
+
+    return retval;
+}
+
+/*
+ * A function, which can be used as a core part of the test programs,
+ * intended to detect various problems with the help of fuzzing input
+ * to pixman API (according to some templates, aka "smart" fuzzing).
+ * Some general information about such testing can be found here:
+ * http://en.wikipedia.org/wiki/Fuzz_testing
+ *
+ * It may help detecting:
+ *  - crashes on bad handling of valid or reasonably invalid input to
+ *    pixman API.
+ *  - deviations from the behavior of older pixman releases.
+ *  - deviations from the behavior of the same pixman release, but
+ *    configured in a different way (for example with SIMD optimizations
+ *    disabled), or running on a different OS or hardware.
+ *
+ * The test is performed by calling a callback function a huge number
+ * of times. The callback function is expected to run some snippet of
+ * pixman code with pseudorandom variations to the data feeded to
+ * pixman API. A result of running each callback function should be
+ * some deterministic value which depends on test number (test number
+ * can be used as a seed for PRNG). When 'verbose' argument is nonzero,
+ * callback function is expected to print to stdout some information
+ * about what it does.
+ *
+ * Return values from many small tests are accumulated together and
+ * used as final checksum, which can be compared to some expected
+ * value. Running the tests not individually, but in a batch helps
+ * to reduce process start overhead and also allows to parallelize
+ * testing and utilize multiple CPU cores.
+ *
+ * The resulting executable can be run without any arguments. In
+ * this case it runs a batch of tests starting from 1 and up to
+ * 'default_number_of_iterations'. The resulting checksum is
+ * compared with 'expected_checksum' and FAIL or PASS verdict
+ * depends on the result of this comparison.
+ *
+ * If the executable is run with 2 numbers provided as command line
+ * arguments, they specify the starting and ending numbers for a test
+ * batch.
+ *
+ * If the executable is run with only one number provided as a command
+ * line argument, then this number is used to call the callback function
+ * once, and also with verbose flag set.
+ */
+int
+fuzzer_test_main (const char *test_name,
+		  int         default_number_of_iterations,
+		  uint32_t    expected_checksum,
+		  uint32_t    (*test_function)(int testnum, int verbose),
+		  int         argc,
+		  const char *argv[])
+{
+    int i, n1 = 1, n2 = 0;
+    uint32_t checksum = 0;
+    int verbose = getenv ("VERBOSE") != NULL;
+
+    if (argc >= 3)
+    {
+	n1 = atoi (argv[1]);
+	n2 = atoi (argv[2]);
+	if (n2 < n1)
+	{
+	    printf ("invalid test range\n");
+	    return 1;
+	}
+    }
+    else if (argc >= 2)
+    {
+	n2 = atoi (argv[1]);
+
+	checksum = call_test_function (test_function, n2, 1);
+
+	printf ("%d: checksum=%08X\n", n2, checksum);
+	return 0;
+    }
+    else
+    {
+	n1 = 1;
+	n2 = default_number_of_iterations;
+    }
+
+#ifdef USE_OPENMP
+    #pragma omp parallel for reduction(+:checksum) default(none) \
+					shared(n1, n2, test_function, verbose)
+#endif
+    for (i = n1; i <= n2; i++)
+    {
+	uint32_t crc = call_test_function (test_function, i, 0);
+	if (verbose)
+	    printf ("%d: %08X\n", i, crc);
+	checksum += crc;
+    }
+
+    if (n1 == 1 && n2 == default_number_of_iterations)
+    {
+	if (checksum == expected_checksum)
+	{
+	    printf ("%s test passed (checksum=%08X)\n",
+		    test_name, checksum);
+	}
+	else
+	{
+	    printf ("%s test failed! (checksum=%08X, expected %08X)\n",
+		    test_name, checksum, expected_checksum);
+	    return 1;
+	}
+    }
+    else
+    {
+	printf ("%d-%d: checksum=%08X\n", n1, n2, checksum);
+    }
+
+    return 0;
+}
+
+/* Try to obtain current time in seconds */
+double
+gettime (void)
+{
+#ifdef HAVE_GETTIMEOFDAY
+    struct timeval tv;
+
+    gettimeofday (&tv, NULL);
+    return (double)((int64_t)tv.tv_sec * 1000000 + tv.tv_usec) / 1000000.;
+#else
+    return (double)clock() / (double)CLOCKS_PER_SEC;
+#endif
+}
+
+uint32_t
+get_random_seed (void)
+{
+    union { double d; uint32_t u32; } t;
+    t.d = gettime();
+    prng_srand (t.u32);
+
+    return prng_rand ();
+}
+
+#ifdef HAVE_SIGACTION
+#ifdef HAVE_ALARM
+static const char *global_msg;
+
+static void
+on_alarm (int signo)
+{
+    printf ("%s\n", global_msg);
+    exit (1);
+}
+#endif
+#endif
+
+void
+fail_after (int seconds, const char *msg)
+{
+#ifdef HAVE_SIGACTION
+#ifdef HAVE_ALARM
+    struct sigaction action;
+
+    global_msg = msg;
+
+    memset (&action, 0, sizeof (action));
+    action.sa_handler = on_alarm;
+
+    alarm (seconds);
+
+    sigaction (SIGALRM, &action, NULL);
+#endif
+#endif
+}
+
+void
+enable_divbyzero_exceptions (void)
+{
+#ifdef HAVE_FENV_H
+#ifdef HAVE_FEENABLEEXCEPT
+    feenableexcept (FE_DIVBYZERO);
+#endif
+#endif
+}
+
+void *
+aligned_malloc (size_t align, size_t size)
+{
+    void *result;
+
+#ifdef HAVE_POSIX_MEMALIGN
+    if (posix_memalign (&result, align, size) != 0)
+      result = NULL;
+#else
+    result = malloc (size);
+#endif
+
+    return result;
+}
+
+#define CONVERT_15(c, is_rgb)						\
+    (is_rgb?								\
+     ((((c) >> 3) & 0x001f) |						\
+      (((c) >> 6) & 0x03e0) |						\
+      (((c) >> 9) & 0x7c00)) :						\
+     (((((c) >> 16) & 0xff) * 153 +					\
+       (((c) >>  8) & 0xff) * 301 +					\
+       (((c)      ) & 0xff) * 58) >> 2))
+
+double
+convert_srgb_to_linear (double c)
+{
+    if (c <= 0.04045)
+        return c / 12.92;
+    else
+        return pow ((c + 0.055) / 1.055, 2.4);
+}
+
+double
+convert_linear_to_srgb (double c)
+{
+    if (c <= 0.0031308)
+        return c * 12.92;
+    else
+        return 1.055 * pow (c, 1.0/2.4) - 0.055;
+}
+
+void
+initialize_palette (pixman_indexed_t *palette, uint32_t depth, int is_rgb)
+{
+    int i;
+    uint32_t mask = (1 << depth) - 1;
+
+    for (i = 0; i < 32768; ++i)
+	palette->ent[i] = prng_rand() & mask;
+
+    memset (palette->rgba, 0, sizeof (palette->rgba));
+
+    for (i = 0; i < mask + 1; ++i)
+    {
+	uint32_t rgba24;
+ 	pixman_bool_t retry;
+	uint32_t i15;
+
+	/* We filled the rgb->index map with random numbers, but we
+	 * do need the ability to round trip, that is if some indexed
+	 * color expands to an argb24, then the 15 bit version of that
+	 * color must map back to the index. Anything else, we don't
+	 * care about too much.
+	 */
+	do
+	{
+	    uint32_t old_idx;
+
+	    rgba24 = prng_rand();
+	    i15 = CONVERT_15 (rgba24, is_rgb);
+
+	    old_idx = palette->ent[i15];
+	    if (CONVERT_15 (palette->rgba[old_idx], is_rgb) == i15)
+		retry = 1;
+	    else
+		retry = 0;
+	} while (retry);
+
+	palette->rgba[i] = rgba24;
+	palette->ent[i15] = i;
+    }
+
+    for (i = 0; i < mask + 1; ++i)
+    {
+	assert (palette->ent[CONVERT_15 (palette->rgba[i], is_rgb)] == i);
+    }
+}
+
+const char *
+operator_name (pixman_op_t op)
+{
+    switch (op)
+    {
+    case PIXMAN_OP_CLEAR: return "PIXMAN_OP_CLEAR";
+    case PIXMAN_OP_SRC: return "PIXMAN_OP_SRC";
+    case PIXMAN_OP_DST: return "PIXMAN_OP_DST";
+    case PIXMAN_OP_OVER: return "PIXMAN_OP_OVER";
+    case PIXMAN_OP_OVER_REVERSE: return "PIXMAN_OP_OVER_REVERSE";
+    case PIXMAN_OP_IN: return "PIXMAN_OP_IN";
+    case PIXMAN_OP_IN_REVERSE: return "PIXMAN_OP_IN_REVERSE";
+    case PIXMAN_OP_OUT: return "PIXMAN_OP_OUT";
+    case PIXMAN_OP_OUT_REVERSE: return "PIXMAN_OP_OUT_REVERSE";
+    case PIXMAN_OP_ATOP: return "PIXMAN_OP_ATOP";
+    case PIXMAN_OP_ATOP_REVERSE: return "PIXMAN_OP_ATOP_REVERSE";
+    case PIXMAN_OP_XOR: return "PIXMAN_OP_XOR";
+    case PIXMAN_OP_ADD: return "PIXMAN_OP_ADD";
+    case PIXMAN_OP_SATURATE: return "PIXMAN_OP_SATURATE";
+
+    case PIXMAN_OP_DISJOINT_CLEAR: return "PIXMAN_OP_DISJOINT_CLEAR";
+    case PIXMAN_OP_DISJOINT_SRC: return "PIXMAN_OP_DISJOINT_SRC";
+    case PIXMAN_OP_DISJOINT_DST: return "PIXMAN_OP_DISJOINT_DST";
+    case PIXMAN_OP_DISJOINT_OVER: return "PIXMAN_OP_DISJOINT_OVER";
+    case PIXMAN_OP_DISJOINT_OVER_REVERSE: return "PIXMAN_OP_DISJOINT_OVER_REVERSE";
+    case PIXMAN_OP_DISJOINT_IN: return "PIXMAN_OP_DISJOINT_IN";
+    case PIXMAN_OP_DISJOINT_IN_REVERSE: return "PIXMAN_OP_DISJOINT_IN_REVERSE";
+    case PIXMAN_OP_DISJOINT_OUT: return "PIXMAN_OP_DISJOINT_OUT";
+    case PIXMAN_OP_DISJOINT_OUT_REVERSE: return "PIXMAN_OP_DISJOINT_OUT_REVERSE";
+    case PIXMAN_OP_DISJOINT_ATOP: return "PIXMAN_OP_DISJOINT_ATOP";
+    case PIXMAN_OP_DISJOINT_ATOP_REVERSE: return "PIXMAN_OP_DISJOINT_ATOP_REVERSE";
+    case PIXMAN_OP_DISJOINT_XOR: return "PIXMAN_OP_DISJOINT_XOR";
+
+    case PIXMAN_OP_CONJOINT_CLEAR: return "PIXMAN_OP_CONJOINT_CLEAR";
+    case PIXMAN_OP_CONJOINT_SRC: return "PIXMAN_OP_CONJOINT_SRC";
+    case PIXMAN_OP_CONJOINT_DST: return "PIXMAN_OP_CONJOINT_DST";
+    case PIXMAN_OP_CONJOINT_OVER: return "PIXMAN_OP_CONJOINT_OVER";
+    case PIXMAN_OP_CONJOINT_OVER_REVERSE: return "PIXMAN_OP_CONJOINT_OVER_REVERSE";
+    case PIXMAN_OP_CONJOINT_IN: return "PIXMAN_OP_CONJOINT_IN";
+    case PIXMAN_OP_CONJOINT_IN_REVERSE: return "PIXMAN_OP_CONJOINT_IN_REVERSE";
+    case PIXMAN_OP_CONJOINT_OUT: return "PIXMAN_OP_CONJOINT_OUT";
+    case PIXMAN_OP_CONJOINT_OUT_REVERSE: return "PIXMAN_OP_CONJOINT_OUT_REVERSE";
+    case PIXMAN_OP_CONJOINT_ATOP: return "PIXMAN_OP_CONJOINT_ATOP";
+    case PIXMAN_OP_CONJOINT_ATOP_REVERSE: return "PIXMAN_OP_CONJOINT_ATOP_REVERSE";
+    case PIXMAN_OP_CONJOINT_XOR: return "PIXMAN_OP_CONJOINT_XOR";
+
+    case PIXMAN_OP_MULTIPLY: return "PIXMAN_OP_MULTIPLY";
+    case PIXMAN_OP_SCREEN: return "PIXMAN_OP_SCREEN";
+    case PIXMAN_OP_OVERLAY: return "PIXMAN_OP_OVERLAY";
+    case PIXMAN_OP_DARKEN: return "PIXMAN_OP_DARKEN";
+    case PIXMAN_OP_LIGHTEN: return "PIXMAN_OP_LIGHTEN";
+    case PIXMAN_OP_COLOR_DODGE: return "PIXMAN_OP_COLOR_DODGE";
+    case PIXMAN_OP_COLOR_BURN: return "PIXMAN_OP_COLOR_BURN";
+    case PIXMAN_OP_HARD_LIGHT: return "PIXMAN_OP_HARD_LIGHT";
+    case PIXMAN_OP_SOFT_LIGHT: return "PIXMAN_OP_SOFT_LIGHT";
+    case PIXMAN_OP_DIFFERENCE: return "PIXMAN_OP_DIFFERENCE";
+    case PIXMAN_OP_EXCLUSION: return "PIXMAN_OP_EXCLUSION";
+    case PIXMAN_OP_HSL_HUE: return "PIXMAN_OP_HSL_HUE";
+    case PIXMAN_OP_HSL_SATURATION: return "PIXMAN_OP_HSL_SATURATION";
+    case PIXMAN_OP_HSL_COLOR: return "PIXMAN_OP_HSL_COLOR";
+    case PIXMAN_OP_HSL_LUMINOSITY: return "PIXMAN_OP_HSL_LUMINOSITY";
+
+    case PIXMAN_OP_NONE:
+	return "<invalid operator 'none'>";
+    };
+
+    return "<unknown operator>";
+}
+
+const char *
+format_name (pixman_format_code_t format)
+{
+    switch (format)
+    {
+/* 32bpp formats */
+    case PIXMAN_a8r8g8b8: return "a8r8g8b8";
+    case PIXMAN_x8r8g8b8: return "x8r8g8b8";
+    case PIXMAN_a8b8g8r8: return "a8b8g8r8";
+    case PIXMAN_x8b8g8r8: return "x8b8g8r8";
+    case PIXMAN_b8g8r8a8: return "b8g8r8a8";
+    case PIXMAN_b8g8r8x8: return "b8g8r8x8";
+    case PIXMAN_r8g8b8a8: return "r8g8b8a8";
+    case PIXMAN_r8g8b8x8: return "r8g8b8x8";
+    case PIXMAN_x14r6g6b6: return "x14r6g6b6";
+    case PIXMAN_x2r10g10b10: return "x2r10g10b10";
+    case PIXMAN_a2r10g10b10: return "a2r10g10b10";
+    case PIXMAN_x2b10g10r10: return "x2b10g10r10";
+    case PIXMAN_a2b10g10r10: return "a2b10g10r10";
+
+/* sRGB formats */
+    case PIXMAN_a8r8g8b8_sRGB: return "a8r8g8b8_sRGB";
+
+/* 24bpp formats */
+    case PIXMAN_r8g8b8: return "r8g8b8";
+    case PIXMAN_b8g8r8: return "b8g8r8";
+
+/* 16bpp formats */
+    case PIXMAN_r5g6b5: return "r5g6b5";
+    case PIXMAN_b5g6r5: return "b5g6r5";
+
+    case PIXMAN_a1r5g5b5: return "a1r5g5b5";
+    case PIXMAN_x1r5g5b5: return "x1r5g5b5";
+    case PIXMAN_a1b5g5r5: return "a1b5g5r5";
+    case PIXMAN_x1b5g5r5: return "x1b5g5r5";
+    case PIXMAN_a4r4g4b4: return "a4r4g4b4";
+    case PIXMAN_x4r4g4b4: return "x4r4g4b4";
+    case PIXMAN_a4b4g4r4: return "a4b4g4r4";
+    case PIXMAN_x4b4g4r4: return "x4b4g4r4";
+
+/* 8bpp formats */
+    case PIXMAN_a8: return "a8";
+    case PIXMAN_r3g3b2: return "r3g3b2";
+    case PIXMAN_b2g3r3: return "b2g3r3";
+    case PIXMAN_a2r2g2b2: return "a2r2g2b2";
+    case PIXMAN_a2b2g2r2: return "a2b2g2r2";
+
+#if 0
+    case PIXMAN_x4c4: return "x4c4";
+    case PIXMAN_g8: return "g8";
+#endif
+    case PIXMAN_c8: return "x4c4 / c8";
+    case PIXMAN_x4g4: return "x4g4 / g8";
+
+    case PIXMAN_x4a4: return "x4a4";
+
+/* 4bpp formats */
+    case PIXMAN_a4: return "a4";
+    case PIXMAN_r1g2b1: return "r1g2b1";
+    case PIXMAN_b1g2r1: return "b1g2r1";
+    case PIXMAN_a1r1g1b1: return "a1r1g1b1";
+    case PIXMAN_a1b1g1r1: return "a1b1g1r1";
+
+    case PIXMAN_c4: return "c4";
+    case PIXMAN_g4: return "g4";
+
+/* 1bpp formats */
+    case PIXMAN_a1: return "a1";
+
+    case PIXMAN_g1: return "g1";
+
+/* YUV formats */
+    case PIXMAN_yuy2: return "yuy2";
+    case PIXMAN_yv12: return "yv12";
+    };
+
+    /* Fake formats.
+     *
+     * This is separate switch to prevent GCC from complaining
+     * that the values are not in the pixman_format_code_t enum.
+     */
+    switch ((uint32_t)format)
+    {
+    case PIXMAN_null: return "null"; 
+    case PIXMAN_solid: return "solid"; 
+    case PIXMAN_pixbuf: return "pixbuf"; 
+    case PIXMAN_rpixbuf: return "rpixbuf"; 
+    case PIXMAN_unknown: return "unknown"; 
+    };
+
+    return "<unknown format>";
+};
+
+static double
+calc_op (pixman_op_t op, double src, double dst, double srca, double dsta)
+{
+#define mult_chan(src, dst, Fa, Fb) MIN ((src) * (Fa) + (dst) * (Fb), 1.0)
+
+    double Fa, Fb;
+
+    switch (op)
+    {
+    case PIXMAN_OP_CLEAR:
+    case PIXMAN_OP_DISJOINT_CLEAR:
+    case PIXMAN_OP_CONJOINT_CLEAR:
+	return mult_chan (src, dst, 0.0, 0.0);
+
+    case PIXMAN_OP_SRC:
+    case PIXMAN_OP_DISJOINT_SRC:
+    case PIXMAN_OP_CONJOINT_SRC:
+	return mult_chan (src, dst, 1.0, 0.0);
+
+    case PIXMAN_OP_DST:
+    case PIXMAN_OP_DISJOINT_DST:
+    case PIXMAN_OP_CONJOINT_DST:
+	return mult_chan (src, dst, 0.0, 1.0);
+
+    case PIXMAN_OP_OVER:
+	return mult_chan (src, dst, 1.0, 1.0 - srca);
+
+    case PIXMAN_OP_OVER_REVERSE:
+	return mult_chan (src, dst, 1.0 - dsta, 1.0);
+
+    case PIXMAN_OP_IN:
+	return mult_chan (src, dst, dsta, 0.0);
+
+    case PIXMAN_OP_IN_REVERSE:
+	return mult_chan (src, dst, 0.0, srca);
+
+    case PIXMAN_OP_OUT:
+	return mult_chan (src, dst, 1.0 - dsta, 0.0);
+
+    case PIXMAN_OP_OUT_REVERSE:
+	return mult_chan (src, dst, 0.0, 1.0 - srca);
+
+    case PIXMAN_OP_ATOP:
+	return mult_chan (src, dst, dsta, 1.0 - srca);
+
+    case PIXMAN_OP_ATOP_REVERSE:
+	return mult_chan (src, dst, 1.0 - dsta,  srca);
+
+    case PIXMAN_OP_XOR:
+	return mult_chan (src, dst, 1.0 - dsta, 1.0 - srca);
+
+    case PIXMAN_OP_ADD:
+	return mult_chan (src, dst, 1.0, 1.0);
+
+    case PIXMAN_OP_SATURATE:
+    case PIXMAN_OP_DISJOINT_OVER_REVERSE:
+	if (srca == 0.0)
+	    Fa = 1.0;
+	else
+	    Fa = MIN (1.0, (1.0 - dsta) / srca);
+	return mult_chan (src, dst, Fa, 1.0);
+
+    case PIXMAN_OP_DISJOINT_OVER:
+	if (dsta == 0.0)
+	    Fb = 1.0;
+	else
+	    Fb = MIN (1.0, (1.0 - srca) / dsta);
+	return mult_chan (src, dst, 1.0, Fb);
+
+    case PIXMAN_OP_DISJOINT_IN:
+	if (srca == 0.0)
+	    Fa = 0.0;
+	else
+	    Fa = MAX (0.0, 1.0 - (1.0 - dsta) / srca);
+	return mult_chan (src, dst, Fa, 0.0);
+
+    case PIXMAN_OP_DISJOINT_IN_REVERSE:
+	if (dsta == 0.0)
+	    Fb = 0.0;
+	else
+	    Fb = MAX (0.0, 1.0 - (1.0 - srca) / dsta);
+	return mult_chan (src, dst, 0.0, Fb);
+
+    case PIXMAN_OP_DISJOINT_OUT:
+	if (srca == 0.0)
+	    Fa = 1.0;
+	else
+	    Fa = MIN (1.0, (1.0 - dsta) / srca);
+	return mult_chan (src, dst, Fa, 0.0);
+
+    case PIXMAN_OP_DISJOINT_OUT_REVERSE:
+	if (dsta == 0.0)
+	    Fb = 1.0;
+	else
+	    Fb = MIN (1.0, (1.0 - srca) / dsta);
+	return mult_chan (src, dst, 0.0, Fb);
+
+    case PIXMAN_OP_DISJOINT_ATOP:
+	if (srca == 0.0)
+	    Fa = 0.0;
+	else
+	    Fa = MAX (0.0, 1.0 - (1.0 - dsta) / srca);
+	if (dsta == 0.0)
+	    Fb = 1.0;
+	else
+	    Fb = MIN (1.0, (1.0 - srca) / dsta);
+	return mult_chan (src, dst, Fa, Fb);
+
+    case PIXMAN_OP_DISJOINT_ATOP_REVERSE:
+	if (srca == 0.0)
+	    Fa = 1.0;
+	else
+	    Fa = MIN (1.0, (1.0 - dsta) / srca);
+	if (dsta == 0.0)
+	    Fb = 0.0;
+	else
+	    Fb = MAX (0.0, 1.0 - (1.0 - srca) / dsta);
+	return mult_chan (src, dst, Fa, Fb);
+
+    case PIXMAN_OP_DISJOINT_XOR:
+	if (srca == 0.0)
+	    Fa = 1.0;
+	else
+	    Fa = MIN (1.0, (1.0 - dsta) / srca);
+	if (dsta == 0.0)
+	    Fb = 1.0;
+	else
+	    Fb = MIN (1.0, (1.0 - srca) / dsta);
+	return mult_chan (src, dst, Fa, Fb);
+
+    case PIXMAN_OP_CONJOINT_OVER:
+	if (dsta == 0.0)
+	    Fb = 0.0;
+	else
+	    Fb = MAX (0.0, 1.0 - srca / dsta);
+	return mult_chan (src, dst, 1.0, Fb);
+
+    case PIXMAN_OP_CONJOINT_OVER_REVERSE:
+	if (srca == 0.0)
+	    Fa = 0.0;
+	else
+	    Fa = MAX (0.0, 1.0 - dsta / srca);
+	return mult_chan (src, dst, Fa, 1.0);
+
+    case PIXMAN_OP_CONJOINT_IN:
+	if (srca == 0.0)
+	    Fa = 1.0;
+	else
+	    Fa = MIN (1.0, dsta / srca);
+	return mult_chan (src, dst, Fa, 0.0);
+
+    case PIXMAN_OP_CONJOINT_IN_REVERSE:
+	if (dsta == 0.0)
+	    Fb = 1.0;
+	else
+	    Fb = MIN (1.0, srca / dsta);
+	return mult_chan (src, dst, 0.0, Fb);
+
+    case PIXMAN_OP_CONJOINT_OUT:
+	if (srca == 0.0)
+	    Fa = 0.0;
+	else
+	    Fa = MAX (0.0, 1.0 - dsta / srca);
+	return mult_chan (src, dst, Fa, 0.0);
+
+    case PIXMAN_OP_CONJOINT_OUT_REVERSE:
+	if (dsta == 0.0)
+	    Fb = 0.0;
+	else
+	    Fb = MAX (0.0, 1.0 - srca / dsta);
+	return mult_chan (src, dst, 0.0, Fb);
+
+    case PIXMAN_OP_CONJOINT_ATOP:
+	if (srca == 0.0)
+	    Fa = 1.0;
+	else
+	    Fa = MIN (1.0, dsta / srca);
+	if (dsta == 0.0)
+	    Fb = 0.0;
+	else
+	    Fb = MAX (0.0, 1.0 - srca / dsta);
+	return mult_chan (src, dst, Fa, Fb);
+
+    case PIXMAN_OP_CONJOINT_ATOP_REVERSE:
+	if (srca == 0.0)
+	    Fa = 0.0;
+	else
+	    Fa = MAX (0.0, 1.0 - dsta / srca);
+	if (dsta == 0.0)
+	    Fb = 1.0;
+	else
+	    Fb = MIN (1.0, srca / dsta);
+	return mult_chan (src, dst, Fa, Fb);
+
+    case PIXMAN_OP_CONJOINT_XOR:
+	if (srca == 0.0)
+	    Fa = 0.0;
+	else
+	    Fa = MAX (0.0, 1.0 - dsta / srca);
+	if (dsta == 0.0)
+	    Fb = 0.0;
+	else
+	    Fb = MAX (0.0, 1.0 - srca / dsta);
+	return mult_chan (src, dst, Fa, Fb);
+
+    case PIXMAN_OP_MULTIPLY:
+    case PIXMAN_OP_SCREEN:
+    case PIXMAN_OP_OVERLAY:
+    case PIXMAN_OP_DARKEN:
+    case PIXMAN_OP_LIGHTEN:
+    case PIXMAN_OP_COLOR_DODGE:
+    case PIXMAN_OP_COLOR_BURN:
+    case PIXMAN_OP_HARD_LIGHT:
+    case PIXMAN_OP_SOFT_LIGHT:
+    case PIXMAN_OP_DIFFERENCE:
+    case PIXMAN_OP_EXCLUSION:
+    case PIXMAN_OP_HSL_HUE:
+    case PIXMAN_OP_HSL_SATURATION:
+    case PIXMAN_OP_HSL_COLOR:
+    case PIXMAN_OP_HSL_LUMINOSITY:
+    default:
+	abort();
+	return 0; /* silence MSVC */
+    }
+#undef mult_chan
+}
+
+void
+do_composite (pixman_op_t op,
+	      const color_t *src,
+	      const color_t *mask,
+	      const color_t *dst,
+	      color_t *result,
+	      pixman_bool_t component_alpha)
+{
+    color_t srcval, srcalpha;
+
+    if (mask == NULL)
+    {
+	srcval = *src;
+
+	srcalpha.r = src->a;
+	srcalpha.g = src->a;
+	srcalpha.b = src->a;
+	srcalpha.a = src->a;
+    }
+    else if (component_alpha)
+    {
+	srcval.r = src->r * mask->r;
+	srcval.g = src->g * mask->g;
+	srcval.b = src->b * mask->b;
+	srcval.a = src->a * mask->a;
+
+	srcalpha.r = src->a * mask->r;
+	srcalpha.g = src->a * mask->g;
+	srcalpha.b = src->a * mask->b;
+	srcalpha.a = src->a * mask->a;
+    }
+    else
+    {
+	srcval.r = src->r * mask->a;
+	srcval.g = src->g * mask->a;
+	srcval.b = src->b * mask->a;
+	srcval.a = src->a * mask->a;
+
+	srcalpha.r = src->a * mask->a;
+	srcalpha.g = src->a * mask->a;
+	srcalpha.b = src->a * mask->a;
+	srcalpha.a = src->a * mask->a;
+    }
+
+    result->r = calc_op (op, srcval.r, dst->r, srcalpha.r, dst->a);
+    result->g = calc_op (op, srcval.g, dst->g, srcalpha.g, dst->a);
+    result->b = calc_op (op, srcval.b, dst->b, srcalpha.b, dst->a);
+    result->a = calc_op (op, srcval.a, dst->a, srcalpha.a, dst->a);
+}
+
+static double
+round_channel (double p, int m)
+{
+    int t;
+    double r;
+
+    t = p * ((1 << m));
+    t -= t >> m;
+
+    r = t / (double)((1 << m) - 1);
+
+    return r;
+}
+
+void
+round_color (pixman_format_code_t format, color_t *color)
+{
+    if (PIXMAN_FORMAT_R (format) == 0)
+    {
+	color->r = 0.0;
+	color->g = 0.0;
+	color->b = 0.0;
+    }
+    else
+    {
+	color->r = round_channel (color->r, PIXMAN_FORMAT_R (format));
+	color->g = round_channel (color->g, PIXMAN_FORMAT_G (format));
+	color->b = round_channel (color->b, PIXMAN_FORMAT_B (format));
+    }
+
+    if (PIXMAN_FORMAT_A (format) == 0)
+	color->a = 1;
+    else
+	color->a = round_channel (color->a, PIXMAN_FORMAT_A (format));
+}
+
+/* Check whether @pixel is a valid quantization of the a, r, g, b
+ * parameters. Some slack is permitted.
+ */
+void
+pixel_checker_init (pixel_checker_t *checker, pixman_format_code_t format)
+{
+    assert (PIXMAN_FORMAT_VIS (format));
+
+    checker->format = format;
+
+    switch (PIXMAN_FORMAT_TYPE (format))
+    {
+    case PIXMAN_TYPE_A:
+	checker->bs = 0;
+	checker->gs = 0;
+	checker->rs = 0;
+	checker->as = 0;
+	break;
+
+    case PIXMAN_TYPE_ARGB:
+    case PIXMAN_TYPE_ARGB_SRGB:
+	checker->bs = 0;
+	checker->gs = checker->bs + PIXMAN_FORMAT_B (format);
+	checker->rs = checker->gs + PIXMAN_FORMAT_G (format);
+	checker->as = checker->rs + PIXMAN_FORMAT_R (format);
+	break;
+
+    case PIXMAN_TYPE_ABGR:
+	checker->rs = 0;
+	checker->gs = checker->rs + PIXMAN_FORMAT_R (format);
+	checker->bs = checker->gs + PIXMAN_FORMAT_G (format);
+	checker->as = checker->bs + PIXMAN_FORMAT_B (format);
+	break;
+
+    case PIXMAN_TYPE_BGRA:
+	/* With BGRA formats we start counting at the high end of the pixel */
+	checker->bs = PIXMAN_FORMAT_BPP (format) - PIXMAN_FORMAT_B (format);
+	checker->gs = checker->bs - PIXMAN_FORMAT_B (format);
+	checker->rs = checker->gs - PIXMAN_FORMAT_G (format);
+	checker->as = checker->rs - PIXMAN_FORMAT_R (format);
+	break;
+
+    case PIXMAN_TYPE_RGBA:
+	/* With BGRA formats we start counting at the high end of the pixel */
+	checker->rs = PIXMAN_FORMAT_BPP (format) - PIXMAN_FORMAT_R (format);
+	checker->gs = checker->rs - PIXMAN_FORMAT_R (format);
+	checker->bs = checker->gs - PIXMAN_FORMAT_G (format);
+	checker->as = checker->bs - PIXMAN_FORMAT_B (format);
+	break;
+
+    default:
+	assert (0);
+	break;
+    }
+
+    checker->am = ((1 << PIXMAN_FORMAT_A (format)) - 1) << checker->as;
+    checker->rm = ((1 << PIXMAN_FORMAT_R (format)) - 1) << checker->rs;
+    checker->gm = ((1 << PIXMAN_FORMAT_G (format)) - 1) << checker->gs;
+    checker->bm = ((1 << PIXMAN_FORMAT_B (format)) - 1) << checker->bs;
+
+    checker->aw = PIXMAN_FORMAT_A (format);
+    checker->rw = PIXMAN_FORMAT_R (format);
+    checker->gw = PIXMAN_FORMAT_G (format);
+    checker->bw = PIXMAN_FORMAT_B (format);
+}
+
+void
+pixel_checker_split_pixel (const pixel_checker_t *checker, uint32_t pixel,
+			   int *a, int *r, int *g, int *b)
+{
+    *a = (pixel & checker->am) >> checker->as;
+    *r = (pixel & checker->rm) >> checker->rs;
+    *g = (pixel & checker->gm) >> checker->gs;
+    *b = (pixel & checker->bm) >> checker->bs;
+}
+
+void
+pixel_checker_get_masks (const pixel_checker_t *checker,
+                         uint32_t              *am,
+                         uint32_t              *rm,
+                         uint32_t              *gm,
+                         uint32_t              *bm)
+{
+    if (am)
+        *am = checker->am;
+    if (rm)
+        *rm = checker->rm;
+    if (gm)
+        *gm = checker->gm;
+    if (bm)
+        *bm = checker->bm;
+}
+
+void
+pixel_checker_convert_pixel_to_color (const pixel_checker_t *checker,
+                                      uint32_t pixel, color_t *color)
+{
+    int a, r, g, b;
+
+    pixel_checker_split_pixel (checker, pixel, &a, &r, &g, &b);
+
+    if (checker->am == 0)
+        color->a = 1.0;
+    else
+        color->a = a / (double)(checker->am >> checker->as);
+
+    if (checker->rm == 0)
+        color->r = 0.0;
+    else
+        color->r = r / (double)(checker->rm >> checker->rs);
+
+    if (checker->gm == 0)
+        color->g = 0.0;
+    else
+        color->g = g / (double)(checker->gm >> checker->gs);
+
+    if (checker->bm == 0)
+        color->b = 0.0;
+    else
+        color->b = b / (double)(checker->bm >> checker->bs);
+
+    if (PIXMAN_FORMAT_TYPE (checker->format) == PIXMAN_TYPE_ARGB_SRGB)
+    {
+	color->r = convert_srgb_to_linear (color->r);
+	color->g = convert_srgb_to_linear (color->g);
+	color->b = convert_srgb_to_linear (color->b);
+    }
+}
+
+static int32_t
+convert (double v, uint32_t width, uint32_t mask, uint32_t shift, double def)
+{
+    int32_t r;
+
+    if (!mask)
+	v = def;
+
+    r = (v * ((mask >> shift) + 1));
+    r -= r >> width;
+
+    return r;
+}
+
+static void
+get_limits (const pixel_checker_t *checker, double limit,
+	    color_t *color,
+	    int *ao, int *ro, int *go, int *bo)
+{
+    color_t tmp;
+
+    if (PIXMAN_FORMAT_TYPE (checker->format) == PIXMAN_TYPE_ARGB_SRGB)
+    {
+	tmp.a = color->a;
+	tmp.r = convert_linear_to_srgb (color->r);
+	tmp.g = convert_linear_to_srgb (color->g);
+	tmp.b = convert_linear_to_srgb (color->b);
+
+	color = &tmp;
+    }
+    
+    *ao = convert (color->a + limit, checker->aw, checker->am, checker->as, 1.0);
+    *ro = convert (color->r + limit, checker->rw, checker->rm, checker->rs, 0.0);
+    *go = convert (color->g + limit, checker->gw, checker->gm, checker->gs, 0.0);
+    *bo = convert (color->b + limit, checker->bw, checker->bm, checker->bs, 0.0);
+}
+
+/* The acceptable deviation in units of [0.0, 1.0]
+ */
+#define DEVIATION (0.0064)
+
+void
+pixel_checker_get_max (const pixel_checker_t *checker, color_t *color,
+		       int *am, int *rm, int *gm, int *bm)
+{
+    get_limits (checker, DEVIATION, color, am, rm, gm, bm);
+}
+
+void
+pixel_checker_get_min (const pixel_checker_t *checker, color_t *color,
+		       int *am, int *rm, int *gm, int *bm)
+{
+    get_limits (checker, - DEVIATION, color, am, rm, gm, bm);
+}
+
+pixman_bool_t
+pixel_checker_check (const pixel_checker_t *checker, uint32_t pixel,
+		     color_t *color)
+{
+    int32_t a_lo, a_hi, r_lo, r_hi, g_lo, g_hi, b_lo, b_hi;
+    int32_t ai, ri, gi, bi;
+    pixman_bool_t result;
+
+    pixel_checker_get_min (checker, color, &a_lo, &r_lo, &g_lo, &b_lo);
+    pixel_checker_get_max (checker, color, &a_hi, &r_hi, &g_hi, &b_hi);
+    pixel_checker_split_pixel (checker, pixel, &ai, &ri, &gi, &bi);
+
+    result =
+	a_lo <= ai && ai <= a_hi	&&
+	r_lo <= ri && ri <= r_hi	&&
+	g_lo <= gi && gi <= g_hi	&&
+	b_lo <= bi && bi <= b_hi;
+
+    return result;
+}
diff --git a/pixman/test/utils.h b/pixman/test/utils.h
index fb1ccec48..ebb14d9e4 100644
--- a/pixman/test/utils.h
+++ b/pixman/test/utils.h
@@ -1,32 +1,61 @@
-#include <stdlib.h>
+#ifdef HAVE_CONFIG_H
 #include <config.h>
+#endif
+
+#include <assert.h>
 #include "pixman-private.h" /* For 'inline' definition */
+#include "utils-prng.h"
+
+#if defined(_MSC_VER)
+#define snprintf _snprintf
+#define strcasecmp _stricmp
+#endif
+
+#define ARRAY_LENGTH(A) ((int) (sizeof (A) / sizeof ((A) [0])))
 
 /* A primitive pseudorandom number generator,
  * taken from POSIX.1-2001 example
  */
 
-extern uint32_t lcg_seed;
+extern prng_t prng_state_data;
+extern prng_t *prng_state;
+#ifdef USE_OPENMP
+#pragma omp threadprivate(prng_state_data)
+#pragma omp threadprivate(prng_state)
+#endif
 
 static inline uint32_t
-lcg_rand (void)
+prng_rand (void)
 {
-    lcg_seed = lcg_seed * 1103515245 + 12345;
-    return ((uint32_t)(lcg_seed / 65536) % 32768);
+    return prng_rand_r (prng_state);
 }
 
 static inline void
-lcg_srand (uint32_t seed)
+prng_srand (uint32_t seed)
 {
-    lcg_seed = seed;
+    if (!prng_state)
+    {
+        /* Without setting a seed, PRNG does not work properly (is just
+         * returning zeros). So we only initialize the pointer here to
+         * make sure that 'prng_srand' is always called before any
+         * other 'prng_*' function. The wrongdoers violating this order
+         * will get a segfault. */
+        prng_state = &prng_state_data;
+    }
+    prng_srand_r (prng_state, seed);
 }
 
 static inline uint32_t
-lcg_rand_n (int max)
+prng_rand_n (int max)
 {
-    return lcg_rand () % max;
+    return prng_rand () % max;
 }
 
+static inline void
+prng_randmemset (void *buffer, size_t size, prng_randmemset_flags_t flags)
+{
+    prng_randmemset_r (prng_state, buffer, size, flags);
+}
 
 /* CRC 32 computation
  */
@@ -35,11 +64,184 @@ compute_crc32 (uint32_t    in_crc32,
 	       const void *buf,
 	       size_t      buf_len);
 
+uint32_t
+compute_crc32_for_image (uint32_t        in_crc32,
+			 pixman_image_t *image);
+
+/* Print the image in hexadecimal */
+void
+print_image (pixman_image_t *image);
+
+/* Returns TRUE if running on a little endian system
+ */
+static force_inline pixman_bool_t
+is_little_endian (void)
+{
+    unsigned long endian_check_var = 1;
+    return *(unsigned char *)&endian_check_var == 1;
+}
+
 /* perform endian conversion of pixel data
  */
 void
-image_endian_swap (pixman_image_t *img, int bpp);
+image_endian_swap (pixman_image_t *img);
+
+/* Allocate memory that is bounded by protected pages,
+ * so that out-of-bounds access will cause segfaults
+ */
+void *
+fence_malloc (int64_t len);
+
+void
+fence_free (void *data);
 
-/* Generate n_bytes random bytes in malloced memory */
+/* Generate n_bytes random bytes in fence_malloced memory */
 uint8_t *
 make_random_bytes (int n_bytes);
+
+/* Return current time in seconds */
+double
+gettime (void);
+
+uint32_t
+get_random_seed (void);
+
+/* main body of the fuzzer test */
+int
+fuzzer_test_main (const char *test_name,
+		  int         default_number_of_iterations,
+		  uint32_t    expected_checksum,
+		  uint32_t    (*test_function)(int testnum, int verbose),
+		  int         argc,
+		  const char *argv[]);
+
+void
+fail_after (int seconds, const char *msg);
+
+/* If possible, enable traps for floating point exceptions */
+void enable_divbyzero_exceptions(void);
+
+/* Converts a8r8g8b8 pixels to pixels that
+ *  - are not premultiplied,
+ *  - are stored in this order in memory: R, G, B, A, regardless of
+ *    the endianness of the computer.
+ * It is allowed for @src and @dst to point to the same memory buffer.
+ */
+void
+a8r8g8b8_to_rgba_np (uint32_t *dst, uint32_t *src, int n_pixels);
+
+pixman_bool_t
+write_png (pixman_image_t *image, const char *filename);
+
+void
+draw_checkerboard (pixman_image_t *image,
+		   int check_size,
+		   uint32_t color1, uint32_t color2);
+
+/* A pair of macros which can help to detect corruption of
+ * floating point registers after a function call. This may
+ * happen if _mm_empty() call is forgotten in MMX/SSE2 fast
+ * path code, or ARM NEON assembly optimized function forgets
+ * to save/restore d8-d15 registers before use.
+ */
+
+#define FLOAT_REGS_CORRUPTION_DETECTOR_START()                 \
+    static volatile double frcd_volatile_constant1 = 123451;   \
+    static volatile double frcd_volatile_constant2 = 123452;   \
+    static volatile double frcd_volatile_constant3 = 123453;   \
+    static volatile double frcd_volatile_constant4 = 123454;   \
+    static volatile double frcd_volatile_constant5 = 123455;   \
+    static volatile double frcd_volatile_constant6 = 123456;   \
+    static volatile double frcd_volatile_constant7 = 123457;   \
+    static volatile double frcd_volatile_constant8 = 123458;   \
+    double frcd_canary_variable1 = frcd_volatile_constant1;    \
+    double frcd_canary_variable2 = frcd_volatile_constant2;    \
+    double frcd_canary_variable3 = frcd_volatile_constant3;    \
+    double frcd_canary_variable4 = frcd_volatile_constant4;    \
+    double frcd_canary_variable5 = frcd_volatile_constant5;    \
+    double frcd_canary_variable6 = frcd_volatile_constant6;    \
+    double frcd_canary_variable7 = frcd_volatile_constant7;    \
+    double frcd_canary_variable8 = frcd_volatile_constant8;
+
+#define FLOAT_REGS_CORRUPTION_DETECTOR_FINISH()                \
+    assert (frcd_canary_variable1 == frcd_volatile_constant1); \
+    assert (frcd_canary_variable2 == frcd_volatile_constant2); \
+    assert (frcd_canary_variable3 == frcd_volatile_constant3); \
+    assert (frcd_canary_variable4 == frcd_volatile_constant4); \
+    assert (frcd_canary_variable5 == frcd_volatile_constant5); \
+    assert (frcd_canary_variable6 == frcd_volatile_constant6); \
+    assert (frcd_canary_variable7 == frcd_volatile_constant7); \
+    assert (frcd_canary_variable8 == frcd_volatile_constant8);
+
+/* Try to get an aligned memory chunk */
+void *
+aligned_malloc (size_t align, size_t size);
+
+double
+convert_srgb_to_linear (double component);
+
+double
+convert_linear_to_srgb (double component);
+
+void
+initialize_palette (pixman_indexed_t *palette, uint32_t depth, int is_rgb);
+
+const char *
+operator_name (pixman_op_t op);
+
+const char *
+format_name (pixman_format_code_t format);
+
+typedef struct
+{
+    double r, g, b, a;
+} color_t;
+
+void
+do_composite (pixman_op_t op,
+	      const color_t *src,
+	      const color_t *mask,
+	      const color_t *dst,
+	      color_t *result,
+	      pixman_bool_t component_alpha);
+
+void
+round_color (pixman_format_code_t format, color_t *color);
+
+typedef struct
+{
+    pixman_format_code_t format;
+    uint32_t am, rm, gm, bm;
+    uint32_t as, rs, gs, bs;
+    uint32_t aw, rw, gw, bw;
+} pixel_checker_t;
+
+void
+pixel_checker_init (pixel_checker_t *checker, pixman_format_code_t format);
+
+void
+pixel_checker_split_pixel (const pixel_checker_t *checker, uint32_t pixel,
+			   int *a, int *r, int *g, int *b);
+
+void
+pixel_checker_get_max (const pixel_checker_t *checker, color_t *color,
+		       int *a, int *r, int *g, int *b);
+
+void
+pixel_checker_get_min (const pixel_checker_t *checker, color_t *color,
+		       int *a, int *r, int *g, int *b);
+
+pixman_bool_t
+pixel_checker_check (const pixel_checker_t *checker,
+		     uint32_t pixel, color_t *color);
+
+void
+pixel_checker_convert_pixel_to_color (const pixel_checker_t *checker,
+                                      uint32_t pixel, color_t *color);
+
+void
+pixel_checker_get_masks (const pixel_checker_t *checker,
+                         uint32_t              *am,
+                         uint32_t              *rm,
+                         uint32_t              *gm,
+                         uint32_t              *bm);
diff --git a/pixman/test/window-test.c b/pixman/test/window-test.c
deleted file mode 100644
index 919fc16ed..000000000
--- a/pixman/test/window-test.c
+++ /dev/null
@@ -1,173 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <config.h>
-#include "pixman-private.h"
-#include "pixman.h"
-
-#define FALSE 0
-#define TRUE 1
-
-/* Randomly decide between 32 and 16 bit
- *
- * Allocate bits with random width, stride and height
- *
- * Then make up some random offset (dx, dy)
- *
- * Then make an image with those values.
- *
- * Do this for both source and destination
- *
- * Composite them together using OVER.
- *
- * The bits in the source and the destination should have
- * recognizable colors so that the result can be verified.
- *
- * Ie., walk the bits and verify that they have been composited.
- */
-
-static int
-get_rand (int bound)
-{
-    return rand () % bound;
-}
-
-static pixman_image_t *
-make_image (int width, int height, pixman_bool_t src, int *rx, int *ry)
-{
-    pixman_format_code_t format;
-    pixman_image_t *image;
-    pixman_region32_t region;
-    uint8_t *bits;
-    int stride;
-    int bpp;
-    int dx, dy;
-    int i, j;
-
-    if (src)
-	format = PIXMAN_a8r8g8b8;
-    else
-	format = PIXMAN_r5g6b5;
-
-    bpp = PIXMAN_FORMAT_BPP (format) / 8;
-
-    stride = width + get_rand (width);
-    stride += (stride & 1);		/* Make it an even number */
-
-    bits = malloc (height * stride * bpp);
-
-    for (j = 0; j < height; ++j)
-    {
-	for (i = 0; i < width; ++i)
-	{
-	    uint8_t *pixel = bits + (stride * j + i) * bpp;
-
-	    if (src)
-		*(uint32_t *)pixel = 0x7f00007f;
-	    else
-		*(uint16_t *)pixel = 0xf100;
-	}
-    }
-
-    dx = dy = 0;
-
-    dx = get_rand (500);
-    dy = get_rand (500);
-
-    if (!src)
-    {
-	/* Now simulate the bogus X server translations */
-	bits -= (dy * stride + dx) * bpp;
-    }
-
-    image = pixman_image_create_bits (
-	format, width, height, (uint32_t *)bits, stride * bpp);
-
-    if (!src)
-    {
-	/* And add the bogus clip region */
-	pixman_region32_init_rect (&region, dx, dy, dx + width, dy + height);
-
-	pixman_image_set_clip_region32 (image, &region);
-    }
-
-    pixman_image_set_source_clipping (image, TRUE);
-
-    if (src)
-    {
-	pixman_transform_t trans;
-
-	pixman_transform_init_identity (&trans);
-
-	pixman_transform_translate (&trans,
-				    NULL,
-				    - pixman_int_to_fixed (width / 2),
-				    - pixman_int_to_fixed (height / 2));
-
-	pixman_transform_scale (&trans,
-				NULL,
-				pixman_double_to_fixed (0.5),
-				pixman_double_to_fixed (0.5));
-
-	pixman_transform_translate (&trans,
-				    NULL,
-				    pixman_int_to_fixed (width / 2),
-				    pixman_int_to_fixed (height / 2));
-
-	pixman_image_set_transform (image, &trans);
-	pixman_image_set_filter (image, PIXMAN_FILTER_BILINEAR, NULL, 0);
-	pixman_image_set_repeat (image, PIXMAN_REPEAT_PAD);
-    }
-
-    if (!src)
-    {
-	*rx = dx;
-	*ry = dy;
-    }
-    else
-    {
-	*rx = *ry = 0;
-    }
-
-    return image;
-}
-
-int
-main ()
-{
-    pixman_image_t *src, *dest;
-    int src_x, src_y, dest_x, dest_y;
-    int i, j;
-    int width = get_rand (499) + 1;
-    int height = get_rand (499) + 1;
-
-    src = make_image (width, height, TRUE, &src_x, &src_y);
-    dest = make_image (width, height, FALSE, &dest_x, &dest_y);
-
-    pixman_image_composite (
-	PIXMAN_OP_OVER, src, NULL, dest,
-	src_x, src_y,
-	-1, -1,
-	dest_x, dest_y,
-	width, height);
-
-    for (i = 0; i < height; ++i)
-    {
-	for (j = 0; j < width; ++j)
-	{
-	    uint8_t *bits = (uint8_t *)dest->bits.bits;
-	    int bpp = PIXMAN_FORMAT_BPP (dest->bits.format) / 8;
-	    int stride = dest->bits.rowstride * 4;
-
-	    uint8_t *pixel =
-		bits + (i + dest_y) * stride + (j + dest_x) * bpp;
-
-	    if (*(uint16_t *)pixel != 0x788f)
-	    {
-		printf ("bad pixel %x\n", *(uint16_t *)pixel);
-		assert (*(uint16_t *)pixel == 0x788f);
-	    }
-	}
-    }
-
-    return 0;
-}
diff --git a/po/Makefile b/po/Makefile
index 669f8654a..b271f79ba 100644
--- a/po/Makefile
+++ b/po/Makefile
@@ -4,6 +4,14 @@
 # Set SRC_PATH for in-tree builds without configuration.
 SRC_PATH=..
 
+# The default target must come before any include statements.
+all:
+
+.PHONY:	all build clean install update
+
+%.mo: %.po
+	$(call quiet-command, msgfmt -o $@ $<, "  GEN   $@")
+
 -include ../config-host.mak
 include $(SRC_PATH)/rules.mak
 
@@ -33,9 +41,6 @@ install: $(OBJS)
 	    $(INSTALL) -m644 $$obj $(DESTDIR)$(prefix)/share/locale/$$base/LC_MESSAGES/qemu.mo; \
 	done
 
-%.mo: %.po
-	$(call quiet-command, msgfmt -o $@ $<, "  GEN   $@")
-
 $(PO_PATH)/messages.po: $(SRC_PATH)/ui/gtk.c
 	$(call quiet-command, ( cd $(SRC_PATH) && \
           xgettext -o - --from-code=UTF-8 --foreign-user \
@@ -45,5 +50,3 @@ $(PO_PATH)/messages.po: $(SRC_PATH)/ui/gtk.c
 
 $(PO_PATH)/%.po: $(PO_PATH)/messages.po
 	$(call quiet-command, msgmerge -q $@ $< > $@.bak && mv $@.bak $@, "  GEN   $@")
-
-.PHONY: clean all
diff --git a/po/zh_CN.po b/po/zh_CN.po
new file mode 100644
index 000000000..2b1d42e97
--- /dev/null
+++ b/po/zh_CN.po
@@ -0,0 +1,86 @@
+# Chinese translation for QEMU.
+# This file is put in the public domain.
+#
+# Fam Zheng <famz@redhat.com>, 2014
+msgid ""
+msgstr ""
+"Project-Id-Version: QEMU 2.2\n"
+"Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
+"POT-Creation-Date: 2014-07-31 10:03+0800\n"
+"PO-Revision-Date: 2014-07-31 10:00+0800\n"
+"Last-Translator: Fam Zheng <famz@redhat.com>\n"
+"Language-Team: Chinese <zh@li.org>\n"
+"Language: zh\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=2; plural=n != 1;\n"
+"X-Generator: Lokalize 1.4\n"
+
+#: ui/gtk.c:321
+msgid " - Press Ctrl+Alt+G to release grab"
+msgstr " - 按下 Ctrl+Alt+G 取消捕获"
+
+#: ui/gtk.c:325
+msgid " [Paused]"
+msgstr " [已暂停]"
+
+#: ui/gtk.c:1601
+msgid "_Pause"
+msgstr "暂停(_P)"
+
+#: ui/gtk.c:1607
+msgid "_Reset"
+msgstr "重置(_R)"
+
+#: ui/gtk.c:1610
+msgid "Power _Down"
+msgstr "关闭电源(_D)"
+
+#: ui/gtk.c:1616
+msgid "_Quit"
+msgstr "退出(_Q)"
+
+#: ui/gtk.c:1692
+msgid "_Fullscreen"
+msgstr "全屏(_F)"
+
+#: ui/gtk.c:1702
+msgid "Zoom _In"
+msgstr "放大(_I)"
+
+#: ui/gtk.c:1709
+msgid "Zoom _Out"
+msgstr "缩小(_O)"
+
+#: ui/gtk.c:1716
+msgid "Best _Fit"
+msgstr "最合适大小(_F)"
+
+#: ui/gtk.c:1723
+msgid "Zoom To _Fit"
+msgstr "缩放以适应大小(_F)"
+
+#: ui/gtk.c:1729
+msgid "Grab On _Hover"
+msgstr "鼠标经过时捕获(_H)"
+
+#: ui/gtk.c:1732
+msgid "_Grab Input"
+msgstr "捕获输入(_G)"
+
+#: ui/gtk.c:1761
+msgid "Show _Tabs"
+msgstr "显示标签页(_T)"
+
+#: ui/gtk.c:1764
+msgid "Detach Tab"
+msgstr "分离标签页"
+
+#: ui/gtk.c:1778
+msgid "_Machine"
+msgstr "虚拟机(_M)"
+
+#: ui/gtk.c:1783
+msgid "_View"
+msgstr "视图(_V)"
diff --git a/qapi-schema.json b/qapi-schema.json
index b11aad206..9ffdcf8e8 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -11,6 +11,9 @@
 # QAPI event definitions
 { 'include': 'qapi/event.json' }
 
+# Tracing commands
+{ 'include': 'qapi/trace.json' }
+
 ##
 # LostTickPolicy:
 #
@@ -1186,13 +1189,13 @@
 ##
 # @inject-nmi:
 #
-# Injects an Non-Maskable Interrupt into all guest's VCPUs.
+# Injects a Non-Maskable Interrupt into the default CPU (x86/s390) or all CPUs (ppc64).
 #
 # Returns:  If successful, nothing
 #
 # Since:  0.14.0
 #
-# Notes: Only x86 Virtual Machines support this command.
+# Note: prior to 2.1, this command was only supported for x86 and s390 VMs
 ##
 { 'command': 'inject-nmi' }
 
@@ -1612,11 +1615,13 @@
 #
 # @name: the name of the property
 # @type: the typename of the property
+# @description: #optional if specified, the description of the property.
+#               (since 2.2)
 #
 # Since: 1.2
 ##
 { 'type': 'DevicePropertyInfo',
-  'data': { 'name': 'str', 'type': 'str' } }
+  'data': { 'name': 'str', 'type': 'str', '*description': 'str' } }
 
 ##
 # @device-list-properties:
@@ -2648,14 +2653,19 @@
 # @nodelay: #optional set TCP_NODELAY socket option (default: false)
 # @telnet: #optional enable telnet protocol on server
 #          sockets (default: false)
+# @reconnect: #optional For a client socket, if a socket is disconnected,
+#          then attempt a reconnect after the given number of seconds.
+#          Setting this to zero disables this function. (default: 0)
+#          (Since: 2.2)
 #
 # Since: 1.4
 ##
-{ 'type': 'ChardevSocket', 'data': { 'addr'     : 'SocketAddress',
-                                     '*server'  : 'bool',
-                                     '*wait'    : 'bool',
-                                     '*nodelay' : 'bool',
-                                     '*telnet'  : 'bool' } }
+{ 'type': 'ChardevSocket', 'data': { 'addr'       : 'SocketAddress',
+                                     '*server'    : 'bool',
+                                     '*wait'      : 'bool',
+                                     '*nodelay'   : 'bool',
+                                     '*telnet'    : 'bool',
+                                     '*reconnect' : 'int' } }
 
 ##
 # @ChardevUdp:
@@ -2749,7 +2759,7 @@
 #
 # Configuration info for the new chardev backend.
 #
-# Since: 1.4
+# Since: 1.4 (testdev since 2.2)
 ##
 { 'type': 'ChardevDummy', 'data': { } }
 
@@ -2764,6 +2774,7 @@
                                        'mux'    : 'ChardevMux',
                                        'msmouse': 'ChardevDummy',
                                        'braille': 'ChardevDummy',
+                                       'testdev': 'ChardevDummy',
                                        'stdio'  : 'ChardevStdio',
                                        'console': 'ChardevDummy',
                                        'spicevmc' : 'ChardevSpiceChannel',
@@ -3220,6 +3231,11 @@
 #
 # Input event union.
 #
+# @key: Input event of Keyboard
+# @btn: Input event of pointer buttons
+# @rel: Input event of relative pointer motion
+# @abs: Input event of absolute pointer motion
+#
 # Since: 2.0
 ##
 { 'union' : 'InputEvent',
@@ -3229,6 +3245,25 @@
               'abs'     : 'InputMoveEvent' } }
 
 ##
+# @x-input-send-event
+#
+# Send input event(s) to guest.
+#
+# @console: #optional console to send event(s) to.
+#
+# @events: List of InputEvent union.
+#
+# Returns: Nothing on success.
+#
+# Since: 2.2
+#
+# Note: this command is experimental, and not a stable API.
+#
+##
+{ 'command': 'x-input-send-event',
+  'data': { '*console':'int', 'events': [ 'InputEvent' ] } }
+
+##
 # @NumaOptions
 #
 # A discriminated record of NUMA options. (for OptsVisitor)
diff --git a/qapi/Makefile.objs b/qapi/Makefile.objs
index d14b769cf..227897069 100644
--- a/qapi/Makefile.objs
+++ b/qapi/Makefile.objs
@@ -1,6 +1,6 @@
 util-obj-y = qapi-visit-core.o qapi-dealloc-visitor.o qmp-input-visitor.o
 util-obj-y += qmp-output-visitor.o qmp-registry.o qmp-dispatch.o
 util-obj-y += string-input-visitor.o string-output-visitor.o
-
 util-obj-y += opts-visitor.o
 util-obj-y += qmp-event.o
+util-obj-y += qapi-util.o
diff --git a/qapi/block-core.json b/qapi/block-core.json
index e378653a7..a14e6ab1f 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -38,12 +38,16 @@
 #
 # @lazy-refcounts: #optional on or off; only valid for compat >= 1.1
 #
+# @corrupt: #optional true if the image has been marked corrupt; only valid for
+#           compat >= 1.1 (since 2.2)
+#
 # Since: 1.7
 ##
 { 'type': 'ImageInfoSpecificQCow2',
   'data': {
       'compat': 'str',
-      '*lazy-refcounts': 'bool'
+      '*lazy-refcounts': 'bool',
+      '*corrupt': 'bool'
   } }
 
 ##
@@ -194,6 +198,7 @@
 #       'file', 'file', 'ftp', 'ftps', 'host_cdrom', 'host_device',
 #       'host_floppy', 'http', 'https', 'nbd', 'parallels', 'qcow',
 #       'qcow2', 'raw', 'tftp', 'vdi', 'vmdk', 'vpc', 'vvfat'
+#       2.2: 'archipelago' added, 'cow' dropped
 #
 # @backing_file: #optional the name of the backing file (for copy-on-write)
 #
@@ -332,6 +337,7 @@
 #
 # @io-status: #optional @BlockDeviceIoStatus. Only present if the device
 #             supports it and the VM is configured to stop on errors
+#             (supported device models: virtio-blk, ide, scsi-disk)
 #
 # @inserted: #optional @BlockDeviceInfo describing the device if media is
 #            present
@@ -505,12 +511,14 @@
 #
 # @io-status: the status of the job (since 1.3)
 #
+# @ready: true if the job may be completed (since 2.2)
+#
 # Since: 1.1
 ##
 { 'type': 'BlockJobInfo',
   'data': {'type': 'str', 'device': 'str', 'len': 'int',
            'offset': 'int', 'busy': 'bool', 'paused': 'bool', 'speed': 'int',
-           'io-status': 'BlockDeviceIoStatus'} }
+           'io-status': 'BlockDeviceIoStatus', 'ready': 'bool'} }
 
 ##
 # @query-block-jobs:
@@ -1143,10 +1151,11 @@
 # Since: 2.0
 ##
 { 'enum': 'BlockdevDriver',
-  'data': [ 'file', 'host_device', 'host_cdrom', 'host_floppy',
-            'http', 'https', 'ftp', 'ftps', 'tftp', 'vvfat', 'blkdebug',
-            'blkverify', 'bochs', 'cloop', 'cow', 'dmg', 'parallels', 'qcow',
-            'qcow2', 'qed', 'raw', 'vdi', 'vhdx', 'vmdk', 'vpc', 'quorum' ] }
+  'data': [ 'archipelago', 'blkdebug', 'blkverify', 'bochs', 'cloop',
+            'dmg', 'file', 'ftp', 'ftps', 'host_cdrom', 'host_device',
+            'host_floppy', 'http', 'https', 'null-aio', 'null-co', 'parallels',
+            'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'tftp', 'vdi', 'vhdx',
+            'vmdk', 'vpc', 'vvfat' ] }
 
 ##
 # @BlockdevOptionsBase
@@ -1199,6 +1208,18 @@
   'data': { 'filename': 'str' } }
 
 ##
+# @BlockdevOptionsNull
+#
+# Driver specific block device options for the null backend.
+#
+# @size:    #optional size of the device in bytes.
+#
+# Since: 2.2
+##
+{ 'type': 'BlockdevOptionsNull',
+  'data': { '*size': 'int' } }
+
+##
 # @BlockdevOptionsVVFAT
 #
 # Driver specific block device options for the vvfat protocol.
@@ -1246,6 +1267,67 @@
   'data': { '*backing': 'BlockdevRef' } }
 
 ##
+# @Qcow2OverlapCheckMode
+#
+# General overlap check modes.
+#
+# @none:        Do not perform any checks
+#
+# @constant:    Perform only checks which can be done in constant time and
+#               without reading anything from disk
+#
+# @cached:      Perform only checks which can be done without reading anything
+#               from disk
+#
+# @all:         Perform all available overlap checks
+#
+# Since: 2.2
+##
+{ 'enum': 'Qcow2OverlapCheckMode',
+  'data': [ 'none', 'constant', 'cached', 'all' ] }
+
+##
+# @Qcow2OverlapCheckFlags
+#
+# Structure of flags for each metadata structure. Setting a field to 'true'
+# makes qemu guard that structure against unintended overwriting. The default
+# value is chosen according to the template given.
+#
+# @template: Specifies a template mode which can be adjusted using the other
+#            flags, defaults to 'cached'
+#
+# Since: 2.2
+##
+{ 'type': 'Qcow2OverlapCheckFlags',
+  'data': { '*template':       'Qcow2OverlapCheckMode',
+            '*main-header':    'bool',
+            '*active-l1':      'bool',
+            '*active-l2':      'bool',
+            '*refcount-table': 'bool',
+            '*refcount-block': 'bool',
+            '*snapshot-table': 'bool',
+            '*inactive-l1':    'bool',
+            '*inactive-l2':    'bool' } }
+
+##
+# @Qcow2OverlapChecks
+#
+# Specifies which metadata structures should be guarded against unintended
+# overwriting.
+#
+# @flags:   set of flags for separate specification of each metadata structure
+#           type
+#
+# @mode:    named mode which chooses a specific set of flags
+#
+# Since: 2.2
+##
+{ 'union': 'Qcow2OverlapChecks',
+  'discriminator': {},
+  'data': { 'flags': 'Qcow2OverlapCheckFlags',
+            'mode':  'Qcow2OverlapCheckMode' } }
+
+##
 # @BlockdevOptionsQcow2
 #
 # Driver specific block device options for qcow2.
@@ -1264,6 +1346,18 @@
 #                         should be issued on other occasions where a cluster
 #                         gets freed
 #
+# @overlap-check:         #optional which overlap checks to perform for writes
+#                         to the image, defaults to 'cached' (since 2.2)
+#
+# @cache-size:            #optional the maximum total size of the L2 table and
+#                         refcount block caches in bytes (since 2.2)
+#
+# @l2-cache-size:         #optional the maximum size of the L2 table cache in
+#                         bytes (since 2.2)
+#
+# @refcount-cache-size:   #optional the maximum size of the refcount block cache
+#                         in bytes (since 2.2)
+#
 # Since: 1.7
 ##
 { 'type': 'BlockdevOptionsQcow2',
@@ -1271,7 +1365,42 @@
   'data': { '*lazy-refcounts': 'bool',
             '*pass-discard-request': 'bool',
             '*pass-discard-snapshot': 'bool',
-            '*pass-discard-other': 'bool' } }
+            '*pass-discard-other': 'bool',
+            '*overlap-check': 'Qcow2OverlapChecks',
+            '*cache-size': 'int',
+            '*l2-cache-size': 'int',
+            '*refcount-cache-size': 'int' } }
+
+
+##
+# @BlockdevOptionsArchipelago
+#
+# Driver specific block device options for Archipelago.
+#
+# @volume:              Name of the Archipelago volume image
+#
+# @mport:               #optional The port number on which mapperd is
+#                       listening. This is optional
+#                       and if not specified, QEMU will make Archipelago
+#                       use the default port (1001).
+#
+# @vport:               #optional The port number on which vlmcd is
+#                       listening. This is optional
+#                       and if not specified, QEMU will make Archipelago
+#                       use the default port (501).
+#
+# @segment:             #optional The name of the shared memory segment
+#                       Archipelago stack is using. This is optional
+#                       and if not specified, QEMU will make Archipelago
+#                       use the default value, 'archipelago'.
+# Since: 2.2
+##
+{ 'type': 'BlockdevOptionsArchipelago',
+  'data': { 'volume': 'str',
+            '*mport': 'int',
+            '*vport': 'int',
+            '*segment': 'str' } }
+
 
 ##
 # @BlkdebugEvent
@@ -1290,7 +1419,9 @@
             'refblock_alloc.write_blocks', 'refblock_alloc.write_table',
             'refblock_alloc.switch_table', 'cluster_alloc',
             'cluster_alloc_bytes', 'cluster_free', 'flush_to_os',
-            'flush_to_disk' ] }
+            'flush_to_disk', 'pwritev_rmw.head', 'pwritev_rmw.after_head',
+            'pwritev_rmw.tail', 'pwritev_rmw.after_tail', 'pwritev',
+            'pwritev_zero', 'pwritev_done', 'empty_image_prepare' ] }
 
 ##
 # @BlkdebugInjectErrorOptions
@@ -1384,6 +1515,19 @@
             'raw': 'BlockdevRef' } }
 
 ##
+# @QuorumReadPattern
+#
+# An enumeration of quorum read patterns.
+#
+# @quorum: read all the children and do a quorum vote on reads
+#
+# @fifo: read only from the first child that has not failed
+#
+# Since: 2.2
+##
+{ 'enum': 'QuorumReadPattern', 'data': [ 'quorum', 'fifo' ] }
+
+##
 # @BlockdevOptionsQuorum
 #
 # Driver specific block device options for Quorum
@@ -1398,12 +1542,17 @@
 # @rewrite-corrupted: #optional rewrite corrupted data when quorum is reached
 #                     (Since 2.1)
 #
+# @read-pattern: #optional choose read pattern and set to quorum by default
+#                (Since 2.2)
+#
 # Since: 2.0
 ##
 { 'type': 'BlockdevOptionsQuorum',
   'data': { '*blkverify': 'bool',
             'children': [ 'BlockdevRef' ],
-            'vote-threshold': 'int', '*rewrite-corrupted': 'bool' } }
+            'vote-threshold': 'int',
+            '*rewrite-corrupted': 'bool',
+            '*read-pattern': 'QuorumReadPattern' } }
 
 ##
 # @BlockdevOptions
@@ -1416,39 +1565,41 @@
   'base': 'BlockdevOptionsBase',
   'discriminator': 'driver',
   'data': {
+      'archipelago':'BlockdevOptionsArchipelago',
+      'blkdebug':   'BlockdevOptionsBlkdebug',
+      'blkverify':  'BlockdevOptionsBlkverify',
+      'bochs':      'BlockdevOptionsGenericFormat',
+      'cloop':      'BlockdevOptionsGenericFormat',
+      'dmg':        'BlockdevOptionsGenericFormat',
       'file':       'BlockdevOptionsFile',
-      'host_device':'BlockdevOptionsFile',
+      'ftp':        'BlockdevOptionsFile',
+      'ftps':       'BlockdevOptionsFile',
+# TODO gluster: Wait for structured options
       'host_cdrom': 'BlockdevOptionsFile',
+      'host_device':'BlockdevOptionsFile',
       'host_floppy':'BlockdevOptionsFile',
       'http':       'BlockdevOptionsFile',
       'https':      'BlockdevOptionsFile',
-      'ftp':        'BlockdevOptionsFile',
-      'ftps':       'BlockdevOptionsFile',
-      'tftp':       'BlockdevOptionsFile',
-# TODO gluster: Wait for structured options
 # TODO iscsi: Wait for structured options
 # TODO nbd: Should take InetSocketAddress for 'host'?
 # TODO nfs: Wait for structured options
-# TODO rbd: Wait for structured options
-# TODO sheepdog: Wait for structured options
-# TODO ssh: Should take InetSocketAddress for 'host'?
-      'vvfat':      'BlockdevOptionsVVFAT',
-      'blkdebug':   'BlockdevOptionsBlkdebug',
-      'blkverify':  'BlockdevOptionsBlkverify',
-      'bochs':      'BlockdevOptionsGenericFormat',
-      'cloop':      'BlockdevOptionsGenericFormat',
-      'cow':        'BlockdevOptionsGenericCOWFormat',
-      'dmg':        'BlockdevOptionsGenericFormat',
+      'null-aio':   'BlockdevOptionsNull',
+      'null-co':    'BlockdevOptionsNull',
       'parallels':  'BlockdevOptionsGenericFormat',
-      'qcow':       'BlockdevOptionsGenericCOWFormat',
       'qcow2':      'BlockdevOptionsQcow2',
+      'qcow':       'BlockdevOptionsGenericCOWFormat',
       'qed':        'BlockdevOptionsGenericCOWFormat',
+      'quorum':     'BlockdevOptionsQuorum',
       'raw':        'BlockdevOptionsGenericFormat',
+# TODO rbd: Wait for structured options
+# TODO sheepdog: Wait for structured options
+# TODO ssh: Should take InetSocketAddress for 'host'?
+      'tftp':       'BlockdevOptionsFile',
       'vdi':        'BlockdevOptionsGenericFormat',
       'vhdx':       'BlockdevOptionsGenericFormat',
       'vmdk':       'BlockdevOptionsGenericCOWFormat',
       'vpc':        'BlockdevOptionsGenericFormat',
-      'quorum':     'BlockdevOptionsQuorum'
+      'vvfat':      'BlockdevOptionsVVFAT'
   } }
 
 ##
@@ -1500,7 +1651,7 @@
 ##
 # @BLOCK_IMAGE_CORRUPTED
 #
-# Emitted when a disk image is being marked corrupt
+# Emitted when a corruption has been detected in a disk image
 #
 # @device: device name
 #
@@ -1514,13 +1665,18 @@
 # @size: #optional, if the corruption resulted from an image access, this is
 #        the access size
 #
+# fatal: if set, the image is marked corrupt and therefore unusable after this
+#        event and must be repaired (Since 2.2; before, every
+#        BLOCK_IMAGE_CORRUPTED event was fatal)
+#
 # Since: 1.7
 ##
 { 'event': 'BLOCK_IMAGE_CORRUPTED',
   'data': { 'device' : 'str',
             'msg'    : 'str',
             '*offset': 'int',
-            '*size'  : 'int' } }
+            '*size'  : 'int',
+            'fatal'  : 'bool' } }
 
 ##
 # @BLOCK_IO_ERROR
@@ -1533,6 +1689,15 @@
 #
 # @action: action that has been taken
 #
+# @nospace: #optional true if I/O error was caused due to a no-space
+#           condition. This key is only present if query-block's
+#           io-status is present, please see query-block documentation
+#           for more information (since: 2.2)
+#
+# @reason: human readable string describing the error cause.
+#          (This field is a debugging aid for humans, it should not
+#           be parsed by applications) (since: 2.2)
+#
 # Note: If action is "stop", a STOP event will eventually follow the
 # BLOCK_IO_ERROR event
 #
@@ -1540,7 +1705,8 @@
 ##
 { 'event': 'BLOCK_IO_ERROR',
   'data': { 'device': 'str', 'operation': 'IoOperationType',
-            'action': 'BlockErrorAction' } }
+            'action': 'BlockErrorAction', '*nospace': 'bool',
+            'reason': 'str' } }
 
 ##
 # @BLOCK_JOB_COMPLETED
@@ -1643,3 +1809,20 @@
             'len'   : 'int',
             'offset': 'int',
             'speed' : 'int' } }
+
+# @PreallocMode
+#
+# Preallocation mode of QEMU image file
+#
+# @off: no preallocation
+# @metadata: preallocate only for metadata
+# @falloc: like @full preallocation but allocate disk space by
+#          posix_fallocate() rather than writing zeros.
+# @full: preallocate all data by writing zeros to device to ensure disk
+#        space is really available. @full preallocation also sets up
+#        metadata correctly.
+#
+# Since 2.2
+##
+{ 'enum': 'PreallocMode',
+  'data': [ 'off', 'metadata', 'falloc', 'full' ] }
diff --git a/qapi/common.json b/qapi/common.json
index 4e9a21f2f..63ef3b472 100644
--- a/qapi/common.json
+++ b/qapi/common.json
@@ -87,3 +87,18 @@
 ##
 { 'command': 'query-commands', 'returns': ['CommandInfo'] }
 
+##
+# @OnOffAuto
+#
+# An enumeration of three options: on, off, and auto
+#
+# @auto: QEMU selects the value between on and off
+#
+# @on: Enabled
+#
+# @off: Disabled
+#
+# Since: 2.2
+##
+{ 'enum': 'OnOffAuto',
+  'data': [ 'auto', 'on', 'off' ] }
diff --git a/qapi/qapi-dealloc-visitor.c b/qapi/qapi-dealloc-visitor.c
index dc53545fa..a14a1c714 100644
--- a/qapi/qapi-dealloc-visitor.c
+++ b/qapi/qapi-dealloc-visitor.c
@@ -162,6 +162,31 @@ static void qapi_dealloc_type_enum(Visitor *v, int *obj, const char *strings[],
 {
 }
 
+/* If there's no data present, the dealloc visitor has nothing to free.
+ * Thus, indicate to visitor code that the subsequent union fields can
+ * be skipped. This is not an error condition, since the cleanup of the
+ * rest of an object can continue unhindered, so leave errp unset in
+ * these cases.
+ *
+ * NOTE: In cases where we're attempting to deallocate an object that
+ * may have missing fields, the field indicating the union type may
+ * be missing. In such a case, it's possible we don't have enough
+ * information to differentiate data_present == false from a case where
+ * data *is* present but happens to be a scalar with a value of 0.
+ * This is okay, since in the case of the dealloc visitor there's no
+ * work that needs to done in either situation.
+ *
+ * The current inability in QAPI code to more thoroughly verify a union
+ * type in such cases will likely need to be addressed if we wish to
+ * implement this interface for other types of visitors in the future,
+ * however.
+ */
+static bool qapi_dealloc_start_union(Visitor *v, bool data_present,
+                                     Error **errp)
+{
+    return data_present;
+}
+
 Visitor *qapi_dealloc_get_visitor(QapiDeallocVisitor *v)
 {
     return &v->visitor;
@@ -191,6 +216,7 @@ QapiDeallocVisitor *qapi_dealloc_visitor_new(void)
     v->visitor.type_str = qapi_dealloc_type_str;
     v->visitor.type_number = qapi_dealloc_type_number;
     v->visitor.type_size = qapi_dealloc_type_size;
+    v->visitor.start_union = qapi_dealloc_start_union;
 
     QTAILQ_INIT(&v->stack);
 
diff --git a/qapi/qapi-util.c b/qapi/qapi-util.c
new file mode 100644
index 000000000..1d8fb96ef
--- /dev/null
+++ b/qapi/qapi-util.c
@@ -0,0 +1,34 @@
+/*
+ * QAPI util functions
+ *
+ * Authors:
+ *  Hu Tao       <hutao@cn.fujitsu.com>
+ *  Peter Lieven <pl@kamp.de>
+ * 
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "qemu-common.h"
+#include "qapi/error.h"
+#include "qapi/util.h"
+
+int qapi_enum_parse(const char *lookup[], const char *buf,
+                    int max, int def, Error **errp)
+{
+    int i;
+
+    if (!buf) {
+        return def;
+    }
+
+    for (i = 0; i < max; i++) {
+        if (!strcmp(buf, lookup[i])) {
+            return i;
+        }
+    }
+
+    error_setg(errp, "invalid parameter value: %s", buf);
+    return def;
+}
diff --git a/qapi/qapi-visit-core.c b/qapi/qapi-visit-core.c
index 55f8d4068..b66b93ae2 100644
--- a/qapi/qapi-visit-core.c
+++ b/qapi/qapi-visit-core.c
@@ -58,6 +58,21 @@ void visit_end_list(Visitor *v, Error **errp)
     v->end_list(v, errp);
 }
 
+bool visit_start_union(Visitor *v, bool data_present, Error **errp)
+{
+    if (v->start_union) {
+        return v->start_union(v, data_present, errp);
+    }
+    return true;
+}
+
+void visit_end_union(Visitor *v, bool data_present, Error **errp)
+{
+    if (v->end_union) {
+        v->end_union(v, data_present, errp);
+    }
+}
+
 void visit_optional(Visitor *v, bool *present, const char *name,
                     Error **errp)
 {
diff --git a/qapi/trace.json b/qapi/trace.json
new file mode 100644
index 000000000..06c613c21
--- /dev/null
+++ b/qapi/trace.json
@@ -0,0 +1,65 @@
+# -*- mode: python -*-
+#
+# Copyright (C) 2011-2014 Lluís Vilanova <vilanova@ac.upc.edu>
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or later.
+# See the COPYING file in the top-level directory.
+
+
+##
+# @TraceEventState:
+#
+# State of a tracing event.
+#
+# @unavailable: The event is statically disabled.
+#
+# @disabled: The event is dynamically disabled.
+#
+# @enabled: The event is dynamically enabled.
+#
+# Since 2.2
+##
+{ 'enum': 'TraceEventState',
+  'data': ['unavailable', 'disabled', 'enabled'] }
+
+##
+# @TraceEventInfo:
+#
+# Information of a tracing event.
+#
+# @name: Event name.
+# @state: Tracing state.
+#
+# Since 2.2
+##
+{ 'type': 'TraceEventInfo',
+  'data': {'name': 'str', 'state': 'TraceEventState'} }
+
+##
+# @trace-event-get-state:
+#
+# Query the state of events.
+#
+# @name: Event name pattern (case-sensitive glob).
+#
+# Returns: a list of @TraceEventInfo for the matching events
+#
+# Since 2.2
+##
+{ 'command': 'trace-event-get-state',
+  'data': {'name': 'str'},
+  'returns': ['TraceEventInfo'] }
+
+##
+# @trace-event-set-state:
+#
+# Set the dynamic tracing state of events.
+#
+# @name: Event name pattern (case-sensitive glob).
+# @enable: Whether to enable tracing.
+# @ignore-unavailable: #optional Do not match unavailable events with @name.
+#
+# Since 2.2
+##
+{ 'command': 'trace-event-set-state',
+  'data': {'name': 'str', 'enable': 'bool', '*ignore-unavailable': 'bool'} }
diff --git a/qdev-monitor.c b/qdev-monitor.c
index f87f3d89c..ebfa701a9 100644
--- a/qdev-monitor.c
+++ b/qdev-monitor.c
@@ -180,11 +180,50 @@ static const char *find_typename_by_alias(const char *alias)
     return NULL;
 }
 
+static DeviceClass *qdev_get_device_class(const char **driver, Error **errp)
+{
+    ObjectClass *oc;
+    DeviceClass *dc;
+
+    oc = object_class_by_name(*driver);
+    if (!oc) {
+        const char *typename = find_typename_by_alias(*driver);
+
+        if (typename) {
+            *driver = typename;
+            oc = object_class_by_name(*driver);
+        }
+    }
+
+    if (!object_class_dynamic_cast(oc, TYPE_DEVICE)) {
+        error_setg(errp, "'%s' is not a valid device model name", *driver);
+        return NULL;
+    }
+
+    if (object_class_is_abstract(oc)) {
+        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "driver",
+                  "non-abstract device type");
+        return NULL;
+    }
+
+    dc = DEVICE_CLASS(oc);
+    if (dc->cannot_instantiate_with_device_add_yet ||
+        (qdev_hotplug && !dc->hotpluggable)) {
+        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "driver",
+                  "pluggable device type");
+        return NULL;
+    }
+
+    return dc;
+}
+
+
 int qdev_device_help(QemuOpts *opts)
 {
+    Error *local_err = NULL;
     const char *driver;
-    Property *prop;
-    ObjectClass *klass;
+    DevicePropertyInfoList *prop_list;
+    DevicePropertyInfoList *prop;
 
     driver = qemu_opt_get(opts, "driver");
     if (driver && is_help_option(driver)) {
@@ -196,35 +235,33 @@ int qdev_device_help(QemuOpts *opts)
         return 0;
     }
 
-    klass = object_class_by_name(driver);
-    if (!klass) {
-        const char *typename = find_typename_by_alias(driver);
-
-        if (typename) {
-            driver = typename;
-            klass = object_class_by_name(driver);
-        }
+    qdev_get_device_class(&driver, &local_err);
+    if (local_err) {
+        goto error;
     }
 
-    if (!object_class_dynamic_cast(klass, TYPE_DEVICE)) {
-        return 0;
+    prop_list = qmp_device_list_properties(driver, &local_err);
+    if (local_err) {
+        goto error;
     }
-    do {
-        for (prop = DEVICE_CLASS(klass)->props; prop && prop->name; prop++) {
-            /*
-             * TODO Properties without a parser are just for dirty hacks.
-             * qdev_prop_ptr is the only such PropertyInfo.  It's marked
-             * for removal.  This conditional should be removed along with
-             * it.
-             */
-            if (!prop->info->set) {
-                continue;           /* no way to set it, don't show */
-            }
-            error_printf("%s.%s=%s\n", driver, prop->name,
-                         prop->info->legacy_name ?: prop->info->name);
+
+    for (prop = prop_list; prop; prop = prop->next) {
+        error_printf("%s.%s=%s", driver,
+                     prop->value->name,
+                     prop->value->type);
+        if (prop->value->has_description) {
+            error_printf(" (%s)\n", prop->value->description);
+        } else {
+            error_printf("\n");
         }
-        klass = object_class_get_parent(klass);
-    } while (klass != object_class_by_name(TYPE_DEVICE));
+    }
+
+    qapi_free_DevicePropertyInfoList(prop_list);
+    return 1;
+
+error:
+    error_printf("%s\n", error_get_pretty(local_err));
+    error_free(local_err);
     return 1;
 }
 
@@ -456,7 +493,6 @@ static BusState *qbus_find(const char *path)
 
 DeviceState *qdev_device_add(QemuOpts *opts)
 {
-    ObjectClass *oc;
     DeviceClass *dc;
     const char *driver, *path, *id;
     DeviceState *dev;
@@ -470,32 +506,10 @@ DeviceState *qdev_device_add(QemuOpts *opts)
     }
 
     /* find driver */
-    oc = object_class_by_name(driver);
-    if (!oc) {
-        const char *typename = find_typename_by_alias(driver);
-
-        if (typename) {
-            driver = typename;
-            oc = object_class_by_name(driver);
-        }
-    }
-
-    if (!object_class_dynamic_cast(oc, TYPE_DEVICE)) {
-        qerror_report(ERROR_CLASS_GENERIC_ERROR,
-                      "'%s' is not a valid device model name", driver);
-        return NULL;
-    }
-
-    if (object_class_is_abstract(oc)) {
-        qerror_report(QERR_INVALID_PARAMETER_VALUE, "driver",
-                      "non-abstract device type");
-        return NULL;
-    }
-
-    dc = DEVICE_CLASS(oc);
-    if (dc->cannot_instantiate_with_device_add_yet) {
-        qerror_report(QERR_INVALID_PARAMETER_VALUE, "driver",
-                      "pluggable device type");
+    dc = qdev_get_device_class(&driver, &err);
+    if (err) {
+        qerror_report_err(err);
+        error_free(err);
         return NULL;
     }
 
@@ -521,7 +535,7 @@ DeviceState *qdev_device_add(QemuOpts *opts)
             return NULL;
         }
     }
-    if (qdev_hotplug && bus && !bus->allow_hotplug) {
+    if (qdev_hotplug && bus && !qbus_is_hotpluggable(bus)) {
         qerror_report(QERR_BUS_NO_HOTPLUG, bus->name);
         return NULL;
     }
@@ -691,15 +705,20 @@ int do_device_add(Monitor *mon, const QDict *qdict, QObject **ret_data)
 
 void qmp_device_del(const char *id, Error **errp)
 {
-    DeviceState *dev;
+    Object *obj;
+    char *root_path = object_get_canonical_path(qdev_get_peripheral());
+    char *path = g_strdup_printf("%s/%s", root_path, id);
+
+    g_free(root_path);
+    obj = object_resolve_path_type(path, TYPE_DEVICE, NULL);
+    g_free(path);
 
-    dev = qdev_find_recursive(sysbus_get_default(), id);
-    if (NULL == dev) {
+    if (!obj) {
         error_set(errp, QERR_DEVICE_NOT_FOUND, id);
         return;
     }
 
-    qdev_unplug(dev, errp);
+    qdev_unplug(DEVICE(obj), errp);
 }
 
 void qdev_machine_init(void)
diff --git a/qemu-char.c b/qemu-char.c
index 956be49ec..a8b01da3e 100644
--- a/qemu-char.c
+++ b/qemu-char.c
@@ -28,6 +28,9 @@
 #include "sysemu/char.h"
 #include "hw/usb.h"
 #include "qmp-commands.h"
+#include "qapi/qmp-input-visitor.h"
+#include "qapi/qmp-output-visitor.h"
+#include "qapi-visit.h"
 
 #include <unistd.h>
 #include <fcntl.h>
@@ -84,6 +87,95 @@
 
 #define READ_BUF_LEN 4096
 #define READ_RETRIES 10
+#define CHR_MAX_FILENAME_SIZE 256
+#define TCP_MAX_FDS 16
+
+/***********************************************************/
+/* Socket address helpers */
+static void qapi_copy_SocketAddress(SocketAddress **p_dest,
+                                    SocketAddress *src)
+{
+    QmpOutputVisitor *qov;
+    QmpInputVisitor *qiv;
+    Visitor *ov, *iv;
+    QObject *obj;
+
+    *p_dest = NULL;
+
+    qov = qmp_output_visitor_new();
+    ov = qmp_output_get_visitor(qov);
+    visit_type_SocketAddress(ov, &src, NULL, &error_abort);
+    obj = qmp_output_get_qobject(qov);
+    qmp_output_visitor_cleanup(qov);
+    if (!obj) {
+        return;
+    }
+
+    qiv = qmp_input_visitor_new(obj);
+    iv = qmp_input_get_visitor(qiv);
+    visit_type_SocketAddress(iv, p_dest, NULL, &error_abort);
+    qmp_input_visitor_cleanup(qiv);
+    qobject_decref(obj);
+}
+
+static int SocketAddress_to_str(char *dest, int max_len,
+                                const char *prefix, SocketAddress *addr,
+                                bool is_listen, bool is_telnet)
+{
+    switch (addr->kind) {
+    case SOCKET_ADDRESS_KIND_INET:
+        return snprintf(dest, max_len, "%s%s:%s:%s%s", prefix,
+                        is_telnet ? "telnet" : "tcp", addr->inet->host,
+                        addr->inet->port, is_listen ? ",server" : "");
+        break;
+    case SOCKET_ADDRESS_KIND_UNIX:
+        return snprintf(dest, max_len, "%sunix:%s%s", prefix,
+                        addr->q_unix->path, is_listen ? ",server" : "");
+        break;
+    case SOCKET_ADDRESS_KIND_FD:
+        return snprintf(dest, max_len, "%sfd:%s%s", prefix, addr->fd->str,
+                        is_listen ? ",server" : "");
+        break;
+    default:
+        abort();
+    }
+}
+
+static int sockaddr_to_str(char *dest, int max_len,
+                           struct sockaddr_storage *ss, socklen_t ss_len,
+                           struct sockaddr_storage *ps, socklen_t ps_len,
+                           bool is_listen, bool is_telnet)
+{
+    char shost[NI_MAXHOST], sserv[NI_MAXSERV];
+    char phost[NI_MAXHOST], pserv[NI_MAXSERV];
+    const char *left = "", *right = "";
+
+    switch (ss->ss_family) {
+#ifndef _WIN32
+    case AF_UNIX:
+        return snprintf(dest, max_len, "unix:%s%s",
+                        ((struct sockaddr_un *)(ss))->sun_path,
+                        is_listen ? ",server" : "");
+#endif
+    case AF_INET6:
+        left  = "[";
+        right = "]";
+        /* fall through */
+    case AF_INET:
+        getnameinfo((struct sockaddr *) ss, ss_len, shost, sizeof(shost),
+                    sserv, sizeof(sserv), NI_NUMERICHOST | NI_NUMERICSERV);
+        getnameinfo((struct sockaddr *) ps, ps_len, phost, sizeof(phost),
+                    pserv, sizeof(pserv), NI_NUMERICHOST | NI_NUMERICSERV);
+        return snprintf(dest, max_len, "%s:%s%s%s:%s%s <-> %s%s%s:%s",
+                        is_telnet ? "telnet" : "tcp",
+                        left, shost, right, sserv,
+                        is_listen ? ",server" : "",
+                        left, phost, right, pserv);
+
+    default:
+        return snprintf(dest, max_len, "unknown");
+    }
+}
 
 /***********************************************************/
 /* character device */
@@ -373,7 +465,7 @@ static const char * const mux_help[] = {
     "% h    print this help\n\r",
     "% x    exit emulator\n\r",
     "% s    save disk data back to file (if -snapshot)\n\r",
-    "% t    toggle console timestamps\n\r"
+    "% t    toggle console timestamps\n\r",
     "% b    send break (magic sysrq)\n\r",
     "% c    switch between console and monitor\n\r",
     "% %  sends %\n\r",
@@ -975,7 +1067,7 @@ static CharDriverState *qemu_chr_open_fd(int fd_in, int fd_out)
     s = g_malloc0(sizeof(FDCharDriver));
     s->fd_in = io_channel_from_fd(fd_in);
     s->fd_out = io_channel_from_fd(fd_out);
-    fcntl(fd_out, F_SETFL, O_NONBLOCK);
+    qemu_set_nonblock(fd_out);
     s->chr = chr;
     chr->opaque = s;
     chr->chr_add_watch = fd_chr_add_watch;
@@ -989,7 +1081,8 @@ static CharDriverState *qemu_chr_open_fd(int fd_in, int fd_out)
 static CharDriverState *qemu_chr_open_pipe(ChardevHostdev *opts)
 {
     int fd_in, fd_out;
-    char filename_in[256], filename_out[256];
+    char filename_in[CHR_MAX_FILENAME_SIZE];
+    char filename_out[CHR_MAX_FILENAME_SIZE];
     const char *filename = opts->device;
 
     if (filename == NULL) {
@@ -997,8 +1090,8 @@ static CharDriverState *qemu_chr_open_pipe(ChardevHostdev *opts)
         return NULL;
     }
 
-    snprintf(filename_in, 256, "%s.in", filename);
-    snprintf(filename_out, 256, "%s.out", filename);
+    snprintf(filename_in, CHR_MAX_FILENAME_SIZE, "%s.in", filename);
+    snprintf(filename_out, CHR_MAX_FILENAME_SIZE, "%s.out", filename);
     TFR(fd_in = qemu_open(filename_in, O_RDWR | O_BINARY));
     TFR(fd_out = qemu_open(filename_out, O_RDWR | O_BINARY));
     if (fd_in < 0 || fd_out < 0) {
@@ -1017,6 +1110,7 @@ static CharDriverState *qemu_chr_open_pipe(ChardevHostdev *opts)
 /* init terminal so that we can grab keys */
 static struct termios oldtty;
 static int old_fd0_flags;
+static bool stdio_in_use;
 static bool stdio_allow_signal;
 
 static void term_exit(void)
@@ -1060,9 +1154,16 @@ static CharDriverState *qemu_chr_open_stdio(ChardevStdio *opts)
         error_report("cannot use stdio with -daemonize");
         return NULL;
     }
+
+    if (stdio_in_use) {
+        error_report("cannot use stdio by multiple character devices");
+        exit(1);
+    }
+
+    stdio_in_use = true;
     old_fd0_flags = fcntl(0, F_GETFL);
-    tcgetattr (0, &oldtty);
-    fcntl(0, F_SETFL, O_NONBLOCK);
+    tcgetattr(0, &oldtty);
+    qemu_set_nonblock(0);
     atexit(term_exit);
 
     chr = qemu_chr_open_fd(0, 1);
@@ -1160,7 +1261,9 @@ static int pty_chr_write(CharDriverState *chr, const uint8_t *buf, int len)
     if (!s->connected) {
         /* guest sends data, check for (re-)connect */
         pty_chr_update_read_handler_locked(chr);
-        return 0;
+        if (!s->connected) {
+            return 0;
+        }
     }
     return io_channel_send(s->fd, buf, len);
 }
@@ -1966,7 +2069,7 @@ static int win_chr_pipe_init(CharDriverState *chr, const char *filename)
     OVERLAPPED ov;
     int ret;
     DWORD size;
-    char openname[256];
+    char openname[CHR_MAX_FILENAME_SIZE];
 
     s->fpipe = TRUE;
 
@@ -2383,20 +2486,6 @@ static CharDriverState *qemu_chr_open_udp_fd(int fd)
     return chr;
 }
 
-static CharDriverState *qemu_chr_open_udp(QemuOpts *opts)
-{
-    Error *local_err = NULL;
-    int fd = -1;
-
-    fd = inet_dgram_opts(opts, &local_err);
-    if (fd < 0) {
-        qerror_report_err(local_err);
-        error_free(local_err);
-        return NULL;
-    }
-    return qemu_chr_open_udp_fd(fd);
-}
-
 /***********************************************************/
 /* TCP Net console */
 
@@ -2414,8 +2503,39 @@ typedef struct {
     int read_msgfds_num;
     int *write_msgfds;
     int write_msgfds_num;
+
+    SocketAddress *addr;
+    bool is_listen;
+    bool is_telnet;
+
+    guint reconnect_timer;
+    int64_t reconnect_time;
+    bool connect_err_reported;
 } TCPCharDriver;
 
+static gboolean socket_reconnect_timeout(gpointer opaque);
+
+static void qemu_chr_socket_restart_timer(CharDriverState *chr)
+{
+    TCPCharDriver *s = chr->opaque;
+    assert(s->connected == 0);
+    s->reconnect_timer = g_timeout_add_seconds(s->reconnect_time,
+                                               socket_reconnect_timeout, chr);
+}
+
+static void check_report_connect_error(CharDriverState *chr,
+                                       Error *err)
+{
+    TCPCharDriver *s = chr->opaque;
+
+    if (!s->connect_err_reported) {
+        error_report("Unable to connect character device %s: %s",
+                     chr->label, error_get_pretty(err));
+        s->connect_err_reported = true;
+    }
+    qemu_chr_socket_restart_timer(chr);
+}
+
 static gboolean tcp_chr_accept(GIOChannel *chan, GIOCondition cond, void *opaque);
 
 #ifndef _WIN32
@@ -2549,6 +2669,8 @@ static int tcp_get_msgfds(CharDriverState *chr, int *fds, int num)
     TCPCharDriver *s = chr->opaque;
     int to_copy = (s->read_msgfds_num < num) ? s->read_msgfds_num : num;
 
+    assert(num <= TCP_MAX_FDS);
+
     if (to_copy) {
         int i;
 
@@ -2643,7 +2765,7 @@ static ssize_t tcp_chr_recv(CharDriverState *chr, char *buf, size_t len)
     struct iovec iov[1];
     union {
         struct cmsghdr cmsg;
-        char control[CMSG_SPACE(sizeof(int))];
+        char control[CMSG_SPACE(sizeof(int) * TCP_MAX_FDS)];
     } msg_control;
     int flags = 0;
     ssize_t ret;
@@ -2694,7 +2816,12 @@ static void tcp_chr_disconnect(CharDriverState *chr)
     s->chan = NULL;
     closesocket(s->fd);
     s->fd = -1;
+    SocketAddress_to_str(chr->filename, CHR_MAX_FILENAME_SIZE,
+                         "disconnected:", s->addr, s->is_listen, s->is_telnet);
     qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
+    if (s->reconnect_time) {
+        qemu_chr_socket_restart_timer(chr);
+    }
 }
 
 static gboolean tcp_chr_read(GIOChannel *chan, GIOCondition cond, void *opaque)
@@ -2765,6 +2892,21 @@ static void tcp_chr_connect(void *opaque)
 {
     CharDriverState *chr = opaque;
     TCPCharDriver *s = chr->opaque;
+    struct sockaddr_storage ss, ps;
+    socklen_t ss_len = sizeof(ss), ps_len = sizeof(ps);
+
+    memset(&ss, 0, ss_len);
+    if (getsockname(s->fd, (struct sockaddr *) &ss, &ss_len) != 0) {
+        snprintf(chr->filename, CHR_MAX_FILENAME_SIZE,
+                 "Error in getsockname: %s\n", strerror(errno));
+    } else if (getpeername(s->fd, (struct sockaddr *) &ps, &ps_len) != 0) {
+        snprintf(chr->filename, CHR_MAX_FILENAME_SIZE,
+                 "Error in getpeername: %s\n", strerror(errno));
+    } else {
+        sockaddr_to_str(chr->filename, CHR_MAX_FILENAME_SIZE,
+                        &ss, ss_len, &ps, ps_len,
+                        s->is_listen, s->is_telnet);
+    }
 
     s->connected = 1;
     if (s->chan) {
@@ -2863,6 +3005,12 @@ static void tcp_chr_close(CharDriverState *chr)
 {
     TCPCharDriver *s = chr->opaque;
     int i;
+
+    if (s->reconnect_timer) {
+        g_source_remove(s->reconnect_timer);
+        s->reconnect_timer = 0;
+    }
+    qapi_free_SocketAddress(s->addr);
     if (s->fd >= 0) {
         remove_fd_in_watch(chr);
         if (s->chan) {
@@ -2893,79 +3041,15 @@ static void tcp_chr_close(CharDriverState *chr)
     qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
 }
 
-static CharDriverState *qemu_chr_open_socket_fd(int fd, bool do_nodelay,
-                                                bool is_listen, bool is_telnet,
-                                                bool is_waitconnect,
-                                                Error **errp)
+static void qemu_chr_finish_socket_connection(CharDriverState *chr, int fd)
 {
-    CharDriverState *chr = NULL;
-    TCPCharDriver *s = NULL;
-    char host[NI_MAXHOST], serv[NI_MAXSERV];
-    const char *left = "", *right = "";
-    struct sockaddr_storage ss;
-    socklen_t ss_len = sizeof(ss);
-
-    memset(&ss, 0, ss_len);
-    if (getsockname(fd, (struct sockaddr *) &ss, &ss_len) != 0) {
-        error_setg_errno(errp, errno, "getsockname");
-        return NULL;
-    }
-
-    chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(TCPCharDriver));
-
-    s->connected = 0;
-    s->fd = -1;
-    s->listen_fd = -1;
-    s->read_msgfds = 0;
-    s->read_msgfds_num = 0;
-    s->write_msgfds = 0;
-    s->write_msgfds_num = 0;
-
-    chr->filename = g_malloc(256);
-    switch (ss.ss_family) {
-#ifndef _WIN32
-    case AF_UNIX:
-        s->is_unix = 1;
-        snprintf(chr->filename, 256, "unix:%s%s",
-                 ((struct sockaddr_un *)(&ss))->sun_path,
-                 is_listen ? ",server" : "");
-        break;
-#endif
-    case AF_INET6:
-        left  = "[";
-        right = "]";
-        /* fall through */
-    case AF_INET:
-        s->do_nodelay = do_nodelay;
-        getnameinfo((struct sockaddr *) &ss, ss_len, host, sizeof(host),
-                    serv, sizeof(serv), NI_NUMERICHOST | NI_NUMERICSERV);
-        snprintf(chr->filename, 256, "%s:%s%s%s:%s%s",
-                 is_telnet ? "telnet" : "tcp",
-                 left, host, right, serv,
-                 is_listen ? ",server" : "");
-        break;
-    }
-
-    chr->opaque = s;
-    chr->chr_write = tcp_chr_write;
-    chr->chr_sync_read = tcp_chr_sync_read;
-    chr->chr_close = tcp_chr_close;
-    chr->get_msgfds = tcp_get_msgfds;
-    chr->set_msgfds = tcp_set_msgfds;
-    chr->chr_add_client = tcp_chr_add_client;
-    chr->chr_add_watch = tcp_chr_add_watch;
-    chr->chr_update_read_handler = tcp_chr_update_read_handler;
-    /* be isn't opened until we get a connection */
-    chr->explicit_be_open = true;
+    TCPCharDriver *s = chr->opaque;
 
-    if (is_listen) {
+    if (s->is_listen) {
         s->listen_fd = fd;
         s->listen_chan = io_channel_from_socket(s->listen_fd);
-        s->listen_tag = g_io_add_watch(s->listen_chan, G_IO_IN, tcp_chr_accept, chr);
-        if (is_telnet) {
-            s->do_telnetopt = 1;
-        }
+        s->listen_tag = g_io_add_watch(s->listen_chan, G_IO_IN,
+                                       tcp_chr_accept, chr);
     } else {
         s->connected = 1;
         s->fd = fd;
@@ -2973,69 +3057,41 @@ static CharDriverState *qemu_chr_open_socket_fd(int fd, bool do_nodelay,
         s->chan = io_channel_from_socket(s->fd);
         tcp_chr_connect(chr);
     }
-
-    if (is_listen && is_waitconnect) {
-        fprintf(stderr, "QEMU waiting for connection on: %s\n",
-                chr->filename);
-        tcp_chr_accept(s->listen_chan, G_IO_IN, chr);
-        qemu_set_nonblock(s->listen_fd);
-    }
-    return chr;
 }
 
-static CharDriverState *qemu_chr_open_socket(QemuOpts *opts)
+static void qemu_chr_socket_connected(int fd, Error *err, void *opaque)
 {
-    CharDriverState *chr = NULL;
-    Error *local_err = NULL;
-    int fd = -1;
-
-    bool is_listen      = qemu_opt_get_bool(opts, "server", false);
-    bool is_waitconnect = is_listen && qemu_opt_get_bool(opts, "wait", true);
-    bool is_telnet      = qemu_opt_get_bool(opts, "telnet", false);
-    bool do_nodelay     = !qemu_opt_get_bool(opts, "delay", true);
-    bool is_unix        = qemu_opt_get(opts, "path") != NULL;
+    CharDriverState *chr = opaque;
+    TCPCharDriver *s = chr->opaque;
 
-    if (is_unix) {
-        if (is_listen) {
-            fd = unix_listen_opts(opts, &local_err);
-        } else {
-            fd = unix_connect_opts(opts, &local_err, NULL, NULL);
-        }
-    } else {
-        if (is_listen) {
-            fd = inet_listen_opts(opts, 0, &local_err);
-        } else {
-            fd = inet_connect_opts(opts, &local_err, NULL, NULL);
-        }
-    }
     if (fd < 0) {
-        goto fail;
+        check_report_connect_error(chr, err);
+        return;
     }
 
-    if (!is_waitconnect)
-        qemu_set_nonblock(fd);
-
-    chr = qemu_chr_open_socket_fd(fd, do_nodelay, is_listen, is_telnet,
-                                  is_waitconnect, &local_err);
-    if (local_err) {
-        goto fail;
-    }
-    return chr;
+    s->connect_err_reported = false;
+    qemu_chr_finish_socket_connection(chr, fd);
+}
 
+static bool qemu_chr_open_socket_fd(CharDriverState *chr, Error **errp)
+{
+    TCPCharDriver *s = chr->opaque;
+    int fd;
 
- fail:
-    if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
-    }
-    if (fd >= 0) {
-        closesocket(fd);
+    if (s->is_listen) {
+        fd = socket_listen(s->addr, errp);
+    } else if (s->reconnect_time) {
+        fd = socket_connect(s->addr, errp, qemu_chr_socket_connected, chr);
+        return fd >= 0;
+    } else {
+        fd = socket_connect(s->addr, errp, NULL, NULL);
     }
-    if (chr) {
-        g_free(chr->opaque);
-        g_free(chr);
+    if (fd < 0) {
+        return false;
     }
-    return NULL;
+
+    qemu_chr_finish_socket_connection(chr, fd);
+    return true;
 }
 
 /*********************************************************/
@@ -3256,6 +3312,7 @@ QemuOpts *qemu_chr_parse_compat(const char *label, const char *filename)
         strcmp(filename, "pty")     == 0 ||
         strcmp(filename, "msmouse") == 0 ||
         strcmp(filename, "braille") == 0 ||
+        strcmp(filename, "testdev") == 0 ||
         strcmp(filename, "stdio")   == 0) {
         qemu_opt_set(opts, "backend", filename);
         return opts;
@@ -3447,29 +3504,124 @@ static void qemu_chr_parse_mux(QemuOpts *opts, ChardevBackend *backend,
     backend->mux->chardev = g_strdup(chardev);
 }
 
+static void qemu_chr_parse_socket(QemuOpts *opts, ChardevBackend *backend,
+                                  Error **errp)
+{
+    bool is_listen      = qemu_opt_get_bool(opts, "server", false);
+    bool is_waitconnect = is_listen && qemu_opt_get_bool(opts, "wait", true);
+    bool is_telnet      = qemu_opt_get_bool(opts, "telnet", false);
+    bool do_nodelay     = !qemu_opt_get_bool(opts, "delay", true);
+    int64_t reconnect   = qemu_opt_get_number(opts, "reconnect", 0);
+    const char *path = qemu_opt_get(opts, "path");
+    const char *host = qemu_opt_get(opts, "host");
+    const char *port = qemu_opt_get(opts, "port");
+    SocketAddress *addr;
+
+    if (!path) {
+        if (!host) {
+            error_setg(errp, "chardev: socket: no host given");
+            return;
+        }
+        if (!port) {
+            error_setg(errp, "chardev: socket: no port given");
+            return;
+        }
+    }
+
+    backend->socket = g_new0(ChardevSocket, 1);
+
+    backend->socket->has_nodelay = true;
+    backend->socket->nodelay = do_nodelay;
+    backend->socket->has_server = true;
+    backend->socket->server = is_listen;
+    backend->socket->has_telnet = true;
+    backend->socket->telnet = is_telnet;
+    backend->socket->has_wait = true;
+    backend->socket->wait = is_waitconnect;
+    backend->socket->has_reconnect = true;
+    backend->socket->reconnect = reconnect;
+
+    addr = g_new0(SocketAddress, 1);
+    if (path) {
+        addr->kind = SOCKET_ADDRESS_KIND_UNIX;
+        addr->q_unix = g_new0(UnixSocketAddress, 1);
+        addr->q_unix->path = g_strdup(path);
+    } else {
+        addr->kind = SOCKET_ADDRESS_KIND_INET;
+        addr->inet = g_new0(InetSocketAddress, 1);
+        addr->inet->host = g_strdup(host);
+        addr->inet->port = g_strdup(port);
+        addr->inet->has_to = qemu_opt_get(opts, "to");
+        addr->inet->to = qemu_opt_get_number(opts, "to", 0);
+        addr->inet->has_ipv4 = qemu_opt_get(opts, "ipv4");
+        addr->inet->ipv4 = qemu_opt_get_bool(opts, "ipv4", 0);
+        addr->inet->has_ipv6 = qemu_opt_get(opts, "ipv6");
+        addr->inet->ipv6 = qemu_opt_get_bool(opts, "ipv6", 0);
+    }
+    backend->socket->addr = addr;
+}
+
+static void qemu_chr_parse_udp(QemuOpts *opts, ChardevBackend *backend,
+                               Error **errp)
+{
+    const char *host = qemu_opt_get(opts, "host");
+    const char *port = qemu_opt_get(opts, "port");
+    const char *localaddr = qemu_opt_get(opts, "localaddr");
+    const char *localport = qemu_opt_get(opts, "localport");
+    bool has_local = false;
+    SocketAddress *addr;
+
+    if (host == NULL || strlen(host) == 0) {
+        host = "localhost";
+    }
+    if (port == NULL || strlen(port) == 0) {
+        error_setg(errp, "chardev: udp: remote port not specified");
+        return;
+    }
+    if (localport == NULL || strlen(localport) == 0) {
+        localport = "0";
+    } else {
+        has_local = true;
+    }
+    if (localaddr == NULL || strlen(localaddr) == 0) {
+        localaddr = "";
+    } else {
+        has_local = true;
+    }
+
+    backend->udp = g_new0(ChardevUdp, 1);
+
+    addr = g_new0(SocketAddress, 1);
+    addr->kind = SOCKET_ADDRESS_KIND_INET;
+    addr->inet = g_new0(InetSocketAddress, 1);
+    addr->inet->host = g_strdup(host);
+    addr->inet->port = g_strdup(port);
+    addr->inet->has_ipv4 = qemu_opt_get(opts, "ipv4");
+    addr->inet->ipv4 = qemu_opt_get_bool(opts, "ipv4", 0);
+    addr->inet->has_ipv6 = qemu_opt_get(opts, "ipv6");
+    addr->inet->ipv6 = qemu_opt_get_bool(opts, "ipv6", 0);
+    backend->udp->remote = addr;
+
+    if (has_local) {
+        backend->udp->has_local = true;
+        addr = g_new0(SocketAddress, 1);
+        addr->kind = SOCKET_ADDRESS_KIND_INET;
+        addr->inet = g_new0(InetSocketAddress, 1);
+        addr->inet->host = g_strdup(localaddr);
+        addr->inet->port = g_strdup(localport);
+        backend->udp->local = addr;
+    }
+}
+
 typedef struct CharDriver {
     const char *name;
-    /* old, pre qapi */
-    CharDriverState *(*open)(QemuOpts *opts);
-    /* new, qapi-based */
     ChardevBackendKind kind;
     void (*parse)(QemuOpts *opts, ChardevBackend *backend, Error **errp);
 } CharDriver;
 
 static GSList *backends;
 
-void register_char_driver(const char *name, CharDriverState *(*open)(QemuOpts *))
-{
-    CharDriver *s;
-
-    s = g_malloc0(sizeof(*s));
-    s->name = g_strdup(name);
-    s->open = open;
-
-    backends = g_slist_append(backends, s);
-}
-
-void register_char_driver_qapi(const char *name, ChardevBackendKind kind,
+void register_char_driver(const char *name, ChardevBackendKind kind,
         void (*parse)(QemuOpts *opts, ChardevBackend *backend, Error **errp))
 {
     CharDriver *s;
@@ -3490,8 +3642,12 @@ CharDriverState *qemu_chr_new_from_opts(QemuOpts *opts,
     CharDriver *cd;
     CharDriverState *chr;
     GSList *i;
+    ChardevReturn *ret = NULL;
+    ChardevBackend *backend;
+    const char *id = qemu_opts_id(opts);
+    char *bid = NULL;
 
-    if (qemu_opts_id(opts) == NULL) {
+    if (id == NULL) {
         error_setg(errp, "chardev: no id specified");
         goto err;
     }
@@ -3514,89 +3670,49 @@ CharDriverState *qemu_chr_new_from_opts(QemuOpts *opts,
         goto err;
     }
 
-    if (!cd->open) {
-        /* using new, qapi init */
-        ChardevBackend *backend = g_new0(ChardevBackend, 1);
-        ChardevReturn *ret = NULL;
-        const char *id = qemu_opts_id(opts);
-        char *bid = NULL;
+    backend = g_new0(ChardevBackend, 1);
 
-        if (qemu_opt_get_bool(opts, "mux", 0)) {
-            bid = g_strdup_printf("%s-base", id);
-        }
+    if (qemu_opt_get_bool(opts, "mux", 0)) {
+        bid = g_strdup_printf("%s-base", id);
+    }
 
-        chr = NULL;
-        backend->kind = cd->kind;
-        if (cd->parse) {
-            cd->parse(opts, backend, &local_err);
-            if (local_err) {
-                error_propagate(errp, local_err);
-                goto qapi_out;
-            }
-        }
-        ret = qmp_chardev_add(bid ? bid : id, backend, errp);
-        if (!ret) {
+    chr = NULL;
+    backend->kind = cd->kind;
+    if (cd->parse) {
+        cd->parse(opts, backend, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
             goto qapi_out;
         }
-
-        if (bid) {
-            qapi_free_ChardevBackend(backend);
-            qapi_free_ChardevReturn(ret);
-            backend = g_new0(ChardevBackend, 1);
-            backend->mux = g_new0(ChardevMux, 1);
-            backend->kind = CHARDEV_BACKEND_KIND_MUX;
-            backend->mux->chardev = g_strdup(bid);
-            ret = qmp_chardev_add(id, backend, errp);
-            if (!ret) {
-                chr = qemu_chr_find(bid);
-                qemu_chr_delete(chr);
-                chr = NULL;
-                goto qapi_out;
-            }
-        }
-
-        chr = qemu_chr_find(id);
-        chr->opts = opts;
-
-    qapi_out:
-        qapi_free_ChardevBackend(backend);
-        qapi_free_ChardevReturn(ret);
-        g_free(bid);
-        return chr;
     }
-
-    chr = cd->open(opts);
-    if (!chr) {
-        error_setg(errp, "chardev: opening backend \"%s\" failed",
-                   qemu_opt_get(opts, "backend"));
-        goto err;
+    ret = qmp_chardev_add(bid ? bid : id, backend, errp);
+    if (!ret) {
+        goto qapi_out;
     }
 
-    if (!chr->filename)
-        chr->filename = g_strdup(qemu_opt_get(opts, "backend"));
-    chr->init = init;
-    /* if we didn't create the chardev via qmp_chardev_add, we
-     * need to send the OPENED event here
-     */
-    if (!chr->explicit_be_open) {
-        qemu_chr_be_event(chr, CHR_EVENT_OPENED);
+    if (bid) {
+        qapi_free_ChardevBackend(backend);
+        qapi_free_ChardevReturn(ret);
+        backend = g_new0(ChardevBackend, 1);
+        backend->mux = g_new0(ChardevMux, 1);
+        backend->kind = CHARDEV_BACKEND_KIND_MUX;
+        backend->mux->chardev = g_strdup(bid);
+        ret = qmp_chardev_add(id, backend, errp);
+        if (!ret) {
+            chr = qemu_chr_find(bid);
+            qemu_chr_delete(chr);
+            chr = NULL;
+            goto qapi_out;
+        }
     }
-    QTAILQ_INSERT_TAIL(&chardevs, chr, next);
 
-    if (qemu_opt_get_bool(opts, "mux", 0)) {
-        CharDriverState *base = chr;
-        int len = strlen(qemu_opts_id(opts)) + 6;
-        base->label = g_malloc(len);
-        snprintf(base->label, len, "%s-base", qemu_opts_id(opts));
-        chr = qemu_chr_open_mux(base);
-        chr->filename = base->filename;
-        chr->avail_connections = MAX_MUX;
-        QTAILQ_INSERT_TAIL(&chardevs, chr, next);
-    } else {
-        chr->avail_connections = 1;
-    }
-    chr->label = g_strdup(qemu_opts_id(opts));
+    chr = qemu_chr_find(id);
     chr->opts = opts;
+
+qapi_out:
+    qapi_free_ChardevBackend(backend);
+    qapi_free_ChardevReturn(ret);
+    g_free(bid);
     return chr;
 
 err:
@@ -3823,6 +3939,9 @@ QemuOptsList qemu_chardev_opts = {
             .name = "delay",
             .type = QEMU_OPT_BOOL,
         },{
+            .name = "reconnect",
+            .type = QEMU_OPT_NUMBER,
+        },{
             .name = "telnet",
             .type = QEMU_OPT_BOOL,
         },{
@@ -3966,26 +4085,95 @@ static CharDriverState *qmp_chardev_open_parallel(ChardevHostdev *parallel,
 
 #endif /* WIN32 */
 
+static void socket_try_connect(CharDriverState *chr)
+{
+    Error *err = NULL;
+
+    if (!qemu_chr_open_socket_fd(chr, &err)) {
+        check_report_connect_error(chr, err);
+    }
+}
+
+static gboolean socket_reconnect_timeout(gpointer opaque)
+{
+    CharDriverState *chr = opaque;
+    TCPCharDriver *s = chr->opaque;
+
+    s->reconnect_timer = 0;
+
+    if (chr->be_open) {
+        return false;
+    }
+
+    socket_try_connect(chr);
+
+    return false;
+}
+
 static CharDriverState *qmp_chardev_open_socket(ChardevSocket *sock,
                                                 Error **errp)
 {
+    CharDriverState *chr;
+    TCPCharDriver *s;
     SocketAddress *addr = sock->addr;
     bool do_nodelay     = sock->has_nodelay ? sock->nodelay : false;
     bool is_listen      = sock->has_server  ? sock->server  : true;
     bool is_telnet      = sock->has_telnet  ? sock->telnet  : false;
     bool is_waitconnect = sock->has_wait    ? sock->wait    : false;
-    int fd;
+    int64_t reconnect   = sock->has_reconnect ? sock->reconnect : 0;
+
+    chr = qemu_chr_alloc();
+    s = g_malloc0(sizeof(TCPCharDriver));
+
+    s->fd = -1;
+    s->listen_fd = -1;
+    s->is_unix = addr->kind == SOCKET_ADDRESS_KIND_UNIX;
+    s->is_listen = is_listen;
+    s->is_telnet = is_telnet;
+    s->do_nodelay = do_nodelay;
+    qapi_copy_SocketAddress(&s->addr, sock->addr);
+
+    chr->opaque = s;
+    chr->chr_write = tcp_chr_write;
+    chr->chr_sync_read = tcp_chr_sync_read;
+    chr->chr_close = tcp_chr_close;
+    chr->get_msgfds = tcp_get_msgfds;
+    chr->set_msgfds = tcp_set_msgfds;
+    chr->chr_add_client = tcp_chr_add_client;
+    chr->chr_add_watch = tcp_chr_add_watch;
+    chr->chr_update_read_handler = tcp_chr_update_read_handler;
+    /* be isn't opened until we get a connection */
+    chr->explicit_be_open = true;
+
+    chr->filename = g_malloc(CHR_MAX_FILENAME_SIZE);
+    SocketAddress_to_str(chr->filename, CHR_MAX_FILENAME_SIZE, "disconnected:",
+                         addr, is_listen, is_telnet);
 
     if (is_listen) {
-        fd = socket_listen(addr, errp);
-    } else {
-        fd = socket_connect(addr, errp, NULL, NULL);
+        if (is_telnet) {
+            s->do_telnetopt = 1;
+        }
+    } else if (reconnect > 0) {
+        s->reconnect_time = reconnect;
     }
-    if (fd < 0) {
+
+    if (s->reconnect_time) {
+        socket_try_connect(chr);
+    } else if (!qemu_chr_open_socket_fd(chr, errp)) {
+        g_free(s);
+        g_free(chr->filename);
+        g_free(chr);
         return NULL;
     }
-    return qemu_chr_open_socket_fd(fd, do_nodelay, is_listen,
-                                   is_telnet, is_waitconnect, errp);
+
+    if (is_listen && is_waitconnect) {
+        fprintf(stderr, "QEMU waiting for connection on: %s\n",
+                chr->filename);
+        tcp_chr_accept(s->listen_chan, G_IO_IN, chr);
+        qemu_set_nonblock(s->listen_fd);
+    }
+
+    return chr;
 }
 
 static CharDriverState *qmp_chardev_open_udp(ChardevUdp *udp,
@@ -4057,6 +4245,9 @@ ChardevReturn *qmp_chardev_add(const char *id, ChardevBackend *backend,
         chr = chr_baum_init();
         break;
 #endif
+    case CHARDEV_BACKEND_KIND_TESTDEV:
+        chr = chr_testdev_init();
+        break;
     case CHARDEV_BACKEND_KIND_STDIO:
         chr = qemu_chr_open_stdio(backend->stdio);
         break;
@@ -4117,7 +4308,7 @@ void qmp_chardev_remove(const char *id, Error **errp)
     CharDriverState *chr;
 
     chr = qemu_chr_find(id);
-    if (NULL == chr) {
+    if (chr == NULL) {
         error_setg(errp, "Chardev '%s' not found", id);
         return;
     }
@@ -4131,32 +4322,32 @@ void qmp_chardev_remove(const char *id, Error **errp)
 
 static void register_types(void)
 {
-    register_char_driver_qapi("null", CHARDEV_BACKEND_KIND_NULL, NULL);
-    register_char_driver("socket", qemu_chr_open_socket);
-    register_char_driver("udp", qemu_chr_open_udp);
-    register_char_driver_qapi("ringbuf", CHARDEV_BACKEND_KIND_RINGBUF,
-                              qemu_chr_parse_ringbuf);
-    register_char_driver_qapi("file", CHARDEV_BACKEND_KIND_FILE,
-                              qemu_chr_parse_file_out);
-    register_char_driver_qapi("stdio", CHARDEV_BACKEND_KIND_STDIO,
-                              qemu_chr_parse_stdio);
-    register_char_driver_qapi("serial", CHARDEV_BACKEND_KIND_SERIAL,
-                              qemu_chr_parse_serial);
-    register_char_driver_qapi("tty", CHARDEV_BACKEND_KIND_SERIAL,
-                              qemu_chr_parse_serial);
-    register_char_driver_qapi("parallel", CHARDEV_BACKEND_KIND_PARALLEL,
-                              qemu_chr_parse_parallel);
-    register_char_driver_qapi("parport", CHARDEV_BACKEND_KIND_PARALLEL,
-                              qemu_chr_parse_parallel);
-    register_char_driver_qapi("pty", CHARDEV_BACKEND_KIND_PTY, NULL);
-    register_char_driver_qapi("console", CHARDEV_BACKEND_KIND_CONSOLE, NULL);
-    register_char_driver_qapi("pipe", CHARDEV_BACKEND_KIND_PIPE,
-                              qemu_chr_parse_pipe);
-    register_char_driver_qapi("mux", CHARDEV_BACKEND_KIND_MUX,
-                              qemu_chr_parse_mux);
+    register_char_driver("null", CHARDEV_BACKEND_KIND_NULL, NULL);
+    register_char_driver("socket", CHARDEV_BACKEND_KIND_SOCKET,
+                         qemu_chr_parse_socket);
+    register_char_driver("udp", CHARDEV_BACKEND_KIND_UDP, qemu_chr_parse_udp);
+    register_char_driver("ringbuf", CHARDEV_BACKEND_KIND_RINGBUF,
+                         qemu_chr_parse_ringbuf);
+    register_char_driver("file", CHARDEV_BACKEND_KIND_FILE,
+                         qemu_chr_parse_file_out);
+    register_char_driver("stdio", CHARDEV_BACKEND_KIND_STDIO,
+                         qemu_chr_parse_stdio);
+    register_char_driver("serial", CHARDEV_BACKEND_KIND_SERIAL,
+                         qemu_chr_parse_serial);
+    register_char_driver("tty", CHARDEV_BACKEND_KIND_SERIAL,
+                         qemu_chr_parse_serial);
+    register_char_driver("parallel", CHARDEV_BACKEND_KIND_PARALLEL,
+                         qemu_chr_parse_parallel);
+    register_char_driver("parport", CHARDEV_BACKEND_KIND_PARALLEL,
+                         qemu_chr_parse_parallel);
+    register_char_driver("pty", CHARDEV_BACKEND_KIND_PTY, NULL);
+    register_char_driver("console", CHARDEV_BACKEND_KIND_CONSOLE, NULL);
+    register_char_driver("pipe", CHARDEV_BACKEND_KIND_PIPE,
+                         qemu_chr_parse_pipe);
+    register_char_driver("mux", CHARDEV_BACKEND_KIND_MUX, qemu_chr_parse_mux);
     /* Bug-compatibility: */
-    register_char_driver_qapi("memory", CHARDEV_BACKEND_KIND_MEMORY,
-                              qemu_chr_parse_ringbuf);
+    register_char_driver("memory", CHARDEV_BACKEND_KIND_MEMORY,
+                         qemu_chr_parse_ringbuf);
     /* this must be done after machine init, since we register FEs with muxes
      * as part of realize functions like serial_isa_realizefn when -nographic
      * is specified
diff --git a/qemu-coroutine-io.c b/qemu-coroutine-io.c
index 054ca7062..d4049260d 100644
--- a/qemu-coroutine-io.c
+++ b/qemu-coroutine-io.c
@@ -34,13 +34,15 @@ qemu_co_sendv_recvv(int sockfd, struct iovec *iov, unsigned iov_cnt,
 {
     size_t done = 0;
     ssize_t ret;
+    int err;
     while (done < bytes) {
         ret = iov_send_recv(sockfd, iov, iov_cnt,
                             offset + done, bytes - done, do_send);
         if (ret > 0) {
             done += ret;
         } else if (ret < 0) {
-            if (errno == EAGAIN) {
+            err = socket_error();
+            if (err == EAGAIN || err == EWOULDBLOCK) {
                 qemu_coroutine_yield();
             } else if (done == 0) {
                 return -1;
diff --git a/qemu-coroutine-sleep.c b/qemu-coroutine-sleep.c
index ad78fbaa2..9abb7fdf3 100644
--- a/qemu-coroutine-sleep.c
+++ b/qemu-coroutine-sleep.c
@@ -27,18 +27,6 @@ static void co_sleep_cb(void *opaque)
     qemu_coroutine_enter(sleep_cb->co, NULL);
 }
 
-void coroutine_fn co_sleep_ns(QEMUClockType type, int64_t ns)
-{
-    CoSleepCB sleep_cb = {
-        .co = qemu_coroutine_self(),
-    };
-    sleep_cb.ts = timer_new(type, SCALE_NS, co_sleep_cb, &sleep_cb);
-    timer_mod(sleep_cb.ts, qemu_clock_get_ns(type) + ns);
-    qemu_coroutine_yield();
-    timer_del(sleep_cb.ts);
-    timer_free(sleep_cb.ts);
-}
-
 void coroutine_fn co_aio_sleep_ns(AioContext *ctx, QEMUClockType type,
                                   int64_t ns)
 {
diff --git a/qemu-coroutine.c b/qemu-coroutine.c
index 470852100..bd574aa1b 100644
--- a/qemu-coroutine.c
+++ b/qemu-coroutine.c
@@ -19,14 +19,14 @@
 #include "block/coroutine_int.h"
 
 enum {
-    /* Maximum free pool size prevents holding too many freed coroutines */
-    POOL_MAX_SIZE = 64,
+    POOL_DEFAULT_SIZE = 64,
 };
 
 /** Free list to speed up creation */
 static QemuMutex pool_lock;
 static QSLIST_HEAD(, Coroutine) pool = QSLIST_HEAD_INITIALIZER(pool);
 static unsigned int pool_size;
+static unsigned int pool_max_size = POOL_DEFAULT_SIZE;
 
 Coroutine *qemu_coroutine_create(CoroutineEntry *entry)
 {
@@ -55,7 +55,7 @@ static void coroutine_delete(Coroutine *co)
 {
     if (CONFIG_COROUTINE_POOL) {
         qemu_mutex_lock(&pool_lock);
-        if (pool_size < POOL_MAX_SIZE) {
+        if (pool_size < pool_max_size) {
             QSLIST_INSERT_HEAD(&pool, co, pool_next);
             co->caller = NULL;
             pool_size++;
@@ -137,3 +137,23 @@ void coroutine_fn qemu_coroutine_yield(void)
     self->caller = NULL;
     coroutine_swap(self, to);
 }
+
+void qemu_coroutine_adjust_pool_size(int n)
+{
+    qemu_mutex_lock(&pool_lock);
+
+    pool_max_size += n;
+
+    /* Callers should never take away more than they added */
+    assert(pool_max_size >= POOL_DEFAULT_SIZE);
+
+    /* Trim oversized pool down to new max */
+    while (pool_size > pool_max_size) {
+        Coroutine *co = QSLIST_FIRST(&pool);
+        QSLIST_REMOVE_HEAD(&pool, pool_next);
+        pool_size--;
+        qemu_coroutine_delete(co);
+    }
+
+    qemu_mutex_unlock(&pool_lock);
+}
diff --git a/qemu-doc.texi b/qemu-doc.texi
index 2b232ae8b..ad418f851 100644
--- a/qemu-doc.texi
+++ b/qemu-doc.texi
@@ -527,6 +527,15 @@ Linux or NTFS on Windows), then only the written sectors will reserve
 space. Use @code{qemu-img info} to know the real size used by the
 image or @code{ls -ls} on Unix/Linux.
 
+Supported options:
+@table @code
+@item preallocation
+Preallocation mode (allowed values: @code{off}, @code{falloc}, @code{full}).
+@code{falloc} mode preallocates space for image by calling posix_fallocate().
+@code{full} mode preallocates space for image by writing zeros to underlying
+storage.
+@end table
+
 @item qcow2
 QEMU image format, the most versatile format. Use it to have smaller
 images (useful if your filesystem does not supports holes, for example
@@ -575,9 +584,11 @@ sizes can improve the image file size whereas larger cluster sizes generally
 provide better performance.
 
 @item preallocation
-Preallocation mode (allowed values: off, metadata). An image with preallocated
-metadata is initially larger but can improve performance when the image needs
-to grow.
+Preallocation mode (allowed values: @code{off}, @code{metadata}, @code{falloc},
+@code{full}). An image with preallocated metadata is initially larger but can
+improve performance when the image needs to grow. @code{falloc} and @code{full}
+preallocations are like the same options of @code{raw} format, but sets up
+metadata also.
 
 @item lazy_refcounts
 If this option is set to @code{on}, reference count updates are postponed with
@@ -643,15 +654,6 @@ File name of a base image (see @option{create} subcommand)
 If this option is set to @code{on}, the image is encrypted.
 @end table
 
-@item cow
-User Mode Linux Copy On Write image format. It is supported only for
-compatibility with previous versions.
-Supported options:
-@table @code
-@item backing_file
-File name of a base image (see @option{create} subcommand)
-@end table
-
 @item vdi
 VirtualBox 1.1 compatible image format.
 Supported options:
@@ -1629,7 +1631,7 @@ EOF
 # certtool --generate-certificate \
            --load-ca-certificate ca-cert.pem \
            --load-ca-privkey ca-key.pem \
-           --load-privkey server server-key.pem \
+           --load-privkey server-key.pem \
            --template server.info \
            --outfile server-cert.pem
 @end example
@@ -1652,7 +1654,7 @@ the secure CA private key:
 country = GB
 state = London
 locality = London
-organiazation = Name of your organization
+organization = Name of your organization
 cn = client.foo.example.com
 tls_www_client
 encryption_key
diff --git a/qemu-file-stdio.c b/qemu-file-stdio.c
new file mode 100644
index 000000000..285068b30
--- /dev/null
+++ b/qemu-file-stdio.c
@@ -0,0 +1,194 @@
+/*
+ * QEMU System Emulator
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu-common.h"
+#include "block/coroutine.h"
+#include "migration/qemu-file.h"
+
+typedef struct QEMUFileStdio {
+    FILE *stdio_file;
+    QEMUFile *file;
+} QEMUFileStdio;
+
+static int stdio_get_fd(void *opaque)
+{
+    QEMUFileStdio *s = opaque;
+
+    return fileno(s->stdio_file);
+}
+
+static int stdio_put_buffer(void *opaque, const uint8_t *buf, int64_t pos,
+                            int size)
+{
+    QEMUFileStdio *s = opaque;
+    int res;
+
+    res = fwrite(buf, 1, size, s->stdio_file);
+
+    if (res != size) {
+        return -errno;
+    }
+    return res;
+}
+
+static int stdio_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
+{
+    QEMUFileStdio *s = opaque;
+    FILE *fp = s->stdio_file;
+    int bytes;
+
+    for (;;) {
+        clearerr(fp);
+        bytes = fread(buf, 1, size, fp);
+        if (bytes != 0 || !ferror(fp)) {
+            break;
+        }
+        if (errno == EAGAIN) {
+            yield_until_fd_readable(fileno(fp));
+        } else if (errno != EINTR) {
+            break;
+        }
+    }
+    return bytes;
+}
+
+static int stdio_pclose(void *opaque)
+{
+    QEMUFileStdio *s = opaque;
+    int ret;
+    ret = pclose(s->stdio_file);
+    if (ret == -1) {
+        ret = -errno;
+    } else if (!WIFEXITED(ret) || WEXITSTATUS(ret) != 0) {
+        /* close succeeded, but non-zero exit code: */
+        ret = -EIO; /* fake errno value */
+    }
+    g_free(s);
+    return ret;
+}
+
+static int stdio_fclose(void *opaque)
+{
+    QEMUFileStdio *s = opaque;
+    int ret = 0;
+
+    if (qemu_file_is_writable(s->file)) {
+        int fd = fileno(s->stdio_file);
+        struct stat st;
+
+        ret = fstat(fd, &st);
+        if (ret == 0 && S_ISREG(st.st_mode)) {
+            /*
+             * If the file handle is a regular file make sure the
+             * data is flushed to disk before signaling success.
+             */
+            ret = fsync(fd);
+            if (ret != 0) {
+                ret = -errno;
+                return ret;
+            }
+        }
+    }
+    if (fclose(s->stdio_file) == EOF) {
+        ret = -errno;
+    }
+    g_free(s);
+    return ret;
+}
+
+static const QEMUFileOps stdio_pipe_read_ops = {
+    .get_fd =     stdio_get_fd,
+    .get_buffer = stdio_get_buffer,
+    .close =      stdio_pclose
+};
+
+static const QEMUFileOps stdio_pipe_write_ops = {
+    .get_fd =     stdio_get_fd,
+    .put_buffer = stdio_put_buffer,
+    .close =      stdio_pclose
+};
+
+QEMUFile *qemu_popen_cmd(const char *command, const char *mode)
+{
+    FILE *stdio_file;
+    QEMUFileStdio *s;
+
+    if (mode == NULL || (mode[0] != 'r' && mode[0] != 'w') || mode[1] != 0) {
+        fprintf(stderr, "qemu_popen: Argument validity check failed\n");
+        return NULL;
+    }
+
+    stdio_file = popen(command, mode);
+    if (stdio_file == NULL) {
+        return NULL;
+    }
+
+    s = g_malloc0(sizeof(QEMUFileStdio));
+
+    s->stdio_file = stdio_file;
+
+    if (mode[0] == 'r') {
+        s->file = qemu_fopen_ops(s, &stdio_pipe_read_ops);
+    } else {
+        s->file = qemu_fopen_ops(s, &stdio_pipe_write_ops);
+    }
+    return s->file;
+}
+
+static const QEMUFileOps stdio_file_read_ops = {
+    .get_fd =     stdio_get_fd,
+    .get_buffer = stdio_get_buffer,
+    .close =      stdio_fclose
+};
+
+static const QEMUFileOps stdio_file_write_ops = {
+    .get_fd =     stdio_get_fd,
+    .put_buffer = stdio_put_buffer,
+    .close =      stdio_fclose
+};
+
+QEMUFile *qemu_fopen(const char *filename, const char *mode)
+{
+    QEMUFileStdio *s;
+
+    if (qemu_file_mode_is_not_valid(mode)) {
+        return NULL;
+    }
+
+    s = g_malloc0(sizeof(QEMUFileStdio));
+
+    s->stdio_file = fopen(filename, mode);
+    if (!s->stdio_file) {
+        goto fail;
+    }
+
+    if (mode[0] == 'w') {
+        s->file = qemu_fopen_ops(s, &stdio_file_write_ops);
+    } else {
+        s->file = qemu_fopen_ops(s, &stdio_file_read_ops);
+    }
+    return s->file;
+fail:
+    g_free(s);
+    return NULL;
+}
diff --git a/qemu-file-unix.c b/qemu-file-unix.c
new file mode 100644
index 000000000..9682396d9
--- /dev/null
+++ b/qemu-file-unix.c
@@ -0,0 +1,223 @@
+/*
+ * QEMU System Emulator
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu-common.h"
+#include "qemu/iov.h"
+#include "qemu/sockets.h"
+#include "block/coroutine.h"
+#include "migration/qemu-file.h"
+
+typedef struct QEMUFileSocket {
+    int fd;
+    QEMUFile *file;
+} QEMUFileSocket;
+
+static ssize_t socket_writev_buffer(void *opaque, struct iovec *iov, int iovcnt,
+                                    int64_t pos)
+{
+    QEMUFileSocket *s = opaque;
+    ssize_t len;
+    ssize_t size = iov_size(iov, iovcnt);
+
+    len = iov_send(s->fd, iov, iovcnt, 0, size);
+    if (len < size) {
+        len = -socket_error();
+    }
+    return len;
+}
+
+static int socket_get_fd(void *opaque)
+{
+    QEMUFileSocket *s = opaque;
+
+    return s->fd;
+}
+
+static int socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
+{
+    QEMUFileSocket *s = opaque;
+    ssize_t len;
+
+    for (;;) {
+        len = qemu_recv(s->fd, buf, size, 0);
+        if (len != -1) {
+            break;
+        }
+        if (socket_error() == EAGAIN) {
+            yield_until_fd_readable(s->fd);
+        } else if (socket_error() != EINTR) {
+            break;
+        }
+    }
+
+    if (len == -1) {
+        len = -socket_error();
+    }
+    return len;
+}
+
+static int socket_close(void *opaque)
+{
+    QEMUFileSocket *s = opaque;
+    closesocket(s->fd);
+    g_free(s);
+    return 0;
+}
+
+static ssize_t unix_writev_buffer(void *opaque, struct iovec *iov, int iovcnt,
+                                  int64_t pos)
+{
+    QEMUFileSocket *s = opaque;
+    ssize_t len, offset;
+    ssize_t size = iov_size(iov, iovcnt);
+    ssize_t total = 0;
+
+    assert(iovcnt > 0);
+    offset = 0;
+    while (size > 0) {
+        /* Find the next start position; skip all full-sized vector elements  */
+        while (offset >= iov[0].iov_len) {
+            offset -= iov[0].iov_len;
+            iov++, iovcnt--;
+        }
+
+        /* skip `offset' bytes from the (now) first element, undo it on exit */
+        assert(iovcnt > 0);
+        iov[0].iov_base += offset;
+        iov[0].iov_len -= offset;
+
+        do {
+            len = writev(s->fd, iov, iovcnt);
+        } while (len == -1 && errno == EINTR);
+        if (len == -1) {
+            return -errno;
+        }
+
+        /* Undo the changes above */
+        iov[0].iov_base -= offset;
+        iov[0].iov_len += offset;
+
+        /* Prepare for the next iteration */
+        offset += len;
+        total += len;
+        size -= len;
+    }
+
+    return total;
+}
+
+static int unix_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
+{
+    QEMUFileSocket *s = opaque;
+    ssize_t len;
+
+    for (;;) {
+        len = read(s->fd, buf, size);
+        if (len != -1) {
+            break;
+        }
+        if (errno == EAGAIN) {
+            yield_until_fd_readable(s->fd);
+        } else if (errno != EINTR) {
+            break;
+        }
+    }
+
+    if (len == -1) {
+        len = -errno;
+    }
+    return len;
+}
+
+static int unix_close(void *opaque)
+{
+    QEMUFileSocket *s = opaque;
+    close(s->fd);
+    g_free(s);
+    return 0;
+}
+
+static const QEMUFileOps unix_read_ops = {
+    .get_fd =     socket_get_fd,
+    .get_buffer = unix_get_buffer,
+    .close =      unix_close
+};
+
+static const QEMUFileOps unix_write_ops = {
+    .get_fd =     socket_get_fd,
+    .writev_buffer = unix_writev_buffer,
+    .close =      unix_close
+};
+
+QEMUFile *qemu_fdopen(int fd, const char *mode)
+{
+    QEMUFileSocket *s;
+
+    if (mode == NULL ||
+        (mode[0] != 'r' && mode[0] != 'w') ||
+        mode[1] != 'b' || mode[2] != 0) {
+        fprintf(stderr, "qemu_fdopen: Argument validity check failed\n");
+        return NULL;
+    }
+
+    s = g_malloc0(sizeof(QEMUFileSocket));
+    s->fd = fd;
+
+    if (mode[0] == 'r') {
+        s->file = qemu_fopen_ops(s, &unix_read_ops);
+    } else {
+        s->file = qemu_fopen_ops(s, &unix_write_ops);
+    }
+    return s->file;
+}
+
+static const QEMUFileOps socket_read_ops = {
+    .get_fd =     socket_get_fd,
+    .get_buffer = socket_get_buffer,
+    .close =      socket_close
+};
+
+static const QEMUFileOps socket_write_ops = {
+    .get_fd =     socket_get_fd,
+    .writev_buffer = socket_writev_buffer,
+    .close =      socket_close
+};
+
+QEMUFile *qemu_fopen_socket(int fd, const char *mode)
+{
+    QEMUFileSocket *s;
+
+    if (qemu_file_mode_is_not_valid(mode)) {
+        return NULL;
+    }
+
+    s = g_malloc0(sizeof(QEMUFileSocket));
+    s->fd = fd;
+    if (mode[0] == 'w') {
+        qemu_set_block(s->fd);
+        s->file = qemu_fopen_ops(s, &socket_write_ops);
+    } else {
+        s->file = qemu_fopen_ops(s, &socket_read_ops);
+    }
+    return s->file;
+}
diff --git a/qemu-file.c b/qemu-file.c
index a8e39127f..f938e36fe 100644
--- a/qemu-file.c
+++ b/qemu-file.c
@@ -1,3 +1,26 @@
+/*
+ * QEMU System Emulator
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
 #include "qemu-common.h"
 #include "qemu/iov.h"
 #include "qemu/sockets.h"
@@ -28,324 +51,6 @@ struct QEMUFile {
     int last_error;
 };
 
-typedef struct QEMUFileStdio {
-    FILE *stdio_file;
-    QEMUFile *file;
-} QEMUFileStdio;
-
-typedef struct QEMUFileSocket {
-    int fd;
-    QEMUFile *file;
-} QEMUFileSocket;
-
-static ssize_t socket_writev_buffer(void *opaque, struct iovec *iov, int iovcnt,
-                                    int64_t pos)
-{
-    QEMUFileSocket *s = opaque;
-    ssize_t len;
-    ssize_t size = iov_size(iov, iovcnt);
-
-    len = iov_send(s->fd, iov, iovcnt, 0, size);
-    if (len < size) {
-        len = -socket_error();
-    }
-    return len;
-}
-
-static int socket_get_fd(void *opaque)
-{
-    QEMUFileSocket *s = opaque;
-
-    return s->fd;
-}
-
-static int socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
-{
-    QEMUFileSocket *s = opaque;
-    ssize_t len;
-
-    for (;;) {
-        len = qemu_recv(s->fd, buf, size, 0);
-        if (len != -1) {
-            break;
-        }
-        if (socket_error() == EAGAIN) {
-            yield_until_fd_readable(s->fd);
-        } else if (socket_error() != EINTR) {
-            break;
-        }
-    }
-
-    if (len == -1) {
-        len = -socket_error();
-    }
-    return len;
-}
-
-static int socket_close(void *opaque)
-{
-    QEMUFileSocket *s = opaque;
-    closesocket(s->fd);
-    g_free(s);
-    return 0;
-}
-
-static int stdio_get_fd(void *opaque)
-{
-    QEMUFileStdio *s = opaque;
-
-    return fileno(s->stdio_file);
-}
-
-static int stdio_put_buffer(void *opaque, const uint8_t *buf, int64_t pos,
-                            int size)
-{
-    QEMUFileStdio *s = opaque;
-    int res;
-
-    res = fwrite(buf, 1, size, s->stdio_file);
-
-    if (res != size) {
-        return -errno;
-    }
-    return res;
-}
-
-static int stdio_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
-{
-    QEMUFileStdio *s = opaque;
-    FILE *fp = s->stdio_file;
-    int bytes;
-
-    for (;;) {
-        clearerr(fp);
-        bytes = fread(buf, 1, size, fp);
-        if (bytes != 0 || !ferror(fp)) {
-            break;
-        }
-        if (errno == EAGAIN) {
-            yield_until_fd_readable(fileno(fp));
-        } else if (errno != EINTR) {
-            break;
-        }
-    }
-    return bytes;
-}
-
-static int stdio_pclose(void *opaque)
-{
-    QEMUFileStdio *s = opaque;
-    int ret;
-    ret = pclose(s->stdio_file);
-    if (ret == -1) {
-        ret = -errno;
-    } else if (!WIFEXITED(ret) || WEXITSTATUS(ret) != 0) {
-        /* close succeeded, but non-zero exit code: */
-        ret = -EIO; /* fake errno value */
-    }
-    g_free(s);
-    return ret;
-}
-
-static int stdio_fclose(void *opaque)
-{
-    QEMUFileStdio *s = opaque;
-    int ret = 0;
-
-    if (s->file->ops->put_buffer || s->file->ops->writev_buffer) {
-        int fd = fileno(s->stdio_file);
-        struct stat st;
-
-        ret = fstat(fd, &st);
-        if (ret == 0 && S_ISREG(st.st_mode)) {
-            /*
-             * If the file handle is a regular file make sure the
-             * data is flushed to disk before signaling success.
-             */
-            ret = fsync(fd);
-            if (ret != 0) {
-                ret = -errno;
-                return ret;
-            }
-        }
-    }
-    if (fclose(s->stdio_file) == EOF) {
-        ret = -errno;
-    }
-    g_free(s);
-    return ret;
-}
-
-static const QEMUFileOps stdio_pipe_read_ops = {
-    .get_fd =     stdio_get_fd,
-    .get_buffer = stdio_get_buffer,
-    .close =      stdio_pclose
-};
-
-static const QEMUFileOps stdio_pipe_write_ops = {
-    .get_fd =     stdio_get_fd,
-    .put_buffer = stdio_put_buffer,
-    .close =      stdio_pclose
-};
-
-QEMUFile *qemu_popen_cmd(const char *command, const char *mode)
-{
-    FILE *stdio_file;
-    QEMUFileStdio *s;
-
-    if (mode == NULL || (mode[0] != 'r' && mode[0] != 'w') || mode[1] != 0) {
-        fprintf(stderr, "qemu_popen: Argument validity check failed\n");
-        return NULL;
-    }
-
-    stdio_file = popen(command, mode);
-    if (stdio_file == NULL) {
-        return NULL;
-    }
-
-    s = g_malloc0(sizeof(QEMUFileStdio));
-
-    s->stdio_file = stdio_file;
-
-    if (mode[0] == 'r') {
-        s->file = qemu_fopen_ops(s, &stdio_pipe_read_ops);
-    } else {
-        s->file = qemu_fopen_ops(s, &stdio_pipe_write_ops);
-    }
-    return s->file;
-}
-
-static const QEMUFileOps stdio_file_read_ops = {
-    .get_fd =     stdio_get_fd,
-    .get_buffer = stdio_get_buffer,
-    .close =      stdio_fclose
-};
-
-static const QEMUFileOps stdio_file_write_ops = {
-    .get_fd =     stdio_get_fd,
-    .put_buffer = stdio_put_buffer,
-    .close =      stdio_fclose
-};
-
-static ssize_t unix_writev_buffer(void *opaque, struct iovec *iov, int iovcnt,
-                                  int64_t pos)
-{
-    QEMUFileSocket *s = opaque;
-    ssize_t len, offset;
-    ssize_t size = iov_size(iov, iovcnt);
-    ssize_t total = 0;
-
-    assert(iovcnt > 0);
-    offset = 0;
-    while (size > 0) {
-        /* Find the next start position; skip all full-sized vector elements  */
-        while (offset >= iov[0].iov_len) {
-            offset -= iov[0].iov_len;
-            iov++, iovcnt--;
-        }
-
-        /* skip `offset' bytes from the (now) first element, undo it on exit */
-        assert(iovcnt > 0);
-        iov[0].iov_base += offset;
-        iov[0].iov_len -= offset;
-
-        do {
-            len = writev(s->fd, iov, iovcnt);
-        } while (len == -1 && errno == EINTR);
-        if (len == -1) {
-            return -errno;
-        }
-
-        /* Undo the changes above */
-        iov[0].iov_base -= offset;
-        iov[0].iov_len += offset;
-
-        /* Prepare for the next iteration */
-        offset += len;
-        total += len;
-        size -= len;
-    }
-
-    return total;
-}
-
-static int unix_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
-{
-    QEMUFileSocket *s = opaque;
-    ssize_t len;
-
-    for (;;) {
-        len = read(s->fd, buf, size);
-        if (len != -1) {
-            break;
-        }
-        if (errno == EAGAIN) {
-            yield_until_fd_readable(s->fd);
-        } else if (errno != EINTR) {
-            break;
-        }
-    }
-
-    if (len == -1) {
-        len = -errno;
-    }
-    return len;
-}
-
-static int unix_close(void *opaque)
-{
-    QEMUFileSocket *s = opaque;
-    close(s->fd);
-    g_free(s);
-    return 0;
-}
-
-static const QEMUFileOps unix_read_ops = {
-    .get_fd =     socket_get_fd,
-    .get_buffer = unix_get_buffer,
-    .close =      unix_close
-};
-
-static const QEMUFileOps unix_write_ops = {
-    .get_fd =     socket_get_fd,
-    .writev_buffer = unix_writev_buffer,
-    .close =      unix_close
-};
-
-QEMUFile *qemu_fdopen(int fd, const char *mode)
-{
-    QEMUFileSocket *s;
-
-    if (mode == NULL ||
-        (mode[0] != 'r' && mode[0] != 'w') ||
-        mode[1] != 'b' || mode[2] != 0) {
-        fprintf(stderr, "qemu_fdopen: Argument validity check failed\n");
-        return NULL;
-    }
-
-    s = g_malloc0(sizeof(QEMUFileSocket));
-    s->fd = fd;
-
-    if (mode[0] == 'r') {
-        s->file = qemu_fopen_ops(s, &unix_read_ops);
-    } else {
-        s->file = qemu_fopen_ops(s, &unix_write_ops);
-    }
-    return s->file;
-}
-
-static const QEMUFileOps socket_read_ops = {
-    .get_fd =     socket_get_fd,
-    .get_buffer = socket_get_buffer,
-    .close =      socket_close
-};
-
-static const QEMUFileOps socket_write_ops = {
-    .get_fd =     socket_get_fd,
-    .writev_buffer = socket_writev_buffer,
-    .close =      socket_close
-};
-
 bool qemu_file_mode_is_not_valid(const char *mode)
 {
     if (mode == NULL ||
@@ -358,51 +63,6 @@ bool qemu_file_mode_is_not_valid(const char *mode)
     return false;
 }
 
-QEMUFile *qemu_fopen_socket(int fd, const char *mode)
-{
-    QEMUFileSocket *s;
-
-    if (qemu_file_mode_is_not_valid(mode)) {
-        return NULL;
-    }
-
-    s = g_malloc0(sizeof(QEMUFileSocket));
-    s->fd = fd;
-    if (mode[0] == 'w') {
-        qemu_set_block(s->fd);
-        s->file = qemu_fopen_ops(s, &socket_write_ops);
-    } else {
-        s->file = qemu_fopen_ops(s, &socket_read_ops);
-    }
-    return s->file;
-}
-
-QEMUFile *qemu_fopen(const char *filename, const char *mode)
-{
-    QEMUFileStdio *s;
-
-    if (qemu_file_mode_is_not_valid(mode)) {
-        return NULL;
-    }
-
-    s = g_malloc0(sizeof(QEMUFileStdio));
-
-    s->stdio_file = fopen(filename, mode);
-    if (!s->stdio_file) {
-        goto fail;
-    }
-
-    if (mode[0] == 'w') {
-        s->file = qemu_fopen_ops(s, &stdio_file_write_ops);
-    } else {
-        s->file = qemu_fopen_ops(s, &stdio_file_read_ops);
-    }
-    return s->file;
-fail:
-    g_free(s);
-    return NULL;
-}
-
 QEMUFile *qemu_fopen_ops(void *opaque, const QEMUFileOps *ops)
 {
     QEMUFile *f;
@@ -433,7 +93,7 @@ void qemu_file_set_error(QEMUFile *f, int ret)
     }
 }
 
-static inline bool qemu_file_is_writable(QEMUFile *f)
+bool qemu_file_is_writable(QEMUFile *f)
 {
     return f->ops->writev_buffer || f->ops->put_buffer;
 }
@@ -878,3 +538,458 @@ uint64_t qemu_get_be64(QEMUFile *f)
     v |= qemu_get_be32(f);
     return v;
 }
+
+#define QSB_CHUNK_SIZE      (1 << 10)
+#define QSB_MAX_CHUNK_SIZE  (16 * QSB_CHUNK_SIZE)
+
+/**
+ * Create a QEMUSizedBuffer
+ * This type of buffer uses scatter-gather lists internally and
+ * can grow to any size. Any data array in the scatter-gather list
+ * can hold different amount of bytes.
+ *
+ * @buffer: Optional buffer to copy into the QSB
+ * @len: size of initial buffer; if @buffer is given, buffer must
+ *       hold at least len bytes
+ *
+ * Returns a pointer to a QEMUSizedBuffer or NULL on allocation failure
+ */
+QEMUSizedBuffer *qsb_create(const uint8_t *buffer, size_t len)
+{
+    QEMUSizedBuffer *qsb;
+    size_t alloc_len, num_chunks, i, to_copy;
+    size_t chunk_size = (len > QSB_MAX_CHUNK_SIZE)
+                        ? QSB_MAX_CHUNK_SIZE
+                        : QSB_CHUNK_SIZE;
+
+    num_chunks = DIV_ROUND_UP(len ? len : QSB_CHUNK_SIZE, chunk_size);
+    alloc_len = num_chunks * chunk_size;
+
+    qsb = g_try_new0(QEMUSizedBuffer, 1);
+    if (!qsb) {
+        return NULL;
+    }
+
+    qsb->iov = g_try_new0(struct iovec, num_chunks);
+    if (!qsb->iov) {
+        g_free(qsb);
+        return NULL;
+    }
+
+    qsb->n_iov = num_chunks;
+
+    for (i = 0; i < num_chunks; i++) {
+        qsb->iov[i].iov_base = g_try_malloc0(chunk_size);
+        if (!qsb->iov[i].iov_base) {
+            /* qsb_free is safe since g_free can cope with NULL */
+            qsb_free(qsb);
+            return NULL;
+        }
+
+        qsb->iov[i].iov_len = chunk_size;
+        if (buffer) {
+            to_copy = (len - qsb->used) > chunk_size
+                      ? chunk_size : (len - qsb->used);
+            memcpy(qsb->iov[i].iov_base, &buffer[qsb->used], to_copy);
+            qsb->used += to_copy;
+        }
+    }
+
+    qsb->size = alloc_len;
+
+    return qsb;
+}
+
+/**
+ * Free the QEMUSizedBuffer
+ *
+ * @qsb: The QEMUSizedBuffer to free
+ */
+void qsb_free(QEMUSizedBuffer *qsb)
+{
+    size_t i;
+
+    if (!qsb) {
+        return;
+    }
+
+    for (i = 0; i < qsb->n_iov; i++) {
+        g_free(qsb->iov[i].iov_base);
+    }
+    g_free(qsb->iov);
+    g_free(qsb);
+}
+
+/**
+ * Get the number of used bytes in the QEMUSizedBuffer
+ *
+ * @qsb: A QEMUSizedBuffer
+ *
+ * Returns the number of bytes currently used in this buffer
+ */
+size_t qsb_get_length(const QEMUSizedBuffer *qsb)
+{
+    return qsb->used;
+}
+
+/**
+ * Set the length of the buffer; the primary usage of this
+ * function is to truncate the number of used bytes in the buffer.
+ * The size will not be extended beyond the current number of
+ * allocated bytes in the QEMUSizedBuffer.
+ *
+ * @qsb: A QEMUSizedBuffer
+ * @new_len: The new length of bytes in the buffer
+ *
+ * Returns the number of bytes the buffer was truncated or extended
+ * to.
+ */
+size_t qsb_set_length(QEMUSizedBuffer *qsb, size_t new_len)
+{
+    if (new_len <= qsb->size) {
+        qsb->used = new_len;
+    } else {
+        qsb->used = qsb->size;
+    }
+    return qsb->used;
+}
+
+/**
+ * Get the iovec that holds the data for a given position @pos.
+ *
+ * @qsb: A QEMUSizedBuffer
+ * @pos: The index of a byte in the buffer
+ * @d_off: Pointer to an offset that this function will indicate
+ *         at what position within the returned iovec the byte
+ *         is to be found
+ *
+ * Returns the index of the iovec that holds the byte at the given
+ * index @pos in the byte stream; a negative number if the iovec
+ * for the given position @pos does not exist.
+ */
+static ssize_t qsb_get_iovec(const QEMUSizedBuffer *qsb,
+                             off_t pos, off_t *d_off)
+{
+    ssize_t i;
+    off_t curr = 0;
+
+    if (pos > qsb->used) {
+        return -1;
+    }
+
+    for (i = 0; i < qsb->n_iov; i++) {
+        if (curr + qsb->iov[i].iov_len > pos) {
+            *d_off = pos - curr;
+            return i;
+        }
+        curr += qsb->iov[i].iov_len;
+    }
+    return -1;
+}
+
+/*
+ * Convert the QEMUSizedBuffer into a flat buffer.
+ *
+ * Note: If at all possible, try to avoid this function since it
+ *       may unnecessarily copy memory around.
+ *
+ * @qsb: pointer to QEMUSizedBuffer
+ * @start: offset to start at
+ * @count: number of bytes to copy
+ * @buf: a pointer to a buffer to write into (at least @count bytes)
+ *
+ * Returns the number of bytes copied into the output buffer
+ */
+ssize_t qsb_get_buffer(const QEMUSizedBuffer *qsb, off_t start,
+                       size_t count, uint8_t *buffer)
+{
+    const struct iovec *iov;
+    size_t to_copy, all_copy;
+    ssize_t index;
+    off_t s_off;
+    off_t d_off = 0;
+    char *s;
+
+    if (start > qsb->used) {
+        return 0;
+    }
+
+    all_copy = qsb->used - start;
+    if (all_copy > count) {
+        all_copy = count;
+    } else {
+        count = all_copy;
+    }
+
+    index = qsb_get_iovec(qsb, start, &s_off);
+    if (index < 0) {
+        return 0;
+    }
+
+    while (all_copy > 0) {
+        iov = &qsb->iov[index];
+
+        s = iov->iov_base;
+
+        to_copy = iov->iov_len - s_off;
+        if (to_copy > all_copy) {
+            to_copy = all_copy;
+        }
+        memcpy(&buffer[d_off], &s[s_off], to_copy);
+
+        d_off += to_copy;
+        all_copy -= to_copy;
+
+        s_off = 0;
+        index++;
+    }
+
+    return count;
+}
+
+/**
+ * Grow the QEMUSizedBuffer to the given size and allocate
+ * memory for it.
+ *
+ * @qsb: A QEMUSizedBuffer
+ * @new_size: The new size of the buffer
+ *
+ * Return:
+ *    a negative error code in case of memory allocation failure
+ * or
+ *    the new size of the buffer. The returned size may be greater or equal
+ *    to @new_size.
+ */
+static ssize_t qsb_grow(QEMUSizedBuffer *qsb, size_t new_size)
+{
+    size_t needed_chunks, i;
+
+    if (qsb->size < new_size) {
+        struct iovec *new_iov;
+        size_t size_diff = new_size - qsb->size;
+        size_t chunk_size = (size_diff > QSB_MAX_CHUNK_SIZE)
+                             ? QSB_MAX_CHUNK_SIZE : QSB_CHUNK_SIZE;
+
+        needed_chunks = DIV_ROUND_UP(size_diff, chunk_size);
+
+        new_iov = g_try_new(struct iovec, qsb->n_iov + needed_chunks);
+        if (new_iov == NULL) {
+            return -ENOMEM;
+        }
+
+        /* Allocate new chunks as needed into new_iov */
+        for (i = qsb->n_iov; i < qsb->n_iov + needed_chunks; i++) {
+            new_iov[i].iov_base = g_try_malloc0(chunk_size);
+            new_iov[i].iov_len = chunk_size;
+            if (!new_iov[i].iov_base) {
+                size_t j;
+
+                /* Free previously allocated new chunks */
+                for (j = qsb->n_iov; j < i; j++) {
+                    g_free(new_iov[j].iov_base);
+                }
+                g_free(new_iov);
+
+                return -ENOMEM;
+            }
+        }
+
+        /*
+         * Now we can't get any allocation errors, copy over to new iov
+         * and switch.
+         */
+        for (i = 0; i < qsb->n_iov; i++) {
+            new_iov[i] = qsb->iov[i];
+        }
+
+        qsb->n_iov += needed_chunks;
+        g_free(qsb->iov);
+        qsb->iov = new_iov;
+        qsb->size += (needed_chunks * chunk_size);
+    }
+
+    return qsb->size;
+}
+
+/**
+ * Write into the QEMUSizedBuffer at a given position and a given
+ * number of bytes. This function will automatically grow the
+ * QEMUSizedBuffer.
+ *
+ * @qsb: A QEMUSizedBuffer
+ * @source: A byte array to copy data from
+ * @pos: The position within the @qsb to write data to
+ * @size: The number of bytes to copy into the @qsb
+ *
+ * Returns @size or a negative error code in case of memory allocation failure,
+ *           or with an invalid 'pos'
+ */
+ssize_t qsb_write_at(QEMUSizedBuffer *qsb, const uint8_t *source,
+                     off_t pos, size_t count)
+{
+    ssize_t rc = qsb_grow(qsb, pos + count);
+    size_t to_copy;
+    size_t all_copy = count;
+    const struct iovec *iov;
+    ssize_t index;
+    char *dest;
+    off_t d_off, s_off = 0;
+
+    if (rc < 0) {
+        return rc;
+    }
+
+    if (pos + count > qsb->used) {
+        qsb->used = pos + count;
+    }
+
+    index = qsb_get_iovec(qsb, pos, &d_off);
+    if (index < 0) {
+        return -EINVAL;
+    }
+
+    while (all_copy > 0) {
+        iov = &qsb->iov[index];
+
+        dest = iov->iov_base;
+
+        to_copy = iov->iov_len - d_off;
+        if (to_copy > all_copy) {
+            to_copy = all_copy;
+        }
+
+        memcpy(&dest[d_off], &source[s_off], to_copy);
+
+        s_off += to_copy;
+        all_copy -= to_copy;
+
+        d_off = 0;
+        index++;
+    }
+
+    return count;
+}
+
+/**
+ * Create a deep copy of the given QEMUSizedBuffer.
+ *
+ * @qsb: A QEMUSizedBuffer
+ *
+ * Returns a clone of @qsb or NULL on allocation failure
+ */
+QEMUSizedBuffer *qsb_clone(const QEMUSizedBuffer *qsb)
+{
+    QEMUSizedBuffer *out = qsb_create(NULL, qsb_get_length(qsb));
+    size_t i;
+    ssize_t res;
+    off_t pos = 0;
+
+    if (!out) {
+        return NULL;
+    }
+
+    for (i = 0; i < qsb->n_iov; i++) {
+        res =  qsb_write_at(out, qsb->iov[i].iov_base,
+                            pos, qsb->iov[i].iov_len);
+        if (res < 0) {
+            qsb_free(out);
+            return NULL;
+        }
+        pos += res;
+    }
+
+    return out;
+}
+
+typedef struct QEMUBuffer {
+    QEMUSizedBuffer *qsb;
+    QEMUFile *file;
+} QEMUBuffer;
+
+static int buf_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
+{
+    QEMUBuffer *s = opaque;
+    ssize_t len = qsb_get_length(s->qsb) - pos;
+
+    if (len <= 0) {
+        return 0;
+    }
+
+    if (len > size) {
+        len = size;
+    }
+    return qsb_get_buffer(s->qsb, pos, len, buf);
+}
+
+static int buf_put_buffer(void *opaque, const uint8_t *buf,
+                          int64_t pos, int size)
+{
+    QEMUBuffer *s = opaque;
+
+    return qsb_write_at(s->qsb, buf, pos, size);
+}
+
+static int buf_close(void *opaque)
+{
+    QEMUBuffer *s = opaque;
+
+    qsb_free(s->qsb);
+
+    g_free(s);
+
+    return 0;
+}
+
+const QEMUSizedBuffer *qemu_buf_get(QEMUFile *f)
+{
+    QEMUBuffer *p;
+
+    qemu_fflush(f);
+
+    p = f->opaque;
+
+    return p->qsb;
+}
+
+static const QEMUFileOps buf_read_ops = {
+    .get_buffer = buf_get_buffer,
+    .close =      buf_close,
+};
+
+static const QEMUFileOps buf_write_ops = {
+    .put_buffer = buf_put_buffer,
+    .close =      buf_close,
+};
+
+QEMUFile *qemu_bufopen(const char *mode, QEMUSizedBuffer *input)
+{
+    QEMUBuffer *s;
+
+    if (mode == NULL || (mode[0] != 'r' && mode[0] != 'w') ||
+        mode[1] != '\0') {
+        error_report("qemu_bufopen: Argument validity check failed");
+        return NULL;
+    }
+
+    s = g_malloc0(sizeof(QEMUBuffer));
+    if (mode[0] == 'r') {
+        s->qsb = input;
+    }
+
+    if (s->qsb == NULL) {
+        s->qsb = qsb_create(NULL, 0);
+    }
+    if (!s->qsb) {
+        g_free(s);
+        error_report("qemu_bufopen: qsb_create failed");
+        return NULL;
+    }
+
+
+    if (mode[0] == 'r') {
+        s->file = qemu_fopen_ops(s, &buf_read_ops);
+    } else {
+        s->file = qemu_fopen_ops(s, &buf_write_ops);
+    }
+    return s->file;
+}
diff --git a/qemu-img-cmds.hx b/qemu-img-cmds.hx
index d02960921..95677745f 100644
--- a/qemu-img-cmds.hx
+++ b/qemu-img-cmds.hx
@@ -10,9 +10,9 @@ STEXI
 ETEXI
 
 DEF("check", img_check,
-    "check [-q] [-f fmt] [--output=ofmt]  [-r [leaks | all]] filename")
+    "check [-q] [-f fmt] [--output=ofmt] [-r [leaks | all]] [-T src_cache] filename")
 STEXI
-@item check [-q] [-f @var{fmt}] [--output=@var{ofmt}] [-r [leaks | all]] @var{filename}
+@item check [-q] [-f @var{fmt}] [--output=@var{ofmt}] [-r [leaks | all]] [-T @var{src_cache}] @var{filename}
 ETEXI
 
 DEF("create", img_create,
@@ -22,21 +22,21 @@ STEXI
 ETEXI
 
 DEF("commit", img_commit,
-    "commit [-q] [-f fmt] [-t cache] filename")
+    "commit [-q] [-f fmt] [-t cache] [-b base] [-d] [-p] filename")
 STEXI
-@item commit [-q] [-f @var{fmt}] [-t @var{cache}] @var{filename}
+@item commit [-q] [-f @var{fmt}] [-t @var{cache}] [-b @var{base}] [-d] [-p] @var{filename}
 ETEXI
 
 DEF("compare", img_compare,
-    "compare [-f fmt] [-F fmt] [-p] [-q] [-s] filename1 filename2")
+    "compare [-f fmt] [-F fmt] [-T src_cache] [-p] [-q] [-s] filename1 filename2")
 STEXI
-@item compare [-f @var{fmt}] [-F @var{fmt}] [-p] [-q] [-s] @var{filename1} @var{filename2}
+@item compare [-f @var{fmt}] [-F @var{fmt}] [-T @var{src_cache}] [-p] [-q] [-s] @var{filename1} @var{filename2}
 ETEXI
 
 DEF("convert", img_convert,
-    "convert [-c] [-p] [-q] [-n] [-f fmt] [-t cache] [-O output_fmt] [-o options] [-s snapshot_id_or_name] [-l snapshot_param] [-S sparse_size] filename [filename2 [...]] output_filename")
+    "convert [-c] [-p] [-q] [-n] [-f fmt] [-t cache] [-T src_cache] [-O output_fmt] [-o options] [-s snapshot_id_or_name] [-l snapshot_param] [-S sparse_size] filename [filename2 [...]] output_filename")
 STEXI
-@item convert [-c] [-p] [-q] [-n] [-f @var{fmt}] [-t @var{cache}] [-O @var{output_fmt}] [-o @var{options}] [-s @var{snapshot_id_or_name}] [-l @var{snapshot_param}] [-S @var{sparse_size}] @var{filename} [@var{filename2} [...]] @var{output_filename}
+@item convert [-c] [-p] [-q] [-n] [-f @var{fmt}] [-t @var{cache}] [-T @var{src_cache}] [-O @var{output_fmt}] [-o @var{options}] [-s @var{snapshot_id_or_name}] [-l @var{snapshot_param}] [-S @var{sparse_size}] @var{filename} [@var{filename2} [...]] @var{output_filename}
 ETEXI
 
 DEF("info", img_info,
@@ -58,9 +58,9 @@ STEXI
 ETEXI
 
 DEF("rebase", img_rebase,
-    "rebase [-q] [-f fmt] [-t cache] [-p] [-u] -b backing_file [-F backing_fmt] filename")
+    "rebase [-q] [-f fmt] [-t cache] [-T src_cache] [-p] [-u] -b backing_file [-F backing_fmt] filename")
 STEXI
-@item rebase [-q] [-f @var{fmt}] [-t @var{cache}] [-p] [-u] -b @var{backing_file} [-F @var{backing_fmt}] @var{filename}
+@item rebase [-q] [-f @var{fmt}] [-t @var{cache}] [-T @var{src_cache}] [-p] [-u] -b @var{backing_file} [-F @var{backing_fmt}] @var{filename}
 ETEXI
 
 DEF("resize", img_resize,
@@ -70,8 +70,8 @@ STEXI
 ETEXI
 
 DEF("amend", img_amend,
-    "amend [-q] [-f fmt] -o options filename")
+    "amend [-p] [-q] [-f fmt] [-t cache] -o options filename")
 STEXI
-@item amend [-q] [-f @var{fmt}] -o @var{options} @var{filename}
+@item amend [-p] [-q] [-f @var{fmt}] [-t @var{cache}] -o @var{options} @var{filename}
 @end table
 ETEXI
diff --git a/qemu-img.c b/qemu-img.c
index d4518e724..a42335c63 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -29,10 +29,11 @@
 #include "qemu/error-report.h"
 #include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/block-backend.h"
 #include "block/block_int.h"
+#include "block/blockjob.h"
 #include "block/qapi.h"
 #include <getopt.h>
-#include <glib.h>
 
 #define QEMU_IMG_VERSION "qemu-img version " QEMU_VERSION \
                           ", Copyright (c) 2004-2008 Fabrice Bellard\n"
@@ -56,22 +57,9 @@ typedef enum OutputFormat {
 #define BDRV_O_FLAGS BDRV_O_CACHE_WB
 #define BDRV_DEFAULT_CACHE "writeback"
 
-static gint compare_data(gconstpointer a, gconstpointer b, gpointer user)
+static void format_print(void *opaque, const char *name)
 {
-    return g_strcmp0(a, b);
-}
-
-static void print_format(gpointer data, gpointer user)
-{
-    printf(" %s", (char *)data);
-}
-
-static void add_format_to_seq(void *opaque, const char *fmt_name)
-{
-    GSequence *seq = opaque;
-
-    g_sequence_insert_sorted(seq, (gpointer)fmt_name,
-                             compare_data, NULL);
+    printf(" %s", name);
 }
 
 static void QEMU_NORETURN GCC_FMT_ATTR(1, 2) error_exit(const char *fmt, ...)
@@ -109,6 +97,8 @@ static void QEMU_NORETURN help(void)
            "  'cache' is the cache mode used to write the output disk image, the valid\n"
            "    options are: 'none', 'writeback' (default, except for convert), 'writethrough',\n"
            "    'directsync' and 'unsafe' (default for convert)\n"
+           "  'src_cache' is the cache mode used to read input disk images, the valid\n"
+           "    options are the same as for the 'cache' option\n"
            "  'size' is the disk image size in bytes. Optional suffixes\n"
            "    'k' or 'K' (kilobyte, 1024), 'M' (megabyte, 1024k), 'G' (gigabyte, 1024M),\n"
            "    'T' (terabyte, 1024G), 'P' (petabyte, 1024T) and 'E' (exabyte, 1024P)  are\n"
@@ -156,15 +146,10 @@ static void QEMU_NORETURN help(void)
            "  '-f' first image format\n"
            "  '-F' second image format\n"
            "  '-s' run in Strict mode - fail on different image size or sector allocation\n";
-    GSequence *seq;
 
     printf("%s\nSupported formats:", help_msg);
-    seq = g_sequence_new(NULL);
-    bdrv_iterate_format(add_format_to_seq, seq);
-    g_sequence_foreach(seq, print_format, NULL);
+    bdrv_iterate_format(format_print, NULL);
     printf("\n");
-    g_sequence_free(seq);
-
     exit(EXIT_SUCCESS);
 }
 
@@ -185,15 +170,20 @@ static int GCC_FMT_ATTR(2, 3) qprintf(bool quiet, const char *fmt, ...)
 static int read_password(char *buf, int buf_size)
 {
     int c, i;
+
     printf("Password: ");
     fflush(stdout);
     i = 0;
     for(;;) {
         c = getchar();
-        if (c == '\n')
+        if (c < 0) {
+            buf[i] = '\0';
+            return -1;
+        } else if (c == '\n') {
             break;
-        if (i < (buf_size - 1))
+        } else if (i < (buf_size - 1)) {
             buf[i++] = c;
+        }
     }
     buf[i] = '\0';
     return 0;
@@ -295,20 +285,19 @@ static int print_block_option_help(const char *filename, const char *fmt)
     return 0;
 }
 
-static BlockDriverState *bdrv_new_open(const char *id,
-                                       const char *filename,
-                                       const char *fmt,
-                                       int flags,
-                                       bool require_io,
-                                       bool quiet)
+static BlockBackend *img_open(const char *id, const char *filename,
+                              const char *fmt, int flags,
+                              bool require_io, bool quiet)
 {
+    BlockBackend *blk;
     BlockDriverState *bs;
     BlockDriver *drv;
     char password[256];
     Error *local_err = NULL;
     int ret;
 
-    bs = bdrv_new(id, &error_abort);
+    blk = blk_new_with_bs(id, &error_abort);
+    bs = blk_bs(blk);
 
     if (fmt) {
         drv = bdrv_find_format(fmt);
@@ -339,9 +328,9 @@ static BlockDriverState *bdrv_new_open(const char *id,
             goto fail;
         }
     }
-    return bs;
+    return blk;
 fail:
-    bdrv_unref(bs);
+    blk_unref(blk);
     return NULL;
 }
 
@@ -586,7 +575,8 @@ static int img_check(int argc, char **argv)
 {
     int c, ret;
     OutputFormat output_format = OFORMAT_HUMAN;
-    const char *filename, *fmt, *output;
+    const char *filename, *fmt, *output, *cache;
+    BlockBackend *blk;
     BlockDriverState *bs;
     int fix = 0;
     int flags = BDRV_O_FLAGS | BDRV_O_CHECK;
@@ -595,6 +585,7 @@ static int img_check(int argc, char **argv)
 
     fmt = NULL;
     output = NULL;
+    cache = BDRV_DEFAULT_CACHE;
     for(;;) {
         int option_index = 0;
         static const struct option long_options[] = {
@@ -604,7 +595,7 @@ static int img_check(int argc, char **argv)
             {"output", required_argument, 0, OPTION_OUTPUT},
             {0, 0, 0, 0}
         };
-        c = getopt_long(argc, argv, "f:hr:q",
+        c = getopt_long(argc, argv, "hf:r:T:q",
                         long_options, &option_index);
         if (c == -1) {
             break;
@@ -632,6 +623,9 @@ static int img_check(int argc, char **argv)
         case OPTION_OUTPUT:
             output = optarg;
             break;
+        case 'T':
+            cache = optarg;
+            break;
         case 'q':
             quiet = true;
             break;
@@ -651,11 +645,18 @@ static int img_check(int argc, char **argv)
         return 1;
     }
 
-    bs = bdrv_new_open("image", filename, fmt, flags, true, quiet);
-    if (!bs) {
+    ret = bdrv_parse_cache_flags(cache, &flags);
+    if (ret < 0) {
+        error_report("Invalid source cache option: %s", cache);
         return 1;
     }
 
+    blk = img_open("image", filename, fmt, flags, true, quiet);
+    if (!blk) {
+        return 1;
+    }
+    bs = blk_bs(blk);
+
     check = g_new0(ImageCheck, 1);
     ret = collect_image_check(bs, check, filename, fmt, fix);
 
@@ -687,16 +688,23 @@ static int img_check(int argc, char **argv)
         check->corruptions_fixed    = corruptions_fixed;
     }
 
-    switch (output_format) {
-    case OFORMAT_HUMAN:
-        dump_human_image_check(check, quiet);
-        break;
-    case OFORMAT_JSON:
-        dump_json_image_check(check, quiet);
-        break;
+    if (!ret) {
+        switch (output_format) {
+        case OFORMAT_HUMAN:
+            dump_human_image_check(check, quiet);
+            break;
+        case OFORMAT_JSON:
+            dump_json_image_check(check, quiet);
+            break;
+        }
     }
 
     if (ret || check->check_errors) {
+        if (ret) {
+            error_report("Check failed: %s", strerror(-ret));
+        } else {
+            error_report("Check failed");
+        }
         ret = 1;
         goto fail;
     }
@@ -711,22 +719,58 @@ static int img_check(int argc, char **argv)
 
 fail:
     qapi_free_ImageCheck(check);
-    bdrv_unref(bs);
-
+    blk_unref(blk);
     return ret;
 }
 
+typedef struct CommonBlockJobCBInfo {
+    BlockDriverState *bs;
+    Error **errp;
+} CommonBlockJobCBInfo;
+
+static void common_block_job_cb(void *opaque, int ret)
+{
+    CommonBlockJobCBInfo *cbi = opaque;
+
+    if (ret < 0) {
+        error_setg_errno(cbi->errp, -ret, "Block job failed");
+    }
+
+    /* Drop this block job's reference */
+    bdrv_unref(cbi->bs);
+}
+
+static void run_block_job(BlockJob *job, Error **errp)
+{
+    AioContext *aio_context = bdrv_get_aio_context(job->bs);
+
+    do {
+        aio_poll(aio_context, true);
+        qemu_progress_print((float)job->offset / job->len * 100.f, 0);
+    } while (!job->ready);
+
+    block_job_complete_sync(job, errp);
+
+    /* A block job may finish instantaneously without publishing any progress,
+     * so just signal completion here */
+    qemu_progress_print(100.f, 0);
+}
+
 static int img_commit(int argc, char **argv)
 {
     int c, ret, flags;
-    const char *filename, *fmt, *cache;
-    BlockDriverState *bs;
-    bool quiet = false;
+    const char *filename, *fmt, *cache, *base;
+    BlockBackend *blk;
+    BlockDriverState *bs, *base_bs;
+    bool progress = false, quiet = false, drop = false;
+    Error *local_err = NULL;
+    CommonBlockJobCBInfo cbi;
 
     fmt = NULL;
     cache = BDRV_DEFAULT_CACHE;
+    base = NULL;
     for(;;) {
-        c = getopt(argc, argv, "f:ht:q");
+        c = getopt(argc, argv, "f:ht:b:dpq");
         if (c == -1) {
             break;
         }
@@ -741,50 +785,116 @@ static int img_commit(int argc, char **argv)
         case 't':
             cache = optarg;
             break;
+        case 'b':
+            base = optarg;
+            /* -b implies -d */
+            drop = true;
+            break;
+        case 'd':
+            drop = true;
+            break;
+        case 'p':
+            progress = true;
+            break;
         case 'q':
             quiet = true;
             break;
         }
     }
+
+    /* Progress is not shown in Quiet mode */
+    if (quiet) {
+        progress = false;
+    }
+
     if (optind != argc - 1) {
         error_exit("Expecting one image file name");
     }
     filename = argv[optind++];
 
-    flags = BDRV_O_RDWR;
+    flags = BDRV_O_RDWR | BDRV_O_UNMAP;
     ret = bdrv_parse_cache_flags(cache, &flags);
     if (ret < 0) {
         error_report("Invalid cache option: %s", cache);
-        return -1;
+        return 1;
     }
 
-    bs = bdrv_new_open("image", filename, fmt, flags, true, quiet);
-    if (!bs) {
+    blk = img_open("image", filename, fmt, flags, true, quiet);
+    if (!blk) {
         return 1;
     }
-    ret = bdrv_commit(bs);
-    switch(ret) {
-    case 0:
-        qprintf(quiet, "Image committed.\n");
-        break;
-    case -ENOENT:
-        error_report("No disk inserted");
-        break;
-    case -EACCES:
-        error_report("Image is read-only");
-        break;
-    case -ENOTSUP:
-        error_report("Image is already committed");
-        break;
-    default:
-        error_report("Error while committing image");
-        break;
+    bs = blk_bs(blk);
+
+    qemu_progress_init(progress, 1.f);
+    qemu_progress_print(0.f, 100);
+
+    if (base) {
+        base_bs = bdrv_find_backing_image(bs, base);
+        if (!base_bs) {
+            error_set(&local_err, QERR_BASE_NOT_FOUND, base);
+            goto done;
+        }
+    } else {
+        /* This is different from QMP, which by default uses the deepest file in
+         * the backing chain (i.e., the very base); however, the traditional
+         * behavior of qemu-img commit is using the immediate backing file. */
+        base_bs = bs->backing_hd;
+        if (!base_bs) {
+            error_setg(&local_err, "Image does not have a backing file");
+            goto done;
+        }
     }
 
-    bdrv_unref(bs);
-    if (ret) {
+    cbi = (CommonBlockJobCBInfo){
+        .errp = &local_err,
+        .bs   = bs,
+    };
+
+    commit_active_start(bs, base_bs, 0, BLOCKDEV_ON_ERROR_REPORT,
+                        common_block_job_cb, &cbi, &local_err);
+    if (local_err) {
+        goto done;
+    }
+
+    /* The block job will swap base_bs and bs (which is not what we really want
+     * here, but okay) and unref base_bs (after the swap, i.e., the old top
+     * image). In order to still be able to empty that top image afterwards,
+     * increment the reference counter here preemptively. */
+    if (!drop) {
+        bdrv_ref(base_bs);
+    }
+
+    run_block_job(bs->job, &local_err);
+    if (local_err) {
+        goto unref_backing;
+    }
+
+    if (!drop && base_bs->drv->bdrv_make_empty) {
+        ret = base_bs->drv->bdrv_make_empty(base_bs);
+        if (ret) {
+            error_setg_errno(&local_err, -ret, "Could not empty %s",
+                             filename);
+            goto unref_backing;
+        }
+    }
+
+unref_backing:
+    if (!drop) {
+        bdrv_unref(base_bs);
+    }
+
+done:
+    qemu_progress_end();
+
+    blk_unref(blk);
+
+    if (local_err) {
+        qerror_report_err(local_err);
+        error_free(local_err);
         return 1;
     }
+
+    qprintf(quiet, "Image committed.\n");
     return 0;
 }
 
@@ -943,7 +1053,8 @@ static int check_empty_sectors(BlockDriverState *bs, int64_t sect_num,
  */
 static int img_compare(int argc, char **argv)
 {
-    const char *fmt1 = NULL, *fmt2 = NULL, *filename1, *filename2;
+    const char *fmt1 = NULL, *fmt2 = NULL, *cache, *filename1, *filename2;
+    BlockBackend *blk1, *blk2;
     BlockDriverState *bs1, *bs2;
     int64_t total_sectors1, total_sectors2;
     uint8_t *buf1 = NULL, *buf2 = NULL;
@@ -951,15 +1062,16 @@ static int img_compare(int argc, char **argv)
     int allocated1, allocated2;
     int ret = 0; /* return value - 0 Ident, 1 Different, >1 Error */
     bool progress = false, quiet = false, strict = false;
+    int flags;
     int64_t total_sectors;
     int64_t sector_num = 0;
     int64_t nb_sectors;
     int c, pnum;
-    uint64_t bs_sectors;
     uint64_t progress_base;
 
+    cache = BDRV_DEFAULT_CACHE;
     for (;;) {
-        c = getopt(argc, argv, "hpf:F:sq");
+        c = getopt(argc, argv, "hf:F:T:pqs");
         if (c == -1) {
             break;
         }
@@ -974,6 +1086,9 @@ static int img_compare(int argc, char **argv)
         case 'F':
             fmt2 = optarg;
             break;
+        case 'T':
+            cache = optarg;
+            break;
         case 'p':
             progress = true;
             break;
@@ -1001,26 +1116,44 @@ static int img_compare(int argc, char **argv)
     /* Initialize before goto out */
     qemu_progress_init(progress, 2.0);
 
-    bs1 = bdrv_new_open("image 1", filename1, fmt1, BDRV_O_FLAGS, true, quiet);
-    if (!bs1) {
-        error_report("Can't open file %s", filename1);
+    flags = BDRV_O_FLAGS;
+    ret = bdrv_parse_cache_flags(cache, &flags);
+    if (ret < 0) {
+        error_report("Invalid source cache option: %s", cache);
+        ret = 2;
+        goto out3;
+    }
+
+    blk1 = img_open("image_1", filename1, fmt1, flags, true, quiet);
+    if (!blk1) {
         ret = 2;
         goto out3;
     }
+    bs1 = blk_bs(blk1);
 
-    bs2 = bdrv_new_open("image 2", filename2, fmt2, BDRV_O_FLAGS, true, quiet);
-    if (!bs2) {
-        error_report("Can't open file %s", filename2);
+    blk2 = img_open("image_2", filename2, fmt2, flags, true, quiet);
+    if (!blk2) {
         ret = 2;
         goto out2;
     }
+    bs2 = blk_bs(blk2);
 
     buf1 = qemu_blockalign(bs1, IO_BUF_SIZE);
     buf2 = qemu_blockalign(bs2, IO_BUF_SIZE);
-    bdrv_get_geometry(bs1, &bs_sectors);
-    total_sectors1 = bs_sectors;
-    bdrv_get_geometry(bs2, &bs_sectors);
-    total_sectors2 = bs_sectors;
+    total_sectors1 = bdrv_nb_sectors(bs1);
+    if (total_sectors1 < 0) {
+        error_report("Can't get size of %s: %s",
+                     filename1, strerror(-total_sectors1));
+        ret = 4;
+        goto out;
+    }
+    total_sectors2 = bdrv_nb_sectors(bs2);
+    if (total_sectors2 < 0) {
+        error_report("Can't get size of %s: %s",
+                     filename2, strerror(-total_sectors2));
+        ret = 4;
+        goto out;
+    }
     total_sectors = MIN(total_sectors1, total_sectors2);
     progress_base = MAX(total_sectors1, total_sectors2);
 
@@ -1163,11 +1296,11 @@ static int img_compare(int argc, char **argv)
     ret = 0;
 
 out:
-    bdrv_unref(bs2);
     qemu_vfree(buf1);
     qemu_vfree(buf2);
+    blk_unref(blk2);
 out2:
-    bdrv_unref(bs1);
+    blk_unref(blk1);
 out3:
     qemu_progress_end();
     return ret;
@@ -1177,12 +1310,13 @@ static int img_convert(int argc, char **argv)
 {
     int c, n, n1, bs_n, bs_i, compress, cluster_sectors, skip_create;
     int64_t ret = 0;
-    int progress = 0, flags;
-    const char *fmt, *out_fmt, *cache, *out_baseimg, *out_filename;
+    int progress = 0, flags, src_flags;
+    const char *fmt, *out_fmt, *cache, *src_cache, *out_baseimg, *out_filename;
     BlockDriver *drv, *proto_drv;
+    BlockBackend **blk = NULL, *out_blk = NULL;
     BlockDriverState **bs = NULL, *out_bs = NULL;
     int64_t total_sectors, nb_sectors, sector_num, bs_offset;
-    uint64_t bs_sectors;
+    int64_t *bs_sectors = NULL;
     uint8_t * buf = NULL;
     size_t bufsectors = IO_BUF_SIZE / BDRV_SECTOR_SIZE;
     const uint8_t *buf1;
@@ -1200,11 +1334,12 @@ static int img_convert(int argc, char **argv)
     fmt = NULL;
     out_fmt = "raw";
     cache = "unsafe";
+    src_cache = BDRV_DEFAULT_CACHE;
     out_baseimg = NULL;
     compress = 0;
     skip_create = 0;
     for(;;) {
-        c = getopt(argc, argv, "f:O:B:s:hce6o:pS:t:qnl:");
+        c = getopt(argc, argv, "hf:O:B:ce6o:s:l:S:pt:T:qn");
         if (c == -1) {
             break;
         }
@@ -1285,6 +1420,9 @@ static int img_convert(int argc, char **argv)
         case 't':
             cache = optarg;
             break;
+        case 'T':
+            src_cache = optarg;
+            break;
         case 'q':
             quiet = true;
             break;
@@ -1321,24 +1459,39 @@ static int img_convert(int argc, char **argv)
         goto out;
     }
 
+    src_flags = BDRV_O_FLAGS;
+    ret = bdrv_parse_cache_flags(src_cache, &src_flags);
+    if (ret < 0) {
+        error_report("Invalid source cache option: %s", src_cache);
+        goto out;
+    }
+
     qemu_progress_print(0, 100);
 
-    bs = g_malloc0(bs_n * sizeof(BlockDriverState *));
+    blk = g_new0(BlockBackend *, bs_n);
+    bs = g_new0(BlockDriverState *, bs_n);
+    bs_sectors = g_new(int64_t, bs_n);
 
     total_sectors = 0;
     for (bs_i = 0; bs_i < bs_n; bs_i++) {
-        char *id = bs_n > 1 ? g_strdup_printf("source %d", bs_i)
+        char *id = bs_n > 1 ? g_strdup_printf("source_%d", bs_i)
                             : g_strdup("source");
-        bs[bs_i] = bdrv_new_open(id, argv[optind + bs_i], fmt, BDRV_O_FLAGS,
-                                 true, quiet);
+        blk[bs_i] = img_open(id, argv[optind + bs_i], fmt, src_flags,
+                             true, quiet);
         g_free(id);
-        if (!bs[bs_i]) {
-            error_report("Could not open '%s'", argv[optind + bs_i]);
+        if (!blk[bs_i]) {
+            ret = -1;
+            goto out;
+        }
+        bs[bs_i] = blk_bs(blk[bs_i]);
+        bs_sectors[bs_i] = bdrv_nb_sectors(bs[bs_i]);
+        if (bs_sectors[bs_i] < 0) {
+            error_report("Could not get size of %s: %s",
+                         argv[optind + bs_i], strerror(-bs_sectors[bs_i]));
             ret = -1;
             goto out;
         }
-        bdrv_get_geometry(bs[bs_i], &bs_sectors);
-        total_sectors += bs_sectors;
+        total_sectors += bs_sectors[bs_i];
     }
 
     if (sn_opts) {
@@ -1448,15 +1601,15 @@ static int img_convert(int argc, char **argv)
         goto out;
     }
 
-    out_bs = bdrv_new_open("target", out_filename, out_fmt, flags, true, quiet);
-    if (!out_bs) {
+    out_blk = img_open("target", out_filename, out_fmt, flags, true, quiet);
+    if (!out_blk) {
         ret = -1;
         goto out;
     }
+    out_bs = blk_bs(out_blk);
 
     bs_i = 0;
     bs_offset = 0;
-    bdrv_get_geometry(bs[0], &bs_sectors);
 
     /* increase bufsectors from the default 4096 (2M) if opt_transfer_length
      * or discard_alignment of the out_bs is greater. Limit to 32768 (16MB)
@@ -1469,13 +1622,13 @@ static int img_convert(int argc, char **argv)
     buf = qemu_blockalign(out_bs, bufsectors * BDRV_SECTOR_SIZE);
 
     if (skip_create) {
-        int64_t output_length = bdrv_getlength(out_bs);
-        if (output_length < 0) {
+        int64_t output_sectors = bdrv_nb_sectors(out_bs);
+        if (output_sectors < 0) {
             error_report("unable to get output image length: %s\n",
-                         strerror(-output_length));
+                         strerror(-output_sectors));
             ret = -1;
             goto out;
-        } else if (output_length < total_sectors << BDRV_SECTOR_BITS) {
+        } else if (output_sectors < total_sectors) {
             error_report("output file is smaller than input file");
             ret = -1;
             goto out;
@@ -1523,19 +1676,19 @@ static int img_convert(int argc, char **argv)
             buf2 = buf;
             while (remainder > 0) {
                 int nlow;
-                while (bs_num == bs_sectors) {
+                while (bs_num == bs_sectors[bs_i]) {
+                    bs_offset += bs_sectors[bs_i];
                     bs_i++;
                     assert (bs_i < bs_n);
-                    bs_offset += bs_sectors;
-                    bdrv_get_geometry(bs[bs_i], &bs_sectors);
                     bs_num = 0;
                     /* printf("changing part: sector_num=%" PRId64 ", "
                        "bs_i=%d, bs_offset=%" PRId64 ", bs_sectors=%" PRId64
-                       "\n", sector_num, bs_i, bs_offset, bs_sectors); */
+                       "\n", sector_num, bs_i, bs_offset, bs_sectors[bs_i]); */
                 }
-                assert (bs_num < bs_sectors);
+                assert (bs_num < bs_sectors[bs_i]);
 
-                nlow = (remainder > bs_sectors - bs_num) ? bs_sectors - bs_num : remainder;
+                nlow = remainder > bs_sectors[bs_i] - bs_num
+                    ? bs_sectors[bs_i] - bs_num : remainder;
 
                 ret = bdrv_read(bs[bs_i], bs_num, buf2, nlow);
                 if (ret < 0) {
@@ -1596,14 +1749,13 @@ restart:
                 break;
             }
 
-            while (sector_num - bs_offset >= bs_sectors) {
+            while (sector_num - bs_offset >= bs_sectors[bs_i]) {
+                bs_offset += bs_sectors[bs_i];
                 bs_i ++;
                 assert (bs_i < bs_n);
-                bs_offset += bs_sectors;
-                bdrv_get_geometry(bs[bs_i], &bs_sectors);
                 /* printf("changing part: sector_num=%" PRId64 ", bs_i=%d, "
                   "bs_offset=%" PRId64 ", bs_sectors=%" PRId64 "\n",
-                   sector_num, bs_i, bs_offset, bs_sectors); */
+                   sector_num, bs_i, bs_offset, bs_sectors[bs_i]); */
             }
 
             if ((out_baseimg || has_zero_init) &&
@@ -1656,7 +1808,7 @@ restart:
                 }
             }
 
-            n = MIN(n, bs_sectors - (sector_num - bs_offset));
+            n = MIN(n, bs_sectors[bs_i] - (sector_num - bs_offset));
 
             sectors_read += n;
             if (count_allocated_sectors) {
@@ -1700,20 +1852,16 @@ out:
     qemu_opts_del(opts);
     qemu_opts_free(create_opts);
     qemu_vfree(buf);
-    if (sn_opts) {
-        qemu_opts_del(sn_opts);
-    }
-    if (out_bs) {
-        bdrv_unref(out_bs);
-    }
-    if (bs) {
+    qemu_opts_del(sn_opts);
+    blk_unref(out_blk);
+    g_free(bs);
+    if (blk) {
         for (bs_i = 0; bs_i < bs_n; bs_i++) {
-            if (bs[bs_i]) {
-                bdrv_unref(bs[bs_i]);
-            }
+            blk_unref(blk[bs_i]);
         }
-        g_free(bs);
+        g_free(blk);
     }
+    g_free(bs_sectors);
 fail_getopt:
     g_free(options);
 
@@ -1821,6 +1969,7 @@ static ImageInfoList *collect_image_info_list(const char *filename,
     filenames = g_hash_table_new_full(g_str_hash, str_equal_func, NULL, NULL);
 
     while (filename) {
+        BlockBackend *blk;
         BlockDriverState *bs;
         ImageInfo *info;
         ImageInfoList *elem;
@@ -1832,17 +1981,18 @@ static ImageInfoList *collect_image_info_list(const char *filename,
         }
         g_hash_table_insert(filenames, (gpointer)filename, NULL);
 
-        bs = bdrv_new_open("image", filename, fmt,
-                           BDRV_O_FLAGS | BDRV_O_NO_BACKING, false, false);
-        if (!bs) {
+        blk = img_open("image", filename, fmt,
+                       BDRV_O_FLAGS | BDRV_O_NO_BACKING, false, false);
+        if (!blk) {
             goto err;
         }
+        bs = blk_bs(blk);
 
         bdrv_query_image_info(bs, &info, &err);
         if (err) {
             error_report("%s", error_get_pretty(err));
             error_free(err);
-            bdrv_unref(bs);
+            blk_unref(blk);
             goto err;
         }
 
@@ -1851,7 +2001,7 @@ static ImageInfoList *collect_image_info_list(const char *filename,
         *last = elem;
         last = &elem->next;
 
-        bdrv_unref(bs);
+        blk_unref(blk);
 
         filename = fmt = NULL;
         if (chain) {
@@ -2045,6 +2195,7 @@ static int img_map(int argc, char **argv)
 {
     int c;
     OutputFormat output_format = OFORMAT_HUMAN;
+    BlockBackend *blk;
     BlockDriverState *bs;
     const char *filename, *fmt, *output;
     int64_t length;
@@ -2093,10 +2244,11 @@ static int img_map(int argc, char **argv)
         return 1;
     }
 
-    bs = bdrv_new_open("image", filename, fmt, BDRV_O_FLAGS, true, false);
-    if (!bs) {
+    blk = img_open("image", filename, fmt, BDRV_O_FLAGS, true, false);
+    if (!blk) {
         return 1;
     }
+    bs = blk_bs(blk);
 
     if (output_format == OFORMAT_HUMAN) {
         printf("%-16s%-16s%-16s%s\n", "Offset", "Length", "Mapped to", "File");
@@ -2137,7 +2289,7 @@ static int img_map(int argc, char **argv)
     dump_map_entry(output_format, &curr, NULL);
 
 out:
-    bdrv_unref(bs);
+    blk_unref(blk);
     return ret < 0;
 }
 
@@ -2148,6 +2300,7 @@ out:
 
 static int img_snapshot(int argc, char **argv)
 {
+    BlockBackend *blk;
     BlockDriverState *bs;
     QEMUSnapshotInfo sn;
     char *filename, *snapshot_name = NULL;
@@ -2213,10 +2366,11 @@ static int img_snapshot(int argc, char **argv)
     filename = argv[optind++];
 
     /* Open the image */
-    bs = bdrv_new_open("image", filename, NULL, bdrv_oflags, true, quiet);
-    if (!bs) {
+    blk = img_open("image", filename, NULL, bdrv_oflags, true, quiet);
+    if (!blk) {
         return 1;
     }
+    bs = blk_bs(blk);
 
     /* Perform the requested action */
     switch(action) {
@@ -2259,7 +2413,7 @@ static int img_snapshot(int argc, char **argv)
     }
 
     /* Cleanup */
-    bdrv_unref(bs);
+    blk_unref(blk);
     if (ret) {
         return 1;
     }
@@ -2268,11 +2422,12 @@ static int img_snapshot(int argc, char **argv)
 
 static int img_rebase(int argc, char **argv)
 {
-    BlockDriverState *bs, *bs_old_backing = NULL, *bs_new_backing = NULL;
+    BlockBackend *blk = NULL, *blk_old_backing = NULL, *blk_new_backing = NULL;
+    BlockDriverState *bs = NULL, *bs_old_backing = NULL, *bs_new_backing = NULL;
     BlockDriver *old_backing_drv, *new_backing_drv;
     char *filename;
-    const char *fmt, *cache, *out_basefmt, *out_baseimg;
-    int c, flags, ret;
+    const char *fmt, *cache, *src_cache, *out_basefmt, *out_baseimg;
+    int c, flags, src_flags, ret;
     int unsafe = 0;
     int progress = 0;
     bool quiet = false;
@@ -2281,10 +2436,11 @@ static int img_rebase(int argc, char **argv)
     /* Parse commandline parameters */
     fmt = NULL;
     cache = BDRV_DEFAULT_CACHE;
+    src_cache = BDRV_DEFAULT_CACHE;
     out_baseimg = NULL;
     out_basefmt = NULL;
     for(;;) {
-        c = getopt(argc, argv, "uhf:F:b:pt:q");
+        c = getopt(argc, argv, "hf:F:b:upt:T:q");
         if (c == -1) {
             break;
         }
@@ -2311,6 +2467,9 @@ static int img_rebase(int argc, char **argv)
         case 't':
             cache = optarg;
             break;
+        case 'T':
+            src_cache = optarg;
+            break;
         case 'q':
             quiet = true;
             break;
@@ -2336,7 +2495,14 @@ static int img_rebase(int argc, char **argv)
     ret = bdrv_parse_cache_flags(cache, &flags);
     if (ret < 0) {
         error_report("Invalid cache option: %s", cache);
-        return -1;
+        goto out;
+    }
+
+    src_flags = BDRV_O_FLAGS;
+    ret = bdrv_parse_cache_flags(src_cache, &src_flags);
+    if (ret < 0) {
+        error_report("Invalid source cache option: %s", src_cache);
+        goto out;
     }
 
     /*
@@ -2345,10 +2511,12 @@ static int img_rebase(int argc, char **argv)
      * Ignore the old backing file for unsafe rebase in case we want to correct
      * the reference to a renamed or moved backing file.
      */
-    bs = bdrv_new_open("image", filename, fmt, flags, true, quiet);
-    if (!bs) {
-        return 1;
+    blk = img_open("image", filename, fmt, flags, true, quiet);
+    if (!blk) {
+        ret = -1;
+        goto out;
     }
+    bs = blk_bs(blk);
 
     /* Find the right drivers for the backing files */
     old_backing_drv = NULL;
@@ -2373,16 +2541,13 @@ static int img_rebase(int argc, char **argv)
     }
 
     /* For safe rebasing we need to compare old and new backing file */
-    if (unsafe) {
-        /* Make the compiler happy */
-        bs_old_backing = NULL;
-        bs_new_backing = NULL;
-    } else {
+    if (!unsafe) {
         char backing_name[1024];
 
-        bs_old_backing = bdrv_new("old_backing", &error_abort);
+        blk_old_backing = blk_new_with_bs("old_backing", &error_abort);
+        bs_old_backing = blk_bs(blk_old_backing);
         bdrv_get_backing_filename(bs, backing_name, sizeof(backing_name));
-        ret = bdrv_open(&bs_old_backing, backing_name, NULL, NULL, BDRV_O_FLAGS,
+        ret = bdrv_open(&bs_old_backing, backing_name, NULL, NULL, src_flags,
                         old_backing_drv, &local_err);
         if (ret) {
             error_report("Could not open old backing file '%s': %s",
@@ -2391,9 +2556,10 @@ static int img_rebase(int argc, char **argv)
             goto out;
         }
         if (out_baseimg[0]) {
-            bs_new_backing = bdrv_new("new_backing", &error_abort);
-            ret = bdrv_open(&bs_new_backing, out_baseimg, NULL, NULL,
-                            BDRV_O_FLAGS, new_backing_drv, &local_err);
+            blk_new_backing = blk_new_with_bs("new_backing", &error_abort);
+            bs_new_backing = blk_bs(blk_new_backing);
+            ret = bdrv_open(&bs_new_backing, out_baseimg, NULL, NULL, src_flags,
+                            new_backing_drv, &local_err);
             if (ret) {
                 error_report("Could not open new backing file '%s': %s",
                              out_baseimg, error_get_pretty(local_err));
@@ -2413,9 +2579,9 @@ static int img_rebase(int argc, char **argv)
      * the image is the same as the original one at any time.
      */
     if (!unsafe) {
-        uint64_t num_sectors;
-        uint64_t old_backing_num_sectors;
-        uint64_t new_backing_num_sectors = 0;
+        int64_t num_sectors;
+        int64_t old_backing_num_sectors;
+        int64_t new_backing_num_sectors = 0;
         uint64_t sector;
         int n;
         uint8_t * buf_old;
@@ -2425,10 +2591,31 @@ static int img_rebase(int argc, char **argv)
         buf_old = qemu_blockalign(bs, IO_BUF_SIZE);
         buf_new = qemu_blockalign(bs, IO_BUF_SIZE);
 
-        bdrv_get_geometry(bs, &num_sectors);
-        bdrv_get_geometry(bs_old_backing, &old_backing_num_sectors);
+        num_sectors = bdrv_nb_sectors(bs);
+        if (num_sectors < 0) {
+            error_report("Could not get size of '%s': %s",
+                         filename, strerror(-num_sectors));
+            ret = -1;
+            goto out;
+        }
+        old_backing_num_sectors = bdrv_nb_sectors(bs_old_backing);
+        if (old_backing_num_sectors < 0) {
+            char backing_name[1024];
+
+            bdrv_get_backing_filename(bs, backing_name, sizeof(backing_name));
+            error_report("Could not get size of '%s': %s",
+                         backing_name, strerror(-old_backing_num_sectors));
+            ret = -1;
+            goto out;
+        }
         if (bs_new_backing) {
-            bdrv_get_geometry(bs_new_backing, &new_backing_num_sectors);
+            new_backing_num_sectors = bdrv_nb_sectors(bs_new_backing);
+            if (new_backing_num_sectors < 0) {
+                error_report("Could not get size of '%s': %s",
+                             out_baseimg, strerror(-new_backing_num_sectors));
+                ret = -1;
+                goto out;
+            }
         }
 
         if (num_sectors != 0) {
@@ -2545,15 +2732,11 @@ out:
     qemu_progress_end();
     /* Cleanup */
     if (!unsafe) {
-        if (bs_old_backing != NULL) {
-            bdrv_unref(bs_old_backing);
-        }
-        if (bs_new_backing != NULL) {
-            bdrv_unref(bs_new_backing);
-        }
+        blk_unref(blk_old_backing);
+        blk_unref(blk_new_backing);
     }
 
-    bdrv_unref(bs);
+    blk_unref(blk);
     if (ret) {
         return 1;
     }
@@ -2566,6 +2749,7 @@ static int img_resize(int argc, char **argv)
     const char *filename, *fmt, *size;
     int64_t n, total_size;
     bool quiet = false;
+    BlockBackend *blk = NULL;
     BlockDriverState *bs = NULL;
     QemuOpts *param;
     static QemuOptsList resize_options = {
@@ -2642,12 +2826,13 @@ static int img_resize(int argc, char **argv)
     n = qemu_opt_get_size(param, BLOCK_OPT_SIZE, 0);
     qemu_opts_del(param);
 
-    bs = bdrv_new_open("image", filename, fmt, BDRV_O_FLAGS | BDRV_O_RDWR,
-                       true, quiet);
-    if (!bs) {
+    blk = img_open("image", filename, fmt, BDRV_O_FLAGS | BDRV_O_RDWR,
+                   true, quiet);
+    if (!blk) {
         ret = -1;
         goto out;
     }
+    bs = blk_bs(blk);
 
     if (relative) {
         total_size = bdrv_getlength(bs) + n * relative;
@@ -2676,27 +2861,34 @@ static int img_resize(int argc, char **argv)
         break;
     }
 out:
-    if (bs) {
-        bdrv_unref(bs);
-    }
+    blk_unref(blk);
     if (ret) {
         return 1;
     }
     return 0;
 }
 
+static void amend_status_cb(BlockDriverState *bs,
+                            int64_t offset, int64_t total_work_size)
+{
+    qemu_progress_print(100.f * offset / total_work_size, 0);
+}
+
 static int img_amend(int argc, char **argv)
 {
     int c, ret = 0;
     char *options = NULL;
     QemuOptsList *create_opts = NULL;
     QemuOpts *opts = NULL;
-    const char *fmt = NULL, *filename;
-    bool quiet = false;
+    const char *fmt = NULL, *filename, *cache;
+    int flags;
+    bool quiet = false, progress = false;
+    BlockBackend *blk = NULL;
     BlockDriverState *bs = NULL;
 
+    cache = BDRV_DEFAULT_CACHE;
     for (;;) {
-        c = getopt(argc, argv, "hqf:o:");
+        c = getopt(argc, argv, "ho:f:t:pq");
         if (c == -1) {
             break;
         }
@@ -2723,6 +2915,12 @@ static int img_amend(int argc, char **argv)
             case 'f':
                 fmt = optarg;
                 break;
+            case 't':
+                cache = optarg;
+                break;
+            case 'p':
+                progress = true;
+                break;
             case 'q':
                 quiet = true;
                 break;
@@ -2733,6 +2931,11 @@ static int img_amend(int argc, char **argv)
         error_exit("Must specify options (-o)");
     }
 
+    if (quiet) {
+        progress = false;
+    }
+    qemu_progress_init(progress, 1.0);
+
     filename = (optind == argc - 1) ? argv[argc - 1] : NULL;
     if (fmt && has_help_option(options)) {
         /* If a format is explicitly specified (and possibly no filename is
@@ -2742,16 +2945,24 @@ static int img_amend(int argc, char **argv)
     }
 
     if (optind != argc - 1) {
-        error_exit("Expecting one image file name");
+        error_report("Expecting one image file name");
+        ret = -1;
+        goto out;
     }
 
-    bs = bdrv_new_open("image", filename, fmt,
-                       BDRV_O_FLAGS | BDRV_O_RDWR, true, quiet);
-    if (!bs) {
-        error_report("Could not open image '%s'", filename);
+    flags = BDRV_O_FLAGS | BDRV_O_RDWR;
+    ret = bdrv_parse_cache_flags(cache, &flags);
+    if (ret < 0) {
+        error_report("Invalid cache option: %s", cache);
+        goto out;
+    }
+
+    blk = img_open("image", filename, fmt, flags, true, quiet);
+    if (!blk) {
         ret = -1;
         goto out;
     }
+    bs = blk_bs(blk);
 
     fmt = bs->drv->format_name;
 
@@ -2769,16 +2980,19 @@ static int img_amend(int argc, char **argv)
         goto out;
     }
 
-    ret = bdrv_amend_options(bs, opts);
+    /* In case the driver does not call amend_status_cb() */
+    qemu_progress_print(0.f, 0);
+    ret = bdrv_amend_options(bs, opts, &amend_status_cb);
+    qemu_progress_print(100.f, 0);
     if (ret < 0) {
         error_report("Error while amending options: %s", strerror(-ret));
         goto out;
     }
 
 out:
-    if (bs) {
-        bdrv_unref(bs);
-    }
+    qemu_progress_end();
+
+    blk_unref(blk);
     qemu_opts_del(opts);
     qemu_opts_free(create_opts);
     g_free(options);
@@ -2802,6 +3016,7 @@ int main(int argc, char **argv)
 {
     const img_cmd_t *cmd;
     const char *cmdname;
+    Error *local_error = NULL;
     int c;
     static const struct option long_options[] = {
         {"help", no_argument, 0, 'h'},
@@ -2816,7 +3031,12 @@ int main(int argc, char **argv)
     error_set_progname(argv[0]);
     qemu_init_exec_dir(argv[0]);
 
-    qemu_init_main_loop();
+    if (qemu_init_main_loop(&local_error)) {
+        error_report("%s", error_get_pretty(local_error));
+        error_free(local_error);
+        exit(EXIT_FAILURE);
+    }
+
     bdrv_init();
     if (argc < 2) {
         error_exit("Not enough arguments");
diff --git a/qemu-img.texi b/qemu-img.texi
index 514be90f7..0a1ab3598 100644
--- a/qemu-img.texi
+++ b/qemu-img.texi
@@ -72,6 +72,10 @@ down to the nearest 512 bytes. You may use the common size suffixes like
 specifies the cache mode that should be used with the (destination) file. See
 the documentation of the emulator's @code{-drive cache=...} option for allowed
 values.
+@item -T @var{src_cache}
+specifies the cache mode that should be used with the source file(s). See
+the documentation of the emulator's @code{-drive cache=...} option for allowed
+values.
 @end table
 
 Parameters to snapshot subcommand:
@@ -113,7 +117,7 @@ Skip the creation of the target volume
 Command description:
 
 @table @option
-@item check [-f @var{fmt}] [--output=@var{ofmt}] [-r [leaks | all]] @var{filename}
+@item check [-f @var{fmt}] [--output=@var{ofmt}] [-r [leaks | all]] [-T @var{src_cache}] @var{filename}
 
 Perform a consistency check on the disk image @var{filename}. The command can
 output in the format @var{ofmt} which is either @code{human} or @code{json}.
@@ -163,7 +167,7 @@ this case. @var{backing_file} will never be modified unless you use the
 The size can also be specified using the @var{size} option with @code{-o},
 it doesn't need to be specified separately in this case.
 
-@item commit [-f @var{fmt}] [-t @var{cache}] @var{filename}
+@item commit [-q] [-f @var{fmt}] [-t @var{cache}] [-b @var{base}] [-d] [-p] @var{filename}
 
 Commit the changes recorded in @var{filename} in its base image or backing file.
 If the backing file is smaller than the snapshot, then the backing file will be
@@ -172,7 +176,21 @@ the backing file, the backing file will not be truncated.  If you want the
 backing file to match the size of the smaller snapshot, you can safely truncate
 it yourself once the commit operation successfully completes.
 
-@item compare [-f @var{fmt}] [-F @var{fmt}] [-p] [-s] [-q] @var{filename1} @var{filename2}
+The image @var{filename} is emptied after the operation has succeeded. If you do
+not need @var{filename} afterwards and intend to drop it, you may skip emptying
+@var{filename} by specifying the @code{-d} flag.
+
+If the backing chain of the given image file @var{filename} has more than one
+layer, the backing file into which the changes will be committed may be
+specified as @var{base} (which has to be part of @var{filename}'s backing
+chain). If @var{base} is not specified, the immediate backing file of the top
+image (which is @var{filename}) will be used. For reasons of consistency,
+explicitly specifying @var{base} will always imply @code{-d} (since emptying an
+image after committing to an indirect backing file would lead to different data
+being read from the image due to content in the intermediate backing chain
+overruling the commit target).
+
+@item compare [-f @var{fmt}] [-F @var{fmt}] [-T @var{src_cache}] [-p] [-s] [-q] @var{filename1} @var{filename2}
 
 Check if two images have the same content. You can compare images with
 different format or settings.
@@ -213,7 +231,7 @@ Error on reading data
 
 @end table
 
-@item convert [-c] [-p] [-n] [-f @var{fmt}] [-t @var{cache}] [-O @var{output_fmt}] [-o @var{options}] [-s @var{snapshot_id_or_name}] [-l @var{snapshot_param}] [-S @var{sparse_size}] @var{filename} [@var{filename2} [...]] @var{output_filename}
+@item convert [-c] [-p] [-n] [-f @var{fmt}] [-t @var{cache}] [-T @var{src_cache}] [-O @var{output_fmt}] [-o @var{options}] [-s @var{snapshot_id_or_name}] [-l @var{snapshot_param}] [-S @var{sparse_size}] @var{filename} [@var{filename2} [...]] @var{output_filename}
 
 Convert the disk image @var{filename} or a snapshot @var{snapshot_param}(@var{snapshot_id_or_name} is deprecated)
 to disk image @var{output_filename} using format @var{output_fmt}. It can be optionally compressed (@code{-c}
@@ -224,8 +242,8 @@ compression is read-only. It means that if a compressed sector is
 rewritten, then it is rewritten as uncompressed data.
 
 Image conversion is also useful to get smaller image when using a
-growable format such as @code{qcow} or @code{cow}: the empty sectors
-are detected and suppressed from the destination image.
+growable format such as @code{qcow}: the empty sectors are detected and
+suppressed from the destination image.
 
 @var{sparse_size} indicates the consecutive number of bytes (defaults to 4k)
 that must contain only zeros for qemu-img to create a sparse image during
@@ -325,7 +343,7 @@ source code.
 
 List, apply, create or delete snapshots in image @var{filename}.
 
-@item rebase [-f @var{fmt}] [-t @var{cache}] [-p] [-u] -b @var{backing_file} [-F @var{backing_fmt}] @var{filename}
+@item rebase [-f @var{fmt}] [-t @var{cache}] [-T @var{src_cache}] [-p] [-u] -b @var{backing_file} [-F @var{backing_fmt}] @var{filename}
 
 Changes the backing file of an image. Only the formats @code{qcow2} and
 @code{qed} support changing the backing file.
@@ -336,6 +354,9 @@ The backing file is changed to @var{backing_file} and (if the image format of
 string), then the image is rebased onto no backing file (i.e. it will exist
 independently of any backing file).
 
+@var{cache} specifies the cache mode to be used for @var{filename}, whereas
+@var{src_cache} specifies the cache mode for reading backing files.
+
 There are two different modes in which @code{rebase} can operate:
 @table @option
 @item Safe mode
@@ -391,7 +412,7 @@ After using this command to grow a disk image, you must use file system and
 partitioning tools inside the VM to actually begin using the new space on the
 device.
 
-@item amend [-f @var{fmt}] -o @var{options} @var{filename}
+@item amend [-p] [-f @var{fmt}] [-t @var{cache}] -o @var{options} @var{filename}
 
 Amends the image format specific @var{options} for the image file
 @var{filename}. Not all file formats support this operation.
@@ -412,6 +433,15 @@ Linux or NTFS on Windows), then only the written sectors will reserve
 space. Use @code{qemu-img info} to know the real size used by the
 image or @code{ls -ls} on Unix/Linux.
 
+Supported options:
+@table @code
+@item preallocation
+Preallocation mode (allowed values: @code{off}, @code{falloc}, @code{full}).
+@code{falloc} mode preallocates space for image by calling posix_fallocate().
+@code{full} mode preallocates space for image by writing zeros to underlying
+storage.
+@end table
+
 @item qcow2
 QEMU image format, the most versatile format. Use it to have smaller
 images (useful if your filesystem does not supports holes, for example
@@ -460,9 +490,11 @@ sizes can improve the image file size whereas larger cluster sizes generally
 provide better performance.
 
 @item preallocation
-Preallocation mode (allowed values: off, metadata). An image with preallocated
-metadata is initially larger but can improve performance when the image needs
-to grow.
+Preallocation mode (allowed values: @code{off}, @code{metadata}, @code{falloc},
+@code{full}). An image with preallocated metadata is initially larger but can
+improve performance when the image needs to grow. @code{falloc} and @code{full}
+preallocations are like the same options of @code{raw} format, but sets up
+metadata also.
 
 @item lazy_refcounts
 If this option is set to @code{on}, reference count updates are postponed with
diff --git a/qemu-io-cmds.c b/qemu-io-cmds.c
index c503fc66a..d94fb1e2e 100644
--- a/qemu-io-cmds.c
+++ b/qemu-io-cmds.c
@@ -29,7 +29,7 @@ static int compare_cmdname(const void *a, const void *b)
 
 void qemuio_add_command(const cmdinfo_t *ci)
 {
-    cmdtab = g_realloc(cmdtab, ++ncmds * sizeof(*cmdtab));
+    cmdtab = g_renew(cmdinfo_t, cmdtab, ++ncmds);
     cmdtab[ncmds - 1] = *ci;
     qsort(cmdtab, ncmds, sizeof(*cmdtab), compare_cmdname);
 }
@@ -114,23 +114,14 @@ static char **breakline(char *input, int *count)
 {
     int c = 0;
     char *p;
-    char **rval = g_malloc0(sizeof(char *));
-    char **tmp;
+    char **rval = g_new0(char *, 1);
 
     while (rval && (p = qemu_strsep(&input, " ")) != NULL) {
         if (!*p) {
             continue;
         }
         c++;
-        tmp = g_realloc(rval, sizeof(*rval) * (c + 1));
-        if (!tmp) {
-            g_free(rval);
-            rval = NULL;
-            c = 0;
-            break;
-        } else {
-            rval = tmp;
-        }
+        rval = g_renew(char *, rval, (c + 1));
         rval[c - 1] = p;
         rval[c] = NULL;
     }
@@ -1264,9 +1255,9 @@ static int multiwrite_f(BlockDriverState *bs, int argc, char **argv)
         }
     }
 
-    reqs = g_malloc0(nr_reqs * sizeof(*reqs));
-    buf = g_malloc0(nr_reqs * sizeof(*buf));
-    qiovs = g_malloc(nr_reqs * sizeof(*qiovs));
+    reqs = g_new0(BlockRequest, nr_reqs);
+    buf = g_new0(char *, nr_reqs);
+    qiovs = g_new(QEMUIOVector, nr_reqs);
 
     for (i = 0; i < nr_reqs && optind < argc; i++) {
         int j;
@@ -1909,7 +1900,7 @@ static int map_is_allocated(BlockDriverState *bs, int64_t sector_num,
 
         num_checked = MIN(nb_sectors, INT_MAX);
         ret = bdrv_is_allocated(bs, sector_num, num_checked, &num);
-        if (ret == firstret) {
+        if (ret == firstret && num) {
             *pnum += num;
         } else {
             break;
@@ -1936,6 +1927,9 @@ static int map_f(BlockDriverState *bs, int argc, char **argv)
         if (ret < 0) {
             error_report("Failed to get allocation status: %s", strerror(-ret));
             return 0;
+        } else if (!num) {
+            error_report("Unexpected end of image");
+            return 0;
         }
 
         retstr = ret ? "    allocated" : "not allocated";
diff --git a/qemu-io.c b/qemu-io.c
index b55a550f2..60f84dd72 100644
--- a/qemu-io.c
+++ b/qemu-io.c
@@ -19,6 +19,7 @@
 #include "qemu/option.h"
 #include "qemu/config-file.h"
 #include "qemu/readline.h"
+#include "sysemu/block-backend.h"
 #include "block/block_int.h"
 #include "trace/control.h"
 
@@ -26,6 +27,7 @@
 
 static char *progname;
 
+static BlockBackend *qemuio_blk;
 static BlockDriverState *qemuio_bs;
 
 /* qemu-io commands passed using -c */
@@ -36,8 +38,9 @@ static ReadLineState *readline_state;
 
 static int close_f(BlockDriverState *bs, int argc, char **argv)
 {
-    bdrv_unref(bs);
+    blk_unref(qemuio_blk);
     qemuio_bs = NULL;
+    qemuio_blk = NULL;
     return 0;
 }
 
@@ -58,30 +61,22 @@ static int openfile(char *name, int flags, int growable, QDict *opts)
         return 1;
     }
 
+    qemuio_blk = blk_new_with_bs("hda", &error_abort);
+    qemuio_bs = blk_bs(qemuio_blk);
+
     if (growable) {
-        if (bdrv_open(&qemuio_bs, name, NULL, opts, flags | BDRV_O_PROTOCOL,
-                      NULL, &local_err))
-        {
-            fprintf(stderr, "%s: can't open%s%s: %s\n", progname,
-                    name ? " device " : "", name ?: "",
-                    error_get_pretty(local_err));
-            error_free(local_err);
-            return 1;
-        }
-    } else {
-        qemuio_bs = bdrv_new("hda", &error_abort);
-
-        if (bdrv_open(&qemuio_bs, name, NULL, opts, flags, NULL, &local_err)
-            < 0)
-        {
-            fprintf(stderr, "%s: can't open%s%s: %s\n", progname,
-                    name ? " device " : "", name ?: "",
-                    error_get_pretty(local_err));
-            error_free(local_err);
-            bdrv_unref(qemuio_bs);
-            qemuio_bs = NULL;
-            return 1;
-        }
+        flags |= BDRV_O_PROTOCOL;
+    }
+
+    if (bdrv_open(&qemuio_bs, name, NULL, opts, flags, NULL, &local_err) < 0) {
+        fprintf(stderr, "%s: can't open%s%s: %s\n", progname,
+                name ? " device " : "", name ?: "",
+                error_get_pretty(local_err));
+        error_free(local_err);
+        blk_unref(qemuio_blk);
+        qemuio_bs = NULL;
+        qemuio_blk = NULL;
+        return 1;
     }
 
     return 0;
@@ -356,7 +351,7 @@ static void command_loop(void)
 
 static void add_user_command(char *optarg)
 {
-    cmdline = g_realloc(cmdline, ++ncmdline * sizeof(char *));
+    cmdline = g_renew(char *, cmdline, ++ncmdline);
     cmdline[ncmdline-1] = optarg;
 }
 
@@ -389,6 +384,7 @@ int main(int argc, char **argv)
     int c;
     int opt_index = 0;
     int flags = BDRV_O_UNMAP;
+    Error *local_error = NULL;
 
 #ifdef CONFIG_POSIX
     signal(SIGPIPE, SIG_IGN);
@@ -454,7 +450,11 @@ int main(int argc, char **argv)
         exit(1);
     }
 
-    qemu_init_main_loop();
+    if (qemu_init_main_loop(&local_error)) {
+        error_report("%s", error_get_pretty(local_error));
+        error_free(local_error);
+        exit(1);
+    }
     bdrv_init();
 
     /* initialize commands */
@@ -486,9 +486,7 @@ int main(int argc, char **argv)
      */
     bdrv_drain_all();
 
-    if (qemuio_bs) {
-        bdrv_unref(qemuio_bs);
-    }
+    blk_unref(qemuio_blk);
     g_free(readline_state);
     return 0;
 }
diff --git a/qemu-nbd.c b/qemu-nbd.c
index 626e5844f..5cd6c6d18 100644
--- a/qemu-nbd.c
+++ b/qemu-nbd.c
@@ -17,12 +17,14 @@
  */
 
 #include "qemu-common.h"
-#include "block/block.h"
+#include "sysemu/block-backend.h"
+#include "block/block_int.h"
 #include "block/nbd.h"
 #include "qemu/main-loop.h"
 #include "qemu/sockets.h"
 #include "qemu/error-report.h"
 #include "block/snapshot.h"
+#include "qapi/util.h"
 
 #include <stdarg.h>
 #include <stdio.h>
@@ -37,10 +39,11 @@
 #include <libgen.h>
 #include <pthread.h>
 
-#define SOCKET_PATH          "/var/lock/qemu-nbd-%s"
-#define QEMU_NBD_OPT_CACHE   1
-#define QEMU_NBD_OPT_AIO     2
-#define QEMU_NBD_OPT_DISCARD 3
+#define SOCKET_PATH                "/var/lock/qemu-nbd-%s"
+#define QEMU_NBD_OPT_CACHE         1
+#define QEMU_NBD_OPT_AIO           2
+#define QEMU_NBD_OPT_DISCARD       3
+#define QEMU_NBD_OPT_DETECT_ZEROES 4
 
 static NBDExport *exp;
 static int verbose;
@@ -57,45 +60,47 @@ static void usage(const char *name)
 "Usage: %s [OPTIONS] FILE\n"
 "QEMU Disk Network Block Device Server\n"
 "\n"
-"  -h, --help           display this help and exit\n"
-"  -V, --version        output version information and exit\n"
+"  -h, --help                display this help and exit\n"
+"  -V, --version             output version information and exit\n"
 "\n"
 "Connection properties:\n"
-"  -p, --port=PORT      port to listen on (default `%d')\n"
-"  -b, --bind=IFACE     interface to bind to (default `0.0.0.0')\n"
-"  -k, --socket=PATH    path to the unix socket\n"
-"                       (default '"SOCKET_PATH"')\n"
-"  -e, --shared=NUM     device can be shared by NUM clients (default '1')\n"
-"  -t, --persistent     don't exit on the last connection\n"
-"  -v, --verbose        display extra debugging information\n"
+"  -p, --port=PORT           port to listen on (default `%d')\n"
+"  -b, --bind=IFACE          interface to bind to (default `0.0.0.0')\n"
+"  -k, --socket=PATH         path to the unix socket\n"
+"                            (default '"SOCKET_PATH"')\n"
+"  -e, --shared=NUM          device can be shared by NUM clients (default '1')\n"
+"  -t, --persistent          don't exit on the last connection\n"
+"  -v, --verbose             display extra debugging information\n"
 "\n"
 "Exposing part of the image:\n"
-"  -o, --offset=OFFSET  offset into the image\n"
-"  -P, --partition=NUM  only expose partition NUM\n"
+"  -o, --offset=OFFSET       offset into the image\n"
+"  -P, --partition=NUM       only expose partition NUM\n"
 "\n"
 #ifdef __linux__
 "Kernel NBD client support:\n"
-"  -c, --connect=DEV    connect FILE to the local NBD device DEV\n"
-"  -d, --disconnect     disconnect the specified device\n"
+"  -c, --connect=DEV         connect FILE to the local NBD device DEV\n"
+"  -d, --disconnect          disconnect the specified device\n"
 "\n"
 #endif
 "\n"
 "Block device options:\n"
-"  -f, --format=FORMAT  set image format (raw, qcow2, ...)\n"
-"  -r, --read-only      export read-only\n"
-"  -s, --snapshot       use FILE as an external snapshot, create a temporary\n"
-"                       file with backing_file=FILE, redirect the write to\n"
-"                       the temporary one\n"
+"  -f, --format=FORMAT       set image format (raw, qcow2, ...)\n"
+"  -r, --read-only           export read-only\n"
+"  -s, --snapshot            use FILE as an external snapshot, create a temporary\n"
+"                            file with backing_file=FILE, redirect the write to\n"
+"                            the temporary one\n"
 "  -l, --load-snapshot=SNAPSHOT_PARAM\n"
-"                       load an internal snapshot inside FILE and export it\n"
-"                       as an read-only device, SNAPSHOT_PARAM format is\n"
-"                       'snapshot.id=[ID],snapshot.name=[NAME]', or\n"
-"                       '[ID_OR_NAME]'\n"
-"  -n, --nocache        disable host cache\n"
-"      --cache=MODE     set cache mode (none, writeback, ...)\n"
+"                            load an internal snapshot inside FILE and export it\n"
+"                            as an read-only device, SNAPSHOT_PARAM format is\n"
+"                            'snapshot.id=[ID],snapshot.name=[NAME]', or\n"
+"                            '[ID_OR_NAME]'\n"
+"  -n, --nocache             disable host cache\n"
+"      --cache=MODE          set cache mode (none, writeback, ...)\n"
 #ifdef CONFIG_LINUX_AIO
-"      --aio=MODE       set AIO mode (native or threads)\n"
+"      --aio=MODE            set AIO mode (native or threads)\n"
 #endif
+"      --discard=MODE        set discard mode (ignore, unmap)\n"
+"      --detect-zeroes=MODE  set detect-zeroes mode (off, on, discard)\n"
 "\n"
 "Report bugs to <qemu-devel@nongnu.org>\n"
     , name, NBD_DEFAULT_PORT, "DEVICE");
@@ -379,6 +384,7 @@ static void nbd_accept(void *opaque)
 
 int main(int argc, char **argv)
 {
+    BlockBackend *blk;
     BlockDriverState *bs;
     BlockDriver *drv;
     off_t dev_offset = 0;
@@ -410,6 +416,7 @@ int main(int argc, char **argv)
         { "aio", 1, NULL, QEMU_NBD_OPT_AIO },
 #endif
         { "discard", 1, NULL, QEMU_NBD_OPT_DISCARD },
+        { "detect-zeroes", 1, NULL, QEMU_NBD_OPT_DETECT_ZEROES },
         { "shared", 1, NULL, 'e' },
         { "format", 1, NULL, 'f' },
         { "persistent", 0, NULL, 't' },
@@ -432,6 +439,7 @@ int main(int argc, char **argv)
     pthread_t client_thread;
     const char *fmt = NULL;
     Error *local_err = NULL;
+    BlockdevDetectZeroesOptions detect_zeroes = BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF;
 
     /* The client thread uses SIGTERM to interrupt the server.  A signal
      * handler ensures that "qemu-nbd -v -c" exits with a nice status code.
@@ -483,6 +491,23 @@ int main(int argc, char **argv)
                 errx(EXIT_FAILURE, "Invalid discard mode `%s'", optarg);
             }
             break;
+        case QEMU_NBD_OPT_DETECT_ZEROES:
+            detect_zeroes =
+                qapi_enum_parse(BlockdevDetectZeroesOptions_lookup,
+                                optarg,
+                                BLOCKDEV_DETECT_ZEROES_OPTIONS_MAX,
+                                BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF,
+                                &local_err);
+            if (local_err) {
+                errx(EXIT_FAILURE, "Failed to parse detect_zeroes mode: %s", 
+                     error_get_pretty(local_err));
+            }
+            if (detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP &&
+                !(flags & BDRV_O_UNMAP)) {
+                errx(EXIT_FAILURE, "setting detect-zeroes to unmap is not allowed "
+                                   "without setting discard operation to unmap"); 
+            }
+            break;
         case 'b':
             bindto = optarg;
             break;
@@ -522,15 +547,18 @@ int main(int argc, char **argv)
             break;
         case 'P':
             partition = strtol(optarg, &end, 0);
-            if (*end)
+            if (*end) {
                 errx(EXIT_FAILURE, "Invalid partition `%s'", optarg);
-            if (partition < 1 || partition > 8)
+            }
+            if (partition < 1 || partition > 8) {
                 errx(EXIT_FAILURE, "Invalid partition %d", partition);
+            }
             break;
         case 'k':
             sockpath = optarg;
-            if (sockpath[0] != '/')
+            if (sockpath[0] != '/') {
                 errx(EXIT_FAILURE, "socket path must be absolute\n");
+            }
             break;
         case 'd':
             disconnect = true;
@@ -550,9 +578,9 @@ int main(int argc, char **argv)
         case 'f':
             fmt = optarg;
             break;
-	case 't':
-	    persistent = 1;
-	    break;
+        case 't':
+            persistent = 1;
+            break;
         case 'v':
             verbose = 1;
             break;
@@ -587,7 +615,7 @@ int main(int argc, char **argv)
 
         printf("%s disconnected\n", argv[optind]);
 
-	return 0;
+        return 0;
     }
 
     if (device && !verbose) {
@@ -647,7 +675,11 @@ int main(int argc, char **argv)
         snprintf(sockpath, 128, SOCKET_PATH, basename(device));
     }
 
-    qemu_init_main_loop();
+    if (qemu_init_main_loop(&local_err)) {
+        error_report("%s", error_get_pretty(local_err));
+        error_free(local_err);
+        exit(EXIT_FAILURE);
+    }
     bdrv_init();
     atexit(bdrv_close_all);
 
@@ -660,7 +692,8 @@ int main(int argc, char **argv)
         drv = NULL;
     }
 
-    bs = bdrv_new("hda", &error_abort);
+    blk = blk_new_with_bs("hda", &error_abort);
+    bs = blk_bs(blk);
 
     srcpath = argv[optind];
     ret = bdrv_open(&bs, srcpath, NULL, NULL, flags, drv, &local_err);
@@ -686,6 +719,7 @@ int main(int argc, char **argv)
             error_get_pretty(local_err));
     }
 
+    bs->detect_zeroes = detect_zeroes;
     fd_size = bdrv_getlength(bs);
 
     if (partition != -1) {
@@ -741,14 +775,12 @@ int main(int argc, char **argv)
         }
     } while (state != TERMINATED);
 
-    bdrv_close(bs);
+    blk_unref(blk);
     if (sockpath) {
         unlink(sockpath);
     }
 
-    if (sn_opts) {
-        qemu_opts_del(sn_opts);
-    }
+    qemu_opts_del(sn_opts);
 
     if (device) {
         void *ret;
diff --git a/qemu-options.hx b/qemu-options.hx
index 154962558..64af16d64 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -33,9 +33,11 @@ DEF("machine", HAS_ARG, QEMU_OPTION_machine, \
     "                property accel=accel1[:accel2[:...]] selects accelerator\n"
     "                supported accelerators are kvm, xen, tcg (default: tcg)\n"
     "                kernel_irqchip=on|off controls accelerated irqchip support\n"
+    "                vmport=on|off|auto controls emulation of vmport (default: auto)\n"
     "                kvm_shadow_mem=size of KVM shadow MMU\n"
     "                dump-guest-core=on|off include guest memory in a core dump (default=on)\n"
-    "                mem-merge=on|off controls memory merge support (default: on)\n",
+    "                mem-merge=on|off controls memory merge support (default: on)\n"
+    "                iommu=on|off controls emulated Intel IOMMU (VT-d) support (default=off)\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -machine [type=]@var{name}[,prop=@var{value}[,...]]
@@ -50,6 +52,10 @@ than one accelerator specified, the next one is used if the previous one fails
 to initialize.
 @item kernel_irqchip=on|off
 Enables in-kernel irqchip support for the chosen accelerator when available.
+@item vmport=on|off|auto
+Enables emulation of VMWare IO port, for vmmouse etc. auto says to select the
+value based on accel. For accel=xen the default is off otherwise the default
+is on.
 @item kvm_shadow_mem=size
 Defines the size of the KVM shadow MMU.
 @item dump-guest-core=on|off
@@ -58,6 +64,8 @@ Include guest memory in a core dump. The default is on.
 Enables or disables memory merge support. This feature, when supported by
 the host, de-duplicates identical memory pages among VMs instances
 (enabled by default).
+@item iommu=on|off
+Enables or disables emulated Intel IOMMU (VT-d) support. The default is off.
 @end table
 ETEXI
 
@@ -225,7 +233,8 @@ DEF("m", HAS_ARG, QEMU_OPTION_m,
     "                size: initial amount of guest memory (default: "
     stringify(DEFAULT_RAM_SIZE) "MiB)\n"
     "                slots: number of hotplug slots (default: none)\n"
-    "                maxmem: maximum amount of guest memory (default: none)\n",
+    "                maxmem: maximum amount of guest memory (default: none)\n"
+    "NOTE: Some architectures might enforce a specific granularity\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -m [size=]@var{megs}
@@ -427,7 +436,7 @@ DEF("drive", HAS_ARG, QEMU_OPTION_drive,
     "       [,serial=s][,addr=A][,rerror=ignore|stop|report]\n"
     "       [,werror=ignore|stop|report|enospc][,id=name][,aio=threads|native]\n"
     "       [,readonly=on|off][,copy-on-read=on|off]\n"
-    "       [,detect-zeroes=on|off|unmap]\n"
+    "       [,discard=ignore|unmap][,detect-zeroes=on|off|unmap]\n"
     "       [[,bps=b]|[[,bps_rd=r][,bps_wr=w]]]\n"
     "       [[,iops=i]|[[,iops_rd=r][,iops_wr=w]]]\n"
     "       [[,bps_max=bm]|[[,bps_rd_max=rm][,bps_wr_max=wm]]]\n"
@@ -1444,7 +1453,7 @@ DEF("net", HAS_ARG, QEMU_OPTION_net,
     "                use 'src=' to specify source address\n"
     "                use 'dst=' to specify destination address\n"
     "                use 'udp=on' to specify udp encapsulation\n"
-    "                use 'dstport=' to specify destination udp port\n"
+    "                use 'srcport=' to specify source udp port\n"
     "                use 'dstport=' to specify destination udp port\n"
     "                use 'ipv6=on' to force v6\n"
     "                L2TPv3 uses cookies to prevent misconfiguration as\n"
@@ -1926,9 +1935,9 @@ ETEXI
 
 DEF("chardev", HAS_ARG, QEMU_OPTION_chardev,
     "-chardev null,id=id[,mux=on|off]\n"
-    "-chardev socket,id=id[,host=host],port=host[,to=to][,ipv4][,ipv6][,nodelay]\n"
-    "         [,server][,nowait][,telnet][,mux=on|off] (tcp)\n"
-    "-chardev socket,id=id,path=path[,server][,nowait][,telnet],[mux=on|off] (unix)\n"
+    "-chardev socket,id=id[,host=host],port=port[,to=to][,ipv4][,ipv6][,nodelay][,reconnect=seconds]\n"
+    "         [,server][,nowait][,telnet][,reconnect=seconds][,mux=on|off] (tcp)\n"
+    "-chardev socket,id=id,path=path[,server][,nowait][,telnet][,reconnect=seconds][,mux=on|off] (unix)\n"
     "-chardev udp,id=id[,host=host],port=port[,localaddr=localaddr]\n"
     "         [,localport=localport][,ipv4][,ipv6][,mux=on|off]\n"
     "-chardev msmouse,id=id[,mux=on|off]\n"
@@ -2000,7 +2009,7 @@ Options to each backend are described below.
 A void device. This device will not emit any data, and will drop any data it
 receives. The null backend does not take any options.
 
-@item -chardev socket ,id=@var{id} [@var{TCP options} or @var{unix options}] [,server] [,nowait] [,telnet]
+@item -chardev socket ,id=@var{id} [@var{TCP options} or @var{unix options}] [,server] [,nowait] [,telnet] [,reconnect=@var{seconds}]
 
 Create a two-way stream socket, which can be either a TCP or a unix socket. A
 unix socket will be created if @option{path} is specified. Behaviour is
@@ -2014,6 +2023,10 @@ connect to a listening socket.
 @option{telnet} specifies that traffic on the socket should interpret telnet
 escape sequences.
 
+@option{reconnect} sets the timeout for reconnecting on non-server sockets when
+the remote end goes away.  qemu will delay this many seconds and then attempt
+to reconnect.  Zero disables reconnecting, and is the default.
+
 TCP and unix socket options are given below:
 
 @table @option
@@ -2351,6 +2364,16 @@ multiple of 512 bytes. It defaults to 256k.
 @item sslverify
 Whether to verify the remote server's certificate when connecting over SSL. It
 can have the value 'on' or 'off'. It defaults to 'on'.
+
+@item cookie
+Send this cookie (it can also be a list of cookies separated by ';') with
+each outgoing request.  Only supported when using protocols such as HTTP
+which support cookies, otherwise ignored.
+
+@item timeout
+Set the timeout in seconds of the CURL connection. This timeout is the time
+that CURL waits for a response from the remote server to get the size of the
+image to be downloaded. If not set, the default timeout of 5 seconds is used.
 @end table
 
 Note that when passing options to qemu explicitly, @option{driver} is the value
@@ -2372,9 +2395,10 @@ qemu-system-x86_64 -drive file=/tmp/Fedora-x86_64-20-20131211.1-sda.qcow2,copy-o
 @end example
 
 Example: boot from an image stored on a VMware vSphere server with a self-signed
-certificate using a local overlay for writes and a readahead of 64k
+certificate using a local overlay for writes, a readahead of 64k and a timeout
+of 10 seconds.
 @example
-qemu-img create -f qcow2 -o backing_file='json:@{"file.driver":"https",, "file.url":"https://user:password@@vsphere.example.com/folder/test/test-flat.vmdk?dcPath=Datacenter&dsName=datastore1",, "file.sslverify":"off",, "file.readahead":"64k"@}' /tmp/test.qcow2
+qemu-img create -f qcow2 -o backing_file='json:@{"file.driver":"https",, "file.url":"https://user:password@@vsphere.example.com/folder/test/test-flat.vmdk?dcPath=Datacenter&dsName=datastore1",, "file.sslverify":"off",, "file.readahead":"64k",, "file.timeout":10@}' /tmp/test.qcow2
 
 qemu-system-x86_64 -drive file=/tmp/test.qcow2
 @end example
@@ -2672,14 +2696,16 @@ telnet on port 5555 to access the QEMU port.
 localhost 5555
 @end table
 
-@item tcp:[@var{host}]:@var{port}[,@var{server}][,nowait][,nodelay]
+@item tcp:[@var{host}]:@var{port}[,@var{server}][,nowait][,nodelay][,reconnect=@var{seconds}]
 The TCP Net Console has two modes of operation.  It can send the serial
 I/O to a location or wait for a connection from a location.  By default
 the TCP Net Console is sent to @var{host} at the @var{port}.  If you use
 the @var{server} option QEMU will wait for a client socket application
 to connect to the port before continuing, unless the @code{nowait}
 option was specified.  The @code{nodelay} option disables the Nagle buffering
-algorithm.  If @var{host} is omitted, 0.0.0.0 is assumed. Only
+algorithm.  The @code{reconnect} option only applies if @var{noserver} is
+set, if the connection goes down it will attempt to reconnect at the
+given interval.  If @var{host} is omitted, 0.0.0.0 is assumed. Only
 one TCP connection at a time is accepted. You can use @code{telnet} to
 connect to the corresponding character device.
 @table @code
@@ -2700,7 +2726,7 @@ MAGIC_SYSRQ sequence if you use a telnet that supports sending the break
 sequence.  Typically in unix telnet you do it with Control-] and then
 type "send break" followed by pressing the enter key.
 
-@item unix:@var{path}[,server][,nowait]
+@item unix:@var{path}[,server][,nowait][,reconnect=@var{seconds}]
 A unix domain socket is used instead of a tcp socket.  The option works the
 same as if you had specified @code{-serial tcp} except the unix domain socket
 @var{path} is used for connections.
@@ -2968,16 +2994,8 @@ Load the contents of @var{file} as an option ROM.
 This option is useful to load things like EtherBoot.
 ETEXI
 
-DEF("clock", HAS_ARG, QEMU_OPTION_clock, \
-    "-clock          force the use of the given methods for timer alarm.\n" \
-    "                To see what timers are available use '-clock help'\n",
-    QEMU_ARCH_ALL)
-STEXI
-@item -clock @var{method}
-@findex -clock
-Force the use of the given methods for timer alarm. To see what timers
-are available use @code{-clock help}.
-ETEXI
+HXCOMM Silently ignored for compatibility
+DEF("clock", HAS_ARG, QEMU_OPTION_clock, "", QEMU_ARCH_ALL)
 
 HXCOMM Options deprecated by -rtc
 DEF("localtime", 0, QEMU_OPTION_localtime, "", QEMU_ARCH_ALL)
@@ -3011,11 +3029,11 @@ re-inject them.
 ETEXI
 
 DEF("icount", HAS_ARG, QEMU_OPTION_icount, \
-    "-icount [N|auto]\n" \
+    "-icount [shift=N|auto][,align=on|off]\n" \
     "                enable virtual instruction counter with 2^N clock ticks per\n" \
-    "                instruction\n", QEMU_ARCH_ALL)
+    "                instruction and enable aligning the host and virtual clocks\n", QEMU_ARCH_ALL)
 STEXI
-@item -icount [@var{N}|auto]
+@item -icount [shift=@var{N}|auto]
 @findex -icount
 Enable virtual instruction counter.  The virtual cpu will execute one
 instruction every 2^@var{N} ns of virtual time.  If @code{auto} is specified
@@ -3026,6 +3044,17 @@ Note that while this option can give deterministic behavior, it does not
 provide cycle accurate emulation.  Modern CPUs contain superscalar out of
 order cores with complex cache hierarchies.  The number of instructions
 executed often has little or no correlation with actual performance.
+
+@option{align=on} will activate the delay algorithm which will try to
+to synchronise the host clock and the virtual clock. The goal is to
+have a guest running at the real frequency imposed by the shift option.
+Whenever the guest clock is behind the host clock and if
+@option{align=on} is specified then we print a messsage to the user
+to inform about the delay.
+Currently this option does not work when @option{shift} is @code{auto}.
+Note: The sync algorithm will work for those shift values for which
+the guest clock runs ahead of the host clock. Typically this happens
+when the shift value is high (how high depends on the host machine).
 ETEXI
 
 DEF("watchdog", HAS_ARG, QEMU_OPTION_watchdog, \
diff --git a/qemu-seccomp.c b/qemu-seccomp.c
index ea8094d04..af6a37512 100644
--- a/qemu-seccomp.c
+++ b/qemu-seccomp.c
@@ -230,7 +230,12 @@ static const struct QemuSeccompSyscall seccomp_whitelist[] = {
     { SCMP_SYS(timerfd_create), 240 },
     { SCMP_SYS(shmctl), 240 },
     { SCMP_SYS(mlock), 240 },
-    { SCMP_SYS(munlock), 240 }
+    { SCMP_SYS(munlock), 240 },
+    { SCMP_SYS(semctl), 240 },
+    { SCMP_SYS(fallocate), 240 },
+    { SCMP_SYS(fadvise64), 240 },
+    { SCMP_SYS(inotify_init1), 240 },
+    { SCMP_SYS(inotify_add_watch), 240 }
 };
 
 int seccomp_start(void)
diff --git a/qemu-timer.c b/qemu-timer.c
index 00a5d35c3..c77de6430 100644
--- a/qemu-timer.c
+++ b/qemu-timer.c
@@ -314,7 +314,14 @@ int qemu_poll_ns(GPollFD *fds, guint nfds, int64_t timeout)
         return ppoll((struct pollfd *)fds, nfds, NULL, NULL);
     } else {
         struct timespec ts;
-        ts.tv_sec = timeout / 1000000000LL;
+        int64_t tvsec = timeout / 1000000000LL;
+        /* Avoid possibly overflowing and specifying a negative number of
+         * seconds, which would turn a very long timeout into a busy-wait.
+         */
+        if (tvsec > (int64_t)INT32_MAX) {
+            tvsec = INT32_MAX;
+        }
+        ts.tv_sec = tvsec;
         ts.tv_nsec = timeout % 1000000000LL;
         return ppoll((struct pollfd *)fds, nfds, &ts, NULL);
     }
diff --git a/qga/channel-posix.c b/qga/channel-posix.c
index e65dda382..8aad4fee9 100644
--- a/qga/channel-posix.c
+++ b/qga/channel-posix.c
@@ -42,7 +42,7 @@ static gboolean ga_channel_listen_accept(GIOChannel *channel,
         g_warning("error converting fd to gsocket: %s", strerror(errno));
         goto out;
     }
-    fcntl(client_fd, F_SETFL, O_NONBLOCK);
+    qemu_set_nonblock(client_fd);
     ret = ga_channel_client_add(c, client_fd);
     if (ret) {
         g_warning("error setting up connection");
diff --git a/qga/commands-posix.c b/qga/commands-posix.c
index 8e6272c5a..f6f3e3cd8 100644
--- a/qga/commands-posix.c
+++ b/qga/commands-posix.c
@@ -18,6 +18,7 @@
 #include <unistd.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <dirent.h>
 #include <stdio.h>
 #include <string.h>
 #include <sys/stat.h>
@@ -575,6 +576,7 @@ static void guest_file_init(void)
 typedef struct FsMount {
     char *dirname;
     char *devtype;
+    unsigned int devmajor, devminor;
     QTAILQ_ENTRY(FsMount) next;
 } FsMount;
 
@@ -596,15 +598,40 @@ static void free_fs_mount_list(FsMountList *mounts)
      }
 }
 
+static int dev_major_minor(const char *devpath,
+                           unsigned int *devmajor, unsigned int *devminor)
+{
+    struct stat st;
+
+    *devmajor = 0;
+    *devminor = 0;
+
+    if (stat(devpath, &st) < 0) {
+        slog("failed to stat device file '%s': %s", devpath, strerror(errno));
+        return -1;
+    }
+    if (S_ISDIR(st.st_mode)) {
+        /* It is bind mount */
+        return -2;
+    }
+    if (S_ISBLK(st.st_mode)) {
+        *devmajor = major(st.st_rdev);
+        *devminor = minor(st.st_rdev);
+        return 0;
+    }
+    return -1;
+}
+
 /*
  * Walk the mount table and build a list of local file systems
  */
-static void build_fs_mount_list(FsMountList *mounts, Error **errp)
+static void build_fs_mount_list_from_mtab(FsMountList *mounts, Error **errp)
 {
     struct mntent *ment;
     FsMount *mount;
     char const *mtab = "/proc/self/mounts";
     FILE *fp;
+    unsigned int devmajor, devminor;
 
     fp = setmntent(mtab, "r");
     if (!fp) {
@@ -624,20 +651,426 @@ static void build_fs_mount_list(FsMountList *mounts, Error **errp)
             (strcmp(ment->mnt_type, "cifs") == 0)) {
             continue;
         }
+        if (dev_major_minor(ment->mnt_fsname, &devmajor, &devminor) == -2) {
+            /* Skip bind mounts */
+            continue;
+        }
 
         mount = g_malloc0(sizeof(FsMount));
         mount->dirname = g_strdup(ment->mnt_dir);
         mount->devtype = g_strdup(ment->mnt_type);
+        mount->devmajor = devmajor;
+        mount->devminor = devminor;
 
         QTAILQ_INSERT_TAIL(mounts, mount, next);
     }
 
     endmntent(fp);
 }
+
+static void decode_mntname(char *name, int len)
+{
+    int i, j = 0;
+    for (i = 0; i <= len; i++) {
+        if (name[i] != '\\') {
+            name[j++] = name[i];
+        } else if (name[i + 1] == '\\') {
+            name[j++] = '\\';
+            i++;
+        } else if (name[i + 1] >= '0' && name[i + 1] <= '3' &&
+                   name[i + 2] >= '0' && name[i + 2] <= '7' &&
+                   name[i + 3] >= '0' && name[i + 3] <= '7') {
+            name[j++] = (name[i + 1] - '0') * 64 +
+                        (name[i + 2] - '0') * 8 +
+                        (name[i + 3] - '0');
+            i += 3;
+        } else {
+            name[j++] = name[i];
+        }
+    }
+}
+
+static void build_fs_mount_list(FsMountList *mounts, Error **errp)
+{
+    FsMount *mount;
+    char const *mountinfo = "/proc/self/mountinfo";
+    FILE *fp;
+    char *line = NULL, *dash;
+    size_t n;
+    char check;
+    unsigned int devmajor, devminor;
+    int ret, dir_s, dir_e, type_s, type_e, dev_s, dev_e;
+
+    fp = fopen(mountinfo, "r");
+    if (!fp) {
+        build_fs_mount_list_from_mtab(mounts, errp);
+        return;
+    }
+
+    while (getline(&line, &n, fp) != -1) {
+        ret = sscanf(line, "%*u %*u %u:%u %*s %n%*s%n%c",
+                     &devmajor, &devminor, &dir_s, &dir_e, &check);
+        if (ret < 3) {
+            continue;
+        }
+        dash = strstr(line + dir_e, " - ");
+        if (!dash) {
+            continue;
+        }
+        ret = sscanf(dash, " - %n%*s%n %n%*s%n%c",
+                     &type_s, &type_e, &dev_s, &dev_e, &check);
+        if (ret < 1) {
+            continue;
+        }
+        line[dir_e] = 0;
+        dash[type_e] = 0;
+        dash[dev_e] = 0;
+        decode_mntname(line + dir_s, dir_e - dir_s);
+        decode_mntname(dash + dev_s, dev_e - dev_s);
+        if (devmajor == 0) {
+            /* btrfs reports major number = 0 */
+            if (strcmp("btrfs", dash + type_s) != 0 ||
+                dev_major_minor(dash + dev_s, &devmajor, &devminor) < 0) {
+                continue;
+            }
+        }
+
+        mount = g_malloc0(sizeof(FsMount));
+        mount->dirname = g_strdup(line + dir_s);
+        mount->devtype = g_strdup(dash + type_s);
+        mount->devmajor = devmajor;
+        mount->devminor = devminor;
+
+        QTAILQ_INSERT_TAIL(mounts, mount, next);
+    }
+    free(line);
+
+    fclose(fp);
+}
 #endif
 
 #if defined(CONFIG_FSFREEZE)
 
+static char *get_pci_driver(char const *syspath, int pathlen, Error **errp)
+{
+    char *path;
+    char *dpath;
+    char *driver = NULL;
+    char buf[PATH_MAX];
+    ssize_t len;
+
+    path = g_strndup(syspath, pathlen);
+    dpath = g_strdup_printf("%s/driver", path);
+    len = readlink(dpath, buf, sizeof(buf) - 1);
+    if (len != -1) {
+        buf[len] = 0;
+        driver = g_strdup(basename(buf));
+    }
+    g_free(dpath);
+    g_free(path);
+    return driver;
+}
+
+static int compare_uint(const void *_a, const void *_b)
+{
+    unsigned int a = *(unsigned int *)_a;
+    unsigned int b = *(unsigned int *)_b;
+
+    return a < b ? -1 : a > b ? 1 : 0;
+}
+
+/* Walk the specified sysfs and build a sorted list of host or ata numbers */
+static int build_hosts(char const *syspath, char const *host, bool ata,
+                       unsigned int *hosts, int hosts_max, Error **errp)
+{
+    char *path;
+    DIR *dir;
+    struct dirent *entry;
+    int i = 0;
+
+    path = g_strndup(syspath, host - syspath);
+    dir = opendir(path);
+    if (!dir) {
+        error_setg_errno(errp, errno, "opendir(\"%s\")", path);
+        g_free(path);
+        return -1;
+    }
+
+    while (i < hosts_max) {
+        entry = readdir(dir);
+        if (!entry) {
+            break;
+        }
+        if (ata && sscanf(entry->d_name, "ata%d", hosts + i) == 1) {
+            ++i;
+        } else if (!ata && sscanf(entry->d_name, "host%d", hosts + i) == 1) {
+            ++i;
+        }
+    }
+
+    qsort(hosts, i, sizeof(hosts[0]), compare_uint);
+
+    g_free(path);
+    closedir(dir);
+    return i;
+}
+
+/* Store disk device info specified by @sysfs into @fs */
+static void build_guest_fsinfo_for_real_device(char const *syspath,
+                                               GuestFilesystemInfo *fs,
+                                               Error **errp)
+{
+    unsigned int pci[4], host, hosts[8], tgt[3];
+    int i, nhosts = 0, pcilen;
+    GuestDiskAddress *disk;
+    GuestPCIAddress *pciaddr;
+    GuestDiskAddressList *list = NULL;
+    bool has_ata = false, has_host = false, has_tgt = false;
+    char *p, *q, *driver = NULL;
+
+    p = strstr(syspath, "/devices/pci");
+    if (!p || sscanf(p + 12, "%*x:%*x/%x:%x:%x.%x%n",
+                     pci, pci + 1, pci + 2, pci + 3, &pcilen) < 4) {
+        g_debug("only pci device is supported: sysfs path \"%s\"", syspath);
+        return;
+    }
+
+    driver = get_pci_driver(syspath, (p + 12 + pcilen) - syspath, errp);
+    if (!driver) {
+        goto cleanup;
+    }
+
+    p = strstr(syspath, "/target");
+    if (p && sscanf(p + 7, "%*u:%*u:%*u/%*u:%u:%u:%u",
+                    tgt, tgt + 1, tgt + 2) == 3) {
+        has_tgt = true;
+    }
+
+    p = strstr(syspath, "/ata");
+    if (p) {
+        q = p + 4;
+        has_ata = true;
+    } else {
+        p = strstr(syspath, "/host");
+        q = p + 5;
+    }
+    if (p && sscanf(q, "%u", &host) == 1) {
+        has_host = true;
+        nhosts = build_hosts(syspath, p, has_ata, hosts,
+                             sizeof(hosts) / sizeof(hosts[0]), errp);
+        if (nhosts < 0) {
+            goto cleanup;
+        }
+    }
+
+    pciaddr = g_malloc0(sizeof(*pciaddr));
+    pciaddr->domain = pci[0];
+    pciaddr->bus = pci[1];
+    pciaddr->slot = pci[2];
+    pciaddr->function = pci[3];
+
+    disk = g_malloc0(sizeof(*disk));
+    disk->pci_controller = pciaddr;
+
+    list = g_malloc0(sizeof(*list));
+    list->value = disk;
+
+    if (strcmp(driver, "ata_piix") == 0) {
+        /* a host per ide bus, target*:0:<unit>:0 */
+        if (!has_host || !has_tgt) {
+            g_debug("invalid sysfs path '%s' (driver '%s')", syspath, driver);
+            goto cleanup;
+        }
+        for (i = 0; i < nhosts; i++) {
+            if (host == hosts[i]) {
+                disk->bus_type = GUEST_DISK_BUS_TYPE_IDE;
+                disk->bus = i;
+                disk->unit = tgt[1];
+                break;
+            }
+        }
+        if (i >= nhosts) {
+            g_debug("no host for '%s' (driver '%s')", syspath, driver);
+            goto cleanup;
+        }
+    } else if (strcmp(driver, "sym53c8xx") == 0) {
+        /* scsi(LSI Logic): target*:0:<unit>:0 */
+        if (!has_tgt) {
+            g_debug("invalid sysfs path '%s' (driver '%s')", syspath, driver);
+            goto cleanup;
+        }
+        disk->bus_type = GUEST_DISK_BUS_TYPE_SCSI;
+        disk->unit = tgt[1];
+    } else if (strcmp(driver, "virtio-pci") == 0) {
+        if (has_tgt) {
+            /* virtio-scsi: target*:0:0:<unit> */
+            disk->bus_type = GUEST_DISK_BUS_TYPE_SCSI;
+            disk->unit = tgt[2];
+        } else {
+            /* virtio-blk: 1 disk per 1 device */
+            disk->bus_type = GUEST_DISK_BUS_TYPE_VIRTIO;
+        }
+    } else if (strcmp(driver, "ahci") == 0) {
+        /* ahci: 1 host per 1 unit */
+        if (!has_host || !has_tgt) {
+            g_debug("invalid sysfs path '%s' (driver '%s')", syspath, driver);
+            goto cleanup;
+        }
+        for (i = 0; i < nhosts; i++) {
+            if (host == hosts[i]) {
+                disk->unit = i;
+                disk->bus_type = GUEST_DISK_BUS_TYPE_SATA;
+                break;
+            }
+        }
+        if (i >= nhosts) {
+            g_debug("no host for '%s' (driver '%s')", syspath, driver);
+            goto cleanup;
+        }
+    } else {
+        g_debug("unknown driver '%s' (sysfs path '%s')", driver, syspath);
+        goto cleanup;
+    }
+
+    list->next = fs->disk;
+    fs->disk = list;
+    g_free(driver);
+    return;
+
+cleanup:
+    if (list) {
+        qapi_free_GuestDiskAddressList(list);
+    }
+    g_free(driver);
+}
+
+static void build_guest_fsinfo_for_device(char const *devpath,
+                                          GuestFilesystemInfo *fs,
+                                          Error **errp);
+
+/* Store a list of slave devices of virtual volume specified by @syspath into
+ * @fs */
+static void build_guest_fsinfo_for_virtual_device(char const *syspath,
+                                                  GuestFilesystemInfo *fs,
+                                                  Error **errp)
+{
+    DIR *dir;
+    char *dirpath;
+    struct dirent *entry;
+
+    dirpath = g_strdup_printf("%s/slaves", syspath);
+    dir = opendir(dirpath);
+    if (!dir) {
+        error_setg_errno(errp, errno, "opendir(\"%s\")", dirpath);
+        g_free(dirpath);
+        return;
+    }
+
+    for (;;) {
+        errno = 0;
+        entry = readdir(dir);
+        if (entry == NULL) {
+            if (errno) {
+                error_setg_errno(errp, errno, "readdir(\"%s\")", dirpath);
+            }
+            break;
+        }
+
+        if (entry->d_type == DT_LNK) {
+            char *path;
+
+            g_debug(" slave device '%s'", entry->d_name);
+            path = g_strdup_printf("%s/slaves/%s", syspath, entry->d_name);
+            build_guest_fsinfo_for_device(path, fs, errp);
+            g_free(path);
+
+            if (*errp) {
+                break;
+            }
+        }
+    }
+
+    g_free(dirpath);
+    closedir(dir);
+}
+
+/* Dispatch to functions for virtual/real device */
+static void build_guest_fsinfo_for_device(char const *devpath,
+                                          GuestFilesystemInfo *fs,
+                                          Error **errp)
+{
+    char *syspath = realpath(devpath, NULL);
+
+    if (!syspath) {
+        error_setg_errno(errp, errno, "realpath(\"%s\")", devpath);
+        return;
+    }
+
+    if (!fs->name) {
+        fs->name = g_strdup(basename(syspath));
+    }
+
+    g_debug("  parse sysfs path '%s'", syspath);
+
+    if (strstr(syspath, "/devices/virtual/block/")) {
+        build_guest_fsinfo_for_virtual_device(syspath, fs, errp);
+    } else {
+        build_guest_fsinfo_for_real_device(syspath, fs, errp);
+    }
+
+    free(syspath);
+}
+
+/* Return a list of the disk device(s)' info which @mount lies on */
+static GuestFilesystemInfo *build_guest_fsinfo(struct FsMount *mount,
+                                               Error **errp)
+{
+    GuestFilesystemInfo *fs = g_malloc0(sizeof(*fs));
+    char *devpath = g_strdup_printf("/sys/dev/block/%u:%u",
+                                    mount->devmajor, mount->devminor);
+
+    fs->mountpoint = g_strdup(mount->dirname);
+    fs->type = g_strdup(mount->devtype);
+    build_guest_fsinfo_for_device(devpath, fs, errp);
+
+    g_free(devpath);
+    return fs;
+}
+
+GuestFilesystemInfoList *qmp_guest_get_fsinfo(Error **errp)
+{
+    FsMountList mounts;
+    struct FsMount *mount;
+    GuestFilesystemInfoList *new, *ret = NULL;
+    Error *local_err = NULL;
+
+    QTAILQ_INIT(&mounts);
+    build_fs_mount_list(&mounts, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return NULL;
+    }
+
+    QTAILQ_FOREACH(mount, &mounts, next) {
+        g_debug("Building guest fsinfo for '%s'", mount->dirname);
+
+        new = g_malloc0(sizeof(*ret));
+        new->value = build_guest_fsinfo(mount, &local_err);
+        new->next = ret;
+        ret = new;
+        if (local_err) {
+            error_propagate(errp, local_err);
+            qapi_free_GuestFilesystemInfoList(ret);
+            ret = NULL;
+            break;
+        }
+    }
+
+    free_fs_mount_list(&mounts);
+    return ret;
+}
+
+
 typedef enum {
     FSFREEZE_HOOK_THAW = 0,
     FSFREEZE_HOOK_FREEZE,
@@ -710,13 +1143,21 @@ GuestFsfreezeStatus qmp_guest_fsfreeze_status(Error **errp)
     return GUEST_FSFREEZE_STATUS_THAWED;
 }
 
+int64_t qmp_guest_fsfreeze_freeze(Error **errp)
+{
+    return qmp_guest_fsfreeze_freeze_list(false, NULL, errp);
+}
+
 /*
  * Walk list of mounted file systems in the guest, and freeze the ones which
  * are real local file systems.
  */
-int64_t qmp_guest_fsfreeze_freeze(Error **errp)
+int64_t qmp_guest_fsfreeze_freeze_list(bool has_mountpoints,
+                                       strList *mountpoints,
+                                       Error **errp)
 {
     int ret = 0, i = 0;
+    strList *list;
     FsMountList mounts;
     struct FsMount *mount;
     Error *local_err = NULL;
@@ -741,6 +1182,19 @@ int64_t qmp_guest_fsfreeze_freeze(Error **errp)
     ga_set_frozen(ga_state);
 
     QTAILQ_FOREACH_REVERSE(mount, &mounts, FsMountList, next) {
+        /* To issue fsfreeze in the reverse order of mounts, check if the
+         * mount is listed in the list here */
+        if (has_mountpoints) {
+            for (list = mountpoints; list; list = list->next) {
+                if (strcmp(list->value, mount->dirname) == 0) {
+                    break;
+                }
+            }
+            if (!list) {
+                continue;
+            }
+        }
+
         fd = qemu_open(mount->dirname, O_RDONLY);
         if (fd == -1) {
             error_setg_errno(errp, errno, "failed to open %s", mount->dirname);
@@ -1460,6 +1914,12 @@ int64_t qmp_guest_set_vcpus(GuestLogicalProcessorList *vcpus, Error **errp)
 
 #if !defined(CONFIG_FSFREEZE)
 
+GuestFilesystemInfoList *qmp_guest_get_fsinfo(Error **errp)
+{
+    error_set(errp, QERR_UNSUPPORTED);
+    return NULL;
+}
+
 GuestFsfreezeStatus qmp_guest_fsfreeze_status(Error **errp)
 {
     error_set(errp, QERR_UNSUPPORTED);
@@ -1474,6 +1934,15 @@ int64_t qmp_guest_fsfreeze_freeze(Error **errp)
     return 0;
 }
 
+int64_t qmp_guest_fsfreeze_freeze_list(bool has_mountpoints,
+                                       strList *mountpoints,
+                                       Error **errp)
+{
+    error_set(errp, QERR_UNSUPPORTED);
+
+    return 0;
+}
+
 int64_t qmp_guest_fsfreeze_thaw(Error **errp)
 {
     error_set(errp, QERR_UNSUPPORTED);
@@ -1489,6 +1958,44 @@ void qmp_guest_fstrim(bool has_minimum, int64_t minimum, Error **errp)
 }
 #endif
 
+/* add unsupported commands to the blacklist */
+GList *ga_command_blacklist_init(GList *blacklist)
+{
+#if !defined(__linux__)
+    {
+        const char *list[] = {
+            "guest-suspend-disk", "guest-suspend-ram",
+            "guest-suspend-hybrid", "guest-network-get-interfaces",
+            "guest-get-vcpus", "guest-set-vcpus", NULL};
+        char **p = (char **)list;
+
+        while (*p) {
+            blacklist = g_list_append(blacklist, *p++);
+        }
+    }
+#endif
+
+#if !defined(CONFIG_FSFREEZE)
+    {
+        const char *list[] = {
+            "guest-get-fsinfo", "guest-fsfreeze-status",
+            "guest-fsfreeze-freeze", "guest-fsfreeze-freeze-list",
+            "guest-fsfreeze-thaw", "guest-get-fsinfo", NULL};
+        char **p = (char **)list;
+
+        while (*p) {
+            blacklist = g_list_append(blacklist, *p++);
+        }
+    }
+#endif
+
+#if !defined(CONFIG_FSTRIM)
+    blacklist = g_list_append(blacklist, (char *)"guest-fstrim");
+#endif
+
+    return blacklist;
+}
+
 /* register init/cleanup routines for stateful command groups */
 void ga_command_state_init(GAState *s, GACommandState *cs)
 {
diff --git a/qga/commands-win32.c b/qga/commands-win32.c
index e76939651..3bcbeae8f 100644
--- a/qga/commands-win32.c
+++ b/qga/commands-win32.c
@@ -152,6 +152,12 @@ void qmp_guest_file_flush(int64_t handle, Error **errp)
     error_set(errp, QERR_UNSUPPORTED);
 }
 
+GuestFilesystemInfoList *qmp_guest_get_fsinfo(Error **errp)
+{
+    error_set(errp, QERR_UNSUPPORTED);
+    return NULL;
+}
+
 /*
  * Return status of freeze/thaw
  */
@@ -206,6 +212,15 @@ error:
     return 0;
 }
 
+int64_t qmp_guest_fsfreeze_freeze_list(bool has_mountpoints,
+                                       strList *mountpoints,
+                                       Error **errp)
+{
+    error_set(errp, QERR_UNSUPPORTED);
+
+    return 0;
+}
+
 /*
  * Thaw local file systems using Volume Shadow-copy Service.
  */
@@ -431,10 +446,40 @@ int64_t qmp_guest_set_vcpus(GuestLogicalProcessorList *vcpus, Error **errp)
     return -1;
 }
 
+/* add unsupported commands to the blacklist */
+GList *ga_command_blacklist_init(GList *blacklist)
+{
+    const char *list_unsupported[] = {
+        "guest-file-open", "guest-file-close", "guest-file-read",
+        "guest-file-write", "guest-file-seek", "guest-file-flush",
+        "guest-suspend-hybrid", "guest-network-get-interfaces",
+        "guest-get-vcpus", "guest-set-vcpus",
+        "guest-fsfreeze-freeze-list", "guest-get-fsinfo",
+        "guest-fstrim", NULL};
+    char **p = (char **)list_unsupported;
+
+    while (*p) {
+        blacklist = g_list_append(blacklist, *p++);
+    }
+
+    if (!vss_init(true)) {
+        const char *list[] = {
+            "guest-get-fsinfo", "guest-fsfreeze-status",
+            "guest-fsfreeze-freeze", "guest-fsfreeze-thaw", NULL};
+        p = (char **)list;
+
+        while (*p) {
+            blacklist = g_list_append(blacklist, *p++);
+        }
+    }
+
+    return blacklist;
+}
+
 /* register init/cleanup routines for stateful command groups */
 void ga_command_state_init(GAState *s, GACommandState *cs)
 {
-    if (vss_init(true)) {
+    if (!vss_initialized()) {
         ga_command_state_add(cs, NULL, guest_fsfreeze_cleanup);
     }
 }
diff --git a/qga/guest-agent-core.h b/qga/guest-agent-core.h
index e422208b4..e92c6abaf 100644
--- a/qga/guest-agent-core.h
+++ b/qga/guest-agent-core.h
@@ -19,6 +19,7 @@ typedef struct GAState GAState;
 typedef struct GACommandState GACommandState;
 extern GAState *ga_state;
 
+GList *ga_command_blacklist_init(GList *blacklist);
 void ga_command_state_init(GAState *s, GACommandState *cs);
 void ga_command_state_add(GACommandState *cs,
                           void (*init)(void),
diff --git a/qga/main.c b/qga/main.c
index 8b927c9db..9939a2b62 100644
--- a/qga/main.c
+++ b/qga/main.c
@@ -603,8 +603,8 @@ static void process_event(JSONMessageParser *parser, QList *tokens)
             error_free(err);
         }
         ret = send_response(s, QOBJECT(qdict));
-        if (ret) {
-            g_warning("error sending error response: %s", strerror(ret));
+        if (ret < 0) {
+            g_warning("error sending error response: %s", strerror(-ret));
         }
     }
 
@@ -1144,6 +1144,7 @@ int main(int argc, char **argv)
         goto out_bad;
     }
 
+    blacklist = ga_command_blacklist_init(blacklist);
     if (blacklist) {
         s->blacklist = blacklist;
         do {
diff --git a/qga/qapi-schema.json b/qga/qapi-schema.json
index a8cdcb35c..376e79f58 100644
--- a/qga/qapi-schema.json
+++ b/qga/qapi-schema.json
@@ -387,6 +387,23 @@
   'returns': 'int' }
 
 ##
+# @guest-fsfreeze-freeze-list:
+#
+# Sync and freeze specified guest filesystems
+#
+# @mountpoints: #optional an array of mountpoints of filesystems to be frozen.
+#               If omitted, every mounted filesystem is frozen.
+#
+# Returns: Number of file systems currently frozen. On error, all filesystems
+# will be thawed.
+#
+# Since: 2.2
+##
+{ 'command': 'guest-fsfreeze-freeze-list',
+  'data':    { '*mountpoints': ['str'] },
+  'returns': 'int' }
+
+##
 # @guest-fsfreeze-thaw:
 #
 # Unfreeze all frozen guest filesystems
@@ -642,3 +659,82 @@
 { 'command': 'guest-set-vcpus',
   'data':    {'vcpus': ['GuestLogicalProcessor'] },
   'returns': 'int' }
+
+##
+# @GuestDiskBusType
+#
+# An enumeration of bus type of disks
+#
+# @ide: IDE disks
+# @fdc: floppy disks
+# @scsi: SCSI disks
+# @virtio: virtio disks
+# @xen: Xen disks
+# @usb: USB disks
+# @uml: UML disks
+# @sata: SATA disks
+# @sd: SD cards
+#
+# Since: 2.2
+##
+{ 'enum': 'GuestDiskBusType',
+  'data': [ 'ide', 'fdc', 'scsi', 'virtio', 'xen', 'usb', 'uml', 'sata',
+            'sd' ] }
+
+##
+# @GuestPCIAddress:
+#
+# @domain: domain id
+# @bus: bus id
+# @slot: slot id
+# @function: function id
+#
+# Since: 2.2
+##
+{ 'type': 'GuestPCIAddress',
+  'data': {'domain': 'int', 'bus': 'int',
+           'slot': 'int', 'function': 'int'} }
+
+##
+# @GuestDiskAddress:
+#
+# @pci-controller: controller's PCI address
+# @type: bus type
+# @bus: bus id
+# @target: target id
+# @unit: unit id
+#
+# Since: 2.2
+##
+{ 'type': 'GuestDiskAddress',
+  'data': {'pci-controller': 'GuestPCIAddress',
+           'bus-type': 'GuestDiskBusType',
+           'bus': 'int', 'target': 'int', 'unit': 'int'} }
+
+##
+# @GuestFilesystemInfo
+#
+# @name: disk name
+# @mountpoint: mount point path
+# @type: file system type string
+# @disk: an array of disk hardware information that the volume lies on,
+#        which may be empty if the disk type is not supported
+#
+# Since: 2.2
+##
+{ 'type': 'GuestFilesystemInfo',
+  'data': {'name': 'str', 'mountpoint': 'str', 'type': 'str',
+           'disk': ['GuestDiskAddress']} }
+
+##
+# @guest-get-fsinfo:
+#
+# Returns: The list of filesystems information mounted in the guest.
+#          The returned mountpoints may be specified to
+#          @guest-fsfreeze-freeze-list.
+#          Network filesystems (such as CIFS and NFS) are not listed.
+#
+# Since: 2.2
+##
+{ 'command': 'guest-get-fsinfo',
+  'returns': ['GuestFilesystemInfo'] }
diff --git a/qmp-commands.hx b/qmp-commands.hx
index 4be4765f2..718dd92f6 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -477,7 +477,7 @@ SQMP
 inject-nmi
 ----------
 
-Inject an NMI on guest's CPUs.
+Inject an NMI on the default CPU (x86/s390) or all CPUs (ppc64).
 
 Arguments: None.
 
@@ -487,7 +487,6 @@ Example:
 <- { "return": {} }
 
 Note: inject-nmi fails when the guest doesn't support injecting.
-      Currently, only x86 (NMI) and s390x (RESTART) guests do.
 
 EQMP
 
@@ -2082,7 +2081,7 @@ Each json-object contain the following:
          - "file": device file name (json-string)
          - "ro": true if read-only, false otherwise (json-bool)
          - "drv": driver format name (json-string)
-             - Possible values: "blkdebug", "bochs", "cloop", "cow", "dmg",
+             - Possible values: "blkdebug", "bochs", "cloop", "dmg",
                                 "file", "file", "ftp", "ftps", "host_cdrom",
                                 "host_device", "host_floppy", "http", "https",
                                 "nbd", "parallels", "qcow", "qcow2", "raw",
@@ -3753,5 +3752,105 @@ Example:
 
 -> { "execute": "rtc-reset-reinjection" }
 <- { "return": {} }
+EQMP
+
+    {
+        .name       = "trace-event-get-state",
+        .args_type  = "name:s",
+        .mhandler.cmd_new = qmp_marshal_input_trace_event_get_state,
+    },
+
+SQMP
+trace-event-get-state
+---------------------
+
+Query the state of events.
+
+Example:
+
+-> { "execute": "trace-event-get-state", "arguments": { "name": "qemu_memalign" } }
+<- { "return": [ { "name": "qemu_memalign", "state": "disabled" } ] }
+EQMP
+
+    {
+        .name       = "trace-event-set-state",
+        .args_type  = "name:s,enable:b,ignore-unavailable:b?",
+        .mhandler.cmd_new = qmp_marshal_input_trace_event_set_state,
+    },
+
+SQMP
+trace-event-set-state
+---------------------
+
+Set the state of events.
+
+Example:
+
+-> { "execute": "trace-event-set-state", "arguments": { "name": "qemu_memalign", "enable": "true" } }
+<- { "return": {} }
+EQMP
+
+    {
+        .name       = "x-input-send-event",
+        .args_type  = "console:i?,events:q",
+        .mhandler.cmd_new = qmp_marshal_input_x_input_send_event,
+    },
+
+SQMP
+@x-input-send-event
+-----------------
+
+Send input event to guest.
+
+Arguments:
+
+- "console": console index. (json-int, optional)
+- "events": list of input events.
+
+The consoles are visible in the qom tree, under
+/backend/console[$index]. They have a device link and head property, so
+it is possible to map which console belongs to which device and display.
+
+Note: this command is experimental, and not a stable API.
+
+Example (1):
+
+Press left mouse button.
+
+-> { "execute": "x-input-send-event",
+    "arguments": { "console": 0,
+                   "events": [ { "type": "btn",
+                    "data" : { "down": true, "button": "Left" } } } }
+<- { "return": {} }
+
+-> { "execute": "x-input-send-event",
+    "arguments": { "console": 0,
+                   "events": [ { "type": "btn",
+                    "data" : { "down": false, "button": "Left" } } } }
+<- { "return": {} }
+
+Example (2):
+
+Press ctrl-alt-del.
+
+-> { "execute": "x-input-send-event",
+     "arguments": { "console": 0, "events": [
+        { "type": "key", "data" : { "down": true,
+          "key": {"type": "qcode", "data": "ctrl" } } },
+        { "type": "key", "data" : { "down": true,
+          "key": {"type": "qcode", "data": "alt" } } },
+        { "type": "key", "data" : { "down": true,
+          "key": {"type": "qcode", "data": "delete" } } } ] } }
+<- { "return": {} }
+
+Example (3):
+
+Move mouse pointer to absolute coordinates (20000, 400).
+
+-> { "execute": "x-input-send-event" ,
+  "arguments": { "console": 0, "events": [
+               { "type": "abs", "data" : { "axis": "X", "value" : 20000 } },
+               { "type": "abs", "data" : { "axis": "Y", "value" : 400 } } ] } }
+<- { "return": {} }
 
 EQMP
diff --git a/qmp.c b/qmp.c
index 0d2553abf..0b4f13193 100644
--- a/qmp.c
+++ b/qmp.c
@@ -442,7 +442,8 @@ ObjectTypeInfoList *qmp_qom_list_types(bool has_implements,
  */
 static DevicePropertyInfo *make_device_property_info(ObjectClass *klass,
                                                      const char *name,
-                                                     const char *default_type)
+                                                     const char *default_type,
+                                                     const char *description)
 {
     DevicePropertyInfo *info;
     Property *prop;
@@ -465,7 +466,9 @@ static DevicePropertyInfo *make_device_property_info(ObjectClass *klass,
 
             info = g_malloc0(sizeof(*info));
             info->name = g_strdup(prop->name);
-            info->type = g_strdup(prop->info->legacy_name ?: prop->info->name);
+            info->type = g_strdup(prop->info->name);
+            info->has_description = !!prop->info->description;
+            info->description = g_strdup(prop->info->description);
             return info;
         }
         klass = object_class_get_parent(klass);
@@ -475,6 +478,9 @@ static DevicePropertyInfo *make_device_property_info(ObjectClass *klass,
     info = g_malloc0(sizeof(*info));
     info->name = g_strdup(name);
     info->type = g_strdup(default_type);
+    info->has_description = !!description;
+    info->description = g_strdup(description);
+
     return info;
 }
 
@@ -509,6 +515,7 @@ DevicePropertyInfoList *qmp_device_list_properties(const char *typename,
         if (strcmp(prop->name, "type") == 0 ||
             strcmp(prop->name, "realized") == 0 ||
             strcmp(prop->name, "hotpluggable") == 0 ||
+            strcmp(prop->name, "hotplugged") == 0 ||
             strcmp(prop->name, "parent_bus") == 0) {
             continue;
         }
@@ -520,7 +527,8 @@ DevicePropertyInfoList *qmp_device_list_properties(const char *typename,
             continue;
         }
 
-        info = make_device_property_info(klass, prop->name, prop->type);
+        info = make_device_property_info(klass, prop->name, prop->type,
+                                         prop->description);
         if (!info) {
             continue;
         }
diff --git a/qom/cpu.c b/qom/cpu.c
index b32dd0a56..79d22285f 100644
--- a/qom/cpu.c
+++ b/qom/cpu.c
@@ -107,15 +107,6 @@ static void cpu_common_get_memory_mapping(CPUState *cpu,
     error_setg(errp, "Obtaining memory mappings is unsupported on this CPU.");
 }
 
-/* CPU hot-plug notifiers */
-static NotifierList cpu_added_notifiers =
-    NOTIFIER_LIST_INITIALIZER(cpu_add_notifiers);
-
-void qemu_register_cpu_added_notifier(Notifier *notifier)
-{
-    notifier_list_add(&cpu_added_notifiers, notifier);
-}
-
 void cpu_reset_interrupt(CPUState *cpu, int mask)
 {
     cpu->interrupt_request &= ~mask;
@@ -202,6 +193,15 @@ static bool cpu_common_virtio_is_big_endian(CPUState *cpu)
     return target_words_bigendian();
 }
 
+static void cpu_common_noop(CPUState *cpu)
+{
+}
+
+static bool cpu_common_exec_interrupt(CPUState *cpu, int int_req)
+{
+    return false;
+}
+
 void cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
                     int flags)
 {
@@ -303,7 +303,6 @@ static void cpu_common_realizefn(DeviceState *dev, Error **errp)
 
     if (dev->hotplugged) {
         cpu_synchronize_post_init(cpu);
-        notifier_list_notify(&cpu_added_notifiers, dev);
         cpu_resume(cpu);
     }
 }
@@ -340,6 +339,10 @@ static void cpu_class_init(ObjectClass *klass, void *data)
     k->gdb_read_register = cpu_common_gdb_read_register;
     k->gdb_write_register = cpu_common_gdb_write_register;
     k->virtio_is_big_endian = cpu_common_virtio_is_big_endian;
+    k->debug_excp_handler = cpu_common_noop;
+    k->cpu_exec_enter = cpu_common_noop;
+    k->cpu_exec_exit = cpu_common_noop;
+    k->cpu_exec_interrupt = cpu_common_exec_interrupt;
     dc->realize = cpu_common_realizefn;
     /*
      * Reason: CPUs still need special care by board code: wiring up
diff --git a/qom/object.c b/qom/object.c
index 0e8267bc2..1812c7332 100644
--- a/qom/object.c
+++ b/qom/object.c
@@ -369,6 +369,7 @@ static void object_property_del_all(Object *obj)
 
         g_free(prop->name);
         g_free(prop->type);
+        g_free(prop->description);
         g_free(prop);
     }
 }
@@ -387,19 +388,9 @@ static void object_property_del_child(Object *obj, Object *child, Error **errp)
 
 void object_unparent(Object *obj)
 {
-    if (!obj->parent) {
-        return;
-    }
-
-    object_ref(obj);
-    if (obj->class->unparent) {
-        (obj->class->unparent)(obj);
-    }
     if (obj->parent) {
         object_property_del_child(obj->parent, obj, NULL);
-        obj->parent = NULL;
     }
-    object_unref(obj);
 }
 
 static void object_deinit(Object *obj, TypeImpl *type)
@@ -418,8 +409,8 @@ static void object_finalize(void *data)
     Object *obj = data;
     TypeImpl *ti = obj->class->type;
 
-    object_deinit(obj, ti);
     object_property_del_all(obj);
+    object_deinit(obj, ti);
 
     g_assert(obj->ref == 0);
     if (obj->free) {
@@ -678,10 +669,10 @@ void object_class_foreach(void (*fn)(ObjectClass *klass, void *opaque),
 int object_child_foreach(Object *obj, int (*fn)(Object *child, void *opaque),
                          void *opaque)
 {
-    ObjectProperty *prop;
+    ObjectProperty *prop, *next;
     int ret = 0;
 
-    QTAILQ_FOREACH(prop, &obj->properties, node) {
+    QTAILQ_FOREACH_SAFE(prop, &obj->properties, node, next) {
         if (object_property_is_child(prop)) {
             ret = fn(prop->opaque, opaque);
             if (ret != 0) {
@@ -738,6 +729,27 @@ object_property_add(Object *obj, const char *name, const char *type,
                     void *opaque, Error **errp)
 {
     ObjectProperty *prop;
+    size_t name_len = strlen(name);
+
+    if (name_len >= 3 && !memcmp(name + name_len - 3, "[*]", 4)) {
+        int i;
+        ObjectProperty *ret;
+        char *name_no_array = g_strdup(name);
+
+        name_no_array[name_len - 3] = '\0';
+        for (i = 0; ; ++i) {
+            char *full_name = g_strdup_printf("%s[%d]", name_no_array, i);
+
+            ret = object_property_add(obj, full_name, type, get, set,
+                                      release, opaque, NULL);
+            g_free(full_name);
+            if (ret) {
+                break;
+            }
+        }
+        g_free(name_no_array);
+        return ret;
+    }
 
     QTAILQ_FOREACH(prop, &obj->properties, node) {
         if (strcmp(prop->name, name) == 0) {
@@ -792,6 +804,7 @@ void object_property_del(Object *obj, const char *name, Error **errp)
 
     g_free(prop->name);
     g_free(prop->type);
+    g_free(prop->description);
     g_free(prop);
 }
 
@@ -859,9 +872,13 @@ char *object_property_get_str(Object *obj, const char *name,
 void object_property_set_link(Object *obj, Object *value,
                               const char *name, Error **errp)
 {
-    gchar *path = object_get_canonical_path(value);
-    object_property_set_str(obj, path, name, errp);
-    g_free(path);
+    if (value) {
+        gchar *path = object_get_canonical_path(value);
+        object_property_set_str(obj, path, name, errp);
+        g_free(path);
+    } else {
+        object_property_set_str(obj, "", name, errp);
+    }
 }
 
 Object *object_property_get_link(Object *obj, const char *name,
@@ -948,14 +965,18 @@ int object_property_get_enum(Object *obj, const char *name,
 {
     StringOutputVisitor *sov;
     StringInputVisitor *siv;
+    char *str;
     int ret;
 
     sov = string_output_visitor_new(false);
     object_property_get(obj, string_output_get_visitor(sov), name, errp);
-    siv = string_input_visitor_new(string_output_get_string(sov));
+    str = string_output_get_string(sov);
+    siv = string_input_visitor_new(str);
     string_output_visitor_cleanup(sov);
     visit_type_enum(string_input_get_visitor(siv),
                     &ret, strings, NULL, name, errp);
+
+    g_free(str);
     string_input_visitor_cleanup(siv);
 
     return ret;
@@ -966,13 +987,17 @@ void object_property_get_uint16List(Object *obj, const char *name,
 {
     StringOutputVisitor *ov;
     StringInputVisitor *iv;
+    char *str;
 
     ov = string_output_visitor_new(false);
     object_property_get(obj, string_output_get_visitor(ov),
                         name, errp);
-    iv = string_input_visitor_new(string_output_get_string(ov));
+    str = string_output_get_string(ov);
+    iv = string_input_visitor_new(str);
     visit_type_uint16List(string_input_get_visitor(iv),
                           list, NULL, errp);
+
+    g_free(str);
     string_output_visitor_cleanup(ov);
     string_input_visitor_cleanup(iv);
 }
@@ -991,11 +1016,19 @@ char *object_property_print(Object *obj, const char *name, bool human,
                             Error **errp)
 {
     StringOutputVisitor *mo;
-    char *string;
+    char *string = NULL;
+    Error *local_err = NULL;
 
     mo = string_output_visitor_new(human);
-    object_property_get(obj, string_output_get_visitor(mo), name, errp);
+    object_property_get(obj, string_output_get_visitor(mo), name, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto out;
+    }
+
     string = string_output_get_string(mo);
+
+out:
     string_output_visitor_cleanup(mo);
     return string;
 }
@@ -1042,6 +1075,10 @@ static void object_finalize_child_property(Object *obj, const char *name,
 {
     Object *child = opaque;
 
+    if (child->class->unparent) {
+        (child->class->unparent)(child);
+    }
+    child->parent = NULL;
     object_unref(child);
 }
 
@@ -1052,6 +1089,11 @@ void object_property_add_child(Object *obj, const char *name,
     gchar *type;
     ObjectProperty *op;
 
+    if (child->parent != NULL) {
+        error_setg(errp, "child object is already parented");
+        return;
+    }
+
     type = g_strdup_printf("child<%s>", object_get_typename(OBJECT(child)));
 
     op = object_property_add(obj, name, type, object_get_child_property, NULL,
@@ -1063,7 +1105,6 @@ void object_property_add_child(Object *obj, const char *name,
 
     op->resolve = object_resolve_child_property;
     object_ref(child);
-    g_assert(child->parent == NULL);
     child->parent = obj;
 
 out:
@@ -1611,6 +1652,7 @@ void object_property_add_alias(Object *obj, const char *name,
     ObjectProperty *op;
     ObjectProperty *target_prop;
     gchar *prop_type;
+    Error *local_err = NULL;
 
     target_prop = object_property_find(target_obj, target_name, errp);
     if (!target_prop) {
@@ -1632,12 +1674,36 @@ void object_property_add_alias(Object *obj, const char *name,
                              property_get_alias,
                              property_set_alias,
                              property_release_alias,
-                             prop, errp);
+                             prop, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        g_free(prop);
+        goto out;
+    }
     op->resolve = property_resolve_alias;
 
+    object_property_set_description(obj, name,
+                                    target_prop->description,
+                                    &error_abort);
+
+out:
     g_free(prop_type);
 }
 
+void object_property_set_description(Object *obj, const char *name,
+                                     const char *description, Error **errp)
+{
+    ObjectProperty *op;
+
+    op = object_property_find(obj, name, errp);
+    if (!op) {
+        return;
+    }
+
+    g_free(op->description);
+    op->description = g_strdup(description);
+}
+
 static void object_instance_init(Object *obj)
 {
     object_property_add_str(obj, "type", qdev_get_type, NULL, NULL);
diff --git a/qtest.c b/qtest.c
index 04a6dc1f0..2bca04ed4 100644
--- a/qtest.c
+++ b/qtest.c
@@ -17,8 +17,12 @@
 #include "exec/ioport.h"
 #include "exec/memory.h"
 #include "hw/irq.h"
+#include "sysemu/accel.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/cpus.h"
+#include "qemu/config-file.h"
+#include "qemu/option.h"
+#include "qemu/error-report.h"
 
 #define MAX_IRQ 256
 
@@ -197,8 +201,8 @@ static void GCC_FMT_ATTR(2, 3) qtest_send(CharDriverState *chr,
 
 static void qtest_irq_handler(void *opaque, int n, int level)
 {
-    qemu_irq *old_irqs = opaque;
-    qemu_set_irq(old_irqs[n], level);
+    qemu_irq old_irq = *(qemu_irq *)opaque;
+    qemu_set_irq(old_irq, level);
 
     if (irq_levels[n] != level) {
         CharDriverState *chr = qtest_chr;
@@ -260,8 +264,15 @@ static void qtest_process_command(CharDriverState *chr, gchar **words)
                 continue;
             }
             if (words[0][14] == 'o') {
-                qemu_irq_intercept_out(&ngl->out, qtest_irq_handler,
-                                       ngl->num_out);
+                int i;
+                for (i = 0; i < ngl->num_out; ++i) {
+                    qemu_irq *disconnected = g_new0(qemu_irq, 1);
+                    qemu_irq icpt = qemu_allocate_irq(qtest_irq_handler,
+                                                      disconnected, i);
+
+                    *disconnected = qdev_intercept_gpio_out(dev, icpt,
+                                                            ngl->name, i);
+                }
             } else {
                 qemu_irq_intercept_in(ngl->in, qtest_irq_handler,
                                       ngl->num_in);
@@ -509,10 +520,16 @@ static void qtest_event(void *opaque, int event)
     }
 }
 
-int qtest_init_accel(MachineClass *mc)
+static void configure_qtest_icount(const char *options)
 {
-    configure_icount("0");
+    QemuOpts *opts  = qemu_opts_parse(qemu_find_opts("icount"), options, 1);
+    configure_icount(opts, &error_abort);
+    qemu_opts_del(opts);
+}
 
+static int qtest_init_accel(MachineState *ms)
+{
+    configure_qtest_icount("0");
     return 0;
 }
 
@@ -528,11 +545,6 @@ void qtest_init(const char *qtest_chrdev, const char *qtest_log, Error **errp)
         return;
     }
 
-    qemu_chr_add_handlers(chr, qtest_can_read, qtest_read, qtest_event, chr);
-    qemu_chr_fe_set_echo(chr, true);
-
-    inbuf = g_string_new("");
-
     if (qtest_log) {
         if (strcmp(qtest_log, "none") != 0) {
             qtest_log_fp = fopen(qtest_log, "w+");
@@ -541,6 +553,10 @@ void qtest_init(const char *qtest_chrdev, const char *qtest_log, Error **errp)
         qtest_log_fp = stderr;
     }
 
+    qemu_chr_add_handlers(chr, qtest_can_read, qtest_read, qtest_event, chr);
+    qemu_chr_fe_set_echo(chr, true);
+
+    inbuf = g_string_new("");
     qtest_chr = chr;
 }
 
@@ -548,3 +564,27 @@ bool qtest_driver(void)
 {
     return qtest_chr;
 }
+
+static void qtest_accel_class_init(ObjectClass *oc, void *data)
+{
+    AccelClass *ac = ACCEL_CLASS(oc);
+    ac->name = "QTest";
+    ac->available = qtest_available;
+    ac->init_machine = qtest_init_accel;
+    ac->allowed = &qtest_allowed;
+}
+
+#define TYPE_QTEST_ACCEL ACCEL_CLASS_NAME("qtest")
+
+static const TypeInfo qtest_accel_type = {
+    .name = TYPE_QTEST_ACCEL,
+    .parent = TYPE_ACCEL,
+    .class_init = qtest_accel_class_init,
+};
+
+static void qtest_type_init(void)
+{
+    type_register_static(&qtest_accel_type);
+}
+
+type_init(qtest_type_init);
diff --git a/roms/openbios/arch/ppc/qemu/methods.c b/roms/openbios/arch/ppc/qemu/methods.c
index 8219cff90..fd993daa9 100644
--- a/roms/openbios/arch/ppc/qemu/methods.c
+++ b/roms/openbios/arch/ppc/qemu/methods.c
@@ -25,6 +25,7 @@
 #include "qemu/qemu.h"
 #include "libopenbios/ofmem.h"
 #include "arch/ppc/processor.h"
+#include "drivers/usb.h"
 
 /************************************************************************/
 /*	RTAS (run-time abstraction services)				*/
@@ -112,6 +113,7 @@ DECLARE_NODE( ciface, 0, 0, "+/openprom/client-services" );
 static void
 ciface_quiesce( unsigned long args[], unsigned long ret[] )
 {
+	usb_exit();
 #if 0
 	unsigned long msr;
 	/* This seems to be the correct thing to do - but I'm not sure */
diff --git a/roms/openbios/arch/ppc/qemu/tree.fs b/roms/openbios/arch/ppc/qemu/tree.fs
index 1b9c2934f..1ed838397 100644
--- a/roms/openbios/arch/ppc/qemu/tree.fs
+++ b/roms/openbios/arch/ppc/qemu/tree.fs
@@ -12,7 +12,8 @@ include config.fs
 \ -------------------------------------------------------------
 
 " /" find-device
-
+\ Apple calls the root node device-tree
+" device-tree" device-name
 [IFDEF] CONFIG_PPC64 2 [ELSE] 1 [THEN] encode-int " #address-cells" property
 1 encode-int " #size-cells" property
 h# 05f5e100 encode-int " clock-frequency" property
diff --git a/roms/openbios/arch/sparc64/openbios.c b/roms/openbios/arch/sparc64/openbios.c
index 9cc1ac223..3a3614652 100644
--- a/roms/openbios/arch/sparc64/openbios.c
+++ b/roms/openbios/arch/sparc64/openbios.c
@@ -116,6 +116,20 @@ sparc64_reset_all(void)
         : : "r" (val), "r" (addr) : "memory");
 }
 
+/* PCI Target Address Space Register (see UltraSPARC IIi User's Manual
+  section 19.3.0.4) */
+#define PBM_PCI_TARGET_AS              0x2028
+#define PBM_PCI_TARGET_AS_CD_ENABLE    0x40
+
+static void
+sparc64_set_tas_register(unsigned long val)
+{
+    unsigned long addr = APB_SPECIAL_BASE + PBM_PCI_TARGET_AS;
+
+    asm("stxa %0, [%1] 0x15\n\t"
+        : : "r" (val), "r" (addr) : "memory");
+}
+
 static void cpu_generic_init(const struct cpudef *cpu, uint32_t clock_frequency)
 {
     unsigned long iu_version;
@@ -574,6 +588,10 @@ arch_init( void )
 	modules_init();
 #ifdef CONFIG_DRIVER_PCI
         ob_pci_init();
+
+        /* Set TAS register to match the virtual-dma properties
+           set during sabre configure */
+        sparc64_set_tas_register(PBM_PCI_TARGET_AS_CD_ENABLE);
 #endif
         nvconf_init();
         device_end();
diff --git a/roms/openbios/config/examples/ppc_config.xml b/roms/openbios/config/examples/ppc_config.xml
index 621b65d9c..4c14eb6c8 100644
--- a/roms/openbios/config/examples/ppc_config.xml
+++ b/roms/openbios/config/examples/ppc_config.xml
@@ -80,3 +80,6 @@
   <option name="CONFIG_DRIVER_ESCC" type="boolean" value="true"/>
   <option name="CONFIG_DRIVER_FW_CFG" type="boolean" value="true"/>
   <option name="CONFIG_FW_CFG_ADDR" type="integer" value="0xf0000510"/>
+  <option name="CONFIG_DRIVER_USB" type="boolean" value="true"/>
+  <option name="CONFIG_DEBUG_USB" type="boolean" value="false"/>
+  <option name="CONFIG_USB_HID" type="boolean" value="true"/>
diff --git a/roms/openbios/drivers/Kconfig b/roms/openbios/drivers/Kconfig
index 7be334b1c..3bebc0293 100644
--- a/roms/openbios/drivers/Kconfig
+++ b/roms/openbios/drivers/Kconfig
@@ -36,4 +36,24 @@ config DEBUG_IDE
 	help
 	  Debug IDE driver 
 	
+config DRIVER_USB
+	bool "USB Support"
+	default n
+	help
+	  If you want to be able to use USB devices, enable this option.
+
+config DEBUG_USB
+	depends DRIVER_USB
+	bool "Debug USB driver"
+	default n
+	help
+	  Debug USB driver
+
+config USB_HID
+	depends DRIVER_USB
+	bool "USB driver for HID devices"
+	default n
+	help
+	  If you want to be able to use USB keyboard, enable this option.
+
 endmenu
diff --git a/roms/openbios/drivers/build.xml b/roms/openbios/drivers/build.xml
index edec6b512..bd1abd358 100644
--- a/roms/openbios/drivers/build.xml
+++ b/roms/openbios/drivers/build.xml
@@ -22,6 +22,10 @@
   <object source="pc_serial.c" condition="DRIVER_PC_SERIAL"/>
   <object source="escc.c" condition="DRIVER_ESCC"/>
   <object source="fw_cfg.c" condition="DRIVER_FW_CFG"/>
+  <object source="usb.c" condition="DRIVER_USB"/>
+  <object source="usbhid.c" condition="USB_HID"/>
+  <object source="usbohci.c" condition="DRIVER_USB"/>
+  <object source="usbohci_rh.c" condition="DRIVER_USB"/>
  </library>
 
  <dictionary name="openbios" target="forth">
diff --git a/roms/openbios/drivers/ide.c b/roms/openbios/drivers/ide.c
index 0e0f0cf24..327c64a40 100644
--- a/roms/openbios/drivers/ide.c
+++ b/roms/openbios/drivers/ide.c
@@ -1526,6 +1526,10 @@ int macio_ide_init(const char *path, uint32_t addr, int nb_channels)
 	u32 props[8];
 	struct ide_channel *chan;
 
+	/* IDE ports on Macs are numbered from 3.
+	 * Also see comments in macio.c:openpic_init() */
+	current_channel = 3;
+
 	for (i = 0; i < nb_channels; i++, current_channel++) {
 
 		chan = malloc(sizeof(struct ide_channel));
@@ -1574,7 +1578,8 @@ int macio_ide_init(const char *path, uint32_t addr, int nb_channels)
 
 		dnode = find_dev(nodebuff);
 
-		set_property(dnode, "compatible", "heathrow-ata", 13);
+		set_property(dnode, "compatible", (is_oldworld() ?
+			     "heathrow-ata" : "keylargo-ata"), 13);
 
 		props[0] = 0x00000526;
 		props[1] = 0x00000085;
diff --git a/roms/openbios/drivers/macio.c b/roms/openbios/drivers/macio.c
index 00d714990..f54bc86dc 100644
--- a/roms/openbios/drivers/macio.c
+++ b/roms/openbios/drivers/macio.c
@@ -27,20 +27,31 @@
 
 #define NW_IO_NVRAM_SIZE   0x00004000
 #define NW_IO_NVRAM_OFFSET 0xfff04000
-#define NW_IO_NVRAM_SHIFT  1
 
 #define IO_OPENPIC_SIZE    0x00040000
 #define IO_OPENPIC_OFFSET  0x00040000
 
 static char *nvram;
 
+static int macio_nvram_shift(void)
+{
+	int nvram_flat;
+
+        if (is_oldworld())
+                return OW_IO_NVRAM_SHIFT;
+
+	nvram_flat = fw_cfg_read_i32(FW_CFG_PPC_NVRAM_FLAT);
+	return nvram_flat ? 0 : 1;
+}
+
 int
 macio_get_nvram_size(void)
 {
+	int shift = macio_nvram_shift();
         if (is_oldworld())
-                return OW_IO_NVRAM_SIZE >> OW_IO_NVRAM_SHIFT;
+                return OW_IO_NVRAM_SIZE >> shift;
         else
-                return NW_IO_NVRAM_SIZE >> NW_IO_NVRAM_SHIFT;
+                return NW_IO_NVRAM_SIZE >> shift;
 }
 
 static unsigned long macio_nvram_offset(void)
@@ -123,14 +134,9 @@ void
 macio_nvram_put(char *buf)
 {
 	int i;
-        unsigned int it_shift;
+        unsigned int it_shift = macio_nvram_shift();
 
-        if (is_oldworld())
-                it_shift = OW_IO_NVRAM_SHIFT;
-        else
-                it_shift = NW_IO_NVRAM_SHIFT;
-
-	for (i=0; i< arch_nvram_size() ; i++)
+	for (i=0; i < arch_nvram_size(); i++)
 		nvram[i << it_shift] = buf[i];
 #ifdef DUMP_NVRAM
 	printk("new nvram:\n");
@@ -142,12 +148,7 @@ void
 macio_nvram_get(char *buf)
 {
 	int i;
-        unsigned int it_shift;
-
-        if (is_oldworld())
-                it_shift = OW_IO_NVRAM_SHIFT;
-        else
-                it_shift = NW_IO_NVRAM_SHIFT;
+        unsigned int it_shift = macio_nvram_shift();
 
 	for (i=0; i< arch_nvram_size(); i++)
                 buf[i] = nvram[i << it_shift];
@@ -161,7 +162,6 @@ macio_nvram_get(char *buf)
 static void
 openpic_init(const char *path, phys_addr_t addr)
 {
-        phandle_t target_node;
         phandle_t dnode;
         int props[2];
         char buf[128];
@@ -186,42 +186,6 @@ openpic_init(const char *path, phys_addr_t addr)
         set_int_property(dnode, "clock-frequency", 4166666);
 
         fword("finish-device");
-
-        u32 *interrupt_map;
-        int len, i;
-
-        /* patch in interrupt parent */
-        dnode = find_dev(buf);
-
-        target_node = find_dev("/pci/mac-io");
-        set_int_property(target_node, "interrupt-parent", dnode);
-
-        target_node = find_dev("/pci/mac-io/escc/ch-a");
-        set_int_property(target_node, "interrupt-parent", dnode);
-
-        target_node = find_dev("/pci/mac-io/escc/ch-b");
-        set_int_property(target_node, "interrupt-parent", dnode);
-
-        target_node = find_dev("/pci/mac-io/ata-1");
-        set_int_property(target_node, "interrupt-parent", dnode);
-
-        target_node = find_dev("/pci/mac-io/ata-2");
-        set_int_property(target_node, "interrupt-parent", dnode);
-
-        target_node = find_dev("/pci/mac-io/ata-3");
-        set_int_property(target_node, "interrupt-parent", dnode);
-
-        target_node = find_dev("/pci/mac-io/via-cuda");
-        set_int_property(target_node, "interrupt-parent", dnode);
-
-        target_node = find_dev("/pci");
-        set_int_property(target_node, "interrupt-parent", dnode);
-
-        interrupt_map = (u32 *)get_property(target_node, "interrupt-map", &len);
-        for (i = 0; i < 8; i++) {
-            interrupt_map[(i * 7) + PCI_INT_MAP_PIC_HANDLE] = (u32)dnode;
-        }
-        set_property(target_node, "interrupt-map", (char *)interrupt_map, len);
 }
 
 DECLARE_NODE(ob_macio, INSTALL_OPEN, sizeof(int), "Tmac-io");
@@ -311,7 +275,7 @@ ob_macio_keylargo_init(const char *path, phys_addr_t addr)
         /* The NewWorld NVRAM is not located in the MacIO device */
         macio_nvram_init("", 0);
         escc_init(path, addr);
-        macio_ide_init(path, addr, 3);
+        macio_ide_init(path, addr, 2);
         openpic_init(path, addr);
 	ob_unin_init();
 }
diff --git a/roms/openbios/drivers/pc_serial.c b/roms/openbios/drivers/pc_serial.c
index f63fcaa47..a638e1f99 100644
--- a/roms/openbios/drivers/pc_serial.c
+++ b/roms/openbios/drivers/pc_serial.c
@@ -142,22 +142,19 @@ pc_serial_close(void)
 static void
 pc_serial_open(unsigned long *address)
 {
-    int len;
-    phandle_t ph;
-    unsigned long *prop;
-
-    fword("my-self");
-    fword("ihandle>phandle");
-    ph = (phandle_t)POP();
-    prop = (unsigned long *)get_property(ph, "address", &len);
-    *address = *prop;
-
     RET ( -1 );
 }
 
+static void
+pc_serial_init(unsigned long *address)
+{
+    *address = POP();
+}
+
 DECLARE_UNNAMED_NODE(pc_serial, INSTALL_OPEN, sizeof(unsigned long));
 
 NODE_METHODS(pc_serial) = {
+    { "init",               pc_serial_init              },
     { "open",               pc_serial_open              },
     { "close",              pc_serial_close             },
     { "read",               pc_serial_read              },
@@ -177,6 +174,10 @@ ob_pc_serial_init(const char *path, const char *dev_name, uint64_t base,
     push_str(nodebuff);
     fword("find-device");
 
+    PUSH(offset);
+    PUSH(find_package_method("init", get_cur_dev()));
+    fword("execute");
+
     push_str("serial");
     fword("device-type");
 
@@ -191,10 +192,12 @@ ob_pc_serial_init(const char *path, const char *dev_name, uint64_t base,
     push_str("reg");
     fword("property");
     
+#if !defined(CONFIG_SPARC64)
     PUSH(offset);
     fword("encode-int");
     push_str("address");
     fword("property");
+#endif
     
 #if defined(CONFIG_SPARC64)
     set_int_property(get_cur_dev(), "interrupts", 1);
diff --git a/roms/openbios/drivers/pci.c b/roms/openbios/drivers/pci.c
index ca92d63bf..3260354be 100644
--- a/roms/openbios/drivers/pci.c
+++ b/roms/openbios/drivers/pci.c
@@ -34,6 +34,9 @@
 #include "cuda.h"
 #include "macio.h"
 #endif
+#ifdef CONFIG_DRIVER_USB
+#include "drivers/usb.h"
+#endif
 
 #if defined (CONFIG_DEBUG_PCI)
 # define PCI_DPRINTF(format, ...) printk(format, ## __VA_ARGS__)
@@ -414,62 +417,6 @@ static void pci_set_bus_range(const pci_config_t *config)
 	set_property(dev, "bus-range", (char *)props, 2 * sizeof(props[0]));
 }
 
-static void pci_host_set_interrupt_map(const pci_config_t *config)
-{
-/* XXX We currently have a hook in the MPIC init code to fill in its handle.
- *     If you want to have interrupt maps for your PCI host bus, add your
- *     architecture to the #if and make your bridge detect code fill in its
- *     handle too.
- *
- *     It would be great if someone clever could come up with a more universal
- *     mechanism here.
- */
-#if defined(CONFIG_PPC)
-	phandle_t dev = get_cur_dev();
-	u32 props[7 * 8];
-	int i;
-
-#if defined(CONFIG_PPC)
-	/* Oldworld macs do interrupt maps differently */
-	if(!is_newworld())
-		return;
-#endif
-
-	for (i = 0; i < (7*8); i+=7) {
-		props[i+PCI_INT_MAP_PCI0] = 0;
-		props[i+PCI_INT_MAP_PCI1] = 0;
-		props[i+PCI_INT_MAP_PCI2] = 0;
-		props[i+PCI_INT_MAP_PCI_INT] = (i / 7) + 1; // starts at PINA=1
-		props[i+PCI_INT_MAP_PIC_HANDLE] = 0; // gets patched in later
-		props[i+PCI_INT_MAP_PIC_INT] = arch->irqs[i / 7];
-		props[i+PCI_INT_MAP_PIC_POL] = 3;
-	}
-	set_property(dev, "interrupt-map", (char *)props, 7 * 8 * sizeof(props[0]));
-
-	props[PCI_INT_MAP_PCI0] = 0;
-	props[PCI_INT_MAP_PCI1] = 0;
-	props[PCI_INT_MAP_PCI2] = 0;
-	props[PCI_INT_MAP_PCI_INT] = 0x7;
-
-	set_property(dev, "interrupt-map-mask", (char *)props, 4 * sizeof(props[0]));
-#elif defined(CONFIG_SPARC64)
-	phandle_t dev = get_cur_dev();
-	uint32_t props[5];
-
-	props[0] = 0x000001fe;
-	props[1] = 0x020003f8;
-	props[2] = 1;
-	props[3] = find_dev("/");
-	props[4] = 0x2b;
-	set_property(dev, "interrupt-map", (char *)props, 5 * sizeof(props[0]));
-
-	props[0] = 0x000001ff;
-	props[1] = 0xffffffff;
-	props[2] = 3;
-	set_property(dev, "interrupt-map-mask", (char *)props, 3 * sizeof(props[0]));
-#endif
-}
-
 static void pci_host_set_reg(phandle_t phandle)
 {
     phandle_t dev = phandle;
@@ -536,7 +483,6 @@ int host_config_cb(const pci_config_t *config)
 	//XXX this overrides "reg" property
 	pci_host_set_reg(get_cur_dev());
 	pci_host_set_ranges(config);
-	pci_host_set_interrupt_map(config);
 
 	return 0;
 }
@@ -553,6 +499,9 @@ static int sabre_configure(phandle_t dev)
                      sizeof(props[0]));
         set_property(dev, "#virtual-dma-addr-cells", (char *)props,
                      sizeof(props[0]));
+
+        set_property(dev, "no-streaming-cache", (char *)props, 0);
+
         props[0] = 0x000007f0;
         props[1] = 0x000007ee;
         props[2] = 0x000007ef;
@@ -841,10 +790,14 @@ int ebus_config_cb(const pci_config_t *config)
 {
 #ifdef CONFIG_DRIVER_EBUS
     phandle_t dev = get_cur_dev();
-    uint32_t props[5];
+    uint32_t props[12];
+    int ncells;
+    int i;
+    uint32_t mask;
+    int flags, space_code;
 
-    props[0] = 0x000001fe;
-    props[1] = 0x020003f8;
+    props[0] = 0x14;
+    props[1] = 0x3f8;
     props[2] = 1;
     props[3] = find_dev("/");
     props[4] = 0x2b;
@@ -855,14 +808,37 @@ int ebus_config_cb(const pci_config_t *config)
     props[2] = 3;
     set_property(dev, "interrupt-map-mask", (char *)props, 3 * sizeof(props[0]));
 
+    /* Build ranges property from the BARs */
+    ncells = 0;
+    for (i = 0; i < 6; i++) {
+        /* consider only bars with non-zero region size */
+        if (!config->sizes[i])
+            continue;
+
+        pci_decode_pci_addr(config->assigned[i],
+                            &flags, &space_code, &mask);
+
+        props[ncells++] = PCI_BASE_ADDR_0 + (i * sizeof(uint32_t));
+        props[ncells++] = 0x0;
+
+        ncells += pci_encode_phys_addr(props + ncells,
+                                       flags, space_code, config->dev,
+                                       PCI_BASE_ADDR_0 + (i * sizeof(uint32_t)),
+                                       0);
+
+        props[ncells++] = config->sizes[i];
+    }
+
+    set_property(dev, "ranges", (char *)props, ncells * sizeof(props[0]));
+
 #ifdef CONFIG_DRIVER_FLOPPY
     ob_floppy_init(config->path, "fdthree", 0x3f0ULL, 0);
 #endif
 #ifdef CONFIG_DRIVER_PC_SERIAL
-    ob_pc_serial_init(config->path, "su", arch->io_base, 0x3f8ULL, 0);
+    ob_pc_serial_init(config->path, "su", (PCI_BASE_ADDR_1 | 0ULL) << 32, 0x3f8ULL, 0);
 #endif
 #ifdef CONFIG_DRIVER_PC_KBD
-    ob_pc_kbd_init(config->path, "kb_ps2", arch->io_base, 0x60ULL, 0);
+    ob_pc_kbd_init(config->path, "kb_ps2", (PCI_BASE_ADDR_1 | 0ULL) << 32, 0x60ULL, 0);
 #endif
 #endif
     return 0;
@@ -883,6 +859,14 @@ int i82378_config_cb(const pci_config_t *config)
     return 0;
 }
 
+int usb_ohci_config_cb(const pci_config_t *config)
+{
+#ifdef CONFIG_DRIVER_USB
+    ob_usb_ohci_init(config->path, 0x80000000 | config->dev);
+#endif
+    return 0;
+}
+
 static void ob_pci_add_properties(phandle_t phandle,
                                   pci_addr addr, const pci_dev_t *pci_dev,
                                   const pci_config_t *config, int num_bars)
@@ -933,10 +917,7 @@ static void ob_pci_add_properties(phandle_t phandle,
 		OLDWORLD(set_int_property(dev, "AAPL,interrupts",
 					  config->irq_line));
 #if defined(CONFIG_SPARC64)
-                /* direct mapping bssnn (Bus, Slot, interrupt Number */
-                set_int_property(get_cur_dev(), "interrupts",
-                                 ((((config->dev >> 11) << 2)
-                                   + config->irq_pin - 1) & 0x1f));
+                set_int_property(dev, "interrupts", config->irq_pin);
 #else
 		NEWWORLD(set_int_property(dev, "interrupts", config->irq_pin));
 #endif
@@ -1375,6 +1356,132 @@ static void ob_configure_pci_device(const char* parent_path,
     }
 }
 
+static void ob_pci_set_available(phandle_t host, unsigned long mem_base, unsigned long io_base)
+{
+    /* Create an available property for both memory and IO space */
+    uint32_t props[10];
+    int ncells;
+
+    ncells = 0;
+    ncells += pci_encode_phys_addr(props + ncells, 0, MEMORY_SPACE_32, 0, 0, mem_base);
+    ncells += pci_encode_size(props + ncells, arch->mem_len - mem_base);
+    ncells += pci_encode_phys_addr(props + ncells, 0, IO_SPACE, 0, 0, io_base);
+    ncells += pci_encode_size(props + ncells, arch->io_len - io_base);
+
+    set_property(host, "available", (char *)props, ncells * sizeof(props[0]));
+}
+
+/* Convert device/irq pin to interrupt property */
+#define SUN4U_INTERRUPT(dev, irq_pin) \
+            ((((dev >> 11) << 2) + irq_pin - 1) & 0x1f)
+
+static void ob_pci_host_set_interrupt_map(phandle_t host)
+{
+    phandle_t dnode = 0;
+    u32 props[128];
+    int i;
+
+#if defined(CONFIG_PPC)
+    phandle_t target_node;
+
+    /* Oldworld macs do interrupt maps differently */
+    if (!is_newworld())
+        return;
+
+    dnode = dt_iterate_type(0, "open-pic");
+    if (dnode) {
+        /* patch in openpic interrupt-parent properties */
+        target_node = find_dev("/pci/mac-io");
+        set_int_property(target_node, "interrupt-parent", dnode);
+
+        target_node = find_dev("/pci/mac-io/escc/ch-a");
+        set_int_property(target_node, "interrupt-parent", dnode);
+
+        target_node = find_dev("/pci/mac-io/escc/ch-b");
+        set_int_property(target_node, "interrupt-parent", dnode);
+
+        /* QEMU only emulates 2 of the 3 ata buses currently */
+        /* On a new world Mac these are not numbered but named by the
+         * ATA version they support. Thus we have: ata-3, ata-3, ata-4
+         * On g3beige they all called just ide.
+         * We take ata-3 and ata-4 which seems to work for both
+         * at least for clients we care about */
+        target_node = find_dev("/pci/mac-io/ata-3");
+        set_int_property(target_node, "interrupt-parent", dnode);
+
+        target_node = find_dev("/pci/mac-io/ata-4");
+        set_int_property(target_node, "interrupt-parent", dnode);
+
+        target_node = find_dev("/pci/mac-io/via-cuda");
+        set_int_property(target_node, "interrupt-parent", dnode);
+
+        target_node = find_dev("/pci");
+        set_int_property(target_node, "interrupt-parent", dnode);
+
+        /* openpic interrupt mapping */
+        for (i = 0; i < (7*8); i += 7) {
+            props[i + PCI_INT_MAP_PCI0] = 0;
+            props[i + PCI_INT_MAP_PCI1] = 0;
+            props[i + PCI_INT_MAP_PCI2] = 0;
+            props[i + PCI_INT_MAP_PCI_INT] = (i / 7) + 1; // starts at PINA=1
+            props[i + PCI_INT_MAP_PIC_HANDLE] = dnode;
+            props[i + PCI_INT_MAP_PIC_INT] = arch->irqs[i / 7];
+            props[i + PCI_INT_MAP_PIC_POL] = 3;
+        }
+        set_property(host, "interrupt-map", (char *)props, 7 * 8 * sizeof(props[0]));
+
+        props[PCI_INT_MAP_PCI0] = 0;
+        props[PCI_INT_MAP_PCI1] = 0;
+        props[PCI_INT_MAP_PCI2] = 0;
+        props[PCI_INT_MAP_PCI_INT] = 0x7;
+
+        set_property(host, "interrupt-map-mask", (char *)props, 4 * sizeof(props[0]));
+    }
+#elif defined(CONFIG_SPARC64)
+    int ncells, len;
+    u32 *val, addr;
+    char *reg;
+
+    /* Set interrupt-map for PCI devices with an interrupt pin present */
+    ncells = 0;
+
+    PUSH(host);
+    fword("child");
+    dnode = POP();
+    while (dnode) {
+        if (get_int_property(dnode, "interrupts", &len)) {
+            reg = get_property(dnode, "reg", &len);
+            if (reg) {
+                val = (u32 *)reg;
+
+                for (i = 0; i < (len / sizeof(u32)); i += 5) {
+                    addr = val[i];
+
+                    /* Device address is in 1st 32-bit word of encoded PCI address for config space */
+                    if (!(addr & 0x03000000)) {
+                        ncells += pci_encode_phys_addr(props + ncells, 0, 0, addr, 0, 0);
+                        props[ncells++] = 1;    /* always interrupt pin 1 for QEMU */
+                        props[ncells++] = host;
+                        props[ncells++] = SUN4U_INTERRUPT(addr, 1);
+                    }
+                }
+            }
+        }
+
+        PUSH(dnode);
+        fword("peer");
+        dnode = POP();
+    }
+    set_property(host, "interrupt-map", (char *)props, ncells * sizeof(props[0]));
+
+    props[0] = 0x0000f800;
+    props[1] = 0x0;
+    props[2] = 0x0;
+    props[3] = 7;
+    set_property(host, "interrupt-map-mask", (char *)props, 4 * sizeof(props[0]));
+#endif
+}
+
 int ob_pci_init(void)
 {
     int bus, devnum, fn;
@@ -1382,7 +1489,7 @@ int ob_pci_init(void)
     unsigned long mem_base, io_base;
 
     pci_config_t config = {}; /* host bridge */
-    phandle_t phandle_host;
+    phandle_t phandle_host = 0;
 
     PCI_DPRINTF("Initializing PCI host bridge...\n");
 
@@ -1430,6 +1537,11 @@ int ob_pci_init(void)
         break;
     }
 
+    /* create available attributes for the PCI bridge */
+    ob_pci_set_available(phandle_host, mem_base, io_base);
+
+    /* configure the host bridge interrupt map */
+    ob_pci_host_set_interrupt_map(phandle_host);
 
     device_end();
 
diff --git a/roms/openbios/drivers/pci.h b/roms/openbios/drivers/pci.h
index ab8f18437..84a2b2cf6 100644
--- a/roms/openbios/drivers/pci.h
+++ b/roms/openbios/drivers/pci.h
@@ -7,6 +7,7 @@
 #define PCI_COMMAND		0x04
 #define  PCI_COMMAND_IO		0x01
 #define  PCI_COMMAND_MEMORY	0x02
+#define  PCI_COMMAND_BUS_MASTER	0x04
 
 #define PCI_STATUS              0x06    /* 16 bits */
 #define  PCI_STATUS_CAP_LIST    0x10    /* Support Capability List */
diff --git a/roms/openbios/drivers/pci_database.c b/roms/openbios/drivers/pci_database.c
index 0bd854a51..0070a78ba 100644
--- a/roms/openbios/drivers/pci_database.c
+++ b/roms/openbios/drivers/pci_database.c
@@ -845,6 +845,24 @@ static const pci_subclass_t cpu_subclass[] = {
     },
 };
 
+static const pci_dev_t usb_devices[] = {
+#if defined(CONFIG_QEMU)
+    {
+        PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_KEYL_USB,
+        "usb", "usb", NULL,
+	"pci106b,3f\0pciclass,0c0310\0",
+        1, 0, 0,
+        NULL, NULL,
+    },
+#endif
+    {
+        0xFFFF, 0xFFFF,
+        NULL, NULL, NULL, NULL,
+        -1, -1, -1,
+        NULL, NULL,
+    },
+};
+
 static const pci_iface_t usb_iface[] = {
     {
         0x00, "UHCI USB controller", NULL,
@@ -852,7 +870,7 @@ static const pci_iface_t usb_iface[] = {
     },
     {
         0x10, "OHCI USB controller", NULL,
-        NULL, NULL, NULL,
+        usb_devices, &usb_ohci_config_cb, NULL,
     },
     {
         0x20, "EHCI USB controller", NULL,
diff --git a/roms/openbios/drivers/pci_database.h b/roms/openbios/drivers/pci_database.h
index 16b385bf1..7f053d808 100644
--- a/roms/openbios/drivers/pci_database.h
+++ b/roms/openbios/drivers/pci_database.h
@@ -38,6 +38,7 @@ extern int sabre_config_cb(const pci_config_t *config);
 extern int bridge_config_cb(const pci_config_t *config);
 extern int ebus_config_cb(const pci_config_t *config);
 extern int i82378_config_cb(const pci_config_t *config);
+extern int usb_ohci_config_cb(const pci_config_t *config);
 
 static inline int pci_compat_len(const pci_dev_t *dev)
 {
diff --git a/roms/openbios/drivers/tcx.fs b/roms/openbios/drivers/tcx.fs
index e1ec5ad70..af8991fd0 100644
--- a/roms/openbios/drivers/tcx.fs
+++ b/roms/openbios/drivers/tcx.fs
@@ -72,54 +72,53 @@ fcode-version3
 \ Registers
 \
 
-h# 0 constant tcx-off1
-h# 10000 constant /tcx-off1
+h# 0 constant tcx-off-rom
+h# 10000 constant /tcx-off-rom
 
-h# 200000 constant tcx-off-dac
-h# 4000 constant /tcx-off-dac-24
-h# 4 constant /tcx-off-dac-8
+h# 200000 constant tcx-off-cmap
+h# 4000 constant /tcx-off-cmap-24
+h# 4 constant /tcx-off-cmap-8
 
-h# 240000 constant tcx-off3
-h# 4000 constant /tcx-off3-24
-h# 4 constant /tcx-off3-8
+h# 240000 constant tcx-off-dhc
+h# 4000 constant /tcx-off-dhc-24
+h# 4 constant /tcx-off-dhc-8
 
-h# 280000 constant tcx-off4
-h# 8000 constant /tcx-off4-24
-h# 1 constant /tcx-off4-8
+h# 280000 constant tcx-off-alt
+h# 8000 constant /tcx-off-alt-24
+h# 1 constant /tcx-off-alt-8
 
-h# 301000 constant tcx-off5-24
-h# 300000 constant tcx-off5-8
-h# 1000 constant /tcx-off5-24
-h# 81c constant /tcx-off5-8
+h# 301000 constant tcx-off-thc-24
+h# 300000 constant tcx-off-thc-8
+h# 1000 constant /tcx-off-thc-24
+h# 81c constant /tcx-off-thc-8
 
-h# 700000 constant tcx-off6
-h# 1000 constant /tcx-off6
+h# 701000 constant tcx-off-tec
+h# 1000 constant /tcx-off-tec
 
-h# 800000 constant tcx-off-fb
-h# 100000 constant /tcx-off-fb
+h# 800000 constant tcx-off-dfb8
+h# 100000 constant /tcx-off-dfb8
 
-h# 2000000 constant tcx-off8
-h# 400000 constant /tcx-off8-24
-h# 1 constant /tcx-off8-8
+h# 2000000 constant tcx-off-dfb24
+h# 400000 constant /tcx-off-dfb24-24
+h# 1 constant /tcx-off-dfb24-8
 
-h# 4000000 constant tcx-off9
-h# 400000 constant /tcx-off9-24
-h# 1 constant /tcx-off9-8
+h# 4000000 constant tcx-off-stip
+h# 800000 constant /tcx-off-stip
 
-h# 6000000 constant tcx-off10
-h# 800000 constant /tcx-off10
+h# 6000000 constant tcx-off-blit
+h# 800000 constant /tcx-off-blit
 
-h# a000000 constant tcx-off11
-h# 400000 constant /tcx-off11-24
-h# 1 constant /tcx-off11-8
+h# a000000 constant tcx-off-rdfb32
+h# 400000 constant /tcx-off-rdfb32-24
+h# 1 constant /tcx-off-rdfb32-8
 
-h# c000000 constant tcx-off12
-h# 800000 constant /tcx-off12-24
-h# 1 constant /tcx-off12-8
+h# c000000 constant tcx-off-rstip
+h# 800000 constant /tcx-off-rstip-24
+h# 1 constant /tcx-off-rstip-8
 
-h# e000000 constant tcx-off13
-h# 800000 constant /tcx-off13-24
-h# 1 constant /tcx-off13-8
+h# e000000 constant tcx-off-rblit
+h# 800000 constant /tcx-off-rblit-24
+h# 1 constant /tcx-off-rblit-8
 
 : >tcx-reg-spec ( offset size -- encoded-reg )
   >r 0 my-address d+ my-space encode-phys r> encode-int encode+
@@ -127,37 +126,37 @@ h# 1 constant /tcx-off13-8
 
 : tcx-8bit-reg
   \ WARNING: order is important (at least to Solaris)
-  tcx-off-fb /tcx-off-fb >tcx-reg-spec
-  tcx-off8 /tcx-off8-8 >tcx-reg-spec encode+
-  tcx-off9 /tcx-off9-8 >tcx-reg-spec encode+
-  tcx-off10 /tcx-off10 >tcx-reg-spec encode+
-  tcx-off11 /tcx-off11-8 >tcx-reg-spec encode+
-  tcx-off12 /tcx-off12-8 >tcx-reg-spec encode+
-  tcx-off13 /tcx-off13-8 >tcx-reg-spec encode+
-  tcx-off6 /tcx-off6 >tcx-reg-spec encode+
-  tcx-off-dac /tcx-off-dac-8 >tcx-reg-spec encode+
-  tcx-off5-8 /tcx-off5-8 >tcx-reg-spec encode+
-  tcx-off1 /tcx-off1 >tcx-reg-spec encode+
-  tcx-off3 /tcx-off3-8 >tcx-reg-spec encode+
-  tcx-off4 /tcx-off4-8 >tcx-reg-spec encode+
+  tcx-off-dfb8 /tcx-off-dfb8 >tcx-reg-spec
+  tcx-off-dfb24 /tcx-off-dfb24-8 >tcx-reg-spec encode+
+  tcx-off-stip /tcx-off-stip >tcx-reg-spec encode+
+  tcx-off-blit /tcx-off-blit >tcx-reg-spec encode+
+  tcx-off-rdfb32 /tcx-off-rdfb32-8 >tcx-reg-spec encode+
+  tcx-off-rstip /tcx-off-rstip-8 >tcx-reg-spec encode+
+  tcx-off-rblit /tcx-off-rblit-8 >tcx-reg-spec encode+
+  tcx-off-tec /tcx-off-tec >tcx-reg-spec encode+
+  tcx-off-cmap /tcx-off-cmap-8 >tcx-reg-spec encode+
+  tcx-off-thc-8 /tcx-off-thc-8 >tcx-reg-spec encode+
+  tcx-off-rom /tcx-off-rom >tcx-reg-spec encode+
+  tcx-off-dhc /tcx-off-dhc-8 >tcx-reg-spec encode+
+  tcx-off-alt /tcx-off-alt-8 >tcx-reg-spec encode+
   " reg" property
 ;
 
 : tcx-24bit-reg
   \ WARNING: order is important (at least to Solaris)
-  tcx-off-fb /tcx-off-fb >tcx-reg-spec
-  tcx-off8 /tcx-off8-24 >tcx-reg-spec encode+
-  tcx-off9 /tcx-off9-24 >tcx-reg-spec encode+
-  tcx-off10 /tcx-off10 >tcx-reg-spec encode+
-  tcx-off11 /tcx-off11-24 >tcx-reg-spec encode+
-  tcx-off12 /tcx-off12-24 >tcx-reg-spec encode+
-  tcx-off13 /tcx-off13-24 >tcx-reg-spec encode+
-  tcx-off6 /tcx-off6 >tcx-reg-spec encode+
-  tcx-off-dac /tcx-off-dac-24 >tcx-reg-spec encode+
-  tcx-off5-24 /tcx-off5-24 >tcx-reg-spec encode+
-  tcx-off1 /tcx-off1 >tcx-reg-spec encode+
-  tcx-off3 /tcx-off3-24 >tcx-reg-spec encode+
-  tcx-off4 /tcx-off4-24 >tcx-reg-spec encode+
+  tcx-off-dfb8 /tcx-off-dfb8 >tcx-reg-spec
+  tcx-off-dfb24 /tcx-off-dfb24-24 >tcx-reg-spec encode+
+  tcx-off-stip /tcx-off-stip >tcx-reg-spec encode+
+  tcx-off-blit /tcx-off-blit >tcx-reg-spec encode+
+  tcx-off-rdfb32 /tcx-off-rdfb32-24 >tcx-reg-spec encode+
+  tcx-off-rstip /tcx-off-rstip-24 >tcx-reg-spec encode+
+  tcx-off-rblit /tcx-off-rblit-24 >tcx-reg-spec encode+
+  tcx-off-tec /tcx-off-tec >tcx-reg-spec encode+
+  tcx-off-cmap /tcx-off-cmap-24 >tcx-reg-spec encode+
+  tcx-off-thc-24 /tcx-off-thc-24 >tcx-reg-spec encode+
+  tcx-off-rom /tcx-off-rom >tcx-reg-spec encode+
+  tcx-off-dhc /tcx-off-dhc-24 >tcx-reg-spec encode+
+  tcx-off-alt /tcx-off-alt-24 >tcx-reg-spec encode+
   " reg" property
 ;
 
@@ -198,11 +197,11 @@ headerless
 \
 
 : dac-map
-  tcx-off-dac /tcx-dac do-map-in to tcx-dac
+  tcx-off-cmap /tcx-dac do-map-in to tcx-dac
 ;
 
 : fb-map
-  tcx-off-fb h# c0000 do-map-in to fb-addr
+  tcx-off-dfb8 h# c0000 do-map-in to fb-addr
 ;
 
 : map-regs
@@ -228,7 +227,11 @@ headerless
     fb-addr to frame-buffer-adr
     default-font set-font
 
-    frame-buffer-adr encode-int " address" property
+    \ Sun TCX adapters don't have an address property, but it is useful for
+    \ OpenBIOS developers. Unfortunately NetBSD SPARC32 has a bug that causes
+    \ it to fail initialising TCX if the address property is present; so work
+    \ around this by adding an underscore prefix
+    frame-buffer-adr encode-int " _address" property
 
     openbios-video-width openbios-video-height over char-width / over char-height /
     fb8-install
@@ -240,11 +243,11 @@ headerless
   \ Handle differences between 8-bit/24-bit mode
   depth-bits 8 = if
     tcx-8bit-reg
-    /tcx-off-dac-8 to /tcx-dac
+    /tcx-off-cmap-8 to /tcx-dac
     " true" encode-string " tcx-8-bit" property
   else
     tcx-24bit-reg
-    /tcx-off-dac-24 to /tcx-dac
+    /tcx-off-cmap-24 to /tcx-dac
 
     \ Even with a 24-bit enabled TCX card, the control plane is
     \ used in 8-bit mode. So force the video subsystem into 8-bit
@@ -266,7 +269,7 @@ headerless
   openbios-video-width encode-int " width" property
   line-bytes encode-int " linebytes" property
 
-  5 encode-int 0 encode-int encode+ " intr" property
+  h# 39 encode-int 0 encode-int encode+ " intr" property
   5 encode-int " interrupts" property
 
   ['] qemu-tcx-driver-install is-install
diff --git a/roms/openbios/drivers/usb.c b/roms/openbios/drivers/usb.c
new file mode 100644
index 000000000..c6e37174e
--- /dev/null
+++ b/roms/openbios/drivers/usb.c
@@ -0,0 +1,587 @@
+/*
+ * Driver for USB ported from CoreBoot
+ *
+ * Copyright (C) 2014 BALATON Zoltan
+ *
+ * This file was part of the libpayload project.
+ *
+ * Copyright (C) 2008-2010 coresystems GmbH
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "drivers/usb.h"
+#include "usb.h"
+#include "timer.h"
+#include "libc/byteorder.h"
+
+hci_t *usb_hcs = 0;
+
+static void usb_nop_init (usbdev_t *dev);
+
+static void
+usb_nop_destroy (usbdev_t *dev)
+{
+	if (dev->descriptor != 0)
+		free (dev->descriptor);
+	usb_nop_init (dev);
+	dev->address = -1;
+	dev->hub = -1;
+	dev->port = -1;
+}
+
+static void
+usb_nop_poll (usbdev_t *dev)
+{
+	return;
+}
+
+static void
+usb_nop_init (usbdev_t *dev)
+{
+	dev->descriptor = 0;
+	dev->destroy = usb_nop_destroy;
+	dev->poll = usb_nop_poll;
+}
+
+hci_t *
+new_controller (void)
+{
+	hci_t *controller = malloc (sizeof (hci_t));
+
+	if (controller) {
+		/* atomic */
+		controller->next = usb_hcs;
+		usb_hcs = controller;
+		/* atomic end */
+	}
+
+	return controller;
+}
+
+void
+detach_controller (hci_t *controller)
+{
+	if (controller == NULL)
+		return;
+	if (usb_hcs == controller) {
+		usb_hcs = controller->next;
+	} else {
+		hci_t *it = usb_hcs;
+		while (it != NULL) {
+			if (it->next == controller) {
+				it->next = controller->next;
+				return;
+			}
+			it = it->next;
+		}
+	}
+}
+
+/**
+ * Shut down all controllers
+ */
+int
+usb_exit (void)
+{
+	while (usb_hcs != NULL) {
+		usb_hcs->shutdown(usb_hcs);
+	}
+	return 0;
+}
+
+/**
+ * Polls all hubs on all USB controllers, to find out about device changes
+ */
+void
+usb_poll (void)
+{
+	if (usb_hcs == 0)
+		return;
+	hci_t *controller = usb_hcs;
+	while (controller != NULL) {
+		int i;
+		for (i = 0; i < 128; i++) {
+			if (controller->devices[i] != 0) {
+				controller->devices[i]->poll (controller->devices[i]);
+			}
+		}
+		controller = controller->next;
+	}
+}
+
+void
+init_device_entry (hci_t *controller, int i)
+{
+	if (controller->devices[i] != 0)
+		usb_debug("warning: device %d reassigned?\n", i);
+	controller->devices[i] = malloc(sizeof(usbdev_t));
+	controller->devices[i]->controller = controller;
+	controller->devices[i]->address = -1;
+	controller->devices[i]->hub = -1;
+	controller->devices[i]->port = -1;
+	controller->devices[i]->init = usb_nop_init;
+	controller->devices[i]->init (controller->devices[i]);
+}
+
+void
+set_feature (usbdev_t *dev, int endp, int feature, int rtype)
+{
+	dev_req_t dr;
+
+	dr.bmRequestType = rtype;
+	dr.data_dir = host_to_device;
+	dr.bRequest = SET_FEATURE;
+	dr.wValue = __cpu_to_le16(feature);
+	dr.wIndex = __cpu_to_le16(endp);
+	dr.wLength = 0;
+	dev->controller->control (dev, OUT, sizeof (dr), &dr, 0, 0);
+}
+
+void
+get_status (usbdev_t *dev, int intf, int rtype, int len, void *data)
+{
+	dev_req_t dr;
+
+	dr.bmRequestType = rtype;
+	dr.data_dir = device_to_host;
+	dr.bRequest = GET_STATUS;
+	dr.wValue = 0;
+	dr.wIndex = __cpu_to_le16(intf);
+	dr.wLength = __cpu_to_le16(len);
+	dev->controller->control (dev, IN, sizeof (dr), &dr, len, data);
+}
+
+u8 *
+get_descriptor (usbdev_t *dev, unsigned char bmRequestType, int descType,
+		int descIdx, int langID)
+{
+	u8 buf[8];
+	u8 *result;
+	dev_req_t dr;
+	int size;
+
+	dr.bmRequestType = bmRequestType;
+	dr.data_dir = device_to_host;	// always like this for descriptors
+	dr.bRequest = GET_DESCRIPTOR;
+	dr.wValue = __cpu_to_le16((descType << 8) | descIdx);
+	dr.wIndex = __cpu_to_le16(langID);
+	dr.wLength = __cpu_to_le16(8);
+	if (dev->controller->control (dev, IN, sizeof (dr), &dr, 8, buf)) {
+		usb_debug ("getting descriptor size (type %x) failed\n",
+			descType);
+	}
+
+	if (descType == 1) {
+		device_descriptor_t *dd = (device_descriptor_t *) buf;
+		usb_debug ("maxPacketSize0: %x\n", dd->bMaxPacketSize0);
+		if (dd->bMaxPacketSize0 != 0)
+			dev->endpoints[0].maxpacketsize = dd->bMaxPacketSize0;
+	}
+
+	/* special case for configuration descriptors: they carry all their
+	   subsequent descriptors with them, and keep the entire size at a
+	   different location */
+	size = buf[0];
+	if (buf[1] == 2) {
+		int realsize = __le16_to_cpu(((unsigned short *) (buf + 2))[0]);
+		size = realsize;
+	}
+	result = malloc (size);
+	memset (result, 0, size);
+	dr.wLength = __cpu_to_le16(size);
+	if (dev->controller->
+	    control (dev, IN, sizeof (dr), &dr, size, result)) {
+		usb_debug ("getting descriptor (type %x, size %x) failed\n",
+			descType, size);
+	}
+
+	return result;
+}
+
+void
+set_configuration (usbdev_t *dev)
+{
+	dev_req_t dr;
+
+	dr.bmRequestType = 0;
+	dr.bRequest = SET_CONFIGURATION;
+	dr.wValue = __cpu_to_le16(dev->configuration[5]);
+	dr.wIndex = 0;
+	dr.wLength = 0;
+	dev->controller->control (dev, OUT, sizeof (dr), &dr, 0, 0);
+}
+
+int
+clear_feature (usbdev_t *dev, int endp, int feature, int rtype)
+{
+	dev_req_t dr;
+
+	dr.bmRequestType = rtype;
+	dr.data_dir = host_to_device;
+	dr.bRequest = CLEAR_FEATURE;
+	dr.wValue = __cpu_to_le16(feature);
+	dr.wIndex = __cpu_to_le16(endp);
+	dr.wLength = 0;
+	return dev->controller->control (dev, OUT, sizeof (dr), &dr, 0, 0);
+}
+
+int
+clear_stall (endpoint_t *ep)
+{
+	usbdev_t *dev = ep->dev;
+	int endp = ep->endpoint;
+	int rtype = gen_bmRequestType (host_to_device, standard_type,
+					endp ? endp_recp : dev_recp);
+
+	int ret = clear_feature (dev, endp, ENDPOINT_HALT, rtype);
+	ep->toggle = 0;
+	return ret;
+}
+
+/* returns free address or -1 */
+static int
+get_free_address (hci_t *controller)
+{
+	int i;
+	for (i = 1; i < 128; i++) {
+		if (controller->devices[i] == 0)
+			return i;
+	}
+	usb_debug ("no free address found\n");
+	return -1;		// no free address
+}
+
+int
+generic_set_address (hci_t *controller, int speed, int hubport, int hubaddr)
+{
+	int adr = get_free_address (controller);	// address to set
+	dev_req_t dr;
+
+	memset (&dr, 0, sizeof (dr));
+	dr.data_dir = host_to_device;
+	dr.req_type = standard_type;
+	dr.req_recp = dev_recp;
+	dr.bRequest = SET_ADDRESS;
+	dr.wValue = __cpu_to_le16(adr);
+	dr.wIndex = 0;
+	dr.wLength = 0;
+
+	init_device_entry(controller, adr);
+	usbdev_t *dev = controller->devices[adr];
+	// dummy values for registering the address
+	dev->address = 0;
+	dev->hub = hubaddr;
+	dev->port = hubport;
+	dev->speed = speed;
+	dev->endpoints[0].dev = dev;
+	dev->endpoints[0].endpoint = 0;
+	dev->endpoints[0].maxpacketsize = 8;
+	dev->endpoints[0].toggle = 0;
+	dev->endpoints[0].direction = SETUP;
+	mdelay (50);
+	if (dev->controller->control (dev, OUT, sizeof (dr), &dr, 0, 0)) {
+		return -1;
+	}
+	mdelay (50);
+
+	return adr;
+}
+
+/* Normalize bInterval to log2 of microframes */
+static int
+usb_decode_interval(const int speed, const endpoint_type type, const unsigned char bInterval)
+{
+#define LOG2(a) ((sizeof(unsigned) << 3) - __builtin_clz(a) - 1)
+	switch (speed) {
+	case LOW_SPEED:
+		switch (type) {
+		case ISOCHRONOUS: case INTERRUPT:
+			return LOG2(bInterval) + 3;
+		default:
+			return 0;
+		}
+	case FULL_SPEED:
+		switch (type) {
+		case ISOCHRONOUS:
+			return (bInterval - 1) + 3;
+		case INTERRUPT:
+			return LOG2(bInterval) + 3;
+		default:
+			return 0;
+		}
+	case HIGH_SPEED:
+		switch (type) {
+		case ISOCHRONOUS: case INTERRUPT:
+			return bInterval - 1;
+		default:
+			return LOG2(bInterval);
+		}
+	case SUPER_SPEED:
+		switch (type) {
+		case ISOCHRONOUS: case INTERRUPT:
+			return bInterval - 1;
+		default:
+			return 0;
+		}
+	default:
+		return 0;
+	}
+#undef LOG2
+}
+
+static int
+set_address (hci_t *controller, int speed, int hubport, int hubaddr)
+{
+	int adr = controller->set_address(controller, speed, hubport, hubaddr);
+	if (adr < 0 || !controller->devices[adr]) {
+		usb_debug ("set_address failed\n");
+		return -1;
+	}
+	configuration_descriptor_t *cd;
+	device_descriptor_t *dd;
+
+	usbdev_t *dev = controller->devices[adr];
+	dev->address = adr;
+	dev->hub = hubaddr;
+	dev->port = hubport;
+	dev->speed = speed;
+	dev->descriptor = get_descriptor (dev, gen_bmRequestType
+		(device_to_host, standard_type, dev_recp), 1, 0, 0);
+	dd = (device_descriptor_t *) dev->descriptor;
+
+	usb_debug ("* found device (0x%04x:0x%04x, USB %x.%x)",
+		 __le16_to_cpu(dd->idVendor), __le16_to_cpu(dd->idProduct),
+		 __le16_to_cpu(dd->bcdUSB) >> 8, __le16_to_cpu(dd->bcdUSB) & 0xff);
+	dev->quirks = USB_QUIRK_NONE;
+
+	usb_debug ("\ndevice has %x configurations\n", dd->bNumConfigurations);
+	if (dd->bNumConfigurations == 0) {
+		/* device isn't usable */
+		usb_debug ("... no usable configuration!\n");
+		dev->address = 0;
+		return -1;
+	}
+
+	dev->configuration = get_descriptor (dev, gen_bmRequestType
+		(device_to_host, standard_type, dev_recp), 2, 0, 0);
+	cd = (configuration_descriptor_t *) dev->configuration;
+	interface_descriptor_t *interface =
+		(interface_descriptor_t *) (((char *) cd) + cd->bLength);
+	{
+		int i;
+		int num = cd->bNumInterfaces;
+		interface_descriptor_t *current = interface;
+		usb_debug ("device has %x interfaces\n", num);
+		if (num > 1) {
+			usb_debug ("\nNOTICE: This driver defaults to using the first interface.\n"
+				   "This might be the wrong choice and lead to limited functionality\n"
+				   "of the device.\n");
+			 /* we limit to the first interface, as there was no need to
+			 * implement something else for the time being. If you need
+			 * it, see the SetInterface and GetInterface functions in
+			 * the USB specification, and adapt appropriately.
+			 */
+			num = (num > 1) ? 1 : num;
+		}
+		for (i = 0; i < num; i++) {
+			int j;
+			usb_debug (" #%x has %x endpoints, interface %x:%x, protocol %x\n",
+					current->bInterfaceNumber, current->bNumEndpoints, current->bInterfaceClass, current->bInterfaceSubClass, current->bInterfaceProtocol);
+			endpoint_descriptor_t *endp =
+				(endpoint_descriptor_t *) (((char *) current)
+							   + current->bLength);
+			/* Skip any non-endpoint descriptor */
+			if (endp->bDescriptorType != 0x05)
+				endp = (endpoint_descriptor_t *)(((char *)endp) + ((char *)endp)[0]);
+
+			memset (dev->endpoints, 0, sizeof (dev->endpoints));
+			dev->num_endp = 1;	// 0 always exists
+			dev->endpoints[0].dev = dev;
+			dev->endpoints[0].maxpacketsize = dd->bMaxPacketSize0;
+			dev->endpoints[0].direction = SETUP;
+			dev->endpoints[0].type = CONTROL;
+			dev->endpoints[0].interval = usb_decode_interval(dev->speed, CONTROL, endp->bInterval);
+			for (j = 1; j <= current->bNumEndpoints; j++) {
+#ifdef CONFIG_DEBUG_USB
+				static const char *transfertypes[4] = {
+					"control", "isochronous", "bulk", "interrupt"
+				};
+				usb_debug ("   #%x: Endpoint %x (%s), max packet size %x, type %s\n", j, endp->bEndpointAddress & 0x7f, ((endp->bEndpointAddress & 0x80) != 0) ? "in" : "out", __le16_to_cpu(endp->wMaxPacketSize), transfertypes[endp->bmAttributes]);
+#endif
+				endpoint_t *ep =
+					&dev->endpoints[dev->num_endp++];
+				ep->dev = dev;
+				ep->endpoint = endp->bEndpointAddress;
+				ep->toggle = 0;
+				ep->maxpacketsize = __le16_to_cpu(endp->wMaxPacketSize);
+				ep->direction =
+					((endp->bEndpointAddress & 0x80) ==
+					 0) ? OUT : IN;
+				ep->type = endp->bmAttributes;
+				ep->interval = usb_decode_interval(dev->speed, ep->type, endp->bInterval);
+				endp = (endpoint_descriptor_t
+					*) (((char *) endp) + endp->bLength);
+			}
+			current = (interface_descriptor_t *) endp;
+		}
+	}
+
+	if (controller->finish_device_config &&
+			controller->finish_device_config(dev))
+		return adr; /* Device isn't configured correctly,
+			       only control transfers may work. */
+
+	set_configuration(dev);
+
+	int class = dd->bDeviceClass;
+	if (class == 0)
+		class = interface->bInterfaceClass;
+
+	usb_debug(", class: ");
+	switch (class) {
+	case audio_device:
+		usb_debug("audio\n");
+		break;
+	case comm_device:
+		usb_debug("communication\n");
+		break;
+	case hid_device:
+		usb_debug ("HID\n");
+#ifdef CONFIG_USB_HID
+		controller->devices[adr]->init = usb_hid_init;
+		return adr;
+#else
+		usb_debug ("NOTICE: USB HID support not compiled in\n");
+#endif
+		break;
+	case physical_device:
+		usb_debug("physical\n");
+		break;
+	case imaging_device:
+		usb_debug("camera\n");
+		break;
+	case printer_device:
+		usb_debug("printer\n");
+		break;
+	case msc_device:
+		usb_debug ("MSC\n");
+#ifdef CONFIG_USB_MSC
+		controller->devices[adr]->init = usb_msc_init;
+		return adr;
+#else
+		usb_debug ("NOTICE: USB MSC support not compiled in\n");
+#endif
+		break;
+	case hub_device:
+		usb_debug ("hub\n");
+#ifdef CONFIG_USB_HUB
+		controller->devices[adr]->init = usb_hub_init;
+		return adr;
+#else
+		usb_debug ("NOTICE: USB hub support not compiled in.\n");
+#endif
+		break;
+	case cdc_device:
+		usb_debug("CDC\n");
+		break;
+	case ccid_device:
+		usb_debug("smartcard / CCID\n");
+		break;
+	case security_device:
+		usb_debug("content security\n");
+		break;
+	case video_device:
+		usb_debug("video\n");
+		break;
+	case healthcare_device:
+		usb_debug("healthcare\n");
+		break;
+	case diagnostic_device:
+		usb_debug("diagnostic\n");
+		break;
+	case wireless_device:
+		usb_debug("wireless\n");
+		break;
+	default:
+		usb_debug("unsupported class %x\n", class);
+		break;
+	}
+	controller->devices[adr]->init = usb_generic_init;
+	return adr;
+}
+
+/*
+ * Should be called by the hub drivers whenever a physical detach occurs
+ * and can be called by usb class drivers if they are unsatisfied with a
+ * malfunctioning device.
+ */
+void
+usb_detach_device(hci_t *controller, int devno)
+{
+	/* check if device exists, as we may have
+	   been called yet by the usb class driver */
+	if (controller->devices[devno]) {
+		controller->devices[devno]->destroy (controller->devices[devno]);
+		free(controller->devices[devno]);
+		controller->devices[devno] = NULL;
+		if (controller->destroy_device)
+			controller->destroy_device(controller, devno);
+	}
+}
+
+int
+usb_attach_device(hci_t *controller, int hubaddress, int port, int speed)
+{
+#ifdef CONFIG_USB_DEBUG
+	static const char* speeds[] = { "full", "low", "high" };
+	usb_debug ("%sspeed device\n", (speed <= 2) ? speeds[speed] : "invalid value - no");
+#endif
+	int newdev = set_address (controller, speed, port, hubaddress);
+	if (newdev == -1)
+		return -1;
+	usbdev_t *newdev_t = controller->devices[newdev];
+	// determine responsible driver - current done in set_address
+	newdev_t->init (newdev_t);
+	/* init() may have called usb_detach_device() yet, so check */
+	return controller->devices[newdev] ? newdev : -1;
+}
+
+static void
+usb_generic_destroy (usbdev_t *dev)
+{
+	if (usb_generic_remove)
+		usb_generic_remove(dev);
+}
+
+void
+usb_generic_init (usbdev_t *dev)
+{
+	dev->data = NULL;
+	dev->destroy = usb_generic_destroy;
+
+	if (usb_generic_create)
+		usb_generic_create(dev);
+}
diff --git a/roms/openbios/drivers/usb.h b/roms/openbios/drivers/usb.h
new file mode 100644
index 000000000..2e23a1370
--- /dev/null
+++ b/roms/openbios/drivers/usb.h
@@ -0,0 +1,357 @@
+/*
+ * Driver for USB ported from CoreBoot
+ *
+ * Copyright (C) 2014 BALATON Zoltan
+ *
+ * This file was part of the libpayload project.
+ *
+ * Copyright (C) 2008 coresystems GmbH
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef __USB_H
+#define __USB_H
+#include <drivers/pci.h>
+
+typedef enum { host_to_device = 0, device_to_host = 1 } dev_req_dir;
+typedef enum { standard_type = 0, class_type = 1, vendor_type =
+		2, reserved_type = 3
+} dev_req_type;
+typedef enum { dev_recp = 0, iface_recp = 1, endp_recp = 2, other_recp = 3
+} dev_req_recp;
+
+typedef enum {
+	GET_STATUS = 0,
+	CLEAR_FEATURE = 1,
+	SET_FEATURE = 3,
+	SET_ADDRESS = 5,
+	GET_DESCRIPTOR = 6,
+	SET_DESCRIPTOR = 7,
+	GET_CONFIGURATION = 8,
+	SET_CONFIGURATION = 9,
+	GET_INTERFACE = 10,
+	SET_INTERFACE = 11,
+	SYNCH_FRAME = 12
+} bRequest_Codes;
+
+typedef enum {
+	ENDPOINT_HALT = 0,
+	DEVICE_REMOTE_WAKEUP = 1,
+	TEST_MODE = 2
+} feature_selectors;
+
+enum {
+	audio_device      = 0x01,
+	comm_device       = 0x02,
+	hid_device        = 0x03,
+	physical_device   = 0x05,
+	imaging_device    = 0x06,
+	printer_device    = 0x07,
+	msc_device        = 0x08,
+	hub_device        = 0x09,
+	cdc_device        = 0x0a,
+	ccid_device       = 0x0b,
+	security_device   = 0x0d,
+	video_device      = 0x0e,
+	healthcare_device = 0x0f,
+	diagnostic_device = 0xdc,
+	wireless_device   = 0xe0,
+	misc_device       = 0xef,
+};
+
+enum { hid_subclass_none = 0, hid_subclass_boot = 1 };
+
+enum {
+	hid_boot_proto_none = 0,
+	hid_boot_proto_keyboard = 1,
+	hid_boot_proto_mouse = 2
+};
+
+typedef struct {
+	union {
+		struct {
+#ifdef CONFIG_BIG_ENDIAN
+			dev_req_dir data_dir:1;
+			dev_req_type req_type:2;
+			dev_req_recp req_recp:5;
+#else
+			dev_req_recp req_recp:5;
+			dev_req_type req_type:2;
+			dev_req_dir data_dir:1;
+#endif
+		} __attribute__ ((packed));
+		unsigned char bmRequestType;
+	} __attribute__ ((packed));
+	unsigned char bRequest;
+	unsigned short wValue;
+	unsigned short wIndex;
+	unsigned short wLength;
+} __attribute__ ((packed)) dev_req_t;
+
+struct usbdev_hc;
+typedef struct usbdev_hc hci_t;
+
+struct usbdev;
+typedef struct usbdev usbdev_t;
+
+typedef enum { SETUP, IN, OUT } direction_t;
+typedef enum { CONTROL = 0, ISOCHRONOUS = 1, BULK = 2, INTERRUPT = 3
+} endpoint_type;
+
+typedef struct {
+	usbdev_t *dev;
+	int endpoint;
+	direction_t direction;
+	int toggle;
+	int maxpacketsize;
+	endpoint_type type;
+	int interval; /* expressed as binary logarithm of the number
+			 of microframes (i.e. t = 125us * 2^interval) */
+} endpoint_t;
+
+enum { FULL_SPEED = 0, LOW_SPEED = 1, HIGH_SPEED = 2, SUPER_SPEED = 3 };
+
+struct usbdev {
+	hci_t *controller;
+	endpoint_t endpoints[32];
+	int num_endp;
+	int address;		// usb address
+	int hub;		// hub, device is attached to
+	int port;		// port where device is attached
+	int speed;		// 1: lowspeed, 0: fullspeed, 2: highspeed
+	u32 quirks;		// quirks field. got to love usb
+	void *data;
+	u8 *descriptor;
+	u8 *configuration;
+	void (*init) (usbdev_t *dev);
+	void (*destroy) (usbdev_t *dev);
+	void (*poll) (usbdev_t *dev);
+};
+
+typedef enum { OHCI = 0, UHCI = 1, EHCI = 2, XHCI = 3} hc_type;
+
+struct usbdev_hc {
+	hci_t *next;
+	u32 reg_base;
+	hc_type type;
+	usbdev_t *devices[128];	// dev 0 is root hub, 127 is last addressable
+
+	/* start():     Resume operation. */
+	void (*start) (hci_t *controller);
+	/* stop():      Stop operation but keep controller initialized. */
+	void (*stop) (hci_t *controller);
+	/* reset():     Perform a controller reset. The controller needs to
+	                be (re)initialized afterwards to work (again). */
+	void (*reset) (hci_t *controller);
+	/* init():      Initialize a (previously reset) controller
+	                to a working state. */
+	void (*init) (hci_t *controller);
+	/* shutdown():  Stop operation, detach host controller and shutdown
+	                this driver instance. After calling shutdown() any
+			other usage of this hci_t* is invalid. */
+	void (*shutdown) (hci_t *controller);
+
+	int (*bulk) (endpoint_t *ep, int size, u8 *data, int finalize);
+	int (*control) (usbdev_t *dev, direction_t pid, int dr_length,
+			void *devreq, int data_length, u8 *data);
+	void* (*create_intr_queue) (endpoint_t *ep, int reqsize, int reqcount, int reqtiming);
+	void (*destroy_intr_queue) (endpoint_t *ep, void *queue);
+	u8* (*poll_intr_queue) (void *queue);
+	void *instance;
+
+	/* set_address():		Tell the usb device its address and
+					return it. xHCI controllers want to
+					do this by themself. Also, the usbdev
+					structure has to be allocated and
+					initialized. */
+	int (*set_address) (hci_t *controller, int speed, int hubport, int hubaddr);
+	/* finish_device_config():	Another hook for xHCI,
+					returns 0 on success. */
+	int (*finish_device_config) (usbdev_t *dev);
+	/* destroy_device():		Finally, destroy all structures that
+					were allocated during set_address()
+					and finish_device_config(). */
+	void (*destroy_device) (hci_t *controller, int devaddr);
+};
+
+typedef struct {
+	unsigned char bDescLength;
+	unsigned char bDescriptorType;
+	unsigned char bNbrPorts;
+	union {
+		struct {
+#ifdef CONFIG_BIG_ENDIAN
+			unsigned long:8;
+			unsigned long arePortIndicatorsSupported:1;
+			unsigned long ttThinkTime:2;
+			unsigned long overcurrentProtectionMode:2;
+			unsigned long isCompoundDevice:1;
+			unsigned long logicalPowerSwitchingMode:2;
+#else
+			unsigned long logicalPowerSwitchingMode:2;
+			unsigned long isCompoundDevice:1;
+			unsigned long overcurrentProtectionMode:2;
+			unsigned long ttThinkTime:2;
+			unsigned long arePortIndicatorsSupported:1;
+			unsigned long:8;
+#endif
+		} __attribute__ ((packed));
+		unsigned short wHubCharacteristics;
+	} __attribute__ ((packed));
+	unsigned char bPowerOn2PwrGood;
+	unsigned char bHubContrCurrent;
+	char DeviceRemovable[];
+} __attribute__ ((packed)) hub_descriptor_t;
+
+typedef struct {
+	unsigned char bLength;
+	unsigned char bDescriptorType;
+	unsigned short bcdUSB;
+	unsigned char bDeviceClass;
+	unsigned char bDeviceSubClass;
+	unsigned char bDeviceProtocol;
+	unsigned char bMaxPacketSize0;
+	unsigned short idVendor;
+	unsigned short idProduct;
+	unsigned short bcdDevice;
+	unsigned char iManufacturer;
+	unsigned char iProduct;
+	unsigned char iSerialNumber;
+	unsigned char bNumConfigurations;
+} __attribute__ ((packed)) device_descriptor_t;
+
+typedef struct {
+	unsigned char bLength;
+	unsigned char bDescriptorType;
+	unsigned short wTotalLength;
+	unsigned char bNumInterfaces;
+	unsigned char bConfigurationValue;
+	unsigned char iConfiguration;
+	unsigned char bmAttributes;
+	unsigned char bMaxPower;
+} __attribute__ ((packed)) configuration_descriptor_t;
+
+typedef struct {
+	unsigned char bLength;
+	unsigned char bDescriptorType;
+	unsigned char bInterfaceNumber;
+	unsigned char bAlternateSetting;
+	unsigned char bNumEndpoints;
+	unsigned char bInterfaceClass;
+	unsigned char bInterfaceSubClass;
+	unsigned char bInterfaceProtocol;
+	unsigned char iInterface;
+} __attribute__ ((packed)) interface_descriptor_t;
+
+typedef struct {
+	unsigned char bLength;
+	unsigned char bDescriptorType;
+	unsigned char bEndpointAddress;
+	unsigned char bmAttributes;
+	unsigned short wMaxPacketSize;
+	unsigned char bInterval;
+} __attribute__ ((packed)) endpoint_descriptor_t;
+
+typedef struct {
+	unsigned char bLength;
+	unsigned char bDescriptorType;
+	unsigned short bcdHID;
+	unsigned char bCountryCode;
+	unsigned char bNumDescriptors;
+	unsigned char bReportDescriptorType;
+	unsigned short wReportDescriptorLength;
+} __attribute__ ((packed)) hid_descriptor_t;
+
+hci_t *new_controller (void);
+void detach_controller (hci_t *controller);
+void usb_poll (void);
+void init_device_entry (hci_t *controller, int num);
+
+void set_feature (usbdev_t *dev, int endp, int feature, int rtype);
+void get_status (usbdev_t *dev, int endp, int rtype, int len, void *data);
+void set_configuration (usbdev_t *dev);
+int clear_feature (usbdev_t *dev, int endp, int feature, int rtype);
+int clear_stall (endpoint_t *ep);
+
+void usb_hub_init (usbdev_t *dev);
+void usb_hid_init (usbdev_t *dev);
+void usb_msc_init (usbdev_t *dev);
+void usb_generic_init (usbdev_t *dev);
+
+u8 *get_descriptor (usbdev_t *dev, unsigned char bmRequestType,
+		    int descType, int descIdx, int langID);
+
+static inline unsigned char
+gen_bmRequestType (dev_req_dir dir, dev_req_type type, dev_req_recp recp)
+{
+	return (dir << 7) | (type << 5) | recp;
+}
+
+/* default "set address" handler */
+int generic_set_address (hci_t *controller, int speed, int hubport, int hubaddr);
+
+void usb_detach_device(hci_t *controller, int devno);
+int usb_attach_device(hci_t *controller, int hubaddress, int port, int speed);
+
+u32 usb_quirk_check(u16 vendor, u16 device);
+int usb_interface_check(u16 vendor, u16 device);
+
+#define USB_QUIRK_MSC_FORCE_PROTO_SCSI		(1 <<  0)
+#define USB_QUIRK_MSC_FORCE_PROTO_ATAPI		(1 <<  1)
+#define USB_QUIRK_MSC_FORCE_PROTO_UFI		(1 <<  2)
+#define USB_QUIRK_MSC_FORCE_PROTO_RBC		(1 <<  3)
+#define USB_QUIRK_MSC_FORCE_TRANS_BBB		(1 <<  4)
+#define USB_QUIRK_MSC_FORCE_TRANS_CBI		(1 <<  5)
+#define USB_QUIRK_MSC_FORCE_TRANS_CBI_I		(1 <<  6)
+#define USB_QUIRK_MSC_NO_TEST_UNIT_READY	(1 <<  7)
+#define USB_QUIRK_MSC_SHORT_INQUIRY		(1 <<  8)
+#define USB_QUIRK_TEST				(1 << 31)
+#define USB_QUIRK_NONE				 0
+
+#ifdef CONFIG_DEBUG_USB
+#define usb_debug(fmt, args...)  do { printk(fmt , ##args); } while (0)
+#else
+#define usb_debug(fmt, args...)
+#endif
+
+/**
+ * To be implemented by libpayload-client. It's called by the USB stack
+ * when a new USB device is found which isn't claimed by a built in driver,
+ * so the client has the chance to know about it.
+ *
+ * @param dev descriptor for the USB device
+ */
+void __attribute__((weak)) usb_generic_create (usbdev_t *dev);
+
+/**
+ * To be implemented by libpayload-client. It's called by the USB stack
+ * when it finds out that a USB device is removed which wasn't claimed by a
+ * built in driver.
+ *
+ * @param dev descriptor for the USB device
+ */
+void __attribute__((weak)) usb_generic_remove (usbdev_t *dev);
+
+#endif
diff --git a/roms/openbios/drivers/usbhid.c b/roms/openbios/drivers/usbhid.c
new file mode 100644
index 000000000..a423278a8
--- /dev/null
+++ b/roms/openbios/drivers/usbhid.c
@@ -0,0 +1,579 @@
+/*
+ * Driver for HID devices ported from CoreBoot
+ *
+ * Copyright (C) 2014 BALATON Zoltan
+ *
+ * This file was part of the libpayload project.
+ *
+ * Copyright (C) 2008-2010 coresystems GmbH
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "libopenbios/bindings.h"
+#include <libc/string.h>
+#include "libc/byteorder.h"
+#include "libc/vsprintf.h"
+#include "drivers/usb.h"
+#include "usb.h"
+
+DECLARE_UNNAMED_NODE(usb_kbd, INSTALL_OPEN, sizeof(int));
+
+static void
+keyboard_open(int *idx)
+{
+	RET(-1);
+}
+
+static void
+keyboard_close(int *idx)
+{
+}
+
+static void keyboard_read(void);
+
+NODE_METHODS( usb_kbd ) = {
+	{ "open",		keyboard_open		},
+	{ "close",		keyboard_close		},
+	{ "read",               keyboard_read		},
+};
+
+#ifdef CONFIG_USB_DEBUG
+static const char *boot_protos[3] = { "(none)", "keyboard", "mouse" };
+#endif
+typedef enum { hid_proto_boot = 0, hid_proto_report = 1 } hid_proto;
+enum { GET_REPORT = 0x1, GET_IDLE = 0x2, GET_PROTOCOL = 0x3, SET_REPORT =
+		0x9, SET_IDLE = 0xa, SET_PROTOCOL = 0xb
+};
+
+typedef union {
+	struct {
+		u8 modifiers;
+		u8 repeats;
+		u8 keys[6];
+	};
+	u8 buffer[8];
+} usb_hid_keyboard_event_t;
+
+typedef struct {
+	void* queue;
+	hid_descriptor_t *descriptor;
+
+	usb_hid_keyboard_event_t previous;
+	int lastkeypress;
+	int repeat_delay;
+} usbhid_inst_t;
+
+#define HID_INST(dev) ((usbhid_inst_t*)(dev)->data)
+
+static void
+usb_hid_destroy (usbdev_t *dev)
+{
+	if (HID_INST(dev)->queue) {
+		int i;
+		for (i = 0; i <= dev->num_endp; i++) {
+			if (dev->endpoints[i].endpoint == 0)
+				continue;
+			if (dev->endpoints[i].type != INTERRUPT)
+				continue;
+			if (dev->endpoints[i].direction != IN)
+				continue;
+			break;
+		}
+		dev->controller->destroy_intr_queue(
+				&dev->endpoints[i], HID_INST(dev)->queue);
+		HID_INST(dev)->queue = NULL;
+	}
+	free (dev->data);
+}
+
+/* keybuffer is global to all USB keyboards */
+static int keycount;
+#define KEYBOARD_BUFFER_SIZE 16
+static short keybuffer[KEYBOARD_BUFFER_SIZE];
+
+const char *countries[36][2] = {
+	{ "unknown", "us" },
+	{ "Arabic", "ae" },
+	{ "Belgian", "be" },
+	{ "Canadian-Bilingual", "ca" },
+	{ "Canadian-French", "ca" },
+	{ "Czech Republic", "cz" },
+	{ "Danish", "dk" },
+	{ "Finnish", "fi" },
+	{ "French", "fr" },
+	{ "German", "de" },
+	{ "Greek", "gr" },
+	{ "Hebrew", "il" },
+	{ "Hungary", "hu" },
+	{ "International (ISO)", "iso" },
+	{ "Italian", "it" },
+	{ "Japan (Katakana)", "jp" },
+	{ "Korean", "us" },
+	{ "Latin American", "us" },
+	{ "Netherlands/Dutch", "nl" },
+	{ "Norwegian", "no" },
+	{ "Persian (Farsi)", "ir" },
+	{ "Poland", "pl" },
+	{ "Portuguese", "pt" },
+	{ "Russia", "ru" },
+	{ "Slovakia", "sl" },
+	{ "Spanish", "es" },
+	{ "Swedish", "se" },
+	{ "Swiss/French", "ch" },
+	{ "Swiss/German", "ch" },
+	{ "Switzerland", "ch" },
+	{ "Taiwan", "tw" },
+	{ "Turkish-Q", "tr" },
+	{ "UK", "uk" },
+	{ "US", "us" },
+	{ "Yugoslavia", "yu" },
+	{ "Turkish-F", "tr" },
+	/* 36 - 255: Reserved */
+};
+
+
+
+struct layout_maps {
+	const char *country;
+	const short map[4][0x80];
+};
+
+static const struct layout_maps *map;
+
+#define KEY_BREAK     0x101  /* Not on PC KBD */
+#define KEY_DOWN      0x102  /* Down arrow key */
+#define KEY_UP        0x103  /* Up arrow key */
+#define KEY_LEFT      0x104  /* Left arrow key */
+#define KEY_RIGHT     0x105  /* Right arrow key */
+#define KEY_HOME      0x106  /* home key */
+#define KEY_BACKSPACE 0x107  /* not on pc */
+#define KEY_F0        0x108  /* function keys; 64 reserved */
+#define KEY_F(n)      (KEY_F0 + (n))
+
+#define KEY_DC        0x14a  /* delete character */
+#define KEY_IC        0x14b  /* insert char or enter ins mode */
+
+#define KEY_NPAGE     0x152  /* next page */
+#define KEY_PPAGE     0x153  /* previous page */
+
+#define KEY_ENTER     0x157  /* enter or send (unreliable) */
+
+#define KEY_PRINT     0x15a  /* print/copy */
+
+#define KEY_END       0x166  /* end key */
+
+static const struct layout_maps keyboard_layouts[] = {
+// #ifdef CONFIG_PC_KEYBOARD_LAYOUT_US
+{ .country = "us", .map = {
+	{ /* No modifier */
+	-1, -1, -1, -1, 'a', 'b', 'c', 'd',
+	'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
+	/* 0x10 */
+	'm', 'n', 'o', 'p', 'q', 'r', 's', 't',
+	'u', 'v', 'w', 'x', 'y', 'z', '1', '2',
+	/* 0x20 */
+	'3', '4', '5', '6', '7', '8', '9', '0',
+	'\n', '\e', '\b', '\t', ' ', '-', '=', '[',
+	/* 0x30 */
+	']', '\\', -1, ';', '\'', '`', ',', '.',
+	'/', -1 /* CapsLk */, KEY_F(1), KEY_F(2), KEY_F(3), KEY_F(4), KEY_F(5), KEY_F(6),
+	/* 0x40 */
+	KEY_F(7), KEY_F(8), KEY_F(9), KEY_F(10), KEY_F(11), KEY_F(12), KEY_PRINT, -1 /* ScrLk */,
+	KEY_BREAK, KEY_IC, KEY_HOME, KEY_PPAGE, KEY_DC, KEY_END, KEY_NPAGE, KEY_RIGHT,
+	/* 50 */
+	KEY_LEFT, KEY_DOWN, KEY_UP, -1 /*NumLck*/, '/', '*', '-' /* = ? */, '+',
+	KEY_ENTER, KEY_END, KEY_DOWN, KEY_NPAGE, KEY_LEFT, -1, KEY_RIGHT, KEY_HOME,
+	/* 60 */
+	KEY_UP, KEY_PPAGE, -1, KEY_DC, -1 /* < > | */, -1 /* Win Key Right */, -1, -1,
+	-1, -1, -1, -1, -1, -1, -1, -1,
+	/* 70 */
+	-1, -1, -1, -1, -1, -1, -1, -1,
+	-1, -1, -1, -1, -1, -1, -1, -1,
+	 },
+	{ /* Shift modifier */
+	-1, -1, -1, -1, 'A', 'B', 'C', 'D',
+	'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L',
+	/* 0x10 */
+	'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',
+	'U', 'V', 'W', 'X', 'Y', 'Z', '!', '@',
+	/* 0x20 */
+	'#', '$', '%', '^', '&', '*', '(', ')',
+	'\n', '\e', '\b', '\t', ' ', '_', '+', '{',
+	/* 0x30 */
+	'}', '|', -1, ':', '"', '~', '<', '>',
+	'?', -1 /* CapsLk */, KEY_F(1), KEY_F(2), KEY_F(3), KEY_F(4), KEY_F(5), KEY_F(6),
+	/* 0x40 */
+	KEY_F(7), KEY_F(8), KEY_F(9), KEY_F(10), KEY_F(11), KEY_F(12), KEY_PRINT, -1 /* ScrLk */,
+	KEY_BREAK, KEY_IC, KEY_HOME, KEY_PPAGE, KEY_DC, KEY_END, KEY_NPAGE, KEY_RIGHT,
+	/* 50 */
+	KEY_LEFT, KEY_DOWN, KEY_UP, -1 /*NumLck*/, '/', '*', '-' /* = ? */, '+',
+	KEY_ENTER, KEY_END, KEY_DOWN, KEY_NPAGE, KEY_LEFT, -1, KEY_RIGHT, KEY_HOME,
+	/* 60 */
+	KEY_UP, KEY_PPAGE, -1, KEY_DC, -1 /* < > | */, -1 /* Win Key Right */, -1, -1,
+	-1, -1, -1, -1, -1, -1, -1, -1,
+	/* 70 */
+	-1, -1, -1, -1, -1, -1, -1, -1,
+	-1, -1, -1, -1, -1, -1, -1, -1,
+	 },
+	{ /* Alt */
+	-1, -1, -1, -1, 'a', 'b', 'c', 'd',
+	'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
+	/* 0x10 */
+	'm', 'n', 'o', 'p', 'q', 'r', 's', 't',
+	'u', 'v', 'w', 'x', 'y', 'z', '1', '2',
+	/* 0x20 */
+	'3', '4', '5', '6', '7', '8', '9', '0',
+	'\n', '\e', '\b', '\t', ' ', '-', '=', '[',
+	/* 0x30 */
+	']', '\\', -1, ';', '\'', '`', ',', '.',
+	'/', -1 /* CapsLk */, KEY_F(1), KEY_F(2), KEY_F(3), KEY_F(4), KEY_F(5), KEY_F(6),
+	/* 0x40 */
+	KEY_F(7), KEY_F(8), KEY_F(9), KEY_F(10), KEY_F(11), KEY_F(12), KEY_PRINT, -1 /* ScrLk */,
+	KEY_BREAK, KEY_IC, KEY_HOME, KEY_PPAGE, KEY_DC, KEY_END, KEY_NPAGE, KEY_RIGHT,
+	/* 50 */
+	KEY_LEFT, KEY_DOWN, KEY_UP, -1 /*NumLck*/, '/', '*', '-' /* = ? */, '+',
+	KEY_ENTER, KEY_END, KEY_DOWN, KEY_NPAGE, KEY_LEFT, -1, KEY_RIGHT, KEY_HOME,
+	/* 60 */
+	KEY_UP, KEY_PPAGE, -1, KEY_DC, -1 /* < > | */, -1 /* Win Key Right */, -1, -1,
+	-1, -1, -1, -1, -1, -1, -1, -1,
+	/* 70 */
+	-1, -1, -1, -1, -1, -1, -1, -1,
+	-1, -1, -1, -1, -1, -1, -1, -1,
+	 },
+	{ /* Shift+Alt modifier */
+	-1, -1, -1, -1, 'A', 'B', 'C', 'D',
+	'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L',
+	/* 0x10 */
+	'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',
+	'U', 'V', 'W', 'X', 'Y', 'Z', '!', '@',
+	/* 0x20 */
+	'#', '$', '%', '^', '&', '*', '(', ')',
+	'\n', '\e', '\b', '\t', ' ', '-', '=', '[',
+	/* 0x30 */
+	']', '\\', -1, ':', '\'', '`', ',', '.',
+	'/', -1 /* CapsLk */, KEY_F(1), KEY_F(2), KEY_F(3), KEY_F(4), KEY_F(5), KEY_F(6),
+	/* 0x40 */
+	KEY_F(7), KEY_F(8), KEY_F(9), KEY_F(10), KEY_F(11), KEY_F(12), KEY_PRINT, -1 /* ScrLk */,
+	KEY_BREAK, KEY_IC, KEY_HOME, KEY_PPAGE, KEY_DC, KEY_END, KEY_NPAGE, KEY_RIGHT,
+	/* 50 */
+	KEY_LEFT, KEY_DOWN, KEY_UP, -1 /*NumLck*/, '/', '*', '-' /* = ? */, '+',
+	KEY_ENTER, KEY_END, KEY_DOWN, KEY_NPAGE, KEY_LEFT, -1, KEY_RIGHT, KEY_HOME,
+	/* 60 */
+	KEY_UP, KEY_PPAGE, -1, KEY_DC, -1 /* < > | */, -1 /* Win Key Right */, -1, -1,
+	-1, -1, -1, -1, -1, -1, -1, -1,
+	/* 70 */
+	-1, -1, -1, -1, -1, -1, -1, -1,
+	-1, -1, -1, -1, -1, -1, -1, -1,
+	 }
+}},
+//#endif
+};
+
+#define MOD_SHIFT    (1 << 0)
+#define MOD_ALT      (1 << 1)
+#define MOD_CTRL     (1 << 2)
+
+static void usb_hid_keyboard_queue(int ch) {
+	/* ignore key presses if buffer full */
+	if (keycount < KEYBOARD_BUFFER_SIZE)
+		keybuffer[keycount++] = ch;
+}
+
+#define KEYBOARD_REPEAT_MS	30
+#define INITIAL_REPEAT_DELAY	10
+#define REPEAT_DELAY		 2
+
+static void
+usb_hid_process_keyboard_event(usbhid_inst_t *const inst,
+		const usb_hid_keyboard_event_t *const current)
+{
+	const usb_hid_keyboard_event_t *const previous = &inst->previous;
+
+	int i, keypress = 0, modifiers = 0;
+
+	if (current->modifiers & 0x01) /* Left-Ctrl */   modifiers |= MOD_CTRL;
+	if (current->modifiers & 0x02) /* Left-Shift */  modifiers |= MOD_SHIFT;
+	if (current->modifiers & 0x04) /* Left-Alt */    modifiers |= MOD_ALT;
+	if (current->modifiers & 0x08) /* Left-GUI */    ;
+	if (current->modifiers & 0x10) /* Right-Ctrl */  modifiers |= MOD_CTRL;
+	if (current->modifiers & 0x20) /* Right-Shift */ modifiers |= MOD_SHIFT;
+	if (current->modifiers & 0x40) /* Right-AltGr */ modifiers |= MOD_ALT;
+	if (current->modifiers & 0x80) /* Right-GUI */   ;
+
+	/* Did the event change at all? */
+	if (inst->lastkeypress &&
+			!memcmp(current, previous, sizeof(*current))) {
+		/* No. Then it's a key repeat event. */
+		if (inst->repeat_delay) {
+			inst->repeat_delay--;
+		} else {
+			usb_hid_keyboard_queue(inst->lastkeypress);
+			inst->repeat_delay = REPEAT_DELAY;
+		}
+
+		return;
+	}
+
+	inst->lastkeypress = 0;
+
+	for (i=0; i<6; i++) {
+		int j;
+		int skip = 0;
+		// No more keys? skip
+		if (current->keys[i] == 0)
+			return;
+
+		for (j=0; j<6; j++) {
+			if (current->keys[i] == previous->keys[j]) {
+				skip = 1;
+				break;
+			}
+		}
+		if (skip)
+			continue;
+
+
+		/* Mask off MOD_CTRL */
+		keypress = map->map[modifiers & 0x03][current->keys[i]];
+
+		if (modifiers & MOD_CTRL) {
+			switch (keypress) {
+			case 'a' ... 'z':
+				keypress &= 0x1f;
+				break;
+			default:
+				continue;
+			}
+		}
+
+		if (keypress == -1) {
+			/* Debug: Print unknown keys */
+			usb_debug ("usbhid: <%x> %x [ %x %x %x %x %x %x ] %d\n",
+				current->modifiers, current->repeats,
+			current->keys[0], current->keys[1],
+			current->keys[2], current->keys[3],
+			current->keys[4], current->keys[5], i);
+
+			/* Unknown key? Try next one in the queue */
+			continue;
+		}
+
+		usb_hid_keyboard_queue(keypress);
+
+		/* Remember for authentic key repeat */
+		inst->lastkeypress = keypress;
+		inst->repeat_delay = INITIAL_REPEAT_DELAY;
+	}
+}
+
+static void
+usb_hid_poll (usbdev_t *dev)
+{
+	usb_hid_keyboard_event_t current;
+	const u8 *buf;
+
+	while ((buf=dev->controller->poll_intr_queue (HID_INST(dev)->queue))) {
+		memcpy(&current.buffer, buf, 8);
+		usb_hid_process_keyboard_event(HID_INST(dev), &current);
+		HID_INST(dev)->previous = current;
+	}
+}
+
+static void
+usb_hid_set_idle (usbdev_t *dev, interface_descriptor_t *interface, u16 duration)
+{
+	dev_req_t dr;
+	dr.data_dir = host_to_device;
+	dr.req_type = class_type;
+	dr.req_recp = iface_recp;
+	dr.bRequest = SET_IDLE;
+	dr.wValue = __cpu_to_le16((duration >> 2) << 8);
+	dr.wIndex = __cpu_to_le16(interface->bInterfaceNumber);
+	dr.wLength = 0;
+	dev->controller->control (dev, OUT, sizeof (dev_req_t), &dr, 0, 0);
+}
+
+static void
+usb_hid_set_protocol (usbdev_t *dev, interface_descriptor_t *interface, hid_proto proto)
+{
+	dev_req_t dr;
+	dr.data_dir = host_to_device;
+	dr.req_type = class_type;
+	dr.req_recp = iface_recp;
+	dr.bRequest = SET_PROTOCOL;
+	dr.wValue = __cpu_to_le16(proto);
+	dr.wIndex = __cpu_to_le16(interface->bInterfaceNumber);
+	dr.wLength = 0;
+	dev->controller->control (dev, OUT, sizeof (dev_req_t), &dr, 0, 0);
+}
+
+static int usb_hid_set_layout (const char *country)
+{
+	/* FIXME should be per keyboard */
+	int i;
+
+	for (i=0; i<sizeof(keyboard_layouts)/sizeof(keyboard_layouts[0]); i++) {
+		if (strncmp(keyboard_layouts[i].country, country,
+					strlen(keyboard_layouts[i].country)))
+			continue;
+
+		/* Found, changing keyboard layout */
+		map = &keyboard_layouts[i];
+		usb_debug("  Keyboard layout '%s'\n", map->country);
+		return 0;
+	}
+
+	usb_debug("  Keyboard layout '%s' not found, using '%s'\n",
+			country, map->country);
+
+	/* Nothing found, not changed */
+	return -1;
+}
+
+void
+usb_hid_init (usbdev_t *dev)
+{
+	configuration_descriptor_t *cd = (configuration_descriptor_t*)dev->configuration;
+	interface_descriptor_t *interface = (interface_descriptor_t*)(((char *) cd) + cd->bLength);
+
+	if (interface->bInterfaceSubClass == hid_subclass_boot) {
+		u8 countrycode = 0;
+		usb_debug ("  supports boot interface..\n");
+		usb_debug ("  it's a %s\n",
+			boot_protos[interface->bInterfaceProtocol]);
+		switch (interface->bInterfaceProtocol) {
+		case hid_boot_proto_keyboard:
+			dev->data = malloc (sizeof (usbhid_inst_t));
+			if (!dev->data) {
+				printk("Not enough memory for USB HID device.\n");
+				return;
+                        }
+			memset(&HID_INST(dev)->previous, 0x00,
+			       sizeof(HID_INST(dev)->previous));
+			usb_debug ("  configuring...\n");
+			usb_hid_set_protocol(dev, interface, hid_proto_boot);
+			usb_hid_set_idle(dev, interface, KEYBOARD_REPEAT_MS);
+			usb_debug ("  activating...\n");
+#if 0
+			HID_INST (dev)->descriptor =
+				(hid_descriptor_t *)
+					get_descriptor(dev, gen_bmRequestType
+					(device_to_host, standard_type, iface_recp),
+					0x21, 0, 0);
+			countrycode = HID_INST(dev)->descriptor->bCountryCode;
+#endif
+			/* 35 countries defined: */
+			if (countrycode > 35)
+				countrycode = 0;
+			usb_debug ("  Keyboard has %s layout (country code %02x)\n",
+					countries[countrycode][0], countrycode);
+
+			/* Set keyboard layout accordingly */
+			usb_hid_set_layout(countries[countrycode][1]);
+
+			// only add here, because we only support boot-keyboard HID devices
+			dev->destroy = usb_hid_destroy;
+			dev->poll = usb_hid_poll;
+			int i;
+			for (i = 0; i <= dev->num_endp; i++) {
+				if (dev->endpoints[i].endpoint == 0)
+					continue;
+				if (dev->endpoints[i].type != INTERRUPT)
+					continue;
+				if (dev->endpoints[i].direction != IN)
+					continue;
+				break;
+			}
+			usb_debug ("  found endpoint %x for interrupt-in\n", i);
+			/* 20 buffers of 8 bytes, for every 10 msecs */
+			HID_INST(dev)->queue = dev->controller->create_intr_queue (&dev->endpoints[i], 8, 20, 10);
+			keycount = 0;
+			usb_debug ("  configuration done.\n");
+			break;
+		default:
+			usb_debug("NOTICE: HID interface protocol %d%s not supported.\n",
+				 interface->bInterfaceProtocol,
+				 (interface->bInterfaceProtocol == hid_boot_proto_mouse ?
+				 " (USB mouse)" : ""));
+			break;
+		}
+	}
+}
+
+static int usbhid_havechar (void)
+{
+	return (keycount != 0);
+}
+
+static int usbhid_getchar (void)
+{
+	short ret;
+
+	if (keycount == 0)
+		return 0;
+	ret = keybuffer[0];
+	memmove(keybuffer, keybuffer + 1, --keycount);
+
+	return (int)ret;
+}
+
+/* ( addr len -- actual ) */
+static void keyboard_read(void)
+{
+	char *addr;
+	int len, key, i;
+
+	usb_poll();
+	len=POP();
+	addr=(char *)cell2pointer(POP());
+
+	for (i = 0; i < len; i++) {
+	    if (!usbhid_havechar())
+			break;
+		key = usbhid_getchar();
+		*addr++ = (char)key;
+	}
+	PUSH(i);
+}
+
+void ob_usb_hid_add_keyboard(const char *path)
+{
+	char name[128];
+	phandle_t aliases;
+
+	snprintf(name, sizeof(name), "%s/keyboard", path);
+	usb_debug("Found keyboard at %s\n", name);
+	REGISTER_NAMED_NODE(usb_kbd, name);
+
+	push_str(name);
+	fword("find-device");
+
+	push_str("keyboard");
+	fword("device-type");
+
+	aliases = find_dev("/aliases");
+	set_property(aliases, "adb-keyboard", name, strlen(name) + 1);
+}
diff --git a/roms/openbios/drivers/usbohci.c b/roms/openbios/drivers/usbohci.c
new file mode 100644
index 000000000..774164b0b
--- /dev/null
+++ b/roms/openbios/drivers/usbohci.c
@@ -0,0 +1,926 @@
+/*
+ * Driver for USB OHCI ported from CoreBoot
+ *
+ * Copyright (C) 2014 BALATON Zoltan
+ *
+ * This file was part of the libpayload project.
+ *
+ * Copyright (C) 2010 Patrick Georgi
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+//#define USB_DEBUG_ED
+
+#include "config.h"
+#include <asm/io.h>
+#include <libopenbios/ofmem.h>
+#include "timer.h"
+#include "drivers/pci.h"
+#include "pci.h"
+#include <drivers/usb.h>
+#include "usbohci_private.h"
+#include "usbohci.h"
+
+static void ohci_start (hci_t *controller);
+static void ohci_stop (hci_t *controller);
+static void ohci_reset (hci_t *controller);
+static void ohci_shutdown (hci_t *controller);
+static int ohci_bulk (endpoint_t *ep, int size, u8 *data, int finalize);
+static int ohci_control (usbdev_t *dev, direction_t dir, int drlen, void *devreq,
+			 int dalen, u8 *data);
+static void* ohci_create_intr_queue (endpoint_t *ep, int reqsize, int reqcount, int reqtiming);
+static void ohci_destroy_intr_queue (endpoint_t *ep, void *queue);
+static u8* ohci_poll_intr_queue (void *queue);
+static void ohci_process_done_queue(ohci_t *ohci, int spew_debug);
+
+#ifdef USB_DEBUG_ED
+static void
+dump_td (td_t *cur)
+{
+	usb_debug("+---------------------------------------------------+\n");
+	if (((__le32_to_cpu(cur->config) & (3UL << 19)) >> 19) == 0)
+		usb_debug("|..[SETUP]..........................................|\n");
+	else if (((__le32_to_cpu(cur->config) & (3UL << 8)) >> 8) == 2)
+		usb_debug("|..[IN].............................................|\n");
+	else if (((__le32_to_cpu(cur->config) & (3UL << 8)) >> 8) == 1)
+		usb_debug("|..[OUT]............................................|\n");
+	else
+		usb_debug("|..[]...............................................|\n");
+	usb_debug("|:|============ OHCI TD at [0x%08lx] ==========|:|\n", virt_to_phys(cur));
+	usb_debug("|:| ERRORS = [%ld] | CONFIG = [0x%08x] |        |:|\n",
+		  3 - ((__le32_to_cpu(cur->config) & (3UL << 26)) >> 26), __le32_to_cpu(cur->config));
+	usb_debug("|:+-----------------------------------------------+:|\n");
+	usb_debug("|:|   C   | Condition Code               |   [%02ld] |:|\n",
+		 (__le32_to_cpu(cur->config) & (0xFUL << 28)) >> 28);
+	usb_debug("|:|   O   | Direction/PID                |    [%ld] |:|\n",
+		 (__le32_to_cpu(cur->config) & (3UL << 19)) >> 19);
+	usb_debug("|:|   N   | Buffer Rounding              |    [%ld] |:|\n",
+		 (__le32_to_cpu(cur->config) & (1UL << 18)) >> 18);
+	usb_debug("|:|   F   | Delay Intterrupt             |    [%ld] |:|\n",
+		 (__le32_to_cpu(cur->config) & (7UL << 21)) >> 21);
+	usb_debug("|:|   I   | Data Toggle                  |    [%ld] |:|\n",
+		 (__le32_to_cpu(cur->config) & (3UL << 24)) >> 24);
+	usb_debug("|:|   G   | Error Count                  |    [%ld] |:|\n",
+		 (__le32_to_cpu(cur->config) & (3UL << 26)) >> 26);
+	usb_debug("|:+-----------------------------------------------+:|\n");
+	usb_debug("|:| Current Buffer Pointer         [0x%08x]   |:|\n", __le32_to_cpu(cur->current_buffer_pointer));
+	usb_debug("|:+-----------------------------------------------+:|\n");
+	usb_debug("|:| Next TD                        [0x%08x]   |:|\n", __le32_to_cpu(cur->next_td));
+	usb_debug("|:+-----------------------------------------------+:|\n");
+	usb_debug("|:| Current Buffer End             [0x%08x]   |:|\n", __le32_to_cpu(cur->buffer_end));
+	usb_debug("|:|-----------------------------------------------|:|\n");
+	usb_debug("|...................................................|\n");
+	usb_debug("+---------------------------------------------------+\n");
+}
+
+static void
+dump_ed (ed_t *cur)
+{
+	td_t *tmp_td = NULL;
+	usb_debug("+===================================================+\n");
+	usb_debug("| ############# OHCI ED at [0x%08lx] ########### |\n", virt_to_phys(cur));
+	usb_debug("+---------------------------------------------------+\n");
+	usb_debug("| Next Endpoint Descriptor       [0x%08lx]       |\n", __le32_to_cpu(cur->next_ed) & ~0xFUL);
+	usb_debug("+---------------------------------------------------+\n");
+	usb_debug("|        |               @ 0x%08x :             |\n", __le32_to_cpu(cur->config));
+	usb_debug("|   C    | Maximum Packet Length           | [%04ld] |\n",
+		 ((__le32_to_cpu(cur->config) & (0x3fffUL << 16)) >> 16));
+	usb_debug("|   O    | Function Address                | [%04d] |\n",
+		 __le32_to_cpu(cur->config) & 0x7F);
+	usb_debug("|   N    | Endpoint Number                 |   [%02ld] |\n",
+		 (__le32_to_cpu(cur->config) & (0xFUL << 7)) >> 7);
+	usb_debug("|   F    | Endpoint Direction              |    [%ld] |\n",
+		 ((__le32_to_cpu(cur->config) & (3UL << 11)) >> 11));
+	usb_debug("|   I    | Endpoint Speed                  |    [%ld] |\n",
+		 ((__le32_to_cpu(cur->config) & (1UL << 13)) >> 13));
+	usb_debug("|   G    | Skip                            |    [%ld] |\n",
+		 ((__le32_to_cpu(cur->config) & (1UL << 14)) >> 14));
+	usb_debug("|        | Format                          |    [%ld] |\n",
+		 ((__le32_to_cpu(cur->config) & (1UL << 15)) >> 15));
+	usb_debug("+---------------------------------------------------+\n");
+	usb_debug("| TD Queue Tail Pointer          [0x%08lx]       |\n",
+		 __le32_to_cpu(cur->tail_pointer) & ~0xFUL);
+	usb_debug("+---------------------------------------------------+\n");
+	usb_debug("| TD Queue Head Pointer          [0x%08lx]       |\n",
+		 __le32_to_cpu(cur->head_pointer) & ~0xFUL);
+	usb_debug("| CarryToggleBit    [%d]          Halted   [%d]       |\n",
+		 (u16)(__le32_to_cpu(cur->head_pointer) & 0x2UL)>>1, (u16)(__le32_to_cpu(cur->head_pointer) & 0x1UL));
+
+	tmp_td = (td_t *)phys_to_virt((__le32_to_cpu(cur->head_pointer) & ~0xFUL));
+	if ((__le32_to_cpu(cur->head_pointer) & ~0xFUL) != (__le32_to_cpu(cur->tail_pointer) & ~0xFUL)) {
+		usb_debug("|:::::::::::::::::: OHCI TD CHAIN ::::::::::::::::::|\n");
+		while (virt_to_phys(tmp_td) != (__le32_to_cpu(cur->tail_pointer) & ~0xFUL))
+		{
+			dump_td(tmp_td);
+			tmp_td = (td_t *)phys_to_virt((__le32_to_cpu(tmp_td->next_td) & ~0xFUL));
+		}
+		usb_debug("|:::::::::::::::: EOF OHCI TD CHAIN ::::::::::::::::|\n");
+		usb_debug("+---------------------------------------------------+\n");
+	} else {
+		usb_debug("+---------------------------------------------------+\n");
+	}
+}
+#endif
+
+static void
+ohci_reset (hci_t *controller)
+{
+	if (controller == NULL)
+		return;
+
+	OHCI_INST(controller)->opreg->HcCommandStatus = __cpu_to_le32(HostControllerReset);
+	mdelay(2); /* wait 2ms */
+	OHCI_INST(controller)->opreg->HcControl = 0;
+	mdelay(10); /* wait 10ms */
+}
+
+static void
+ohci_reinit (hci_t *controller)
+{
+}
+
+hci_t *
+ohci_init (void *bar)
+{
+	int i;
+
+	hci_t *controller = new_controller ();
+
+	if (!controller) {
+		printk("Could not create USB controller instance.\n");
+		return NULL;
+        }
+
+	controller->instance = malloc (sizeof (ohci_t));
+	if(!controller->instance) {
+		printk("Not enough memory creating USB controller instance.\n");
+                return NULL;
+        }
+
+	controller->type = OHCI;
+
+	controller->start = ohci_start;
+	controller->stop = ohci_stop;
+	controller->reset = ohci_reset;
+	controller->init = ohci_reinit;
+	controller->shutdown = ohci_shutdown;
+	controller->bulk = ohci_bulk;
+	controller->control = ohci_control;
+	controller->set_address = generic_set_address;
+	controller->finish_device_config = NULL;
+	controller->destroy_device = NULL;
+	controller->create_intr_queue = ohci_create_intr_queue;
+	controller->destroy_intr_queue = ohci_destroy_intr_queue;
+	controller->poll_intr_queue = ohci_poll_intr_queue;
+	for (i = 0; i < 128; i++) {
+		controller->devices[i] = 0;
+	}
+	init_device_entry (controller, 0);
+	OHCI_INST (controller)->roothub = controller->devices[0];
+
+	controller->reg_base = (u32)(unsigned long)bar;
+	OHCI_INST (controller)->opreg = (opreg_t*)phys_to_virt(controller->reg_base);
+	usb_debug("OHCI Version %x.%x\n",
+		  (READ_OPREG(OHCI_INST(controller), HcRevision) >> 4) & 0xf,
+		  READ_OPREG(OHCI_INST(controller), HcRevision) & 0xf);
+
+	if ((READ_OPREG(OHCI_INST(controller), HcControl) & HostControllerFunctionalStateMask) == USBReset) {
+		/* cold boot */
+		OHCI_INST (controller)->opreg->HcControl &= __cpu_to_le32(~RemoteWakeupConnected);
+		OHCI_INST (controller)->opreg->HcFmInterval =
+			__cpu_to_le32((11999 * FrameInterval) | ((((11999 - 210)*6)/7) * FSLargestDataPacket));
+		/* TODO: right value for PowerOnToPowerGoodTime ? */
+		OHCI_INST (controller)->opreg->HcRhDescriptorA =
+			__cpu_to_le32(NoPowerSwitching | NoOverCurrentProtection | (10 * PowerOnToPowerGoodTime));
+		OHCI_INST (controller)->opreg->HcRhDescriptorB = __cpu_to_le32(0 * DeviceRemovable);
+		udelay(100); /* TODO: reset asserting according to USB spec */
+	} else if ((READ_OPREG(OHCI_INST(controller), HcControl) & HostControllerFunctionalStateMask) != USBOperational) {
+		OHCI_INST (controller)->opreg->HcControl =
+			__cpu_to_le32((READ_OPREG(OHCI_INST(controller), HcControl) & ~HostControllerFunctionalStateMask)
+			| USBResume);
+		udelay(100); /* TODO: resume time according to USB spec */
+	}
+	int interval = OHCI_INST (controller)->opreg->HcFmInterval;
+
+	OHCI_INST (controller)->opreg->HcCommandStatus = __cpu_to_le32(HostControllerReset);
+	udelay (10); /* at most 10us for reset to complete. State must be set to Operational within 2ms (5.1.1.4) */
+	OHCI_INST (controller)->opreg->HcFmInterval = interval;
+	ofmem_posix_memalign((void **)&(OHCI_INST (controller)->hcca), 256, 256);
+	memset((void*)OHCI_INST (controller)->hcca, 0, 256);
+
+	usb_debug("HCCA addr %p\n", OHCI_INST(controller)->hcca);
+	/* Initialize interrupt table. */
+	u32 *const intr_table = OHCI_INST(controller)->hcca->HccaInterruptTable;
+	ed_t *const periodic_ed;
+        ofmem_posix_memalign((void **)&periodic_ed, sizeof(ed_t), sizeof(ed_t));
+	memset((void *)periodic_ed, 0, sizeof(*periodic_ed));
+	for (i = 0; i < 32; ++i)
+		intr_table[i] = __cpu_to_le32(virt_to_phys(periodic_ed));
+	OHCI_INST (controller)->periodic_ed = periodic_ed;
+
+	OHCI_INST (controller)->opreg->HcHCCA = __cpu_to_le32(virt_to_phys(OHCI_INST(controller)->hcca));
+	/* Make sure periodic schedule is enabled. */
+	OHCI_INST (controller)->opreg->HcControl |= __cpu_to_le32(PeriodicListEnable);
+	OHCI_INST (controller)->opreg->HcControl &= __cpu_to_le32(~IsochronousEnable); // unused by this driver
+	// disable everything, contrary to what OHCI spec says in 5.1.1.4, as we don't need IRQs
+	OHCI_INST (controller)->opreg->HcInterruptEnable = __cpu_to_le32(1<<31);
+	OHCI_INST (controller)->opreg->HcInterruptDisable = __cpu_to_le32(~(1<<31));
+	OHCI_INST (controller)->opreg->HcInterruptStatus = __cpu_to_le32(~0);
+	OHCI_INST (controller)->opreg->HcPeriodicStart =
+		__cpu_to_le32((READ_OPREG(OHCI_INST(controller), HcFmInterval) & FrameIntervalMask) / 10 * 9);
+	OHCI_INST (controller)->opreg->HcControl = __cpu_to_le32((READ_OPREG(OHCI_INST(controller), HcControl)
+								& ~HostControllerFunctionalStateMask) | USBOperational);
+
+	mdelay(100);
+
+	controller->devices[0]->controller = controller;
+	controller->devices[0]->init = ohci_rh_init;
+	controller->devices[0]->init (controller->devices[0]);
+	return controller;
+}
+
+hci_t *
+ohci_pci_init (pci_addr addr)
+{
+	u32 reg_base;
+	uint16_t cmd;
+
+	cmd = pci_config_read16(addr, PCI_COMMAND);
+	cmd |= PCI_COMMAND_BUS_MASTER;
+	pci_config_write16(addr, PCI_COMMAND, cmd);
+
+	/* regarding OHCI spec, Appendix A, BAR_OHCI register description, Table A-4
+	 * BASE ADDRESS only [31-12] bits. All other usually 0, but not all.
+	 * OHCI mandates MMIO, so bit 0 is clear */
+	reg_base = pci_config_read32 (addr, PCI_BASE_ADDR_0) & 0xfffff000;
+
+	return ohci_init((void *)(unsigned long)reg_base);
+}
+
+static void
+ohci_shutdown (hci_t *controller)
+{
+	if (controller == 0)
+		return;
+	detach_controller (controller);
+	ohci_stop(controller);
+	OHCI_INST (controller)->roothub->destroy (OHCI_INST (controller)->
+						  roothub);
+	controller->reset (controller);
+	free ((void *)OHCI_INST (controller)->periodic_ed);
+	free (OHCI_INST (controller));
+	free (controller);
+}
+
+static void
+ohci_start (hci_t *controller)
+{
+// TODO: turn on all operation of OHCI, but assume that it's initialized.
+}
+
+static void
+ohci_stop (hci_t *controller)
+{
+// TODO: turn off all operation of OHCI
+}
+
+static int
+wait_for_ed(usbdev_t *dev, ed_t *head, int pages)
+{
+	usb_debug("Waiting for %d pages on dev %p with head %p\n", pages, dev, head);
+	/* wait for results */
+	/* TOTEST: how long to wait?
+	 *         give 2s per TD (2 pages) plus another 2s for now
+	 */
+	int timeout = pages*1000 + 2000;
+	while (((__le32_to_cpu(head->head_pointer) & ~3) != __le32_to_cpu(head->tail_pointer)) &&
+		!(__le32_to_cpu(head->head_pointer) & 1) &&
+		((__le32_to_cpu((((td_t*)phys_to_virt(__le32_to_cpu(head->head_pointer) & ~3)))->config)
+		& TD_CC_MASK) >= TD_CC_NOACCESS) && timeout--) {
+		/* don't log every ms */
+		if (!(timeout % 100))
+		usb_debug("intst: %x; ctrl: %x; cmdst: %x; head: %x -> %x, tail: %x, condition: %x\n",
+			  READ_OPREG(OHCI_INST(dev->controller), HcInterruptStatus),
+			  READ_OPREG(OHCI_INST(dev->controller), HcControl),
+			  READ_OPREG(OHCI_INST(dev->controller), HcCommandStatus),
+			__le32_to_cpu(head->head_pointer),
+			__le32_to_cpu(((td_t*)phys_to_virt(__le32_to_cpu(head->head_pointer) & ~3))->next_td),
+			__le32_to_cpu(head->tail_pointer),
+			(__le32_to_cpu(((td_t*)phys_to_virt(__le32_to_cpu(head->head_pointer) & ~3))->config) & TD_CC_MASK) >> TD_CC_SHIFT);
+		mdelay(1);
+	}
+	if (timeout < 0)
+		usb_debug("Error: ohci: endpoint "
+			"descriptor processing timed out.\n");
+	/* Clear the done queue. */
+	ohci_process_done_queue(OHCI_INST(dev->controller), 1);
+
+	if (__le32_to_cpu(head->head_pointer) & 1) {
+		usb_debug("HALTED!\n");
+		return 1;
+	}
+	return 0;
+}
+
+static void
+ohci_free_ed (ed_t *const head)
+{
+	/* In case the transfer canceled, we have to free unprocessed TDs. */
+	while ((__le32_to_cpu(head->head_pointer) & ~0x3) != __le32_to_cpu(head->tail_pointer)) {
+		/* Save current TD pointer. */
+		td_t *const cur_td =
+			(td_t*)phys_to_virt(__le32_to_cpu(head->head_pointer) & ~0x3);
+		/* Advance head pointer. */
+		head->head_pointer = cur_td->next_td;
+		/* Free current TD. */
+		free((void *)cur_td);
+	}
+
+	/* Always free the dummy TD */
+	if ((__le32_to_cpu(head->head_pointer) & ~0x3) == __le32_to_cpu(head->tail_pointer))
+		free(phys_to_virt(__le32_to_cpu(head->head_pointer) & ~0x3));
+	/* and the ED. */
+	free((void *)head);
+}
+
+static int
+ohci_control (usbdev_t *dev, direction_t dir, int drlen, void *devreq, int dalen,
+	      unsigned char *data)
+{
+	td_t *cur;
+
+	// pages are specified as 4K in OHCI, so don't use getpagesize()
+	int first_page = (unsigned long)data / 4096;
+	int last_page = (unsigned long)(data+dalen-1)/4096;
+	if (last_page < first_page) last_page = first_page;
+	int pages = (dalen==0)?0:(last_page - first_page + 1);
+
+	/* First TD. */
+	td_t *const first_td;
+        ofmem_posix_memalign((void **)&first_td, sizeof(td_t), sizeof(td_t));
+	memset((void *)first_td, 0, sizeof(*first_td));
+	cur = first_td;
+
+	cur->config = __cpu_to_le32(TD_DIRECTION_SETUP |
+		TD_DELAY_INTERRUPT_NOINTR |
+		TD_TOGGLE_FROM_TD |
+		TD_TOGGLE_DATA0 |
+		TD_CC_NOACCESS);
+	cur->current_buffer_pointer = __cpu_to_le32(virt_to_phys(devreq));
+	cur->buffer_end = __cpu_to_le32(virt_to_phys((char *)devreq + drlen - 1));
+
+	while (pages > 0) {
+		/* One more TD. */
+		td_t *const next;
+		ofmem_posix_memalign((void **)&next, sizeof(td_t), sizeof(td_t));
+		memset((void *)next, 0, sizeof(*next));
+		/* Linked to the previous. */
+		cur->next_td = __cpu_to_le32(virt_to_phys(next));
+		/* Advance to the new TD. */
+		cur = next;
+
+		cur->config = __cpu_to_le32((dir == IN ? TD_DIRECTION_IN : TD_DIRECTION_OUT) |
+			TD_DELAY_INTERRUPT_NOINTR |
+			TD_TOGGLE_FROM_ED |
+			TD_CC_NOACCESS);
+		cur->current_buffer_pointer = __cpu_to_le32(virt_to_phys(data));
+		pages--;
+		int consumed = (4096 - ((unsigned long)data % 4096));
+		if (consumed >= dalen) {
+			// end of data is within same page
+			cur->buffer_end = __cpu_to_le32(virt_to_phys(data + dalen - 1));
+			dalen = 0;
+			/* assert(pages == 0); */
+		} else {
+			dalen -= consumed;
+			data += consumed;
+			pages--;
+			int second_page_size = dalen;
+			if (dalen > 4096) {
+				second_page_size = 4096;
+			}
+			cur->buffer_end = __cpu_to_le32(virt_to_phys(data + second_page_size - 1));
+			dalen -= second_page_size;
+			data += second_page_size;
+		}
+	}
+
+	/* One more TD. */
+	td_t *const next_td;
+	ofmem_posix_memalign((void **)&next_td, sizeof(td_t), sizeof(td_t));
+	memset((void *)next_td, 0, sizeof(*next_td));
+	/* Linked to the previous. */
+	cur->next_td = __cpu_to_le32(virt_to_phys(next_td));
+	/* Advance to the new TD. */
+	cur = next_td;
+	cur->config = __cpu_to_le32((dir == IN ? TD_DIRECTION_OUT : TD_DIRECTION_IN) |
+		TD_DELAY_INTERRUPT_ZERO | /* Write done head after this TD. */
+		TD_TOGGLE_FROM_TD |
+		TD_TOGGLE_DATA1 |
+		TD_CC_NOACCESS);
+	cur->current_buffer_pointer = 0;
+	cur->buffer_end = 0;
+
+	/* Final dummy TD. */
+	td_t *const final_td;
+	ofmem_posix_memalign((void **)&final_td, sizeof(td_t), sizeof(td_t));
+	memset((void *)final_td, 0, sizeof(*final_td));
+	/* Linked to the previous. */
+	cur->next_td = __cpu_to_le32(virt_to_phys(final_td));
+
+	/* Data structures */
+	ed_t *head;
+	ofmem_posix_memalign((void **)&head, sizeof(ed_t), sizeof(ed_t));
+	memset((void*)head, 0, sizeof(*head));
+	head->config = __cpu_to_le32((dev->address << ED_FUNC_SHIFT) |
+		(0 << ED_EP_SHIFT) |
+		(OHCI_FROM_TD << ED_DIR_SHIFT) |
+		(dev->speed?ED_LOWSPEED:0) |
+		(dev->endpoints[0].maxpacketsize << ED_MPS_SHIFT));
+	head->tail_pointer = __cpu_to_le32(virt_to_phys(final_td));
+	head->head_pointer = __cpu_to_le32(virt_to_phys(first_td));
+
+	usb_debug("ohci_control(): doing transfer with %x. first_td at %x\n",
+		__le32_to_cpu(head->config) & ED_FUNC_MASK, __le32_to_cpu(head->head_pointer));
+#ifdef USB_DEBUG_ED
+	dump_ed(head);
+#endif
+
+	/* activate schedule */
+	OHCI_INST(dev->controller)->opreg->HcControlHeadED = __cpu_to_le32(virt_to_phys(head));
+	OHCI_INST(dev->controller)->opreg->HcControl |= __cpu_to_le32(ControlListEnable);
+	OHCI_INST(dev->controller)->opreg->HcCommandStatus = __cpu_to_le32(ControlListFilled);
+
+	int failure = wait_for_ed(dev, head,
+			(dalen==0)?0:(last_page - first_page + 1));
+	/* Wait some frames before and one after disabling list access. */
+	mdelay(4);
+	OHCI_INST(dev->controller)->opreg->HcControl &= __cpu_to_le32(~ControlListEnable);
+	mdelay(1);
+
+	/* free memory */
+	ohci_free_ed(head);
+
+	return failure;
+}
+
+/* finalize == 1: if data is of packet aligned size, add a zero length packet */
+static int
+ohci_bulk (endpoint_t *ep, int dalen, u8 *data, int finalize)
+{
+	int i;
+	usb_debug("bulk: %x bytes from %p, finalize: %x, maxpacketsize: %x\n", dalen, data, finalize, ep->maxpacketsize);
+
+	td_t *cur, *next;
+
+	// pages are specified as 4K in OHCI, so don't use getpagesize()
+	int first_page = (unsigned long)data / 4096;
+	int last_page = (unsigned long)(data+dalen-1)/4096;
+	if (last_page < first_page) last_page = first_page;
+	int pages = (dalen==0)?0:(last_page - first_page + 1);
+	int td_count = (pages+1)/2;
+
+	if (finalize && ((dalen % ep->maxpacketsize) == 0)) {
+		td_count++;
+	}
+
+	/* First TD. */
+	td_t *const first_td;
+	ofmem_posix_memalign((void **)&first_td, sizeof(td_t), sizeof(td_t));
+	memset((void *)first_td, 0, sizeof(*first_td));
+	cur = next = first_td;
+
+	for (i = 0; i < td_count; ++i) {
+		/* Advance to next TD. */
+		cur = next;
+		cur->config = __cpu_to_le32((ep->direction == IN ? TD_DIRECTION_IN : TD_DIRECTION_OUT) |
+                        TD_DELAY_INTERRUPT_NOINTR |
+                        TD_TOGGLE_FROM_ED |
+                        TD_CC_NOACCESS);
+		cur->current_buffer_pointer = __cpu_to_le32(virt_to_phys(data));
+		pages--;
+		if (dalen == 0) {
+			/* magic TD for empty packet transfer */
+			cur->current_buffer_pointer = 0;
+			cur->buffer_end = 0;
+			/* assert((pages == 0) && finalize); */
+		}
+		int consumed = (4096 - ((unsigned long)data % 4096));
+		if (consumed >= dalen) {
+			// end of data is within same page
+			cur->buffer_end = __cpu_to_le32(virt_to_phys(data + dalen - 1));
+			dalen = 0;
+			/* assert(pages == finalize); */
+		} else {
+			dalen -= consumed;
+			data += consumed;
+			pages--;
+			int second_page_size = dalen;
+			if (dalen > 4096) {
+				second_page_size = 4096;
+			}
+			cur->buffer_end = __cpu_to_le32(virt_to_phys(data + second_page_size - 1));
+			dalen -= second_page_size;
+			data += second_page_size;
+		}
+		/* One more TD. */
+		ofmem_posix_memalign((void **)&next, sizeof(td_t), sizeof(td_t));
+		memset((void *)next, 0, sizeof(*next));
+		/* Linked to the previous. */
+		cur->next_td = __cpu_to_le32(virt_to_phys(next));
+	}
+
+	/* Write done head after last TD. */
+	cur->config &= __cpu_to_le32(~TD_DELAY_INTERRUPT_MASK);
+	/* Advance to final, dummy TD. */
+	cur = next;
+
+	/* Data structures */
+	ed_t *head;
+	ofmem_posix_memalign((void **)&head, sizeof(ed_t), sizeof(ed_t));
+	memset((void*)head, 0, sizeof(*head));
+	head->config = __cpu_to_le32((ep->dev->address << ED_FUNC_SHIFT) |
+		((ep->endpoint & 0xf) << ED_EP_SHIFT) |
+		(((ep->direction==IN)?OHCI_IN:OHCI_OUT) << ED_DIR_SHIFT) |
+		(ep->dev->speed?ED_LOWSPEED:0) |
+		(ep->maxpacketsize << ED_MPS_SHIFT));
+	head->tail_pointer = __cpu_to_le32(virt_to_phys(cur));
+	head->head_pointer = __cpu_to_le32(virt_to_phys(first_td) | (ep->toggle?ED_TOGGLE:0));
+
+	usb_debug("doing bulk transfer with %x(%x). first_td at %lx, last %lx\n",
+		__le32_to_cpu(head->config) & ED_FUNC_MASK,
+		(__le32_to_cpu(head->config) & ED_EP_MASK) >> ED_EP_SHIFT,
+		virt_to_phys(first_td), virt_to_phys(cur));
+
+	/* activate schedule */
+	OHCI_INST(ep->dev->controller)->opreg->HcBulkHeadED = __cpu_to_le32(virt_to_phys(head));
+	OHCI_INST(ep->dev->controller)->opreg->HcControl |= __cpu_to_le32(BulkListEnable);
+	OHCI_INST(ep->dev->controller)->opreg->HcCommandStatus = __cpu_to_le32(BulkListFilled);
+
+	int failure = wait_for_ed(ep->dev, head,
+			(dalen==0)?0:(last_page - first_page + 1));
+	/* Wait some frames before and one after disabling list access. */
+	mdelay(4);
+	OHCI_INST(ep->dev->controller)->opreg->HcControl &= __cpu_to_le32(~BulkListEnable);
+	mdelay(1);
+
+	ep->toggle = __le32_to_cpu(head->head_pointer) & ED_TOGGLE;
+
+	/* free memory */
+	ohci_free_ed(head);
+
+	if (failure) {
+		/* try cleanup */
+		clear_stall(ep);
+	}
+
+	return failure;
+}
+
+
+struct _intr_queue;
+
+struct _intrq_td {
+	volatile td_t		td;
+	u8			*data;
+	struct _intrq_td	*next;
+	struct _intr_queue	*intrq;
+};
+
+struct _intr_queue {
+	volatile ed_t		ed;
+	struct _intrq_td	*head;
+	struct _intrq_td	*tail;
+	u8			*data;
+	int			reqsize;
+	endpoint_t		*endp;
+	unsigned int		remaining_tds;
+	int			destroy;
+};
+
+typedef struct _intrq_td intrq_td_t;
+typedef struct _intr_queue intr_queue_t;
+
+#define INTRQ_TD_FROM_TD(x) ((intrq_td_t *)x)
+
+static void
+ohci_fill_intrq_td(intrq_td_t *const td, intr_queue_t *const intrq,
+		   u8 *const data)
+{
+	memset(td, 0, sizeof(*td));
+	td->td.config = __cpu_to_le32(TD_QUEUETYPE_INTR |
+		(intrq->endp->direction == IN ? TD_DIRECTION_IN : TD_DIRECTION_OUT) |
+		TD_DELAY_INTERRUPT_ZERO |
+		TD_TOGGLE_FROM_ED |
+		TD_CC_NOACCESS);
+	td->td.current_buffer_pointer = __cpu_to_le32(virt_to_phys(data));
+	td->td.buffer_end = __cpu_to_le32(virt_to_phys(data) + intrq->reqsize - 1);
+	td->intrq = intrq;
+	td->data = data;
+}
+
+/* create and hook-up an intr queue into device schedule */
+static void *
+ohci_create_intr_queue(endpoint_t *const ep, const int reqsize,
+		       const int reqcount, const int reqtiming)
+{
+	int i;
+	intrq_td_t *first_td = NULL, *last_td = NULL;
+
+	if (reqsize > 4096)
+		return NULL;
+
+	intr_queue_t *const intrq;
+	ofmem_posix_memalign((void **)&intrq, sizeof(intrq->ed), sizeof(*intrq));
+	memset(intrq, 0, sizeof(*intrq));
+	intrq->data = (u8 *)malloc(reqcount * reqsize);
+	intrq->reqsize = reqsize;
+	intrq->endp = ep;
+
+	/* Create #reqcount TDs. */
+	u8 *cur_data = intrq->data;
+	for (i = 0; i < reqcount; ++i) {
+		intrq_td_t *const td;
+		ofmem_posix_memalign((void **)&td, sizeof(td->td), sizeof(*td));
+		++intrq->remaining_tds;
+		ohci_fill_intrq_td(td, intrq, cur_data);
+		cur_data += reqsize;
+		if (!first_td)
+			first_td = td;
+		else
+			last_td->td.next_td = __cpu_to_le32(virt_to_phys(&td->td));
+		last_td = td;
+	}
+
+	/* Create last, dummy TD. */
+	intrq_td_t *dummy_td;
+	ofmem_posix_memalign((void **)&dummy_td, sizeof(dummy_td->td), sizeof(*dummy_td));
+	memset(dummy_td, 0, sizeof(*dummy_td));
+	dummy_td->intrq = intrq;
+	if (last_td)
+		last_td->td.next_td = __cpu_to_le32(virt_to_phys(&dummy_td->td));
+	last_td = dummy_td;
+
+	/* Initialize ED. */
+	intrq->ed.config =  __cpu_to_le32((ep->dev->address << ED_FUNC_SHIFT) |
+		((ep->endpoint & 0xf) << ED_EP_SHIFT) |
+		(((ep->direction == IN) ? OHCI_IN : OHCI_OUT) << ED_DIR_SHIFT) |
+		(ep->dev->speed ? ED_LOWSPEED : 0) |
+		(ep->maxpacketsize << ED_MPS_SHIFT));
+	intrq->ed.tail_pointer = __cpu_to_le32(virt_to_phys(last_td));
+	intrq->ed.head_pointer = __cpu_to_le32(virt_to_phys(first_td) | (ep->toggle ? ED_TOGGLE : 0));
+
+#ifdef USB_DEBUG_ED
+	dump_ed(&intrq->ed);
+#endif
+	/* Insert ED into periodic table. */
+	int nothing_placed	= 1;
+	ohci_t *const ohci	= OHCI_INST(ep->dev->controller);
+	u32 *const intr_table	= ohci->hcca->HccaInterruptTable;
+	const u32 dummy_ptr	= __cpu_to_le32(virt_to_phys(ohci->periodic_ed));
+	for (i = 0; i < 32; i += reqtiming) {
+		/* Advance to the next free position. */
+		while ((i < 32) && (intr_table[i] != dummy_ptr)) ++i;
+		if (i < 32) {
+			usb_debug("Placed endpoint %lx to %d\n", virt_to_phys(&intrq->ed), i);
+			intr_table[i] = __cpu_to_le32(virt_to_phys(&intrq->ed));
+			nothing_placed = 0;
+		}
+	}
+	if (nothing_placed) {
+		usb_debug("Error: Failed to place ohci interrupt endpoint "
+			"descriptor into periodic table: no space left\n");
+		ohci_destroy_intr_queue(ep, intrq);
+		return NULL;
+	}
+
+	return intrq;
+}
+
+/* remove queue from device schedule, dropping all data that came in */
+static void
+ohci_destroy_intr_queue(endpoint_t *const ep, void *const q_)
+{
+	intr_queue_t *const intrq = (intr_queue_t *)q_;
+
+	int i;
+
+	/* Remove interrupt queue from periodic table. */
+	ohci_t *const ohci	= OHCI_INST(ep->dev->controller);
+	u32 *const intr_table	= ohci->hcca->HccaInterruptTable;
+	for (i=0; i < 32; ++i) {
+		if (intr_table[i] == __cpu_to_le32(virt_to_phys(intrq)))
+			intr_table[i] = __cpu_to_le32(virt_to_phys(ohci->periodic_ed));
+	}
+	/* Wait for frame to finish. */
+	mdelay(1);
+
+	/* Free unprocessed TDs. */
+	while ((__le32_to_cpu(intrq->ed.head_pointer) & ~0x3) != __le32_to_cpu(intrq->ed.tail_pointer)) {
+		td_t *const cur_td = (td_t *)phys_to_virt(__le32_to_cpu(intrq->ed.head_pointer) & ~0x3);
+		intrq->ed.head_pointer = cur_td->next_td;
+		free(INTRQ_TD_FROM_TD(cur_td));
+		--intrq->remaining_tds;
+	}
+	/* Free final, dummy TD. */
+	free(phys_to_virt(__le32_to_cpu(intrq->ed.head_pointer) & ~0x3));
+	/* Free data buffer. */
+	free(intrq->data);
+
+	/* Free TDs already fetched from the done queue. */
+	ohci_process_done_queue(ohci, 1);
+	while (intrq->head) {
+		intrq_td_t *const cur_td = (intrq_td_t *const )__le32_to_cpu(intrq->head);
+		intrq->head = intrq->head->next;
+		free(cur_td);
+		--intrq->remaining_tds;
+	}
+
+	/* Mark interrupt queue to be destroyed.
+	   ohci_process_done_queue() will free the remaining TDs
+	   and finish the interrupt queue off once all TDs are gone. */
+	intrq->destroy = 1;
+
+	/* Save data toggle. */
+	ep->toggle = __le32_to_cpu(intrq->ed.head_pointer) & ED_TOGGLE;
+}
+
+/* read one intr-packet from queue, if available. extend the queue for new input.
+   return NULL if nothing new available.
+   Recommended use: while (data=poll_intr_queue(q)) process(data);
+ */
+static u8 *
+ohci_poll_intr_queue(void *const q_)
+{
+	intr_queue_t *const intrq = (intr_queue_t *)q_;
+
+	u8 *data = NULL;
+
+	/* Process done queue first, then check if we have work to do. */
+	ohci_process_done_queue(OHCI_INST(intrq->endp->dev->controller), 0);
+
+	if (intrq->head) {
+		/* Save pointer to processed TD and advance. */
+		intrq_td_t *const cur_td = intrq->head;
+		intrq->head = cur_td->next;
+
+		/* Return data buffer of this TD. */
+		data = cur_td->data;
+
+		/* Requeue this TD (i.e. copy to dummy and requeue as dummy). */
+		intrq_td_t *const dummy_td =
+			INTRQ_TD_FROM_TD(phys_to_virt(__le32_to_cpu(intrq->ed.tail_pointer)));
+		ohci_fill_intrq_td(dummy_td, intrq, data);
+		/* Reset all but intrq pointer (i.e. init as dummy). */
+		memset(cur_td, 0, sizeof(*cur_td));
+		cur_td->intrq = intrq;
+		/* Insert into interrupt queue as dummy. */
+		dummy_td->td.next_td = __le32_to_cpu(virt_to_phys(&cur_td->td));
+		intrq->ed.tail_pointer = __le32_to_cpu(virt_to_phys(&cur_td->td));
+	}
+
+	return data;
+}
+
+static void
+ohci_process_done_queue(ohci_t *const ohci, const int spew_debug)
+{
+	int i, j;
+
+	/* Temporary queue of interrupt queue TDs (to reverse order). */
+	intrq_td_t *temp_tdq = NULL;
+
+	/* Check if done head has been written. */
+	if (!(READ_OPREG(ohci, HcInterruptStatus) & WritebackDoneHead))
+		return;
+	/* Fetch current done head.
+	   Lsb is only interesting for hw interrupts. */
+	u32 phys_done_queue = __le32_to_cpu(ohci->hcca->HccaDoneHead) & ~1;
+	/* Tell host controller, he may overwrite the done head pointer. */
+	ohci->opreg->HcInterruptStatus = __cpu_to_le32(WritebackDoneHead);
+
+	i = 0;
+	/* Process done queue (it's in reversed order). */
+	while (phys_done_queue) {
+		td_t *const done_td = (td_t *)phys_to_virt(phys_done_queue);
+
+		/* Advance pointer to next TD. */
+		phys_done_queue = __le32_to_cpu(done_td->next_td);
+
+		switch (__le32_to_cpu(done_td->config) & TD_QUEUETYPE_MASK) {
+		case TD_QUEUETYPE_ASYNC:
+			/* Free processed async TDs. */
+                  free((void *)done_td);
+			break;
+		case TD_QUEUETYPE_INTR: {
+			intrq_td_t *const td = INTRQ_TD_FROM_TD(done_td);
+			intr_queue_t *const intrq = td->intrq;
+			/* Check if the corresponding interrupt
+			   queue is still beeing processed. */
+			if (intrq->destroy) {
+				/* Free this TD, and */
+				free(td);
+				--intrq->remaining_tds;
+				/* the interrupt queue if it has no more TDs. */
+				if (!intrq->remaining_tds)
+					free(intrq);
+				usb_debug("Freed TD from orphaned interrupt "
+					  "queue, %d TDs remain.\n",
+					  intrq->remaining_tds);
+			} else {
+				/* Save done TD to be processed. */
+				td->next = temp_tdq;
+				temp_tdq = td;
+			}
+			break;
+		}
+		default:
+			break;
+		}
+		++i;
+	}
+	if (spew_debug)
+		usb_debug("Processed %d done TDs.\n", i);
+
+	j = 0;
+	/* Process interrupt queue TDs in right order. */
+	while (temp_tdq) {
+		/* Save pointer of current TD and advance. */
+		intrq_td_t *const cur_td = temp_tdq;
+		temp_tdq = temp_tdq->next;
+
+		/* The interrupt queue for the current TD. */
+		intr_queue_t *const intrq = cur_td->intrq;
+		/* Append to interrupt queue. */
+		if (!intrq->head) {
+			/* First element. */
+			intrq->head = intrq->tail = cur_td;
+		} else {
+			/* Insert at tail. */
+			intrq->tail->next = cur_td;
+			intrq->tail = cur_td;
+		}
+		/* It's always the last element. */
+		cur_td->next = NULL;
+		++j;
+	}
+	if (spew_debug)
+		usb_debug("processed %d done tds, %d intr tds thereof.\n", i, j);
+}
+
+int ob_usb_ohci_init (const char *path, uint32_t addr)
+{
+	hci_t *ctrl;
+	int i;
+
+	usb_debug("ohci_init: %s addr = %x\n", path, addr);
+	ctrl = ohci_pci_init(addr);
+	if (!ctrl)
+		return 0;
+
+	/* Init ports */
+	usb_poll();
+
+	/* Look for a keyboard */
+	for (i = 0; i < 128; i++) {
+		if (ctrl->devices[i] && ctrl->devices[i]->configuration) {
+			configuration_descriptor_t *cd;
+			interface_descriptor_t *intf;
+
+			cd = (configuration_descriptor_t *)ctrl->devices[i]->configuration;
+			intf = (interface_descriptor_t *)(ctrl->devices[i]->configuration + cd->bLength);
+			usb_debug("Device at port %d is class %d\n", i, intf->bInterfaceClass);
+			if (intf->bInterfaceClass == hid_device &&
+			    intf->bInterfaceSubClass == hid_subclass_boot &&
+			    intf->bInterfaceProtocol == hid_boot_proto_keyboard ) {
+				break;
+			}
+		}
+	}
+	if ( i < 128 )
+		ob_usb_hid_add_keyboard(path);
+
+	return 1;
+}
diff --git a/roms/openbios/drivers/usbohci.h b/roms/openbios/drivers/usbohci.h
new file mode 100644
index 000000000..690332871
--- /dev/null
+++ b/roms/openbios/drivers/usbohci.h
@@ -0,0 +1,45 @@
+/*
+ * Driver for USB OHCI ported from CoreBoot
+ *
+ * Copyright (C) 2014 BALATON Zoltan
+ *
+ * This file was part of the libpayload project.
+ *
+ * Copyright (C) 2010 Patrick Georgi
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef __OHCI_H
+#define __OHCI_H
+
+#include "config.h"
+#include "usbohci_private.h"
+
+hci_t *ohci_pci_init (u32 addr);
+hci_t *ohci_init (void *bar);
+
+void ohci_rh_init (usbdev_t *dev);
+
+#endif
diff --git a/roms/openbios/drivers/usbohci_private.h b/roms/openbios/drivers/usbohci_private.h
new file mode 100644
index 000000000..b3a723e21
--- /dev/null
+++ b/roms/openbios/drivers/usbohci_private.h
@@ -0,0 +1,270 @@
+/*
+ * Driver for USB OHCI ported from CoreBoot
+ *
+ * Copyright (C) 2014 BALATON Zoltan
+ *
+ * This file was part of the libpayload project.
+ *
+ * Copyright (C) 2010 Patrick Georgi
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef __OHCI_PRIVATE_H
+#define __OHCI_PRIVATE_H
+
+#include "libc/byteorder.h"
+#include "usb.h"
+
+#define READ_OPREG(ohci, field) (__le32_to_cpu((ohci)->opreg->field))
+#define MASK(startbit, lenbit) (((1<<(lenbit))-1)<<(startbit))
+
+	// FIXME: fake
+	typedef enum { CMD} reg;
+
+	enum {
+		NumberDownstreamPorts = 1<<0,
+		PowerSwitchingMode = 1<<8,
+		NoPowerSwitching = 1<<9,
+		DeviceType = 1<<10,
+		OverCurrentProtectionMode = 1<<11,
+		NoOverCurrentProtection = 1<<12,
+		PowerOnToPowerGoodTime = 1<<24
+	} HcRhDescriptorAReg;
+
+	enum {
+		NumberDownstreamPortsMask = MASK(0, 8),
+		PowerOnToPowerGoodTimeMask = MASK(24, 8)
+	} HcRhDescriptorAMask;
+
+	enum {
+		DeviceRemovable = 1<<0,
+		PortPowerControlMask = 1<<16
+	} HcRhDescriptorBReg;
+
+	enum {
+		CurrentConnectStatus		= 1<<0,
+		PortEnableStatus		= 1<<1,
+		PortSuspendStatus		= 1<<2,
+		PortOverCurrentIndicator	= 1<<3,
+		PortResetStatus			= 1<<4,
+		PortPowerStatus			= 1<<8,
+		LowSpeedDeviceAttached		= 1<<9,
+		ConnectStatusChange		= 1<<16,
+		PortEnableStatusChange		= 1<<17,
+		PortSuspendStatusChange		= 1<<18,
+		PortOverCurrentIndicatorChange	= 1<<19,
+		PortResetStatusChange		= 1<<20
+	} HcRhPortStatusRead;
+	enum {
+		ClearPortEnable			= 1<<0,
+		SetPortEnable			= 1<<1,
+		SetPortSuspend			= 1<<2,
+		ClearSuspendStatus		= 1<<3,
+		SetPortReset			= 1<<4,
+		SetPortPower			= 1<<8,
+		ClearPortPower			= 1<<9,
+	} HcRhPortStatusSet;
+
+	enum {
+		LocalPowerStatus = 1<<0,
+		OverCurrentIndicator = 1<<1,
+		DeviceRemoteWakeupEnable = 1<<15,
+		LocalPowerStatusChange = 1<<16,
+		OverCurrentIndicatorChange = 1<<17,
+		ClearRemoteWakeupEnable = 1<<31
+	} HcRhStatusReg;
+
+	enum {
+		FrameInterval = 1<<0,
+		FSLargestDataPacket = 1<<16,
+		FrameIntervalToggle = 1<<31
+	} HcFmIntervalOffset;
+	enum {
+		FrameIntervalMask = MASK(0, 14),
+		FSLargestDataPacketMask = MASK(16, 15),
+		FrameIntervalToggleMask = MASK(31, 1)
+	} HcFmIntervalMask;
+
+	enum {
+		ControlBulkServiceRatio = 1<<0,
+		PeriodicListEnable = 1<<2,
+		IsochronousEnable = 1<<3,
+		ControlListEnable = 1<<4,
+		BulkListEnable = 1<<5,
+		HostControllerFunctionalState = 1<<6,
+		InterruptRouting = 1<<8,
+		RemoteWakeupConnected = 1<<9,
+		RemoteWakeupEnable = 1<<10
+	} HcControlReg;
+
+	enum {
+		ControlBulkServiceRatioMask = MASK(0, 2),
+		HostControllerFunctionalStateMask = MASK(6, 2)
+	} HcControlMask;
+
+	enum {
+		USBReset = 0*HostControllerFunctionalState,
+		USBResume = 1*HostControllerFunctionalState,
+		USBOperational = 2*HostControllerFunctionalState,
+		USBSuspend = 3*HostControllerFunctionalState
+	};
+
+	enum {
+		HostControllerReset = 1<<0,
+		ControlListFilled = 1<<1,
+		BulkListFilled = 1<<2,
+		OwnershipChangeRequest = 1<<3,
+		SchedulingOverrunCount = 1<<16
+	} HcCommandStatusReg;
+
+	enum {
+		SchedulingOverrunCountMask = MASK(16, 2)
+	} HcCommandStatusMask;
+
+	enum {
+		FrameRemaining = 1<<0,
+		FrameRemainingToggle = 1<<31
+	} HcFmRemainingReg;
+
+	enum {
+		SchedulingOverrung = 1<<0,
+		WritebackDoneHead = 1<<1,
+		StartofFrame = 1<<2,
+		ResumeDetected = 1<<3,
+		UnrecoverableError = 1<<4,
+		FrameNumberOverflow = 1<<5,
+		RootHubStatusChange = 1<<6,
+		OwnershipChange = 1<<30
+	} HcInterruptStatusReg;
+
+     typedef struct {
+	// Control and Status Partition
+	volatile u32 HcRevision;
+	volatile u32 HcControl;
+	volatile u32 HcCommandStatus;
+	volatile u32 HcInterruptStatus;
+	volatile u32 HcInterruptEnable;
+	volatile u32 HcInterruptDisable;
+
+	// Memory Pointer Partition
+	volatile u32 HcHCCA;
+	volatile u32 HcPeriodCurrentED;
+	volatile u32 HcControlHeadED;
+	volatile u32 HcControlCurrentED;
+	volatile u32 HcBulkHeadED;
+	volatile u32 HcBulkCurrentED;
+	volatile u32 HcDoneHead;
+
+	// Frame Counter Partition
+	volatile u32 HcFmInterval;
+	volatile u32 HcFmRemaining;
+	volatile u32 HcFmNumber;
+	volatile u32 HcPeriodicStart;
+	volatile u32 HcLSThreshold;
+
+	// Root Hub Partition
+	volatile u32 HcRhDescriptorA;
+	volatile u32 HcRhDescriptorB;
+	volatile u32 HcRhStatus;
+	/* all bits in HcRhPortStatus registers are R/WC, so
+	   _DO NOT_ use |= to set the bits,
+	   this clears the entire state */
+	volatile u32 HcRhPortStatus[];
+     } __attribute__ ((packed)) opreg_t;
+
+	typedef struct { /* should be 256 bytes according to spec */
+		u32 HccaInterruptTable[32];
+		volatile u16 HccaFrameNumber;
+		volatile u16 HccaPad1;
+		volatile u32 HccaDoneHead;
+		u8 reserved[116]; /* pad according to spec */
+		u8 what[4]; /* really pad to 256 as spec only covers 252 */
+	} __attribute__ ((packed)) hcca_t;
+
+	typedef volatile struct {
+		u32 config;
+		u32 tail_pointer;
+		u32 head_pointer;
+		u32 next_ed;
+	} __attribute__ ((packed)) ed_t;
+#define ED_HALTED 1
+#define ED_TOGGLE 2
+
+#define ED_FUNC_SHIFT 0
+#define ED_FUNC_MASK MASK(0, 7)
+#define ED_EP_SHIFT 7
+#define ED_EP_MASK MASK(7, 4)
+#define ED_DIR_SHIFT 11
+#define ED_DIR_MASK MASK(11, 2)
+#define ED_LOWSPEED (1 << 13)
+#define ED_MPS_SHIFT 16
+
+	typedef volatile struct {
+		u32 config;
+		u32 current_buffer_pointer;
+		u32 next_td;
+		u32 buffer_end;
+	} __attribute__ ((packed)) td_t;
+/*
+ * Bits 0 through 17 of .config won't be interpreted by the host controller
+ * (HC) and, after processing the TD, the HC has to ensure those bits have
+ * the same state as before. So we are free to use those bits for our own
+ * purpose.
+ */
+#define TD_QUEUETYPE_SHIFT	0
+#define TD_QUEUETYPE_MASK	MASK(TD_QUEUETYPE_SHIFT, 2)
+#define TD_QUEUETYPE_ASYNC	(0 << TD_QUEUETYPE_SHIFT)
+#define TD_QUEUETYPE_INTR	(1 << TD_QUEUETYPE_SHIFT)
+
+#define TD_DIRECTION_SHIFT 19
+#define TD_DIRECTION_MASK MASK(TD_DIRECTION_SHIFT, 2)
+#define TD_DIRECTION_SETUP OHCI_SETUP << TD_DIRECTION_SHIFT
+#define TD_DIRECTION_IN OHCI_IN << TD_DIRECTION_SHIFT
+#define TD_DIRECTION_OUT OHCI_OUT << TD_DIRECTION_SHIFT
+#define TD_DELAY_INTERRUPT_SHIFT	21
+#define TD_DELAY_INTERRUPT_MASK		MASK(TD_DELAY_INTERRUPT_SHIFT, 3)
+#define TD_DELAY_INTERRUPT_ZERO		0
+#define TD_DELAY_INTERRUPT_NOINTR	(7 << TD_DELAY_INTERRUPT_SHIFT)
+#define TD_TOGGLE_DATA0 0
+#define TD_TOGGLE_DATA1 (1 << 24)
+#define TD_TOGGLE_FROM_ED 0
+#define TD_TOGGLE_FROM_TD (1 << 25)
+#define TD_CC_SHIFT 28
+#define TD_CC_MASK MASK(TD_CC_SHIFT, 4)
+#define TD_CC_NOERR 0
+#define TD_CC_NOACCESS (14 << TD_CC_SHIFT) /* the lower of the two values, so "no access" can be tested with >= */
+
+#define OHCI_INST(controller) ((ohci_t*)((controller)->instance))
+
+	typedef struct ohci {
+		opreg_t *opreg;
+		hcca_t *hcca;
+		usbdev_t *roothub;
+		ed_t *periodic_ed;
+	} ohci_t;
+
+	typedef enum { OHCI_SETUP=0, OHCI_OUT=1, OHCI_IN=2, OHCI_FROM_TD=3 } ohci_pid_t;
+
+#endif
diff --git a/roms/openbios/drivers/usbohci_rh.c b/roms/openbios/drivers/usbohci_rh.c
new file mode 100644
index 000000000..55503be61
--- /dev/null
+++ b/roms/openbios/drivers/usbohci_rh.c
@@ -0,0 +1,212 @@
+/*
+ * Driver for USB OHCI Root Hubs ported from CoreBoot
+ *
+ * Copyright (C) 2014 BALATON Zoltan
+ *
+ * This file was part of the libpayload project.
+ *
+ * Copyright (C) 2010 Patrick Georgi
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "timer.h"
+#include "usbohci_private.h"
+#include "usbohci.h"
+
+typedef struct {
+	int numports;
+	int *port;
+} rh_inst_t;
+
+#define RH_INST(dev) ((rh_inst_t*)(dev)->data)
+
+static void
+ohci_rh_enable_port (usbdev_t *dev, int port)
+{
+	/* Reset RH port should hold 50ms with pulses of at least 10ms and
+	 * gaps of at most 3ms (usb20 spec 7.1.7.5).
+	 * After reset, the port will be enabled automatically (ohci spec
+	 * 7.4.4).
+	 */
+	int total_delay = 100; /* 100 * 500us == 50ms */
+	while (total_delay > 0) {
+		if (!(READ_OPREG(OHCI_INST(dev->controller), HcRhPortStatus[port])
+					& CurrentConnectStatus))
+			return;
+
+		/* start reset */
+		OHCI_INST (dev->controller)->opreg->HcRhPortStatus[port] =
+			__cpu_to_le32(SetPortReset);
+		int timeout = 200; /* timeout after 200 * 500us == 100ms */
+		while ((READ_OPREG(OHCI_INST(dev->controller), HcRhPortStatus[port])
+					& PortResetStatus)
+				&& timeout--) {
+			udelay(500); total_delay--;
+		}
+		if (READ_OPREG(OHCI_INST(dev->controller), HcRhPortStatus[port])
+				& PortResetStatus) {
+			usb_debug("Warning: root-hub port reset timed out.\n");
+			break;
+		}
+		if ((200-timeout) < 20) {
+			usb_debug("Warning: port reset too short: %dms; "
+					"should be at least 10ms.\n",
+					(200-timeout)/2);
+                        total_delay = 0; /* can happen on QEMU */
+		}
+		/* clear reset status change */
+		OHCI_INST(dev->controller)->opreg->HcRhPortStatus[port] =
+			__cpu_to_le32(PortResetStatusChange);
+		usb_debug ("rh port reset finished after %dms.\n", (200-timeout)/2);
+	}
+}
+
+/* disable root hub */
+static void
+ohci_rh_disable_port (usbdev_t *dev, int port)
+{
+	OHCI_INST (dev->controller)->opreg->HcRhPortStatus[port] =
+		__cpu_to_le32(ClearPortEnable); // disable port
+	int timeout = 50; /* timeout after 50 * 100us == 5ms */
+	while ((READ_OPREG(OHCI_INST (dev->controller), HcRhPortStatus[port])
+				& PortEnableStatus)
+			&& timeout--) {
+		udelay(100);
+	}
+}
+
+static void
+ohci_rh_scanport (usbdev_t *dev, int port)
+{
+	if (port >= RH_INST(dev)->numports) {
+		usb_debug("Invalid port %d\n", port);
+		return;
+	}
+
+	/* device registered, and device change logged, so something must have happened */
+	if (RH_INST (dev)->port[port] != -1) {
+		usb_detach_device(dev->controller, RH_INST (dev)->port[port]);
+		RH_INST (dev)->port[port] = -1;
+	}
+
+	/* no device attached
+	   previously registered devices are detached, nothing left to do */
+	if (!(READ_OPREG(OHCI_INST(dev->controller), HcRhPortStatus[port]) & CurrentConnectStatus))
+		return;
+
+	// clear port state change
+	OHCI_INST(dev->controller)->opreg->HcRhPortStatus[port] = __cpu_to_le32(ConnectStatusChange);
+	ohci_rh_enable_port (dev, port);
+
+	mdelay(100); // wait for signal to stabilize
+
+	if (!(READ_OPREG(OHCI_INST(dev->controller), HcRhPortStatus[port]) & PortEnableStatus)) {
+		usb_debug ("port enable failed\n");
+		return;
+	}
+
+	int speed = (READ_OPREG(OHCI_INST(dev->controller), HcRhPortStatus[port]) & LowSpeedDeviceAttached) != 0;
+	RH_INST (dev)->port[port] = usb_attach_device(dev->controller, dev->address, port, speed);
+}
+
+static int
+ohci_rh_report_port_changes (usbdev_t *dev)
+{
+	ohci_t *const ohcic = OHCI_INST (dev->controller);
+
+	int i;
+
+	for (i = 0; i < RH_INST(dev)->numports; i++) {
+		// maybe detach+attach happened between two scans?
+		if (READ_OPREG(ohcic, HcRhPortStatus[i]) & ConnectStatusChange) {
+			ohcic->opreg->HcRhPortStatus[i] = __cpu_to_le32(ConnectStatusChange);
+			usb_debug("attachment change on port %d\n", i);
+			return i;
+		}
+	}
+
+	// no change
+	return -1;
+}
+
+static void
+ohci_rh_destroy (usbdev_t *dev)
+{
+	int i;
+	for (i = 0; i < RH_INST (dev)->numports; i++)
+		ohci_rh_disable_port (dev, i);
+	free (RH_INST (dev));
+}
+
+static void
+ohci_rh_poll (usbdev_t *dev)
+{
+	ohci_t *const ohcic = OHCI_INST (dev->controller);
+
+	int port;
+
+	/* Check if anything changed. */
+	if (!(READ_OPREG(ohcic, HcInterruptStatus) & RootHubStatusChange))
+		return;
+	ohcic->opreg->HcInterruptStatus = __cpu_to_le32(RootHubStatusChange);
+	usb_debug("root hub status change\n");
+
+	/* Scan ports with changed connection status. */
+	while ((port = ohci_rh_report_port_changes (dev)) != -1)
+		ohci_rh_scanport (dev, port);
+}
+
+void
+ohci_rh_init (usbdev_t *dev)
+{
+	int i;
+
+	dev->destroy = ohci_rh_destroy;
+	dev->poll = ohci_rh_poll;
+
+	dev->data = malloc (sizeof (rh_inst_t));
+	if (!dev->data) {
+		printk("Not enough memory for OHCI RH.\n");
+		return;
+	}
+
+	RH_INST (dev)->numports = READ_OPREG(OHCI_INST(dev->controller), HcRhDescriptorA) & NumberDownstreamPortsMask;
+	RH_INST (dev)->port = malloc(sizeof(int) * RH_INST (dev)->numports);
+	usb_debug("%d ports registered\n", RH_INST (dev)->numports);
+
+	for (i = 0; i < RH_INST (dev)->numports; i++) {
+		ohci_rh_enable_port (dev, i);
+		RH_INST (dev)->port[i] = -1;
+	}
+
+	/* we can set them here because a root hub _really_ shouldn't
+	   appear elsewhere */
+	dev->address = 0;
+	dev->hub = -1;
+	dev->port = -1;
+
+	usb_debug("rh init done\n");
+}
diff --git a/roms/openbios/include/arch/common/fw_cfg.h b/roms/openbios/include/arch/common/fw_cfg.h
index bd4808bbd..df44c2e89 100644
--- a/roms/openbios/include/arch/common/fw_cfg.h
+++ b/roms/openbios/include/arch/common/fw_cfg.h
@@ -46,6 +46,7 @@
 #define FW_CFG_PPC_KVM_PID	(FW_CFG_ARCH_LOCAL + 0x07)
 #define FW_CFG_PPC_NVRAM_ADDR	(FW_CFG_ARCH_LOCAL + 0x08)
 #define FW_CFG_PPC_BUSFREQ      (FW_CFG_ARCH_LOCAL + 0x09)
+#define FW_CFG_PPC_NVRAM_FLAT   (FW_CFG_ARCH_LOCAL + 0x0a)
 
 #define FW_CFG_INVALID          0xffff
 
diff --git a/roms/openbios/include/drivers/pci.h b/roms/openbios/include/drivers/pci.h
index 1f0af6c9f..2eb5685d3 100644
--- a/roms/openbios/include/drivers/pci.h
+++ b/roms/openbios/include/drivers/pci.h
@@ -188,6 +188,7 @@ extern const pci_arch_t *arch;
 #define PCI_DEVICE_ID_APPLE_UNI_N_PCI    0x001f
 #define PCI_DEVICE_ID_APPLE_UNI_N_AGP    0x0020
 #define PCI_DEVICE_ID_APPLE_UNI_N_KEYL   0x0022
+#define PCI_DEVICE_ID_APPLE_KEYL_USB     0x003f
 #define PCI_DEVICE_ID_APPLE_U3_AGP       0x004b
 
 #define PCI_VENDOR_ID_SUN                0x108e
diff --git a/roms/openbios/include/drivers/usb.h b/roms/openbios/include/drivers/usb.h
new file mode 100644
index 000000000..143ed27bc
--- /dev/null
+++ b/roms/openbios/include/drivers/usb.h
@@ -0,0 +1,8 @@
+#ifndef USB_H
+#define USB_H
+
+int ob_usb_ohci_init(const char *path, uint32_t addr);
+void ob_usb_hid_add_keyboard(const char *path);
+int usb_exit(void);
+
+#endif /* USB_H */
diff --git a/roms/openbios/libopenbios/ofmem_common.c b/roms/openbios/libopenbios/ofmem_common.c
index 157ca8741..3b8ca152f 100644
--- a/roms/openbios/libopenbios/ofmem_common.c
+++ b/roms/openbios/libopenbios/ofmem_common.c
@@ -634,8 +634,8 @@ ucell ofmem_claim( ucell addr, ucell size, ucell align )
 	} else {
 		if( align < PAGE_SIZE )
 			align = PAGE_SIZE;
-		phys = ofmem_claim_phys_( addr, size, align, 0, ofmem_arch_get_phys_top(), 1 /* reverse */ );
-		virt = ofmem_claim_virt_( addr, size, align, 0, get_ram_size(), 1 /* reverse */ );
+		phys = ofmem_claim_phys_( -1, size, align, 0, ofmem_arch_get_phys_top(), 1 /* reverse */ );
+		virt = ofmem_claim_virt_( phys, size, 0, 0, 0, 0 );
 		if( phys == -1 || virt == -1 ) {
 			OFMEM_TRACE("ofmem_claim failed\n");
 			return -1;
diff --git a/roms/seabios/.version b/roms/seabios/.version
index 88fc6a6d5..0bd308b59 100644
--- a/roms/seabios/.version
+++ b/roms/seabios/.version
@@ -1 +1 @@
-rel-1.7.5-0-ge51488c
+rel-1.7.5.1-0-g8936dbb
diff --git a/roms/seabios/Makefile b/roms/seabios/Makefile
index 78b598eee..7c2b33c3a 100644
--- a/roms/seabios/Makefile
+++ b/roms/seabios/Makefile
@@ -112,8 +112,8 @@ endif
 # Do a whole file compile by textually including all C code.
 define whole-compile
 @echo "  Compiling whole program $3"
-$(Q)printf '$(foreach i,$2,#include "$(CURDIR)/$i"\n)' > $3.tmp.c
-$(Q)$(CC) $1 $(CFLAGSWHOLE) -c $3.tmp.c -o $3
+$(Q)printf '$(foreach i,$2,#include "$i"\n)' > $3.tmp.c
+$(Q)$(CC) -I. $1 $(CFLAGSWHOLE) -c $3.tmp.c -o $3
 endef
 
 %.strip.o: %.o
diff --git a/roms/seabios/src/boot.c b/roms/seabios/src/boot.c
index 133e2063f..97de89cfa 100644
--- a/roms/seabios/src/boot.c
+++ b/roms/seabios/src/boot.c
@@ -145,7 +145,7 @@ int bootprio_find_scsi_device(struct pci_device *pci, int target, int lun)
     // Find scsi drive - for example: /pci@i0cf8/scsi@5/channel@0/disk@1,0
     char desc[256], *p;
     p = build_pci_path(desc, sizeof(desc), "*", pci);
-    snprintf(p, desc+sizeof(desc)-p, "/*@0/*@%d,%d", target, lun);
+    snprintf(p, desc+sizeof(desc)-p, "/*@0/*@%x,%x", target, lun);
     return find_prio(desc);
 }
 
@@ -189,7 +189,7 @@ int bootprio_find_pci_rom(struct pci_device *pci, int instance)
     char desc[256], *p;
     p = build_pci_path(desc, sizeof(desc), "*", pci);
     if (instance)
-        snprintf(p, desc+sizeof(desc)-p, ":rom%d", instance);
+        snprintf(p, desc+sizeof(desc)-p, ":rom%x", instance);
     return find_prio(desc);
 }
 
@@ -201,7 +201,7 @@ int bootprio_find_named_rom(const char *name, int instance)
     char desc[256], *p;
     p = desc + snprintf(desc, sizeof(desc), "/rom@%s", name);
     if (instance)
-        snprintf(p, desc+sizeof(desc)-p, ":rom%d", instance);
+        snprintf(p, desc+sizeof(desc)-p, ":rom%x", instance);
     return find_prio(desc);
 }
 
@@ -224,7 +224,7 @@ int bootprio_find_usb(struct usbdevice_s *usbdev, int lun)
     char desc[256], *p;
     p = build_pci_path(desc, sizeof(desc), "usb", usbdev->hub->cntl->pci);
     p = build_usb_path(p, desc+sizeof(desc)-p, usbdev->hub);
-    snprintf(p, desc+sizeof(desc)-p, "/storage@%x/*@0/*@0,%d"
+    snprintf(p, desc+sizeof(desc)-p, "/storage@%x/*@0/*@0,%x"
              , usbdev->port+1, lun);
     int ret = find_prio(desc);
     if (ret >= 0)
diff --git a/roms/seabios/src/fw/pciinit.c b/roms/seabios/src/fw/pciinit.c
index 2e6382f98..0e5d51b91 100644
--- a/roms/seabios/src/fw/pciinit.c
+++ b/roms/seabios/src/fw/pciinit.c
@@ -635,6 +635,36 @@ pci_region_create_entry(struct pci_bus *bus, struct pci_device *dev,
     return entry;
 }
 
+static int pci_bus_hotplug_support(struct pci_bus *bus)
+{
+    u8 pcie_cap = pci_find_capability(bus->bus_dev, PCI_CAP_ID_EXP);
+    u8 shpc_cap;
+
+    if (pcie_cap) {
+        u16 pcie_flags = pci_config_readw(bus->bus_dev->bdf,
+                                          pcie_cap + PCI_EXP_FLAGS);
+        u8 port_type = ((pcie_flags & PCI_EXP_FLAGS_TYPE) >>
+                       (__builtin_ffs(PCI_EXP_FLAGS_TYPE) - 1));
+        u8 downstream_port = (port_type == PCI_EXP_TYPE_DOWNSTREAM) ||
+                             (port_type == PCI_EXP_TYPE_ROOT_PORT);
+        /*
+         * PCI Express SPEC, 7.8.2:
+         *   Slot Implemented – When Set, this bit indicates that the Link
+         *   HwInit associated with this Port is connected to a slot (as
+         *   compared to being connected to a system-integrated device or
+         *   being disabled).
+         *   This bit is valid for Downstream Ports. This bit is undefined
+         *   for Upstream Ports.
+         */
+        u16 slot_implemented = pcie_flags & PCI_EXP_FLAGS_SLOT;
+
+        return downstream_port && slot_implemented;
+    }
+
+    shpc_cap = pci_find_capability(bus->bus_dev, PCI_CAP_ID_SHPC);
+    return !!shpc_cap;
+}
+
 static int pci_bios_check_devices(struct pci_bus *busses)
 {
     dprintf(1, "PCI: check devices\n");
@@ -677,7 +707,7 @@ static int pci_bios_check_devices(struct pci_bus *busses)
             continue;
         struct pci_bus *parent = &busses[pci_bdf_to_bus(s->bus_dev->bdf)];
         int type;
-        u8 shpc_cap = pci_find_capability(s->bus_dev, PCI_CAP_ID_SHPC);
+        int hotplug_support = pci_bus_hotplug_support(s);
         for (type = 0; type < PCI_REGION_TYPE_COUNT; type++) {
             u64 align = (type == PCI_REGION_TYPE_IO) ?
                 PCI_BRIDGE_IO_MIN : PCI_BRIDGE_MEM_MIN;
@@ -686,7 +716,7 @@ static int pci_bios_check_devices(struct pci_bus *busses)
             if (pci_region_align(&s->r[type]) > align)
                  align = pci_region_align(&s->r[type]);
             u64 sum = pci_region_sum(&s->r[type]);
-            if (!sum && shpc_cap)
+            if (!sum && hotplug_support)
                 sum = align; /* reserve min size for hot-plug */
             u64 size = ALIGN(sum, align);
             int is64 = pci_bios_bridge_region_is64(&s->r[type],
diff --git a/roms/seabios/src/hw/megasas.c b/roms/seabios/src/hw/megasas.c
index a5dc14fcf..b2a65e48b 100644
--- a/roms/seabios/src/hw/megasas.c
+++ b/roms/seabios/src/hw/megasas.c
@@ -357,6 +357,10 @@ init_megasas(struct pci_device *pci)
     u32 iobase = pci_config_readl(pci->bdf, PCI_BASE_ADDRESS_2)
         & PCI_BASE_ADDRESS_IO_MASK;
 
+    if (!iobase)
+        iobase = pci_config_readl(pci->bdf, PCI_BASE_ADDRESS_0)
+            & PCI_BASE_ADDRESS_IO_MASK;
+
     dprintf(1, "found MegaRAID SAS at %02x:%02x.%x, io @ %x\n",
             pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf),
             pci_bdf_to_fn(bdf), iobase);
diff --git a/roms/seabios/src/hw/usb-ehci.c b/roms/seabios/src/hw/usb-ehci.c
index 9d9427b6c..17a7e8b28 100644
--- a/roms/seabios/src/hw/usb-ehci.c
+++ b/roms/seabios/src/hw/usb-ehci.c
@@ -372,7 +372,7 @@ ehci_desc2pipe(struct ehci_pipe *pipe, struct usbdevice_s *usbdev
         struct ehci_pipe *hpipe = container_of(
             hubdev->defpipe, struct ehci_pipe, pipe);
         if (hpipe->pipe.speed == USB_HIGHSPEED)
-            pipe->qh.info2 |= ((usbdev->port << QH_HUBPORT_SHIFT)
+            pipe->qh.info2 |= (((usbdev->port+1) << QH_HUBPORT_SHIFT)
                                | (hpipe->pipe.devaddr << QH_HUBADDR_SHIFT));
         else
             pipe->qh.info2 = hpipe->qh.info2;
diff --git a/roms/seabios/src/stacks.c b/roms/seabios/src/stacks.c
index 6bcb31940..beccc0fa6 100644
--- a/roms/seabios/src/stacks.c
+++ b/roms/seabios/src/stacks.c
@@ -287,7 +287,7 @@ thread_init(void)
 int
 threads_during_optionroms(void)
 {
-    return CONFIG_THREADS && ThreadControl == 2;
+    return CONFIG_THREADS && ThreadControl == 2 && in_post();
 }
 
 // Switch to next thread stack.
diff --git a/roms/seabios/vgasrc/vgabios.c b/roms/seabios/vgasrc/vgabios.c
index 400e29675..e87b7ebff 100644
--- a/roms/seabios/vgasrc/vgabios.c
+++ b/roms/seabios/vgasrc/vgabios.c
@@ -891,15 +891,15 @@ handle_1011(struct bregs *regs)
 {
     if (CONFIG_VGA_STDVGA_PORTS) {
         switch (regs->al) {
-        case 0x00: handle_101100(regs); break;
-        case 0x01: handle_101101(regs); break;
-        case 0x02: handle_101102(regs); break;
-        case 0x03: handle_101103(regs); break;
-        case 0x04: handle_101104(regs); break;
-        case 0x10: handle_101110(regs); break;
-        case 0x11: handle_101111(regs); break;
-        case 0x12: handle_101112(regs); break;
-        case 0x14: handle_101114(regs); break;
+        case 0x00: handle_101100(regs); return;
+        case 0x01: handle_101101(regs); return;
+        case 0x02: handle_101102(regs); return;
+        case 0x03: handle_101103(regs); return;
+        case 0x04: handle_101104(regs); return;
+        case 0x10: handle_101110(regs); return;
+        case 0x11: handle_101111(regs); return;
+        case 0x12: handle_101112(regs); return;
+        case 0x14: handle_101114(regs); return;
         }
     }
     switch (regs->al) {
diff --git a/rules.mak b/rules.mak
index ba2f4c19a..f500fefdd 100644
--- a/rules.mak
+++ b/rules.mak
@@ -22,6 +22,32 @@ QEMU_DGFLAGS += -MMD -MP -MT $@ -MF $(*D)/$(*F).d
 # Same as -I$(SRC_PATH) -I., but for the nested source/object directories
 QEMU_INCLUDES += -I$(<D) -I$(@D)
 
+WL_U := -Wl,-u,
+find-symbols = $(if $1, $(sort $(shell $(NM) -P -g $1 | $2)))
+defined-symbols = $(call find-symbols,$1,awk '$$2!="U"{print $$1}')
+undefined-symbols = $(call find-symbols,$1,awk '$$2=="U"{print $$1}')
+
+# All the .mo objects in -m variables are also added into corresponding -y
+# variable in unnest-vars, but filtered out here, when LINK is called.
+#
+# The .mo objects are supposed to be linked as a DSO, for module build. So here
+# they are only used as a placeholders to generate those "archive undefined"
+# symbol options (-Wl,-u,$symbol_name), which are the archive functions
+# referenced by the code in the DSO.
+#
+# Also the presence in -y variables will also guarantee they are built before
+# linking executables that will load them. So we can look up symbol reference
+# in LINK.
+#
+# This is necessary because the exectuable itself may not use the function, in
+# which case the function would not be linked in. Then the DSO loading will
+# fail because of the missing symbol.
+process-archive-undefs = $(filter-out %.a %.mo,$1) \
+                $(addprefix $(WL_U), \
+                     $(filter $(call defined-symbols,$(filter %.a, $1)), \
+                              $(call undefined-symbols,$(filter %.mo,$1)))) \
+                $(filter %.a,$1)
+
 extract-libs = $(strip $(foreach o,$1,$($o-libs)))
 expand-objs = $(strip $(sort $(filter %.o,$1)) \
                   $(foreach o,$(filter %.mo,$1),$($o-objs)) \
@@ -38,7 +64,8 @@ LINKPROG = $(or $(CXX),$(CC))
 
 ifeq ($(LIBTOOL),)
 LINK = $(call quiet-command, $(LINKPROG) $(QEMU_CFLAGS) $(CFLAGS) $(LDFLAGS) -o $@ \
-       $1 $(version-obj-y) $(call extract-libs,$1) $(LIBS),"  LINK  $(TARGET_DIR)$@")
+       $(call process-archive-undefs, $1) \
+       $(version-obj-y) $(call extract-libs,$1) $(LIBS),"  LINK  $(TARGET_DIR)$@")
 else
 LIBTOOL += $(if $(V),,--quiet)
 %.lo: %.c
@@ -50,7 +77,8 @@ LIBTOOL += $(if $(V),,--quiet)
 
 LINK = $(call quiet-command,\
        $(if $(filter %.lo %.la,$1),$(LIBTOOL) --mode=link --tag=CC \
-       )$(LINKPROG) $(QEMU_CFLAGS) $(CFLAGS) $(LDFLAGS) -o $@ $1 \
+       )$(LINKPROG) $(QEMU_CFLAGS) $(CFLAGS) $(LDFLAGS) -o $@ \
+       $(call process-archive-undefs, $1)\
        $(if $(filter %.lo %.la,$1),$(version-lobj-y),$(version-obj-y)) \
        $(if $(filter %.lo %.la,$1),$(LIBTOOLFLAGS)) \
        $(call extract-libs,$(1:.lo=.o)) $(LIBS),$(if $(filter %.lo %.la,$1),"lt LINK ", "  LINK  ")"$(TARGET_DIR)$@")
@@ -76,11 +104,17 @@ endif
 
 %$(DSOSUF): CFLAGS += -fPIC -DBUILD_DSO
 %$(DSOSUF): LDFLAGS += $(LDFLAGS_SHARED)
-%$(DSOSUF):
+%$(DSOSUF): %.mo
 	$(call LINK,$^)
 	@# Copy to build root so modules can be loaded when program started without install
 	$(if $(findstring /,$@),$(call quiet-command,cp $@ $(subst /,-,$@), "  CP    $(subst /,-,$@)"))
 
+
+LD_REL := $(CC) -nostdlib -Wl,-r
+
+%.mo:
+	$(call quiet-command,$(LD_REL) -o $@ $^,"  LD -r $(TARGET_DIR)$@")
+
 .PHONY: modules
 modules:
 
@@ -306,6 +340,9 @@ define unnest-vars
         # For module build, build shared libraries during "make modules"
         # For non-module build, add -m to -y
         $(if $(CONFIG_MODULES),
+             $(foreach o,$($v),
+                   $(eval $o: $($o-objs)))
+             $(eval $(patsubst %-m,%-y,$v) += $($v))
              $(eval modules: $($v:%.mo=%$(DSOSUF))),
              $(eval $(patsubst %-m,%-y,$v) += $(call expand-objs, $($v)))))
 
@@ -325,4 +362,9 @@ define unnest-vars
         # Include all the .d files
         $(eval -include $(addsuffix *.d, $(sort $(dir $($v)))))
         $(eval $v := $(filter-out %/,$($v))))
+
+    # For all %.mo objects that are directly added into -y, expand them to %.mo-objs
+    $(foreach v,$2,
+        $(eval $v := $(foreach o,$($v),$(if $($o-objs),$($o-objs),$o))))
+
 endef
diff --git a/savevm.c b/savevm.c
index e19ae0a9b..08ec678dd 100644
--- a/savevm.c
+++ b/savevm.c
@@ -1245,19 +1245,19 @@ int load_vmstate(const char *name)
 
 void do_delvm(Monitor *mon, const QDict *qdict)
 {
-    BlockDriverState *bs, *bs1;
-    Error *err = NULL;
+    BlockDriverState *bs;
+    Error *err;
     const char *name = qdict_get_str(qdict, "name");
 
-    bs = find_vmstate_bs();
-    if (!bs) {
+    if (!find_vmstate_bs()) {
         monitor_printf(mon, "No block device supports snapshots\n");
         return;
     }
 
-    bs1 = NULL;
-    while ((bs1 = bdrv_next(bs1))) {
-        if (bdrv_can_snapshot(bs1)) {
+    bs = NULL;
+    while ((bs = bdrv_next(bs))) {
+        if (bdrv_can_snapshot(bs)) {
+            err = NULL;
             bdrv_snapshot_delete_by_id_or_name(bs, name, &err);
             if (err) {
                 monitor_printf(mon,
diff --git a/scripts/acpi_extract.py b/scripts/acpi_extract.py
index 22ea46810..10c1ffb36 100755
--- a/scripts/acpi_extract.py
+++ b/scripts/acpi_extract.py
@@ -139,13 +139,16 @@ def aml_name_string(offset):
         offset += 1
     return offset;
 
-# Given data offset, find 8 byte buffer offset
-def aml_data_buffer8(offset):
-    #0x08 NameOp NameString DataRef
-    expect = [0x11, 0x0B, 0x0A, 0x08]
+# Given data offset, find variable length byte buffer offset
+def aml_data_buffer(offset, length):
+    #0x11 PkgLength BufferSize ByteList
+    if (length > 63):
+        die( "Name offset 0x%x: expected a one byte PkgLength (length<=63)" %
+             (offset));
+    expect = [0x11, length+3, 0x0A, length]
     if (aml[offset:offset+4] != expect):
         die( "Name offset 0x%x: expected %s actual %s" %
-             (offset, aml[offset:offset+4], expect))
+             (offset, expect, aml[offset:offset+4]))
     return offset + len(expect)
 
 # Given data offset, find dword const offset
@@ -172,9 +175,9 @@ def aml_data_byte_const(offset):
              (offset, aml[offset]));
     return offset + 1;
 
-# Find name'd buffer8
-def aml_name_buffer8(offset):
-    return aml_data_buffer8(aml_name_string(offset) + 4)
+# Find name'd buffer
+def aml_name_buffer(offset, length):
+    return aml_data_buffer(aml_name_string(offset) + 4, length)
 
 # Given name offset, find dword const offset
 def aml_name_dword_const(offset):
@@ -308,7 +311,9 @@ for i in range(len(asl)):
         output[array] = aml
         continue
     if (directive == "ACPI_EXTRACT_NAME_BUFFER8"):
-        offset = aml_name_buffer8(offset)
+        offset = aml_name_buffer(offset, 8)
+    elif (directive == "ACPI_EXTRACT_NAME_BUFFER16"):
+        offset = aml_name_buffer(offset, 16)
     elif (directive == "ACPI_EXTRACT_NAME_DWORD_CONST"):
         offset = aml_name_dword_const(offset)
     elif (directive == "ACPI_EXTRACT_NAME_WORD_CONST"):
diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl
index 9d46e5a10..053e4320f 100755
--- a/scripts/checkpatch.pl
+++ b/scripts/checkpatch.pl
@@ -206,9 +206,13 @@ our $UTF8	= qr {
 	|  \xF4[\x80-\x8F][\x80-\xBF]{2}     # plane 16
 }x;
 
+# There are still some false positives, but this catches most
+# common cases.
 our $typeTypedefs = qr{(?x:
-	(?:__)?(?:u|s|be|le)(?:8|16|32|64)|
-	atomic_t
+        [A-Z][A-Z\d_]*[a-z][A-Za-z\d_]*     # camelcase
+        | [A-Z][A-Z\d_]*AIOCB               # all uppercase
+        | [A-Z][A-Z\d_]*CPU                 # all uppercase
+        | QEMUBH                            # all uppercase
 )};
 
 our $logFunctions = qr{(?x:
diff --git a/scripts/cleanup-trace-events.pl b/scripts/cleanup-trace-events.pl
index cffbf165d..7e808efb6 100755
--- a/scripts/cleanup-trace-events.pl
+++ b/scripts/cleanup-trace-events.pl
@@ -25,7 +25,7 @@ sub out {
 
 while (<>) {
     if (/^(disable )?([a-z_0-9]+)\(/) {
-        open GREP, '-|', 'git', 'grep', '-l', "trace_$2"
+        open GREP, '-|', 'git', 'grep', '-lw', "trace_$2"
             or die "run git grep: $!";
         my $fname;
         while ($fname = <GREP>) {
diff --git a/scripts/get_maintainer.pl b/scripts/get_maintainer.pl
index 38334de87..af68c6c92 100755
--- a/scripts/get_maintainer.pl
+++ b/scripts/get_maintainer.pl
@@ -651,18 +651,26 @@ sub get_maintainers {
 	$email->[0] = deduplicate_email($email->[0]);
     }
 
-    foreach my $file (@files) {
-	if ($email &&
-	    ($email_git || ($email_git_fallback &&
-			    !$exact_pattern_match_hash{$file}))) {
-	    vcs_file_signoffs($file);
-	}
-	if ($email && $email_git_blame) {
-	    vcs_file_blame($file);
+    if ($email) {
+	if (! $interactive) {
+	    $email_git_fallback = 0 if @email_to > 0 || @list_to > 0 || $email_git || $email_git_blame;
+	    if ($email_git_fallback) {
+	        print STDERR "get_maintainer.pl: No maintainers found, printing recent contributors.\n";
+	        print STDERR "get_maintainer.pl: Do not blindly cc: them on patches!  Use common sense.\n";
+	        print STDERR "\n";
+            }
+        }
+
+	foreach my $file (@files) {
+	    if ($email_git || ($email_git_fallback &&
+			       !$exact_pattern_match_hash{$file})) {
+	        vcs_file_signoffs($file);
+	    }
+	    if ($email_git_blame) {
+	        vcs_file_blame($file);
+	    }
 	}
-    }
 
-    if ($email) {
 	foreach my $chief (@penguin_chief) {
 	    if ($chief =~ m/^(.*):(.*)/) {
 		my $email_address;
diff --git a/scripts/kvm/kvm_stat b/scripts/kvm/kvm_stat
index d7e97e748..7b1437ca2 100755
--- a/scripts/kvm/kvm_stat
+++ b/scripts/kvm/kvm_stat
@@ -12,7 +12,7 @@
 # the COPYING file in the top-level directory.
 
 import curses
-import sys, os, time, optparse
+import sys, os, time, optparse, ctypes
 
 class DebugfsProvider(object):
     def __init__(self):
@@ -141,61 +141,97 @@ svm_exit_reasons = {
     0x400: 'NPF',
 }
 
-s390_exit_reasons = {
-	0x000: 'UNKNOWN',
-	0x001: 'EXCEPTION',
-	0x002: 'IO',
-	0x003: 'HYPERCALL',
-	0x004: 'DEBUG',
-	0x005: 'HLT',
-	0x006: 'MMIO',
-	0x007: 'IRQ_WINDOW_OPEN',
-	0x008: 'SHUTDOWN',
-	0x009: 'FAIL_ENTRY',
-	0x010: 'INTR',
-	0x011: 'SET_TPR',
-	0x012: 'TPR_ACCESS',
-	0x013: 'S390_SIEIC',
-	0x014: 'S390_RESET',
-	0x015: 'DCR',
-	0x016: 'NMI',
-	0x017: 'INTERNAL_ERROR',
-	0x018: 'OSI',
-	0x019: 'PAPR_HCALL',
+# From include/uapi/linux/kvm.h, KVM_EXIT_xxx
+userspace_exit_reasons = {
+     0: 'UNKNOWN',
+     1: 'EXCEPTION',
+     2: 'IO',
+     3: 'HYPERCALL',
+     4: 'DEBUG',
+     5: 'HLT',
+     6: 'MMIO',
+     7: 'IRQ_WINDOW_OPEN',
+     8: 'SHUTDOWN',
+     9: 'FAIL_ENTRY',
+    10: 'INTR',
+    11: 'SET_TPR',
+    12: 'TPR_ACCESS',
+    13: 'S390_SIEIC',
+    14: 'S390_RESET',
+    15: 'DCR',
+    16: 'NMI',
+    17: 'INTERNAL_ERROR',
+    18: 'OSI',
+    19: 'PAPR_HCALL',
+    20: 'S390_UCONTROL',
+    21: 'WATCHDOG',
+    22: 'S390_TSCH',
+    23: 'EPR',
 }
 
-vendor_exit_reasons = {
+x86_exit_reasons = {
     'vmx': vmx_exit_reasons,
     'svm': svm_exit_reasons,
-    'IBM/S390': s390_exit_reasons,
 }
 
-syscall_numbers = {
-    'IBM/S390': 331,
-}
-
-sc_perf_evt_open = 298
-
+sc_perf_evt_open = None
 exit_reasons = None
 
-for line in file('/proc/cpuinfo').readlines():
-    if line.startswith('flags') or line.startswith('vendor_id'):
-        for flag in line.split():
-            if flag in vendor_exit_reasons:
-                exit_reasons = vendor_exit_reasons[flag]
-            if flag in syscall_numbers:
-                sc_perf_evt_open = syscall_numbers[flag]
-filters = {
-    'kvm_exit': ('exit_reason', exit_reasons)
+ioctl_numbers = {
+    'SET_FILTER' : 0x40082406,
+    'ENABLE'     : 0x00002400,
+    'DISABLE'    : 0x00002401,
 }
 
+def x86_init(flag):
+    globals().update({
+        'sc_perf_evt_open' : 298,
+        'exit_reasons' : x86_exit_reasons[flag],
+    })
+
+def s390_init():
+    globals().update({
+        'sc_perf_evt_open' : 331
+    })
+
+def ppc_init():
+    globals().update({
+        'sc_perf_evt_open' : 319,
+        'ioctl_numbers' : {
+            'SET_FILTER' : 0x80002406 | (ctypes.sizeof(ctypes.c_char_p) << 16),
+            'ENABLE'     : 0x20002400,
+            'DISABLE'    : 0x20002401,
+        }
+    })
+
+def detect_platform():
+    if os.uname()[4].startswith('ppc'):
+        ppc_init()
+        return
+
+    for line in file('/proc/cpuinfo').readlines():
+        if line.startswith('flags'):
+            for flag in line.split():
+                if flag in x86_exit_reasons:
+                    x86_init(flag)
+                    return
+        elif line.startswith('vendor_id'):
+            for flag in line.split():
+                if flag == 'IBM/S390':
+                    s390_init()
+                    return
+
+detect_platform()
+
 def invert(d):
     return dict((x[1], x[0]) for x in d.iteritems())
 
-for f in filters:
-    filters[f] = (filters[f][0], invert(filters[f][1]))
+filters = {}
+filters['kvm_userspace_exit'] = ('reason', invert(userspace_exit_reasons))
+if exit_reasons:
+    filters['kvm_exit'] = ('exit_reason', invert(exit_reasons))
 
-import ctypes, struct, array
+import struct, array
 
 libc = ctypes.CDLL('libc.so.6')
 syscall = libc.syscall
@@ -285,14 +321,14 @@ class Event(object):
             raise Exception('perf_event_open failed')
         if filter:
             import fcntl
-            fcntl.ioctl(fd, 0x40082406, filter)
+            fcntl.ioctl(fd, ioctl_numbers['SET_FILTER'], filter)
         self.fd = fd
     def enable(self):
         import fcntl
-        fcntl.ioctl(self.fd, 0x00002400, 0)
+        fcntl.ioctl(self.fd, ioctl_numbers['ENABLE'], 0)
     def disable(self):
         import fcntl
-        fcntl.ioctl(self.fd, 0x00002401, 0)
+        fcntl.ioctl(self.fd, ioctl_numbers['DISABLE'], 0)
 
 class TracepointProvider(object):
     def __init__(self):
@@ -311,18 +347,30 @@ class TracepointProvider(object):
         self.select(fields)
     def fields(self):
         return self._fields
+
+    def _online_cpus(self):
+        l = []
+        pattern = r'cpu([0-9]+)'
+        basedir = '/sys/devices/system/cpu'
+        for entry in os.listdir(basedir):
+            match = re.match(pattern, entry)
+            if not match:
+                continue
+            path = os.path.join(basedir, entry, 'online')
+            if os.path.exists(path) and open(path).read().strip() != '1':
+                continue
+            l.append(int(match.group(1)))
+        return l
+
     def _setup(self, _fields):
         self._fields = _fields
-        cpure = r'cpu([0-9]+)'
-        self.cpus = [int(re.match(cpure, x).group(1))
-                     for x in os.listdir('/sys/devices/system/cpu')
-                     if re.match(cpure, x)]
+        cpus = self._online_cpus()
         import resource
-        nfiles = len(self.cpus) * 1000
+        nfiles = len(cpus) * 1000
         resource.setrlimit(resource.RLIMIT_NOFILE, (nfiles, nfiles))
         events = []
         self.group_leaders = []
-        for cpu in self.cpus:
+        for cpu in cpus:
             group = Group(cpu)
             for name in _fields:
                 tracepoint = name
diff --git a/scripts/kvm/vmxcap b/scripts/kvm/vmxcap
index c90eda497..8f0371f49 100755
--- a/scripts/kvm/vmxcap
+++ b/scripts/kvm/vmxcap
@@ -99,7 +99,7 @@ controls = [
     Misc(
         name = 'Basic VMX Information',
         bits = {
-            (0, 31): 'Revision',
+            (0, 30): 'Revision',
             (32,44): 'VMCS size',
             48: 'VMCS restricted to 32 bit addresses',
             49: 'Dual-monitor support',
@@ -169,7 +169,9 @@ controls = [
             12: 'Enable INVPCID',
             13: 'Enable VM functions',
             14: 'VMCS shadowing',
-            18: 'EPT-violation #VE'
+            16: 'RDSEED exiting',
+            18: 'EPT-violation #VE',
+            20: 'Enable XSAVES/XRSTORS',
             },
         cap_msr = MSR_IA32_VMX_PROCBASED_CTLS2,
         ),
@@ -195,7 +197,7 @@ controls = [
         name = 'VM-Entry controls',
         bits = {
             2: 'Load debug controls',
-            9: 'IA-64 mode guest',
+            9: 'IA-32e mode guest',
             10: 'Entry to SMM',
             11: 'Deactivate dual-monitor treatment',
             13: 'Load IA32_PERF_GLOBAL_CTRL',
@@ -216,7 +218,7 @@ controls = [
             8: 'Wait-for-SIPI activity state',
             15: 'IA32_SMBASE support',
             (16,24): 'Number of CR3-target values',
-            (25,27): 'MSR-load/store count recommenation',
+            (25,27): 'MSR-load/store count recommendation',
             28: 'IA32_SMM_MONITOR_CTL[2] can be set to 1',
             29: 'VMWRITE to VM-exit information fields',
             (32,63): 'MSEG revision identifier',
diff --git a/scripts/qapi-types.py b/scripts/qapi-types.py
index b4632324a..d2f815bca 100644
--- a/scripts/qapi-types.py
+++ b/scripts/qapi-types.py
@@ -177,6 +177,8 @@ const int %(name)s_qtypes[QTYPE_MAX] = {
             qtype = "QTYPE_QDICT"
         elif find_union(qapi_type):
             qtype = "QTYPE_QDICT"
+        elif find_enum(qapi_type):
+            qtype = "QTYPE_QSTRING"
         else:
             assert False, "Invalid anonymous union member"
 
diff --git a/scripts/qapi-visit.py b/scripts/qapi-visit.py
index c12969721..8f845a2b2 100644
--- a/scripts/qapi-visit.py
+++ b/scripts/qapi-visit.py
@@ -263,7 +263,8 @@ void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **e
     for key in members:
         assert (members[key] in builtin_types
             or find_struct(members[key])
-            or find_union(members[key])), "Invalid anonymous union member"
+            or find_union(members[key])
+            or find_enum(members[key])), "Invalid anonymous union member"
 
         enum_full_value = generate_enum_full_value(disc_type, key)
         ret += mcgen('''
@@ -357,6 +358,9 @@ void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **e
         if (err) {
             goto out_obj;
         }
+        if (!visit_start_union(m, !!(*obj)->data, &err) || err) {
+            goto out_obj;
+        }
         switch ((*obj)->kind) {
 ''',
                  disc_type = disc_type,
@@ -385,6 +389,9 @@ void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **e
 out_obj:
         error_propagate(errp, err);
         err = NULL;
+        visit_end_union(m, !!(*obj)->data, &err);
+        error_propagate(errp, err);
+        err = NULL;
     }
     visit_end_struct(m, &err);
 out:
diff --git a/scripts/qapi.py b/scripts/qapi.py
index f2c6d1f84..77d46aa99 100644
--- a/scripts/qapi.py
+++ b/scripts/qapi.py
@@ -107,10 +107,10 @@ class QAPISchema:
                                         'Expected a file name (string), got: %s'
                                         % include)
                 include_path = os.path.join(self.input_dir, include)
-                if any(include_path == elem[1]
-                       for elem in self.include_hist):
-                    raise QAPIExprError(expr_info, "Inclusion loop for %s"
-                                        % include)
+                for elem in self.include_hist:
+                    if include_path == elem[1]:
+                        raise QAPIExprError(expr_info, "Inclusion loop for %s"
+                                            % include)
                 # skip multiple include of the same file
                 if include_path in previously_included:
                     continue
diff --git a/scripts/qtest b/scripts/qtest
deleted file mode 100755
index 4ef6c1c50..000000000
--- a/scripts/qtest
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-
-export QTEST_QEMU_BINARY=$1
-shift
-"$@"
diff --git a/scripts/simpletrace.py b/scripts/simpletrace.py
index 1aa9460e4..3916c6d14 100755
--- a/scripts/simpletrace.py
+++ b/scripts/simpletrace.py
@@ -58,8 +58,8 @@ def read_record(edict, fobj):
     rechdr = read_header(fobj, rec_header_fmt)
     return get_record(edict, rechdr, fobj) # return tuple of record elements
 
-def read_trace_file(edict, fobj):
-    """Deserialize trace records from a file, yielding record tuples (event_num, timestamp, pid, arg1, ..., arg6)."""
+def read_trace_header(fobj):
+    """Read and verify trace file header"""
     header = read_header(fobj, log_header_fmt)
     if header is None or \
        header[0] != header_event_id or \
@@ -73,6 +73,8 @@ def read_trace_file(edict, fobj):
         raise ValueError('Log format %d not supported with this QEMU release!'
                          % log_version)
 
+def read_trace_records(edict, fobj):
+    """Deserialize trace records from a file, yielding record tuples (event_num, timestamp, pid, arg1, ..., arg6)."""
     while True:
         rec = read_record(edict, fobj)
         if rec is None:
@@ -102,13 +104,16 @@ class Analyzer(object):
         """Called at the end of the trace."""
         pass
 
-def process(events, log, analyzer):
+def process(events, log, analyzer, read_header=True):
     """Invoke an analyzer on each event in a log."""
     if isinstance(events, str):
         events = _read_events(open(events, 'r'))
     if isinstance(log, str):
         log = open(log, 'rb')
 
+    if read_header:
+        read_trace_header(log)
+
     dropped_event = Event.build("Dropped_Event(uint64_t num_events_dropped)")
     edict = {dropped_event_id: dropped_event}
 
@@ -137,7 +142,7 @@ def process(events, log, analyzer):
 
     analyzer.begin()
     fn_cache = {}
-    for rec in read_trace_file(edict, log):
+    for rec in read_trace_records(edict, log):
         event_num = rec[0]
         event = edict[event_num]
         if event_num not in fn_cache:
@@ -152,12 +157,17 @@ def run(analyzer):
     advanced scripts will want to call process() instead."""
     import sys
 
-    if len(sys.argv) != 3:
-        sys.stderr.write('usage: %s <trace-events> <trace-file>\n' % sys.argv[0])
+    read_header = True
+    if len(sys.argv) == 4 and sys.argv[1] == '--no-header':
+        read_header = False
+        del sys.argv[1]
+    elif len(sys.argv) != 3:
+        sys.stderr.write('usage: %s [--no-header] <trace-events> ' \
+                         '<trace-file>\n' % sys.argv[0])
         sys.exit(1)
 
     events = _read_events(open(sys.argv[1], 'r'))
-    process(events, sys.argv[2], analyzer)
+    process(events, sys.argv[2], analyzer, read_header=read_header)
 
 if __name__ == '__main__':
     class Formatter(Analyzer):
diff --git a/scripts/tracetool/__init__.py b/scripts/tracetool/__init__.py
index e8e8edcc4..181675f00 100644
--- a/scripts/tracetool/__init__.py
+++ b/scripts/tracetool/__init__.py
@@ -15,9 +15,11 @@ __email__      = "stefanha@linux.vnet.ibm.com"
 
 import re
 import sys
+import weakref
 
 import tracetool.format
 import tracetool.backend
+import tracetool.transform
 
 
 def error_write(*lines):
@@ -108,6 +110,18 @@ class Arguments:
         """List of argument types."""
         return [ type_ for type_, _ in self._args ]
 
+    def transform(self, *trans):
+        """Return a new Arguments instance with transformed types.
+
+        The types in the resulting Arguments instance are transformed according
+        to tracetool.transform.transform_type.
+        """
+        res = []
+        for type_, name in self._args:
+            res.append((tracetool.transform.transform_type(type_, *trans),
+                        name))
+        return Arguments(res)
+
 
 class Event(object):
     """Event description.
@@ -122,13 +136,19 @@ class Event(object):
         Properties of the event.
     args : Arguments
         The event arguments.
+
     """
 
-    _CRE = re.compile("((?P<props>.*)\s+)?(?P<name>[^(\s]+)\((?P<args>[^)]*)\)\s*(?P<fmt>\".*)?")
+    _CRE = re.compile("((?P<props>[\w\s]+)\s+)?"
+                      "(?P<name>\w+)"
+                      "\((?P<args>[^)]*)\)"
+                      "\s*"
+                      "(?:(?:(?P<fmt_trans>\".+),)?\s*(?P<fmt>\".+))?"
+                      "\s*")
 
-    _VALID_PROPS = set(["disable"])
+    _VALID_PROPS = set(["disable", "tcg", "tcg-trans", "tcg-exec"])
 
-    def __init__(self, name, props, fmt, args):
+    def __init__(self, name, props, fmt, args, orig=None):
         """
         Parameters
         ----------
@@ -136,20 +156,29 @@ class Event(object):
             Event name.
         props : list of str
             Property names.
-        fmt : str
-            Event printing format.
+        fmt : str, list of str
+            Event printing format (or formats).
         args : Arguments
             Event arguments.
+        orig : Event or None
+            Original Event before transformation.
+
         """
         self.name = name
         self.properties = props
         self.fmt = fmt
         self.args = args
 
+        if orig is None:
+            self.original = weakref.ref(self)
+        else:
+            self.original = orig
+
         unknown_props = set(self.properties) - self._VALID_PROPS
         if len(unknown_props) > 0:
             raise ValueError("Unknown properties: %s"
                              % ", ".join(unknown_props))
+        assert isinstance(self.fmt, str) or len(self.fmt) == 2
 
     def copy(self):
         """Create a new copy."""
@@ -172,34 +201,96 @@ class Event(object):
         name = groups["name"]
         props = groups["props"].split()
         fmt = groups["fmt"]
+        fmt_trans = groups["fmt_trans"]
+        if len(fmt_trans) > 0:
+            fmt = [fmt_trans, fmt]
         args = Arguments.build(groups["args"])
 
+        if "tcg-trans" in props:
+            raise ValueError("Invalid property 'tcg-trans'")
+        if "tcg-exec" in props:
+            raise ValueError("Invalid property 'tcg-exec'")
+        if "tcg" not in props and not isinstance(fmt, str):
+            raise ValueError("Only events with 'tcg' property can have two formats")
+        if "tcg" in props and isinstance(fmt, str):
+            raise ValueError("Events with 'tcg' property must have two formats")
+
         return Event(name, props, fmt, args)
 
     def __repr__(self):
         """Evaluable string representation for this object."""
+        if isinstance(self.fmt, str):
+            fmt = self.fmt
+        else:
+            fmt = "%s, %s" % (self.fmt[0], self.fmt[1])
         return "Event('%s %s(%s) %s')" % (" ".join(self.properties),
                                           self.name,
                                           self.args,
-                                          self.fmt)
+                                          fmt)
+
+    _FMT = re.compile("(%[\d\.]*\w+|%.*PRI\S+)")
+
+    def formats(self):
+        """List of argument print formats."""
+        assert not isinstance(self.fmt, list)
+        return self._FMT.findall(self.fmt)
 
     QEMU_TRACE               = "trace_%(name)s"
+    QEMU_TRACE_TCG           = QEMU_TRACE + "_tcg"
 
     def api(self, fmt=None):
         if fmt is None:
             fmt = Event.QEMU_TRACE
         return fmt % {"name": self.name}
 
+    def transform(self, *trans):
+        """Return a new Event with transformed Arguments."""
+        return Event(self.name,
+                     list(self.properties),
+                     self.fmt,
+                     self.args.transform(*trans),
+                     self)
+
 
 def _read_events(fobj):
-    res = []
+    events = []
     for line in fobj:
         if not line.strip():
             continue
         if line.lstrip().startswith('#'):
             continue
-        res.append(Event.build(line))
-    return res
+
+        event = Event.build(line)
+
+        # transform TCG-enabled events
+        if "tcg" not in event.properties:
+            events.append(event)
+        else:
+            event_trans = event.copy()
+            event_trans.name += "_trans"
+            event_trans.properties += ["tcg-trans"]
+            event_trans.fmt = event.fmt[0]
+            args_trans = []
+            for atrans, aorig in zip(
+                    event_trans.transform(tracetool.transform.TCG_2_HOST).args,
+                    event.args):
+                if atrans == aorig:
+                    args_trans.append(atrans)
+            event_trans.args = Arguments(args_trans)
+            event_trans = event_trans.copy()
+
+            event_exec = event.copy()
+            event_exec.name += "_exec"
+            event_exec.properties += ["tcg-exec"]
+            event_exec.fmt = event.fmt[1]
+            event_exec = event_exec.transform(tracetool.transform.TCG_2_HOST)
+
+            new_event = [event_trans, event_exec]
+            event.event_trans, event.event_exec = new_event
+
+            events.extend(new_event)
+
+    return events
 
 
 class TracetoolError (Exception):
diff --git a/scripts/tracetool/backend/__init__.py b/scripts/tracetool/backend/__init__.py
index 5bfa1efc5..d4b6dab9c 100644
--- a/scripts/tracetool/backend/__init__.py
+++ b/scripts/tracetool/backend/__init__.py
@@ -102,7 +102,8 @@ class Wrapper:
     def __init__(self, backends, format):
         self._backends = [backend.replace("-", "_") for backend in backends]
         self._format = format.replace("-", "_")
-        assert all(exists(backend) for backend in self._backends)
+        for backend in self._backends:
+            assert exists(backend)
         assert tracetool.format.exists(self._format)
 
     def _run_function(self, name, *args, **kwargs):
diff --git a/scripts/tracetool/format/events_h.py b/scripts/tracetool/format/events_h.py
index 25d913bb2..9f114a349 100644
--- a/scripts/tracetool/format/events_h.py
+++ b/scripts/tracetool/format/events_h.py
@@ -40,6 +40,11 @@ def generate(events, backend):
             enabled = 0
         else:
             enabled = 1
+        if "tcg-trans" in e.properties:
+            # a single define for the two "sub-events"
+            out('#define TRACE_%(name)s_ENABLED %(enabled)d',
+                name=e.original.original.name.upper(),
+                enabled=enabled)
         out('#define TRACE_%s_ENABLED %d' % (e.name.upper(), enabled))
 
     out('#include "trace/event-internal.h"',
diff --git a/scripts/tracetool/format/simpletrace_stap.py b/scripts/tracetool/format/simpletrace_stap.py
new file mode 100644
index 000000000..7e44bc181
--- /dev/null
+++ b/scripts/tracetool/format/simpletrace_stap.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Generate .stp file that outputs simpletrace binary traces (DTrace with SystemTAP only).
+"""
+
+__author__     = "Stefan Hajnoczi <redhat.com>"
+__copyright__  = "Copyright (C) 2014, Red Hat, Inc."
+__license__    = "GPL version 2 or (at your option) any later version"
+
+__maintainer__ = "Stefan Hajnoczi"
+__email__      = "stefanha@redhat.com"
+
+
+from tracetool import out
+from tracetool.backend.dtrace import binary, probeprefix
+from tracetool.backend.simple import is_string
+from tracetool.format.stap import stap_escape
+
+
+def generate(events, backend):
+    out('/* This file is autogenerated by tracetool, do not edit. */',
+        '')
+
+    for event_id, e in enumerate(events):
+        if 'disable' in e.properties:
+            continue
+
+        out('probe %(probeprefix)s.simpletrace.%(name)s = %(probeprefix)s.%(name)s ?',
+            '{',
+            probeprefix=probeprefix(),
+            name=e.name)
+
+        # Calculate record size
+        sizes = ['24'] # sizeof(TraceRecord)
+        for type_, name in e.args:
+            name = stap_escape(name)
+            if is_string(type_):
+                out('    try {',
+                    '        arg%(name)s_str = %(name)s ? user_string_n(%(name)s, 512) : "<null>"',
+                    '    } catch {}',
+                    '    arg%(name)s_len = strlen(arg%(name)s_str)',
+                    name=name)
+                sizes.append('4 + arg%s_len' % name)
+            else:
+                sizes.append('8')
+        sizestr = ' + '.join(sizes)
+
+        # Generate format string and value pairs for record header and arguments
+        fields = [('8b', str(event_id)),
+                  ('8b', 'gettimeofday_ns()'),
+                  ('4b', sizestr),
+                  ('4b', 'pid()')]
+        for type_, name in e.args:
+            name = stap_escape(name)
+            if is_string(type_):
+                fields.extend([('4b', 'arg%s_len' % name),
+                               ('.*s', 'arg%s_len, arg%s_str' % (name, name))])
+            else:
+                fields.append(('8b', name))
+
+        # Emit the entire record in a single SystemTap printf()
+        fmt_str = '%'.join(fmt for fmt, _ in fields)
+        arg_str = ', '.join(arg for _, arg in fields)
+        out('    printf("%%%(fmt_str)s", %(arg_str)s)',
+            fmt_str=fmt_str, arg_str=arg_str)
+
+        out('}')
+
+    out()
diff --git a/scripts/tracetool/format/stap.py b/scripts/tracetool/format/stap.py
index e24abf7f1..9e780f1b0 100644
--- a/scripts/tracetool/format/stap.py
+++ b/scripts/tracetool/format/stap.py
@@ -27,6 +27,13 @@ RESERVED_WORDS = (
     )
 
 
+def stap_escape(identifier):
+    # Append underscore to reserved keywords
+    if identifier in RESERVED_WORDS:
+        return identifier + '_'
+    return identifier
+
+
 def generate(events, backend):
     events = [e for e in events
               if "disable" not in e.properties]
@@ -45,9 +52,7 @@ def generate(events, backend):
         i = 1
         if len(e.args) > 0:
             for name in e.args.names():
-                # Append underscore to reserved keywords
-                if name in RESERVED_WORDS:
-                    name += '_'
+                name = stap_escape(name)
                 out('  %s = $arg%d;' % (name, i))
                 i += 1
 
diff --git a/scripts/tracetool/format/tcg_h.py b/scripts/tracetool/format/tcg_h.py
new file mode 100644
index 000000000..f676b6662
--- /dev/null
+++ b/scripts/tracetool/format/tcg_h.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Generate .h file for TCG code generation.
+"""
+
+__author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__license__    = "GPL version 2 or (at your option) any later version"
+
+__maintainer__ = "Stefan Hajnoczi"
+__email__      = "stefanha@linux.vnet.ibm.com"
+
+
+from tracetool import out
+
+
+def generate(events, backend):
+    out('/* This file is autogenerated by tracetool, do not edit. */',
+        '/* You must include this file after the inclusion of helper.h */',
+        '',
+        '#ifndef TRACE__GENERATED_TCG_TRACERS_H',
+        '#define TRACE__GENERATED_TCG_TRACERS_H',
+        '',
+        '#include <stdint.h>',
+        '',
+        '#include "trace.h"',
+        '#include "exec/helper-proto.h"',
+        '',
+        )
+
+    for e in events:
+        # just keep one of them
+        if "tcg-trans" not in e.properties:
+            continue
+
+        # get the original event definition
+        e = e.original.original
+
+        out('static inline void %(name_tcg)s(%(args)s)',
+            '{',
+            name_tcg=e.api(e.QEMU_TRACE_TCG),
+            args=e.args)
+
+        if "disable" not in e.properties:
+            out('    %(name_trans)s(%(argnames_trans)s);',
+                '    gen_helper_%(name_exec)s(%(argnames_exec)s);',
+                name_trans=e.event_trans.api(e.QEMU_TRACE),
+                name_exec=e.event_exec.api(e.QEMU_TRACE),
+                argnames_trans=", ".join(e.event_trans.args.names()),
+                argnames_exec=", ".join(e.event_exec.args.names()))
+
+        out('}')
+
+    out('',
+        '#endif /* TRACE__GENERATED_TCG_TRACERS_H */')
diff --git a/scripts/tracetool/format/tcg_helper_c.py b/scripts/tracetool/format/tcg_helper_c.py
new file mode 100644
index 000000000..96655a059
--- /dev/null
+++ b/scripts/tracetool/format/tcg_helper_c.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Generate trace/generated-helpers.c.
+"""
+
+__author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__license__    = "GPL version 2 or (at your option) any later version"
+
+__maintainer__ = "Stefan Hajnoczi"
+__email__      = "stefanha@linux.vnet.ibm.com"
+
+
+from tracetool import out
+from tracetool.transform import *
+
+
+def generate(events, backend):
+    events = [e for e in events
+              if "disable" not in e.properties]
+
+    out('/* This file is autogenerated by tracetool, do not edit. */',
+        '',
+        '#include "qemu-common.h"',
+        '#include "trace.h"',
+        '#include "exec/helper-proto.h"',
+        '',
+        )
+
+    for e in events:
+        if "tcg-exec" not in e.properties:
+            continue
+
+        # tracetool.generate always transforms types to host
+        e_args = e.original.args
+
+        values = ["(%s)%s" % (t, n)
+                  for t, n in e.args.transform(TCG_2_TCG_HELPER_DEF)]
+
+        out('void %(name_tcg)s(%(args)s)',
+            '{',
+            '    %(name)s(%(values)s);',
+            '}',
+            name_tcg="helper_%s_proxy" % e.api(),
+            name=e.api(),
+            args=e_args.transform(HOST_2_TCG_COMPAT, TCG_2_TCG_HELPER_DEF),
+            values=", ".join(values),
+            )
diff --git a/scripts/tracetool/format/tcg_helper_h.py b/scripts/tracetool/format/tcg_helper_h.py
new file mode 100644
index 000000000..a8ba7ba8e
--- /dev/null
+++ b/scripts/tracetool/format/tcg_helper_h.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Generate trace/generated-helpers.h.
+"""
+
+__author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__license__    = "GPL version 2 or (at your option) any later version"
+
+__maintainer__ = "Stefan Hajnoczi"
+__email__      = "stefanha@linux.vnet.ibm.com"
+
+
+from tracetool import out
+from tracetool.transform import *
+
+
+def generate(events, backend):
+    events = [e for e in events
+              if "disable" not in e.properties]
+
+    out('/* This file is autogenerated by tracetool, do not edit. */',
+        '',
+        )
+
+    for e in events:
+        if "tcg-exec" not in e.properties:
+            continue
+
+        # tracetool.generate always transforms types to host
+        e_args = e.original.args
+
+        # TCG helper proxy declaration
+        fmt = "DEF_HELPER_FLAGS_%(argc)d(%(name)s, %(flags)svoid%(types)s)"
+        args = e_args.transform(HOST_2_TCG_COMPAT, HOST_2_TCG,
+                                TCG_2_TCG_HELPER_DECL)
+        types = ", ".join(args.types())
+        if types != "":
+            types = ", " + types
+
+        flags = "TCG_CALL_NO_RWG, "
+
+        out(fmt,
+            flags=flags,
+            argc=len(args),
+            name=e.api() + "_proxy",
+            types=types,
+            )
diff --git a/scripts/tracetool/format/tcg_helper_wrapper_h.py b/scripts/tracetool/format/tcg_helper_wrapper_h.py
new file mode 100644
index 000000000..cac5a878f
--- /dev/null
+++ b/scripts/tracetool/format/tcg_helper_wrapper_h.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Generate trace/generated-helpers-wrappers.h.
+"""
+
+__author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__license__    = "GPL version 2 or (at your option) any later version"
+
+__maintainer__ = "Stefan Hajnoczi"
+__email__      = "stefanha@linux.vnet.ibm.com"
+
+
+from tracetool import out
+from tracetool.transform import *
+
+
+def generate(events, backend):
+    events = [e for e in events
+              if "disable" not in e.properties]
+
+    out('/* This file is autogenerated by tracetool, do not edit. */',
+        '',
+        '#define tcg_temp_new_nop(v) (v)',
+        '#define tcg_temp_free_nop(v)',
+        '',
+        )
+
+    for e in events:
+        if "tcg-exec" not in e.properties:
+            continue
+
+        # tracetool.generate always transforms types to host
+        e_args = e.original.args
+
+        # mixed-type to TCG helper bridge
+        args_tcg_compat = e_args.transform(HOST_2_TCG_COMPAT)
+
+        code_new = [
+            "%(tcg_type)s __%(name)s = %(tcg_func)s(%(name)s);" %
+            {"tcg_type": transform_type(type_, HOST_2_TCG),
+             "tcg_func": transform_type(type_, HOST_2_TCG_TMP_NEW),
+             "name": name}
+            for (type_, name) in args_tcg_compat
+        ]
+
+        code_free = [
+            "%(tcg_func)s(__%(name)s);" %
+            {"tcg_func": transform_type(type_, HOST_2_TCG_TMP_FREE),
+             "name": name}
+            for (type_, name) in args_tcg_compat
+        ]
+
+        gen_name = "gen_helper_" + e.api()
+
+        out('static inline void %(name)s(%(args)s)',
+            '{',
+            '    %(code_new)s',
+            '    %(proxy_name)s(%(tmp_names)s);',
+            '    %(code_free)s',
+            '}',
+            name=gen_name,
+            args=e_args,
+            proxy_name=gen_name + "_proxy",
+            code_new="\n    ".join(code_new),
+            code_free="\n    ".join(code_free),
+            tmp_names=", ".join(["__%s" % name for _, name in e_args]),
+            )
diff --git a/scripts/tracetool/format/ust_events_h.py b/scripts/tracetool/format/ust_events_h.py
index 510256547..3e8a7cdf1 100644
--- a/scripts/tracetool/format/ust_events_h.py
+++ b/scripts/tracetool/format/ust_events_h.py
@@ -63,13 +63,20 @@ def generate(events, backend):
                 name=e.name,
                 args=", ".join(", ".join(i) for i in e.args))
 
-            for t, n in e.args:
-                if ('int' in t) or ('long' in t) or ('unsigned' in t) or ('size_t' in t):
+            types = e.args.types()
+            names = e.args.names()
+            fmts = e.formats()
+            for t,n,f in zip(types, names, fmts):
+                if ('char *' in t) or ('char*' in t):
+                    out('       ctf_string(' + n + ', ' + n + ')')
+                elif ("%p" in f) or ("x" in f) or ("PRIx" in f):
+                    out('       ctf_integer_hex('+ t + ', ' + n + ', ' + n + ')')
+                elif ("ptr" in t) or ("*" in t):
+                    out('       ctf_integer_hex('+ t + ', ' + n + ', ' + n + ')')
+                elif ('int' in t) or ('long' in t) or ('unsigned' in t) or ('size_t' in t):
                     out('       ctf_integer(' + t + ', ' + n + ', ' + n + ')')
                 elif ('double' in t) or ('float' in t):
                     out('       ctf_float(' + t + ', ' + n + ', ' + n + ')')
-                elif ('char *' in t) or ('char*' in t):
-                    out('       ctf_string(' + n + ', ' + n + ')')
                 elif ('void *' in t) or ('void*' in t):
                     out('       ctf_integer_hex(unsigned long, ' + n + ', ' + n + ')')
 
diff --git a/scripts/tracetool/transform.py b/scripts/tracetool/transform.py
new file mode 100644
index 000000000..fc5e679ed
--- /dev/null
+++ b/scripts/tracetool/transform.py
@@ -0,0 +1,166 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Type-transformation rules.
+"""
+
+__author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__license__    = "GPL version 2 or (at your option) any later version"
+
+__maintainer__ = "Stefan Hajnoczi"
+__email__      = "stefanha@linux.vnet.ibm.com"
+
+
+def _transform_type(type_, trans):
+    if isinstance(trans, str):
+        return trans
+    elif isinstance(trans, dict):
+        if type_ in trans:
+            return _transform_type(type_, trans[type_])
+        elif None in trans:
+            return _transform_type(type_, trans[None])
+        else:
+            return type_
+    elif callable(trans):
+        return trans(type_)
+    else:
+        raise ValueError("Invalid type transformation rule: %s" % trans)
+
+
+def transform_type(type_, *trans):
+    """Return a new type transformed according to the given rules.
+
+    Applies each of the transformation rules in trans in order.
+
+    If an element of trans is a string, return it.
+
+    If an element of trans is a function, call it with type_ as its only
+    argument.
+
+    If an element of trans is a dict, search type_ in its keys. If type_ is
+    a key, use the value as a transformation rule for type_. Otherwise, if
+    None is a key use the value as a transformation rule for type_.
+
+    Otherwise, return type_.
+
+    Parameters
+    ----------
+    type_ : str
+        Type to transform.
+    trans : list of function or dict
+        Type transformation rules.
+    """
+    if len(trans) == 0:
+        raise ValueError
+    res = type_
+    for t in trans:
+        res = _transform_type(res, t)
+    return res
+
+
+##################################################
+# tcg -> host
+
+def _tcg_2_host(type_):
+    if type_ == "TCGv":
+        # force a fixed-size type (target-independent)
+        return "uint64_t"
+    else:
+        return type_
+
+TCG_2_HOST = {
+    "TCGv_i32": "uint32_t",
+    "TCGv_i64": "uint64_t",
+    "TCGv_ptr": "void *",
+    None: _tcg_2_host,
+    }
+
+
+##################################################
+# host -> host compatible with tcg sizes
+
+HOST_2_TCG_COMPAT = {
+    "uint8_t": "uint32_t",
+    }
+
+
+##################################################
+# host/tcg -> tcg
+
+def _host_2_tcg(type_):
+    if type_.startswith("TCGv"):
+        return type_
+    raise ValueError("Don't know how to translate '%s' into a TCG type\n" % type_)
+
+HOST_2_TCG = {
+    "uint32_t": "TCGv_i32",
+    "uint64_t": "TCGv_i64",
+    "void *"  : "TCGv_ptr",
+    None: _host_2_tcg,
+    }
+
+
+##################################################
+# tcg -> tcg helper definition
+
+def _tcg_2_helper_def(type_):
+    if type_ == "TCGv":
+        return "target_ulong"
+    else:
+        return type_
+
+TCG_2_TCG_HELPER_DEF = {
+    "TCGv_i32": "uint32_t",
+    "TCGv_i64": "uint64_t",
+    "TCGv_ptr": "void *",
+    None: _tcg_2_helper_def,
+    }
+
+
+##################################################
+# tcg -> tcg helper declaration
+
+def _tcg_2_tcg_helper_decl_error(type_):
+    raise ValueError("Don't know how to translate type '%s' into a TCG helper declaration type\n" % type_)
+
+TCG_2_TCG_HELPER_DECL = {
+    "TCGv"    : "tl",
+    "TCGv_ptr": "ptr",
+    "TCGv_i32": "i32",
+    "TCGv_i64": "i64",
+    None: _tcg_2_tcg_helper_decl_error,
+    }
+
+
+##################################################
+# host/tcg -> tcg temporal constant allocation
+
+def _host_2_tcg_tmp_new(type_):
+    if type_.startswith("TCGv"):
+        return "tcg_temp_new_nop"
+    raise ValueError("Don't know how to translate type '%s' into a TCG temporal allocation" % type_)
+
+HOST_2_TCG_TMP_NEW = {
+    "uint32_t": "tcg_const_i32",
+    "uint64_t": "tcg_const_i64",
+    "void *"  : "tcg_const_ptr",
+    None: _host_2_tcg_tmp_new,
+    }
+
+
+##################################################
+# host/tcg -> tcg temporal constant deallocation
+
+def _host_2_tcg_tmp_free(type_):
+    if type_.startswith("TCGv"):
+        return "tcg_temp_free_nop"
+    raise ValueError("Don't know how to translate type '%s' into a TCG temporal deallocation" % type_)
+
+HOST_2_TCG_TMP_FREE = {
+    "uint32_t": "tcg_temp_free_i32",
+    "uint64_t": "tcg_temp_free_i64",
+    "void *"  : "tcg_temp_free_ptr",
+    None: _host_2_tcg_tmp_free,
+    }
diff --git a/scripts/vmstate-static-checker.py b/scripts/vmstate-static-checker.py
index 3bae769a3..f7ce3fc48 100755
--- a/scripts/vmstate-static-checker.py
+++ b/scripts/vmstate-static-checker.py
@@ -42,30 +42,44 @@ def check_fields_match(name, s_field, d_field):
     # Some fields changed names between qemu versions.  This list
     # is used to whitelist such changes in each section / description.
     changed_names = {
+        'apic': ['timer', 'timer_expiry'],
         'e1000': ['dev', 'parent_obj'],
         'ehci': ['dev', 'pcidev'],
         'I440FX': ['dev', 'parent_obj'],
         'ich9_ahci': ['card', 'parent_obj'],
+        'ich9-ahci': ['ahci', 'ich9_ahci'],
+        'ioh3420': ['PCIDevice', 'PCIEDevice'],
         'ioh-3240-express-root-port': ['port.br.dev',
                                        'parent_obj.parent_obj.parent_obj',
                                        'port.br.dev.exp.aer_log',
                                 'parent_obj.parent_obj.parent_obj.exp.aer_log'],
+        'lsiscsi': ['dev', 'parent_obj'],
         'mch': ['d', 'parent_obj'],
         'pci_bridge': ['bridge.dev', 'parent_obj', 'bridge.dev.shpc', 'shpc'],
         'pcnet': ['pci_dev', 'parent_obj'],
         'PIIX3': ['pci_irq_levels', 'pci_irq_levels_vmstate'],
         'piix4_pm': ['dev', 'parent_obj', 'pci0_status',
-                     'acpi_pci_hotplug.acpi_pcihp_pci_status[0x0]'],
+                     'acpi_pci_hotplug.acpi_pcihp_pci_status[0x0]',
+                     'pm1a.sts', 'ar.pm1.evt.sts', 'pm1a.en', 'ar.pm1.evt.en',
+                     'pm1_cnt.cnt', 'ar.pm1.cnt.cnt',
+                     'tmr.timer', 'ar.tmr.timer',
+                     'tmr.overflow_time', 'ar.tmr.overflow_time',
+                     'gpe', 'ar.gpe'],
         'rtl8139': ['dev', 'parent_obj'],
         'qxl': ['num_surfaces', 'ssd.num_surfaces'],
+        'usb-ccid': ['abProtocolDataStructure', 'abProtocolDataStructure.data'],
         'usb-host': ['dev', 'parent_obj'],
         'usb-mouse': ['usb-ptr-queue', 'HIDPointerEventQueue'],
         'usb-tablet': ['usb-ptr-queue', 'HIDPointerEventQueue'],
+        'vmware_vga': ['card', 'parent_obj'],
+        'vmware_vga_internal': ['depth', 'new_depth'],
         'xhci': ['pci_dev', 'parent_obj'],
+        'x3130-upstream': ['PCIDevice', 'PCIEDevice'],
         'xio3130-express-downstream-port': ['port.br.dev',
                                             'parent_obj.parent_obj.parent_obj',
                                             'port.br.dev.exp.aer_log',
                                 'parent_obj.parent_obj.parent_obj.exp.aer_log'],
+        'xio3130-downstream': ['PCIDevice', 'PCIEDevice'],
         'xio3130-express-upstream-port': ['br.dev', 'parent_obj.parent_obj',
                                           'br.dev.exp.aer_log',
                                           'parent_obj.parent_obj.exp.aer_log'],
@@ -130,6 +144,7 @@ def check_fields(src_fields, dest_fields, desc, sec):
 
     advance_src = True
     advance_dest = True
+    unused_count = 0
 
     while True:
         if advance_src:
@@ -142,9 +157,10 @@ def check_fields(src_fields, dest_fields, desc, sec):
                 s_iter = s_iter_list.pop()
                 continue
         else:
-            # We want to avoid advancing just once -- when entering a
-            # dest substruct, or when exiting one.
-            advance_src = True
+            if unused_count == 0:
+                # We want to avoid advancing just once -- when entering a
+                # dest substruct, or when exiting one.
+                advance_src = True
 
         if advance_dest:
             try:
@@ -163,7 +179,37 @@ def check_fields(src_fields, dest_fields, desc, sec):
                 advance_src = False
                 continue
         else:
-            advance_dest = True
+            if unused_count == 0:
+                advance_dest = True
+
+        if unused_count > 0:
+            if advance_dest == False:
+                unused_count = unused_count - s_item["size"]
+                if unused_count == 0:
+                    advance_dest = True
+                    continue
+                if unused_count < 0:
+                    print "Section \"" + sec + "\",",
+                    print "Description \"" + desc + "\":",
+                    print "unused size mismatch near \"",
+                    print s_item["field"] + "\""
+                    bump_taint()
+                    break
+                continue
+
+            if advance_src == False:
+                unused_count = unused_count - d_item["size"]
+                if unused_count == 0:
+                    advance_src = True
+                    continue
+                if unused_count < 0:
+                    print "Section \"" + sec + "\",",
+                    print "Description \"" + desc + "\":",
+                    print "unused size mismatch near \"",
+                    print d_item["field"] + "\""
+                    bump_taint()
+                    break
+                continue
 
         if not check_fields_match(desc, s_item["field"], d_item["field"]):
             # Some fields were put in substructs, keeping the
@@ -194,6 +240,20 @@ def check_fields(src_fields, dest_fields, desc, sec):
                 advance_dest = False
                 continue
 
+            if s_item["field"] == "unused" or d_item["field"] == "unused":
+                if s_item["size"] == d_item["size"]:
+                    continue
+
+                if d_item["field"] == "unused":
+                    advance_dest = False
+                    unused_count = d_item["size"] - s_item["size"]
+                    continue
+
+                if s_item["field"] == "unused":
+                    advance_src = False
+                    unused_count = s_item["size"] - d_item["size"]
+                    continue
+
             print "Section \"" + sec + "\",",
             print "Description \"" + desc + "\":",
             print "expected field \"" + s_item["field"] + "\",",
diff --git a/slirp/misc.c b/slirp/misc.c
index b8eb74cab..6543dc777 100644
--- a/slirp/misc.c
+++ b/slirp/misc.c
@@ -54,11 +54,11 @@ int add_exec(struct ex_list **ex_ptr, int do_pty, char *exec,
 	}
 
 	tmp_ptr = *ex_ptr;
-	*ex_ptr = (struct ex_list *)malloc(sizeof(struct ex_list));
+	*ex_ptr = g_new(struct ex_list, 1);
 	(*ex_ptr)->ex_fport = port;
 	(*ex_ptr)->ex_addr = addr;
 	(*ex_ptr)->ex_pty = do_pty;
-	(*ex_ptr)->ex_exec = (do_pty == 3) ? exec : strdup(exec);
+	(*ex_ptr)->ex_exec = (do_pty == 3) ? exec : g_strdup(exec);
 	(*ex_ptr)->ex_next = tmp_ptr;
 	return 0;
 }
@@ -187,7 +187,7 @@ fork_exec(struct socket *so, const char *ex, int do_pty)
 			   bptr++;
 			c = *bptr;
 			*bptr++ = (char)0;
-			argv[i++] = strdup(curarg);
+			argv[i++] = g_strdup(curarg);
 		   } while (c);
 
                 argv[i] = NULL;
@@ -228,20 +228,6 @@ fork_exec(struct socket *so, const char *ex, int do_pty)
 }
 #endif
 
-#ifndef HAVE_STRDUP
-char *
-strdup(str)
-	const char *str;
-{
-	char *bptr;
-
-	bptr = (char *)malloc(strlen(str)+1);
-	strcpy(bptr, str);
-
-	return bptr;
-}
-#endif
-
 void slirp_connection_info(Slirp *slirp, Monitor *mon)
 {
     const char * const tcpstates[] = {
diff --git a/slirp/misc.h b/slirp/misc.h
index ba8beb1b1..41a32583d 100644
--- a/slirp/misc.h
+++ b/slirp/misc.h
@@ -16,10 +16,6 @@ struct ex_list {
 	struct ex_list *ex_next;
 };
 
-#ifndef HAVE_STRDUP
-char *strdup(const char *);
-#endif
-
 #define EMU_NONE 0x0
 
 /* TCP emulations */
diff --git a/slirp/slirp_config.h b/slirp/slirp_config.h
index 18db45c8e..896d8022e 100644
--- a/slirp/slirp_config.h
+++ b/slirp/slirp_config.h
@@ -72,9 +72,6 @@
 /* Define if you have strerror */
 #define HAVE_STRERROR
 
-/* Define if you have strdup() */
-#define HAVE_STRDUP
-
 /* Define according to how time.h should be included */
 #define TIME_WITH_SYS_TIME 0
 #undef HAVE_SYS_TIME_H
diff --git a/slirp/udp.c b/slirp/udp.c
index 8cc6cb66d..f77e00f5a 100644
--- a/slirp/udp.c
+++ b/slirp/udp.c
@@ -152,7 +152,7 @@ udp_input(register struct mbuf *m, int iphlen)
 	 * Locate pcb for datagram.
 	 */
 	so = slirp->udp_last_so;
-	if (so->so_lport != uh->uh_sport ||
+	if (so == &slirp->udb || so->so_lport != uh->uh_sport ||
 	    so->so_laddr.s_addr != ip->ip_src.s_addr) {
 		struct socket *tmp;
 
diff --git a/softmmu_template.h b/softmmu_template.h
index 5a07f991a..6b4e615db 100644
--- a/softmmu_template.h
+++ b/softmmu_template.h
@@ -67,10 +67,10 @@
 #endif
 
 #ifdef SOFTMMU_CODE_ACCESS
-#define READ_ACCESS_TYPE 2
+#define READ_ACCESS_TYPE MMU_INST_FETCH
 #define ADDR_READ addr_code
 #else
-#define READ_ACCESS_TYPE 0
+#define READ_ACCESS_TYPE MMU_DATA_LOAD
 #define ADDR_READ addr_read
 #endif
 
@@ -116,6 +116,31 @@
 # define helper_te_st_name  helper_le_st_name
 #endif
 
+/* macro to check the victim tlb */
+#define VICTIM_TLB_HIT(ty)                                                    \
+({                                                                            \
+    /* we are about to do a page table walk. our last hope is the             \
+     * victim tlb. try to refill from the victim tlb before walking the       \
+     * page table. */                                                         \
+    int vidx;                                                                 \
+    hwaddr tmpiotlb;                                                          \
+    CPUTLBEntry tmptlb;                                                       \
+    for (vidx = CPU_VTLB_SIZE-1; vidx >= 0; --vidx) {                         \
+        if (env->tlb_v_table[mmu_idx][vidx].ty == (addr & TARGET_PAGE_MASK)) {\
+            /* found entry in victim tlb, swap tlb and iotlb */               \
+            tmptlb = env->tlb_table[mmu_idx][index];                          \
+            env->tlb_table[mmu_idx][index] = env->tlb_v_table[mmu_idx][vidx]; \
+            env->tlb_v_table[mmu_idx][vidx] = tmptlb;                         \
+            tmpiotlb = env->iotlb[mmu_idx][index];                            \
+            env->iotlb[mmu_idx][index] = env->iotlb_v[mmu_idx][vidx];         \
+            env->iotlb_v[mmu_idx][vidx] = tmpiotlb;                           \
+            break;                                                            \
+        }                                                                     \
+    }                                                                         \
+    /* return true when there is a vtlb hit, i.e. vidx >=0 */                 \
+    vidx >= 0;                                                                \
+})
+
 #ifndef SOFTMMU_CODE_ACCESS
 static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
                                               hwaddr physaddr,
@@ -161,7 +186,10 @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr, int mmu_idx,
                                  mmu_idx, retaddr);
         }
 #endif
-        tlb_fill(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+        if (!VICTIM_TLB_HIT(ADDR_READ)) {
+            tlb_fill(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
+                     mmu_idx, retaddr);
+        }
         tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
     }
 
@@ -246,7 +274,10 @@ WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr, int mmu_idx,
                                  mmu_idx, retaddr);
         }
 #endif
-        tlb_fill(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+        if (!VICTIM_TLB_HIT(ADDR_READ)) {
+            tlb_fill(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
+                     mmu_idx, retaddr);
+        }
         tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
     }
 
@@ -365,10 +396,13 @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 #ifdef ALIGNED_ONLY
         if ((addr & (DATA_SIZE - 1)) != 0) {
-            cpu_unaligned_access(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
+            cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
+                                 mmu_idx, retaddr);
         }
 #endif
-        tlb_fill(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
+        if (!VICTIM_TLB_HIT(addr_write)) {
+            tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);
+        }
         tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
     }
 
@@ -394,7 +428,8 @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
         int i;
     do_unaligned_access:
 #ifdef ALIGNED_ONLY
-        cpu_unaligned_access(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
+        cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
+                             mmu_idx, retaddr);
 #endif
         /* XXX: not efficient, but simple */
         /* Note: relies on the fact that tlb_fill() does not remove the
@@ -413,7 +448,8 @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
     /* Handle aligned access or unaligned access in the same page.  */
 #ifdef ALIGNED_ONLY
     if ((addr & (DATA_SIZE - 1)) != 0) {
-        cpu_unaligned_access(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
+        cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
+                             mmu_idx, retaddr);
     }
 #endif
 
@@ -441,10 +477,13 @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 #ifdef ALIGNED_ONLY
         if ((addr & (DATA_SIZE - 1)) != 0) {
-            cpu_unaligned_access(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
+            cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
+                                 mmu_idx, retaddr);
         }
 #endif
-        tlb_fill(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
+        if (!VICTIM_TLB_HIT(addr_write)) {
+            tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);
+        }
         tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
     }
 
@@ -470,7 +509,8 @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
         int i;
     do_unaligned_access:
 #ifdef ALIGNED_ONLY
-        cpu_unaligned_access(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
+        cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
+                             mmu_idx, retaddr);
 #endif
         /* XXX: not efficient, but simple */
         /* Note: relies on the fact that tlb_fill() does not remove the
@@ -489,7 +529,8 @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
     /* Handle aligned access or unaligned access in the same page.  */
 #ifdef ALIGNED_ONLY
     if ((addr & (DATA_SIZE - 1)) != 0) {
-        cpu_unaligned_access(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
+        cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
+                             mmu_idx, retaddr);
     }
 #endif
 
diff --git a/spice-qemu-char.c b/spice-qemu-char.c
index 4518a4d97..8106e063c 100644
--- a/spice-qemu-char.c
+++ b/spice-qemu-char.c
@@ -368,10 +368,10 @@ static void qemu_chr_parse_spice_port(QemuOpts *opts, ChardevBackend *backend,
 
 static void register_types(void)
 {
-    register_char_driver_qapi("spicevmc", CHARDEV_BACKEND_KIND_SPICEVMC,
-                              qemu_chr_parse_spice_vmc);
-    register_char_driver_qapi("spiceport", CHARDEV_BACKEND_KIND_SPICEPORT,
-                              qemu_chr_parse_spice_port);
+    register_char_driver("spicevmc", CHARDEV_BACKEND_KIND_SPICEVMC,
+                         qemu_chr_parse_spice_vmc);
+    register_char_driver("spiceport", CHARDEV_BACKEND_KIND_SPICEPORT,
+                         qemu_chr_parse_spice_port);
 }
 
 type_init(register_types);
diff --git a/stubs/Makefile.objs b/stubs/Makefile.objs
index 528e1617b..5e347d04b 100644
--- a/stubs/Makefile.objs
+++ b/stubs/Makefile.objs
@@ -2,6 +2,7 @@ stub-obj-y += arch-query-cpu-def.o
 stub-obj-y += bdrv-commit-all.o
 stub-obj-y += chr-baum-init.o
 stub-obj-y += chr-msmouse.o
+stub-obj-y += chr-testdev.o
 stub-obj-y += clock-warp.o
 stub-obj-y += cpu-get-clock.o
 stub-obj-y += cpu-get-icount.o
diff --git a/stubs/chr-testdev.c b/stubs/chr-testdev.c
new file mode 100644
index 000000000..23112a2c0
--- /dev/null
+++ b/stubs/chr-testdev.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+#include "sysemu/char.h"
+
+CharDriverState *chr_testdev_init(void)
+{
+    return 0;
+}
diff --git a/stubs/fdset-remove-fd.c b/stubs/fdset-remove-fd.c
index b3886d9f4..7f6d61e61 100644
--- a/stubs/fdset-remove-fd.c
+++ b/stubs/fdset-remove-fd.c
@@ -1,7 +1,6 @@
 #include "qemu-common.h"
 #include "monitor/monitor.h"
 
-int monitor_fdset_dup_fd_remove(int dupfd)
+void monitor_fdset_dup_fd_remove(int dupfd)
 {
-    return -1;
 }
diff --git a/target-alpha/cpu-qom.h b/target-alpha/cpu-qom.h
index 0caa362f5..b01c6c82e 100644
--- a/target-alpha/cpu-qom.h
+++ b/target-alpha/cpu-qom.h
@@ -79,6 +79,7 @@ extern const struct VMStateDescription vmstate_alpha_cpu;
 #endif
 
 void alpha_cpu_do_interrupt(CPUState *cpu);
+bool alpha_cpu_exec_interrupt(CPUState *cpu, int int_req);
 void alpha_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                           int flags);
 hwaddr alpha_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
diff --git a/target-alpha/cpu.c b/target-alpha/cpu.c
index 2491f0a30..a98b7d8d7 100644
--- a/target-alpha/cpu.c
+++ b/target-alpha/cpu.c
@@ -284,6 +284,7 @@ static void alpha_cpu_class_init(ObjectClass *oc, void *data)
     cc->class_by_name = alpha_cpu_class_by_name;
     cc->has_work = alpha_cpu_has_work;
     cc->do_interrupt = alpha_cpu_do_interrupt;
+    cc->cpu_exec_interrupt = alpha_cpu_exec_interrupt;
     cc->dump_state = alpha_cpu_dump_state;
     cc->set_pc = alpha_cpu_set_pc;
     cc->gdb_read_register = alpha_cpu_gdb_read_register;
diff --git a/target-alpha/helper.c b/target-alpha/helper.c
index 7c053a3ea..a8aa782a2 100644
--- a/target-alpha/helper.c
+++ b/target-alpha/helper.c
@@ -470,6 +470,50 @@ void alpha_cpu_do_interrupt(CPUState *cs)
 #endif /* !USER_ONLY */
 }
 
+bool alpha_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    AlphaCPU *cpu = ALPHA_CPU(cs);
+    CPUAlphaState *env = &cpu->env;
+    int idx = -1;
+
+    /* We never take interrupts while in PALmode.  */
+    if (env->pal_mode) {
+        return false;
+    }
+
+    /* Fall through the switch, collecting the highest priority
+       interrupt that isn't masked by the processor status IPL.  */
+    /* ??? This hard-codes the OSF/1 interrupt levels.  */
+    switch (env->ps & PS_INT_MASK) {
+    case 0 ... 3:
+        if (interrupt_request & CPU_INTERRUPT_HARD) {
+            idx = EXCP_DEV_INTERRUPT;
+        }
+        /* FALLTHRU */
+    case 4:
+        if (interrupt_request & CPU_INTERRUPT_TIMER) {
+            idx = EXCP_CLK_INTERRUPT;
+        }
+        /* FALLTHRU */
+    case 5:
+        if (interrupt_request & CPU_INTERRUPT_SMP) {
+            idx = EXCP_SMP_INTERRUPT;
+        }
+        /* FALLTHRU */
+    case 6:
+        if (interrupt_request & CPU_INTERRUPT_MCHK) {
+            idx = EXCP_MCHK;
+        }
+    }
+    if (idx >= 0) {
+        cs->exception_index = idx;
+        env->error_code = 0;
+        alpha_cpu_do_interrupt(cs);
+        return true;
+    }
+    return false;
+}
+
 void alpha_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                           int flags)
 {
diff --git a/target-alpha/translate.c b/target-alpha/translate.c
index cc81e774d..76658a074 100644
--- a/target-alpha/translate.c
+++ b/target-alpha/translate.c
@@ -26,6 +26,9 @@
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
 
+#include "trace-tcg.h"
+
+
 #undef ALPHA_DEBUG_DISAS
 #define CONFIG_SOFTFLOAT_INLINE
 
diff --git a/target-arm/Makefile.objs b/target-arm/Makefile.objs
index dcd167e0d..9460b409a 100644
--- a/target-arm/Makefile.objs
+++ b/target-arm/Makefile.objs
@@ -7,5 +7,6 @@ obj-$(call lnot,$(CONFIG_KVM)) += kvm-stub.o
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-y += neon_helper.o iwmmxt_helper.o
 obj-y += gdbstub.o
+obj-$(CONFIG_SOFTMMU) += psci.o
 obj-$(TARGET_AARCH64) += cpu64.o translate-a64.o helper-a64.o gdbstub64.o
 obj-y += crypto_helper.o
diff --git a/target-arm/cpu-qom.h b/target-arm/cpu-qom.h
index ee4fbb1da..dcfda7dfc 100644
--- a/target-arm/cpu-qom.h
+++ b/target-arm/cpu-qom.h
@@ -98,6 +98,13 @@ typedef struct ARMCPU {
 
     /* Should CPU start in PSCI powered-off state? */
     bool start_powered_off;
+    /* CPU currently in PSCI powered-off state */
+    bool powered_off;
+
+    /* PSCI conduit used to invoke PSCI methods
+     * 0 - disabled, 1 - smc, 2 - hvc
+     */
+    uint32_t psci_conduit;
 
     /* [QEMU_]KVM_ARM_TARGET_* constant for this CPU, or
      * QEMU_KVM_ARM_TARGET_NONE if the kernel doesn't support this CPU type.
@@ -148,6 +155,7 @@ typedef struct ARMCPU {
     uint64_t id_aa64isar1;
     uint64_t id_aa64mmfr0;
     uint64_t id_aa64mmfr1;
+    uint32_t dbgdidr;
     uint32_t clidr;
     /* The elements of this array are the CCSIDR values for each cache,
      * in the order L1DCache, L1ICache, L2DCache, L2ICache, etc.
@@ -191,6 +199,7 @@ void init_cpreg_list(ARMCPU *cpu);
 
 void arm_cpu_do_interrupt(CPUState *cpu);
 void arm_v7m_cpu_do_interrupt(CPUState *cpu);
+bool arm_cpu_exec_interrupt(CPUState *cpu, int int_req);
 
 void arm_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                         int flags);
diff --git a/target-arm/cpu.c b/target-arm/cpu.c
index 05e52e0e8..5ce7350ce 100644
--- a/target-arm/cpu.c
+++ b/target-arm/cpu.c
@@ -40,8 +40,13 @@ static void arm_cpu_set_pc(CPUState *cs, vaddr value)
 
 static bool arm_cpu_has_work(CPUState *cs)
 {
-    return cs->interrupt_request &
-        (CPU_INTERRUPT_FIQ | CPU_INTERRUPT_HARD | CPU_INTERRUPT_EXITTB);
+    ARMCPU *cpu = ARM_CPU(cs);
+
+    return !cpu->powered_off
+        && cs->interrupt_request &
+        (CPU_INTERRUPT_FIQ | CPU_INTERRUPT_HARD
+         | CPU_INTERRUPT_VFIQ | CPU_INTERRUPT_VIRQ
+         | CPU_INTERRUPT_EXITTB);
 }
 
 static void cp_reg_reset(gpointer key, gpointer value, gpointer opaque)
@@ -91,6 +96,9 @@ static void arm_cpu_reset(CPUState *s)
     env->vfp.xregs[ARM_VFP_MVFR1] = cpu->mvfr1;
     env->vfp.xregs[ARM_VFP_MVFR2] = cpu->mvfr2;
 
+    cpu->powered_off = cpu->start_powered_off;
+    s->halted = cpu->start_powered_off;
+
     if (arm_feature(env, ARM_FEATURE_IWMMXT)) {
         env->iwmmxt.cregs[ARM_IWMMXT_wCID] = 0x69051000 | 'Q';
     }
@@ -100,8 +108,8 @@ static void arm_cpu_reset(CPUState *s)
         env->aarch64 = 1;
 #if defined(CONFIG_USER_ONLY)
         env->pstate = PSTATE_MODE_EL0t;
-        /* Userspace expects access to CTL_EL0 and the cache ops */
-        env->cp15.c1_sys |= SCTLR_UCT | SCTLR_UCI;
+        /* Userspace expects access to DC ZVA, CTL_EL0 and the cache ops */
+        env->cp15.c1_sys |= SCTLR_UCT | SCTLR_UCI | SCTLR_DZE;
         /* and to the FP/Neon instructions */
         env->cp15.c1_coproc = deposit64(env->cp15.c1_coproc, 20, 2, 3);
 #else
@@ -129,26 +137,38 @@ static void arm_cpu_reset(CPUState *s)
     env->uncached_cpsr = ARM_CPU_MODE_SVC;
     env->daif = PSTATE_D | PSTATE_A | PSTATE_I | PSTATE_F;
     /* On ARMv7-M the CPSR_I is the value of the PRIMASK register, and is
-       clear at reset.  Initial SP and PC are loaded from ROM.  */
+     * clear at reset. Initial SP and PC are loaded from ROM.
+     */
     if (IS_M(env)) {
-        uint32_t pc;
+        uint32_t initial_msp; /* Loaded from 0x0 */
+        uint32_t initial_pc; /* Loaded from 0x4 */
         uint8_t *rom;
+
         env->daif &= ~PSTATE_I;
         rom = rom_ptr(0);
         if (rom) {
-            /* We should really use ldl_phys here, in case the guest
-               modified flash and reset itself.  However images
-               loaded via -kernel have not been copied yet, so load the
-               values directly from there.  */
-            env->regs[13] = ldl_p(rom) & 0xFFFFFFFC;
-            pc = ldl_p(rom + 4);
-            env->thumb = pc & 1;
-            env->regs[15] = pc & ~1;
+            /* Address zero is covered by ROM which hasn't yet been
+             * copied into physical memory.
+             */
+            initial_msp = ldl_p(rom);
+            initial_pc = ldl_p(rom + 4);
+        } else {
+            /* Address zero not covered by a ROM blob, or the ROM blob
+             * is in non-modifiable memory and this is a second reset after
+             * it got copied into memory. In the latter case, rom_ptr
+             * will return a NULL pointer and we should use ldl_phys instead.
+             */
+            initial_msp = ldl_phys(s->as, 0);
+            initial_pc = ldl_phys(s->as, 4);
         }
+
+        env->regs[13] = initial_msp & 0xFFFFFFFC;
+        env->regs[15] = initial_pc & ~1;
+        env->thumb = initial_pc & 1;
     }
 
     if (env->cp15.c1_sys & SCTLR_V) {
-            env->regs[15] = 0xFFFF0000;
+        env->regs[15] = 0xFFFF0000;
     }
 
     env->vfp.xregs[ARM_VFP_FPEXC] = 0;
@@ -161,38 +181,113 @@ static void arm_cpu_reset(CPUState *s)
     set_float_detect_tininess(float_tininess_before_rounding,
                               &env->vfp.standard_fp_status);
     tlb_flush(s, 1);
-    /* Reset is a state change for some CPUARMState fields which we
-     * bake assumptions about into translated code, so we need to
-     * tb_flush().
-     */
-    tb_flush(env);
 
 #ifndef CONFIG_USER_ONLY
     if (kvm_enabled()) {
         kvm_arm_reset_vcpu(cpu);
     }
 #endif
+
+    hw_breakpoint_update_all(cpu);
+    hw_watchpoint_update_all(cpu);
+}
+
+bool arm_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    CPUClass *cc = CPU_GET_CLASS(cs);
+    bool ret = false;
+
+    if (interrupt_request & CPU_INTERRUPT_FIQ
+        && arm_excp_unmasked(cs, EXCP_FIQ)) {
+        cs->exception_index = EXCP_FIQ;
+        cc->do_interrupt(cs);
+        ret = true;
+    }
+    if (interrupt_request & CPU_INTERRUPT_HARD
+        && arm_excp_unmasked(cs, EXCP_IRQ)) {
+        cs->exception_index = EXCP_IRQ;
+        cc->do_interrupt(cs);
+        ret = true;
+    }
+    if (interrupt_request & CPU_INTERRUPT_VIRQ
+        && arm_excp_unmasked(cs, EXCP_VIRQ)) {
+        cs->exception_index = EXCP_VIRQ;
+        cc->do_interrupt(cs);
+        ret = true;
+    }
+    if (interrupt_request & CPU_INTERRUPT_VFIQ
+        && arm_excp_unmasked(cs, EXCP_VFIQ)) {
+        cs->exception_index = EXCP_VFIQ;
+        cc->do_interrupt(cs);
+        ret = true;
+    }
+
+    return ret;
+}
+
+#if !defined(CONFIG_USER_ONLY) || !defined(TARGET_AARCH64)
+static bool arm_v7m_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    CPUClass *cc = CPU_GET_CLASS(cs);
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+    bool ret = false;
+
+
+    if (interrupt_request & CPU_INTERRUPT_FIQ
+        && !(env->daif & PSTATE_F)) {
+        cs->exception_index = EXCP_FIQ;
+        cc->do_interrupt(cs);
+        ret = true;
+    }
+    /* ARMv7-M interrupt return works by loading a magic value
+     * into the PC.  On real hardware the load causes the
+     * return to occur.  The qemu implementation performs the
+     * jump normally, then does the exception return when the
+     * CPU tries to execute code at the magic address.
+     * This will cause the magic PC value to be pushed to
+     * the stack if an interrupt occurred at the wrong time.
+     * We avoid this by disabling interrupts when
+     * pc contains a magic address.
+     */
+    if (interrupt_request & CPU_INTERRUPT_HARD
+        && !(env->daif & PSTATE_I)
+        && (env->regs[15] < 0xfffffff0)) {
+        cs->exception_index = EXCP_IRQ;
+        cc->do_interrupt(cs);
+        ret = true;
+    }
+    return ret;
 }
+#endif
 
 #ifndef CONFIG_USER_ONLY
 static void arm_cpu_set_irq(void *opaque, int irq, int level)
 {
     ARMCPU *cpu = opaque;
+    CPUARMState *env = &cpu->env;
     CPUState *cs = CPU(cpu);
+    static const int mask[] = {
+        [ARM_CPU_IRQ] = CPU_INTERRUPT_HARD,
+        [ARM_CPU_FIQ] = CPU_INTERRUPT_FIQ,
+        [ARM_CPU_VIRQ] = CPU_INTERRUPT_VIRQ,
+        [ARM_CPU_VFIQ] = CPU_INTERRUPT_VFIQ
+    };
 
     switch (irq) {
-    case ARM_CPU_IRQ:
-        if (level) {
-            cpu_interrupt(cs, CPU_INTERRUPT_HARD);
-        } else {
-            cpu_reset_interrupt(cs, CPU_INTERRUPT_HARD);
+    case ARM_CPU_VIRQ:
+    case ARM_CPU_VFIQ:
+        if (!arm_feature(env, ARM_FEATURE_EL2)) {
+            hw_error("%s: Virtual interrupt line %d with no EL2 support\n",
+                     __func__, irq);
         }
-        break;
+        /* fall through */
+    case ARM_CPU_IRQ:
     case ARM_CPU_FIQ:
         if (level) {
-            cpu_interrupt(cs, CPU_INTERRUPT_FIQ);
+            cpu_interrupt(cs, mask[irq]);
         } else {
-            cpu_reset_interrupt(cs, CPU_INTERRUPT_FIQ);
+            cpu_reset_interrupt(cs, mask[irq]);
         }
         break;
     default:
@@ -242,9 +337,12 @@ static void arm_cpu_initfn(Object *obj)
 #ifndef CONFIG_USER_ONLY
     /* Our inbound IRQ and FIQ lines */
     if (kvm_enabled()) {
-        qdev_init_gpio_in(DEVICE(cpu), arm_cpu_kvm_set_irq, 2);
+        /* VIRQ and VFIQ are unused with KVM but we add them to maintain
+         * the same interface as non-KVM CPUs.
+         */
+        qdev_init_gpio_in(DEVICE(cpu), arm_cpu_kvm_set_irq, 4);
     } else {
-        qdev_init_gpio_in(DEVICE(cpu), arm_cpu_set_irq, 2);
+        qdev_init_gpio_in(DEVICE(cpu), arm_cpu_set_irq, 4);
     }
 
     cpu->gt_timer[GTIMER_PHYS] = timer_new(QEMU_CLOCK_VIRTUAL, GTIMER_SCALE,
@@ -263,9 +361,12 @@ static void arm_cpu_initfn(Object *obj)
     cpu->psci_version = 1; /* By default assume PSCI v0.1 */
     cpu->kvm_target = QEMU_KVM_ARM_TARGET_NONE;
 
-    if (tcg_enabled() && !inited) {
-        inited = true;
-        arm_translate_init();
+    if (tcg_enabled()) {
+        cpu->psci_version = 2; /* TCG implements PSCI 0.2 */
+        if (!inited) {
+            inited = true;
+            arm_translate_init();
+        }
     }
 }
 
@@ -447,7 +548,7 @@ static void arm1026_initfn(Object *obj)
         ARMCPRegInfo ifar = {
             .name = "IFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 1,
             .access = PL1_RW,
-            .fieldoffset = offsetofhigh32(CPUARMState, cp15.far_el1),
+            .fieldoffset = offsetofhigh32(CPUARMState, cp15.far_el[1]),
             .resetvalue = 0
         };
         define_one_arm_cp_reg(cpu, &ifar);
@@ -596,11 +697,13 @@ static void cortex_m3_initfn(Object *obj)
 
 static void arm_v7m_class_init(ObjectClass *oc, void *data)
 {
-#ifndef CONFIG_USER_ONLY
     CPUClass *cc = CPU_CLASS(oc);
 
+#ifndef CONFIG_USER_ONLY
     cc->do_interrupt = arm_v7m_cpu_do_interrupt;
 #endif
+
+    cc->cpu_exec_interrupt = arm_v7m_cpu_exec_interrupt;
 }
 
 static const ARMCPRegInfo cortexa8_cp_reginfo[] = {
@@ -640,6 +743,7 @@ static void cortex_a8_initfn(Object *obj)
     cpu->id_isar2 = 0x21232031;
     cpu->id_isar3 = 0x11112131;
     cpu->id_isar4 = 0x00111142;
+    cpu->dbgdidr = 0x15141000;
     cpu->clidr = (1 << 27) | (2 << 24) | 3;
     cpu->ccsidr[0] = 0xe007e01a; /* 16k L1 dcache. */
     cpu->ccsidr[1] = 0x2007e01a; /* 16k L1 icache. */
@@ -712,9 +816,10 @@ static void cortex_a9_initfn(Object *obj)
     cpu->id_isar2 = 0x21232041;
     cpu->id_isar3 = 0x11112131;
     cpu->id_isar4 = 0x00111142;
+    cpu->dbgdidr = 0x35141000;
     cpu->clidr = (1 << 27) | (1 << 24) | 3;
-    cpu->ccsidr[0] = 0xe00fe015; /* 16k L1 dcache. */
-    cpu->ccsidr[1] = 0x200fe015; /* 16k L1 icache. */
+    cpu->ccsidr[0] = 0xe00fe019; /* 16k L1 dcache. */
+    cpu->ccsidr[1] = 0x200fe019; /* 16k L1 icache. */
     define_arm_cp_regs(cpu, cortexa9_cp_reginfo);
 }
 
@@ -773,6 +878,7 @@ static void cortex_a15_initfn(Object *obj)
     cpu->id_isar2 = 0x21232041;
     cpu->id_isar3 = 0x11112131;
     cpu->id_isar4 = 0x10011142;
+    cpu->dbgdidr = 0x3515f021;
     cpu->clidr = 0x0a200023;
     cpu->ccsidr[0] = 0x701fe00a; /* 32K L1 dcache */
     cpu->ccsidr[1] = 0x201fe00a; /* 32K L1 icache */
@@ -1016,6 +1122,7 @@ static const ARMCPUInfo arm_cpus[] = {
 
 static Property arm_cpu_properties[] = {
     DEFINE_PROP_BOOL("start-powered-off", ARMCPU, start_powered_off, false),
+    DEFINE_PROP_UINT32("psci-conduit", ARMCPU, psci_conduit, 0),
     DEFINE_PROP_UINT32("midr", ARMCPU, midr, 0),
     DEFINE_PROP_END_OF_LIST()
 };
@@ -1035,7 +1142,7 @@ static void arm_cpu_class_init(ObjectClass *oc, void *data)
 
     cc->class_by_name = arm_cpu_class_by_name;
     cc->has_work = arm_cpu_has_work;
-    cc->do_interrupt = arm_cpu_do_interrupt;
+    cc->cpu_exec_interrupt = arm_cpu_exec_interrupt;
     cc->dump_state = arm_cpu_dump_state;
     cc->set_pc = arm_cpu_set_pc;
     cc->gdb_read_register = arm_cpu_gdb_read_register;
@@ -1043,11 +1150,14 @@ static void arm_cpu_class_init(ObjectClass *oc, void *data)
 #ifdef CONFIG_USER_ONLY
     cc->handle_mmu_fault = arm_cpu_handle_mmu_fault;
 #else
+    cc->do_interrupt = arm_cpu_do_interrupt;
     cc->get_phys_page_debug = arm_cpu_get_phys_page_debug;
     cc->vmsd = &vmstate_arm_cpu;
 #endif
     cc->gdb_num_core_regs = 26;
     cc->gdb_core_xml_file = "arm-core.xml";
+    cc->gdb_stop_before_watchpoint = true;
+    cc->debug_excp_handler = arm_debug_excp_handler;
 }
 
 static void cpu_register(const ARMCPUInfo *info)
diff --git a/target-arm/cpu.h b/target-arm/cpu.h
index 369d4727a..7f800908f 100644
--- a/target-arm/cpu.h
+++ b/target-arm/cpu.h
@@ -51,6 +51,11 @@
 #define EXCP_EXCEPTION_EXIT  8   /* Return from v7M exception.  */
 #define EXCP_KERNEL_TRAP     9   /* Jumped to kernel code page.  */
 #define EXCP_STREX          10
+#define EXCP_HVC            11   /* HyperVisor Call */
+#define EXCP_HYP_TRAP       12
+#define EXCP_SMC            13   /* Secure Monitor Call */
+#define EXCP_VIRQ           14
+#define EXCP_VFIQ           15
 
 #define ARMV7M_EXCP_RESET   1
 #define ARMV7M_EXCP_NMI     2
@@ -65,6 +70,8 @@
 
 /* ARM-specific interrupt pending bits.  */
 #define CPU_INTERRUPT_FIQ   CPU_INTERRUPT_TGT_EXT_1
+#define CPU_INTERRUPT_VIRQ  CPU_INTERRUPT_TGT_EXT_2
+#define CPU_INTERRUPT_VFIQ  CPU_INTERRUPT_TGT_EXT_3
 
 /* The usual mapping for an AArch64 system register to its AArch32
  * counterpart is for the 32 bit world to have access to the lower
@@ -80,9 +87,11 @@
 #define offsetofhigh32(S, M) (offsetof(S, M) + sizeof(uint32_t))
 #endif
 
-/* Meanings of the ARMCPU object's two inbound GPIO lines */
+/* Meanings of the ARMCPU object's four inbound GPIO lines */
 #define ARM_CPU_IRQ 0
 #define ARM_CPU_FIQ 1
+#define ARM_CPU_VIRQ 2
+#define ARM_CPU_VFIQ 3
 
 typedef void ARMWriteCPFunc(void *opaque, int cp_info,
                             int srcreg, int operand, uint32_t value);
@@ -91,7 +100,7 @@ typedef uint32_t ARMReadCPFunc(void *opaque, int cp_info,
 
 struct arm_boot_info;
 
-#define NB_MMU_MODES 2
+#define NB_MMU_MODES 4
 
 /* We currently assume float and double are IEEE single and double
    precision respectively.
@@ -144,8 +153,8 @@ typedef struct CPUARMState {
 
     /* Banked registers.  */
     uint64_t banked_spsr[8];
-    uint32_t banked_r13[6];
-    uint32_t banked_r14[6];
+    uint32_t banked_r13[8];
+    uint32_t banked_r14[8];
 
     /* These hold r8-r12.  */
     uint32_t usr_regs[5];
@@ -172,7 +181,6 @@ typedef struct CPUARMState {
         uint64_t c1_sys; /* System control register.  */
         uint64_t c1_coproc; /* Coprocessor access register.  */
         uint32_t c1_xscaleauxcr; /* XScale auxiliary control register.  */
-        uint32_t c1_scr; /* secure config register.  */
         uint64_t ttbr0_el1; /* MMU translation table base 0. */
         uint64_t ttbr1_el1; /* MMU translation table base 1. */
         uint64_t c2_control; /* MMU translation table base control.  */
@@ -184,15 +192,17 @@ typedef struct CPUARMState {
                         MPU write buffer control.  */
         uint32_t pmsav5_data_ap; /* PMSAv5 MPU data access permissions */
         uint32_t pmsav5_insn_ap; /* PMSAv5 MPU insn access permissions */
+        uint64_t hcr_el2; /* Hypervisor configuration register */
+        uint64_t scr_el3; /* Secure configuration register.  */
         uint32_t ifsr_el2; /* Fault status registers.  */
-        uint64_t esr_el[2];
+        uint64_t esr_el[4];
         uint32_t c6_region[8]; /* MPU base/size registers.  */
-        uint64_t far_el1; /* Fault address registers.  */
+        uint64_t far_el[4]; /* Fault address registers.  */
         uint64_t par_el1;  /* Translation result. */
         uint32_t c9_insn; /* Cache lockdown registers.  */
         uint32_t c9_data;
-        uint32_t c9_pmcr; /* performance monitor control register */
-        uint32_t c9_pmcnten; /* perf monitor counter enables */
+        uint64_t c9_pmcr; /* performance monitor control register */
+        uint64_t c9_pmcnten; /* perf monitor counter enables */
         uint32_t c9_pmovsr; /* perf monitor overflow status */
         uint32_t c9_pmxevtyper; /* perf monitor event type */
         uint32_t c9_pmuserenr; /* perf monitor user enable */
@@ -220,10 +230,12 @@ typedef struct CPUARMState {
         uint64_t dbgbcr[16]; /* breakpoint control registers */
         uint64_t dbgwvr[16]; /* watchpoint value registers */
         uint64_t dbgwcr[16]; /* watchpoint control registers */
+        uint64_t mdscr_el1;
         /* If the counter is enabled, this stores the last time the counter
          * was reset. Otherwise it stores the counter value
          */
-        uint32_t c15_ccnt;
+        uint64_t c15_ccnt;
+        uint64_t pmccfiltr_el0; /* Performance Monitor Filter Register */
     } cp15;
 
     struct {
@@ -321,6 +333,9 @@ typedef struct CPUARMState {
     int eabi;
 #endif
 
+    struct CPUBreakpoint *cpu_breakpoint[16];
+    struct CPUWatchpoint *cpu_watchpoint[16];
+
     CPU_COMMON
 
     /* These fields after the common ones so they are preserved on reset.  */
@@ -351,6 +366,17 @@ int cpu_arm_signal_handler(int host_signum, void *pinfo,
 int arm_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw,
                              int mmu_idx);
 
+/**
+ * pmccntr_sync
+ * @env: CPUARMState
+ *
+ * Synchronises the counter in the PMCCNTR. This must always be called twice,
+ * once before any action that might affect the timer and again afterwards.
+ * The function is used to swap the state of the register if required.
+ * This only happens when not in user mode (!CONFIG_USER_ONLY)
+ */
+void pmccntr_sync(CPUARMState *env);
+
 /* SCTLR bit meanings. Several bits have been reused in newer
  * versions of the architecture; in that case we define constants
  * for both old and new bit meanings. Code which tests against those
@@ -411,7 +437,13 @@ int arm_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw,
 #define CPSR_E (1U << 9)
 #define CPSR_IT_2_7 (0xfc00U)
 #define CPSR_GE (0xfU << 16)
-#define CPSR_RESERVED (0xfU << 20)
+#define CPSR_IL (1U << 20)
+/* Note that the RESERVED bits include bit 21, which is PSTATE_SS in
+ * an AArch64 SPSR but RES0 in AArch32 SPSR and CPSR. In QEMU we use
+ * env->uncached_cpsr bit 21 to store PSTATE.SS when executing in AArch32,
+ * where it is live state but not accessible to the AArch32 code.
+ */
+#define CPSR_RESERVED (0x7U << 21)
 #define CPSR_J (1U << 24)
 #define CPSR_IT_0_1 (3U << 25)
 #define CPSR_Q (1U << 27)
@@ -428,7 +460,9 @@ int arm_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw,
 /* Bits writable in user mode.  */
 #define CPSR_USER (CPSR_NZCV | CPSR_Q | CPSR_GE)
 /* Execution state bits.  MRS read as zero, MSR writes ignored.  */
-#define CPSR_EXEC (CPSR_T | CPSR_IT | CPSR_J)
+#define CPSR_EXEC (CPSR_T | CPSR_IT | CPSR_J | CPSR_IL)
+/* Mask of bits which may be set by exception return copying them from SPSR */
+#define CPSR_ERET_MASK (~CPSR_RESERVED)
 
 #define TTBCR_N      (7U << 0) /* TTBCR.EAE==0 */
 #define TTBCR_T0SZ   (7U << 0) /* TTBCR.EAE==1 */
@@ -475,6 +509,12 @@ int arm_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw,
 #define PSTATE_MODE_EL1t 4
 #define PSTATE_MODE_EL0t 0
 
+/* Map EL and handler into a PSTATE_MODE.  */
+static inline unsigned int aarch64_pstate_mode(unsigned int el, bool handler)
+{
+    return (el << 2) | handler;
+}
+
 /* Return the current PSTATE value. For the moment we don't support 32<->64 bit
  * interprocessing, so we don't attempt to sync with the cpsr state used by
  * the 32 bit decoder.
@@ -542,6 +582,58 @@ static inline void xpsr_write(CPUARMState *env, uint32_t val, uint32_t mask)
     }
 }
 
+#define HCR_VM        (1ULL << 0)
+#define HCR_SWIO      (1ULL << 1)
+#define HCR_PTW       (1ULL << 2)
+#define HCR_FMO       (1ULL << 3)
+#define HCR_IMO       (1ULL << 4)
+#define HCR_AMO       (1ULL << 5)
+#define HCR_VF        (1ULL << 6)
+#define HCR_VI        (1ULL << 7)
+#define HCR_VSE       (1ULL << 8)
+#define HCR_FB        (1ULL << 9)
+#define HCR_BSU_MASK  (3ULL << 10)
+#define HCR_DC        (1ULL << 12)
+#define HCR_TWI       (1ULL << 13)
+#define HCR_TWE       (1ULL << 14)
+#define HCR_TID0      (1ULL << 15)
+#define HCR_TID1      (1ULL << 16)
+#define HCR_TID2      (1ULL << 17)
+#define HCR_TID3      (1ULL << 18)
+#define HCR_TSC       (1ULL << 19)
+#define HCR_TIDCP     (1ULL << 20)
+#define HCR_TACR      (1ULL << 21)
+#define HCR_TSW       (1ULL << 22)
+#define HCR_TPC       (1ULL << 23)
+#define HCR_TPU       (1ULL << 24)
+#define HCR_TTLB      (1ULL << 25)
+#define HCR_TVM       (1ULL << 26)
+#define HCR_TGE       (1ULL << 27)
+#define HCR_TDZ       (1ULL << 28)
+#define HCR_HCD       (1ULL << 29)
+#define HCR_TRVM      (1ULL << 30)
+#define HCR_RW        (1ULL << 31)
+#define HCR_CD        (1ULL << 32)
+#define HCR_ID        (1ULL << 33)
+#define HCR_MASK      ((1ULL << 34) - 1)
+
+#define SCR_NS                (1U << 0)
+#define SCR_IRQ               (1U << 1)
+#define SCR_FIQ               (1U << 2)
+#define SCR_EA                (1U << 3)
+#define SCR_FW                (1U << 4)
+#define SCR_AW                (1U << 5)
+#define SCR_NET               (1U << 6)
+#define SCR_SMD               (1U << 7)
+#define SCR_HCE               (1U << 8)
+#define SCR_SIF               (1U << 9)
+#define SCR_RW                (1U << 10)
+#define SCR_ST                (1U << 11)
+#define SCR_TWI               (1U << 12)
+#define SCR_TWE               (1U << 13)
+#define SCR_AARCH32_MASK      (0x3fff & ~(SCR_RW | SCR_ST))
+#define SCR_AARCH64_MASK      (0x3fff & ~SCR_NET)
+
 /* Return the current FPSCR value.  */
 uint32_t vfp_get_fpscr(CPUARMState *env);
 void vfp_set_fpscr(CPUARMState *env, uint32_t val);
@@ -661,14 +753,62 @@ static inline int arm_feature(CPUARMState *env, int feature)
     return (env->features & (1ULL << feature)) != 0;
 }
 
+#if !defined(CONFIG_USER_ONLY)
+/* Return true if exception levels below EL3 are in secure state,
+ * or would be following an exception return to that level.
+ * Unlike arm_is_secure() (which is always a question about the
+ * _current_ state of the CPU) this doesn't care about the current
+ * EL or mode.
+ */
+static inline bool arm_is_secure_below_el3(CPUARMState *env)
+{
+    if (arm_feature(env, ARM_FEATURE_EL3)) {
+        return !(env->cp15.scr_el3 & SCR_NS);
+    } else {
+        /* If EL2 is not supported then the secure state is implementation
+         * defined, in which case QEMU defaults to non-secure.
+         */
+        return false;
+    }
+}
+
+/* Return true if the processor is in secure state */
+static inline bool arm_is_secure(CPUARMState *env)
+{
+    if (arm_feature(env, ARM_FEATURE_EL3)) {
+        if (is_a64(env) && extract32(env->pstate, 2, 2) == 3) {
+            /* CPU currently in AArch64 state and EL3 */
+            return true;
+        } else if (!is_a64(env) &&
+                (env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_MON) {
+            /* CPU currently in AArch32 state and monitor mode */
+            return true;
+        }
+    }
+    return arm_is_secure_below_el3(env);
+}
+
+#else
+static inline bool arm_is_secure_below_el3(CPUARMState *env)
+{
+    return false;
+}
+
+static inline bool arm_is_secure(CPUARMState *env)
+{
+    return false;
+}
+#endif
+
 /* Return true if the specified exception level is running in AArch64 state. */
 static inline bool arm_el_is_aa64(CPUARMState *env, int el)
 {
-    /* We don't currently support EL2 or EL3, and this isn't valid for EL0
+    /* We don't currently support EL2, and this isn't valid for EL0
      * (if we're in EL0, is_a64() is what you want, and if we're not in EL0
      * then the state of EL0 isn't well defined.)
      */
-    assert(el == 1);
+    assert(el == 1 || el == 3);
+
     /* AArch64-capable CPUs always run with EL1 in AArch64 mode. This
      * is a QEMU-imposed simplification which we may wish to change later.
      * If we in future support EL2 and/or EL3, then the state of lower
@@ -678,6 +818,7 @@ static inline bool arm_el_is_aa64(CPUARMState *env, int el)
 }
 
 void arm_cpu_list(FILE *f, fprintf_function cpu_fprintf);
+unsigned int arm_excp_target_el(CPUState *cs, unsigned int excp_idx);
 
 /* Interface between CPU and Interrupt controller.  */
 void armv7m_nvic_set_pending(void *opaque, int irq);
@@ -851,19 +992,32 @@ static inline bool cptype_valid(int cptype)
 #define PL1_RW (PL1_R | PL1_W)
 #define PL0_RW (PL0_R | PL0_W)
 
-static inline int arm_current_pl(CPUARMState *env)
+/* Return the current Exception Level (as per ARMv8; note that this differs
+ * from the ARMv7 Privilege Level).
+ */
+static inline int arm_current_el(CPUARMState *env)
 {
-    if (env->aarch64) {
+    if (is_a64(env)) {
         return extract32(env->pstate, 2, 2);
     }
 
-    if ((env->uncached_cpsr & 0x1f) == ARM_CPU_MODE_USR) {
+    switch (env->uncached_cpsr & 0x1f) {
+    case ARM_CPU_MODE_USR:
         return 0;
+    case ARM_CPU_MODE_HYP:
+        return 2;
+    case ARM_CPU_MODE_MON:
+        return 3;
+    default:
+        if (arm_is_secure(env) && !arm_el_is_aa64(env, 3)) {
+            /* If EL3 is 32-bit then all secure privileged modes run in
+             * EL3
+             */
+            return 3;
+        }
+
+        return 1;
     }
-    /* We don't currently implement the Virtualization or TrustZone
-     * extensions, so PL2 and PL3 don't exist for us.
-     */
-    return 1;
 }
 
 typedef struct ARMCPRegInfo ARMCPRegInfo;
@@ -1024,10 +1178,10 @@ static inline bool cpreg_field_is_64bit(const ARMCPRegInfo *ri)
     return (ri->state == ARM_CP_STATE_AA64) || (ri->type & ARM_CP_64BIT);
 }
 
-static inline bool cp_access_ok(int current_pl,
+static inline bool cp_access_ok(int current_el,
                                 const ARMCPRegInfo *ri, int isread)
 {
-    return (ri->access >> ((current_pl * 2) + isread)) & 1;
+    return (ri->access >> ((current_el * 2) + isread)) & 1;
 }
 
 /**
@@ -1088,6 +1242,49 @@ bool write_cpustate_to_list(ARMCPU *cpu);
 #  define TARGET_VIRT_ADDR_SPACE_BITS 32
 #endif
 
+static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx)
+{
+    CPUARMState *env = cs->env_ptr;
+    unsigned int cur_el = arm_current_el(env);
+    unsigned int target_el = arm_excp_target_el(cs, excp_idx);
+    /* FIXME: Use actual secure state.  */
+    bool secure = false;
+    /* If in EL1/0, Physical IRQ routing to EL2 only happens from NS state.  */
+    bool irq_can_hyp = !secure && cur_el < 2 && target_el == 2;
+
+    /* Don't take exceptions if they target a lower EL.  */
+    if (cur_el > target_el) {
+        return false;
+    }
+
+    switch (excp_idx) {
+    case EXCP_FIQ:
+        if (irq_can_hyp && (env->cp15.hcr_el2 & HCR_FMO)) {
+            return true;
+        }
+        return !(env->daif & PSTATE_F);
+    case EXCP_IRQ:
+        if (irq_can_hyp && (env->cp15.hcr_el2 & HCR_IMO)) {
+            return true;
+        }
+        return !(env->daif & PSTATE_I);
+    case EXCP_VFIQ:
+        if (secure || !(env->cp15.hcr_el2 & HCR_FMO)) {
+            /* VFIQs are only taken when hypervized and non-secure.  */
+            return false;
+        }
+        return !(env->daif & PSTATE_F);
+    case EXCP_VIRQ:
+        if (secure || !(env->cp15.hcr_el2 & HCR_IMO)) {
+            /* VIRQs are only taken when hypervized and non-secure.  */
+            return false;
+        }
+        return !(env->daif & PSTATE_I);
+    default:
+        g_assert_not_reached();
+    }
+}
+
 static inline CPUARMState *cpu_init(const char *cpu_model)
 {
     ARMCPU *cpu = cpu_arm_init(cpu_model);
@@ -1108,7 +1305,67 @@ static inline CPUARMState *cpu_init(const char *cpu_model)
 #define MMU_USER_IDX 0
 static inline int cpu_mmu_index (CPUARMState *env)
 {
-    return arm_current_pl(env);
+    return arm_current_el(env);
+}
+
+/* Return the Exception Level targeted by debug exceptions;
+ * currently always EL1 since we don't implement EL2 or EL3.
+ */
+static inline int arm_debug_target_el(CPUARMState *env)
+{
+    return 1;
+}
+
+static inline bool aa64_generate_debug_exceptions(CPUARMState *env)
+{
+    if (arm_current_el(env) == arm_debug_target_el(env)) {
+        if ((extract32(env->cp15.mdscr_el1, 13, 1) == 0)
+            || (env->daif & PSTATE_D)) {
+            return false;
+        }
+    }
+    return true;
+}
+
+static inline bool aa32_generate_debug_exceptions(CPUARMState *env)
+{
+    if (arm_current_el(env) == 0 && arm_el_is_aa64(env, 1)) {
+        return aa64_generate_debug_exceptions(env);
+    }
+    return arm_current_el(env) != 2;
+}
+
+/* Return true if debugging exceptions are currently enabled.
+ * This corresponds to what in ARM ARM pseudocode would be
+ *    if UsingAArch32() then
+ *        return AArch32.GenerateDebugExceptions()
+ *    else
+ *        return AArch64.GenerateDebugExceptions()
+ * We choose to push the if() down into this function for clarity,
+ * since the pseudocode has it at all callsites except for the one in
+ * CheckSoftwareStep(), where it is elided because both branches would
+ * always return the same value.
+ *
+ * Parts of the pseudocode relating to EL2 and EL3 are omitted because we
+ * don't yet implement those exception levels or their associated trap bits.
+ */
+static inline bool arm_generate_debug_exceptions(CPUARMState *env)
+{
+    if (env->aarch64) {
+        return aa64_generate_debug_exceptions(env);
+    } else {
+        return aa32_generate_debug_exceptions(env);
+    }
+}
+
+/* Is single-stepping active? (Note that the "is EL_D AArch64?" check
+ * implicitly means this always returns false in pre-v8 CPUs.)
+ */
+static inline bool arm_singlestep_active(CPUARMState *env)
+{
+    return extract32(env->cp15.mdscr_el1, 0, 1)
+        && arm_el_is_aa64(env, arm_debug_target_el(env))
+        && arm_generate_debug_exceptions(env);
 }
 
 #include "exec/cpu-all.h"
@@ -1136,12 +1393,25 @@ static inline int cpu_mmu_index (CPUARMState *env)
 #define ARM_TBFLAG_BSWAP_CODE_MASK  (1 << ARM_TBFLAG_BSWAP_CODE_SHIFT)
 #define ARM_TBFLAG_CPACR_FPEN_SHIFT 17
 #define ARM_TBFLAG_CPACR_FPEN_MASK  (1 << ARM_TBFLAG_CPACR_FPEN_SHIFT)
+#define ARM_TBFLAG_SS_ACTIVE_SHIFT 18
+#define ARM_TBFLAG_SS_ACTIVE_MASK (1 << ARM_TBFLAG_SS_ACTIVE_SHIFT)
+#define ARM_TBFLAG_PSTATE_SS_SHIFT 19
+#define ARM_TBFLAG_PSTATE_SS_MASK (1 << ARM_TBFLAG_PSTATE_SS_SHIFT)
+/* We store the bottom two bits of the CPAR as TB flags and handle
+ * checks on the other bits at runtime
+ */
+#define ARM_TBFLAG_XSCALE_CPAR_SHIFT 20
+#define ARM_TBFLAG_XSCALE_CPAR_MASK (3 << ARM_TBFLAG_XSCALE_CPAR_SHIFT)
 
 /* Bit usage when in AArch64 state */
 #define ARM_TBFLAG_AA64_EL_SHIFT    0
 #define ARM_TBFLAG_AA64_EL_MASK     (0x3 << ARM_TBFLAG_AA64_EL_SHIFT)
 #define ARM_TBFLAG_AA64_FPEN_SHIFT  2
 #define ARM_TBFLAG_AA64_FPEN_MASK   (1 << ARM_TBFLAG_AA64_FPEN_SHIFT)
+#define ARM_TBFLAG_AA64_SS_ACTIVE_SHIFT 3
+#define ARM_TBFLAG_AA64_SS_ACTIVE_MASK (1 << ARM_TBFLAG_AA64_SS_ACTIVE_SHIFT)
+#define ARM_TBFLAG_AA64_PSTATE_SS_SHIFT 4
+#define ARM_TBFLAG_AA64_PSTATE_SS_MASK (1 << ARM_TBFLAG_AA64_PSTATE_SS_SHIFT)
 
 /* some convenience accessor macros */
 #define ARM_TBFLAG_AARCH64_STATE(F) \
@@ -1162,23 +1432,53 @@ static inline int cpu_mmu_index (CPUARMState *env)
     (((F) & ARM_TBFLAG_BSWAP_CODE_MASK) >> ARM_TBFLAG_BSWAP_CODE_SHIFT)
 #define ARM_TBFLAG_CPACR_FPEN(F) \
     (((F) & ARM_TBFLAG_CPACR_FPEN_MASK) >> ARM_TBFLAG_CPACR_FPEN_SHIFT)
+#define ARM_TBFLAG_SS_ACTIVE(F) \
+    (((F) & ARM_TBFLAG_SS_ACTIVE_MASK) >> ARM_TBFLAG_SS_ACTIVE_SHIFT)
+#define ARM_TBFLAG_PSTATE_SS(F) \
+    (((F) & ARM_TBFLAG_PSTATE_SS_MASK) >> ARM_TBFLAG_PSTATE_SS_SHIFT)
+#define ARM_TBFLAG_XSCALE_CPAR(F) \
+    (((F) & ARM_TBFLAG_XSCALE_CPAR_MASK) >> ARM_TBFLAG_XSCALE_CPAR_SHIFT)
 #define ARM_TBFLAG_AA64_EL(F) \
     (((F) & ARM_TBFLAG_AA64_EL_MASK) >> ARM_TBFLAG_AA64_EL_SHIFT)
 #define ARM_TBFLAG_AA64_FPEN(F) \
     (((F) & ARM_TBFLAG_AA64_FPEN_MASK) >> ARM_TBFLAG_AA64_FPEN_SHIFT)
+#define ARM_TBFLAG_AA64_SS_ACTIVE(F) \
+    (((F) & ARM_TBFLAG_AA64_SS_ACTIVE_MASK) >> ARM_TBFLAG_AA64_SS_ACTIVE_SHIFT)
+#define ARM_TBFLAG_AA64_PSTATE_SS(F) \
+    (((F) & ARM_TBFLAG_AA64_PSTATE_SS_MASK) >> ARM_TBFLAG_AA64_PSTATE_SS_SHIFT)
 
 static inline void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc,
                                         target_ulong *cs_base, int *flags)
 {
-    int fpen = extract32(env->cp15.c1_coproc, 20, 2);
+    int fpen;
+
+    if (arm_feature(env, ARM_FEATURE_V6)) {
+        fpen = extract32(env->cp15.c1_coproc, 20, 2);
+    } else {
+        /* CPACR doesn't exist before v6, so VFP is always accessible */
+        fpen = 3;
+    }
 
     if (is_a64(env)) {
         *pc = env->pc;
         *flags = ARM_TBFLAG_AARCH64_STATE_MASK
-            | (arm_current_pl(env) << ARM_TBFLAG_AA64_EL_SHIFT);
-        if (fpen == 3 || (fpen == 1 && arm_current_pl(env) != 0)) {
+            | (arm_current_el(env) << ARM_TBFLAG_AA64_EL_SHIFT);
+        if (fpen == 3 || (fpen == 1 && arm_current_el(env) != 0)) {
             *flags |= ARM_TBFLAG_AA64_FPEN_MASK;
         }
+        /* The SS_ACTIVE and PSTATE_SS bits correspond to the state machine
+         * states defined in the ARM ARM for software singlestep:
+         *  SS_ACTIVE   PSTATE.SS   State
+         *     0            x       Inactive (the TB flag for SS is always 0)
+         *     1            0       Active-pending
+         *     1            1       Active-not-pending
+         */
+        if (arm_singlestep_active(env)) {
+            *flags |= ARM_TBFLAG_AA64_SS_ACTIVE_MASK;
+            if (env->pstate & PSTATE_SS) {
+                *flags |= ARM_TBFLAG_AA64_PSTATE_SS_MASK;
+            }
+        }
     } else {
         int privmode;
         *pc = env->regs[15];
@@ -1199,9 +1499,24 @@ static inline void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc,
             || arm_el_is_aa64(env, 1)) {
             *flags |= ARM_TBFLAG_VFPEN_MASK;
         }
-        if (fpen == 3 || (fpen == 1 && arm_current_pl(env) != 0)) {
+        if (fpen == 3 || (fpen == 1 && arm_current_el(env) != 0)) {
             *flags |= ARM_TBFLAG_CPACR_FPEN_MASK;
         }
+        /* The SS_ACTIVE and PSTATE_SS bits correspond to the state machine
+         * states defined in the ARM ARM for software singlestep:
+         *  SS_ACTIVE   PSTATE.SS   State
+         *     0            x       Inactive (the TB flag for SS is always 0)
+         *     1            0       Active-pending
+         *     1            1       Active-not-pending
+         */
+        if (arm_singlestep_active(env)) {
+            *flags |= ARM_TBFLAG_SS_ACTIVE_MASK;
+            if (env->uncached_cpsr & PSTATE_SS) {
+                *flags |= ARM_TBFLAG_PSTATE_SS_MASK;
+            }
+        }
+        *flags |= (extract32(env->cp15.c15_cpar, 0, 2)
+                   << ARM_TBFLAG_XSCALE_CPAR_SHIFT);
     }
 
     *cs_base = 0;
@@ -1218,4 +1533,10 @@ static inline void cpu_pc_from_tb(CPUARMState *env, TranslationBlock *tb)
     }
 }
 
+enum {
+    QEMU_PSCI_CONDUIT_DISABLED = 0,
+    QEMU_PSCI_CONDUIT_SMC = 1,
+    QEMU_PSCI_CONDUIT_HVC = 2,
+};
+
 #endif
diff --git a/target-arm/cpu64.c b/target-arm/cpu64.c
index 8b2081c24..bb778b3d9 100644
--- a/target-arm/cpu64.c
+++ b/target-arm/cpu64.c
@@ -123,10 +123,12 @@ static void aarch64_a57_initfn(Object *obj)
     cpu->id_isar2 = 0x21232042;
     cpu->id_isar3 = 0x01112131;
     cpu->id_isar4 = 0x00011142;
+    cpu->id_isar5 = 0x00011121;
     cpu->id_aa64pfr0 = 0x00002222;
     cpu->id_aa64dfr0 = 0x10305106;
-    cpu->id_aa64isar0 = 0x00010000;
+    cpu->id_aa64isar0 = 0x00011120;
     cpu->id_aa64mmfr0 = 0x00001124;
+    cpu->dbgdidr = 0x3516d000;
     cpu->clidr = 0x0a200023;
     cpu->ccsidr[0] = 0x701fe00a; /* 32KB L1 dcache */
     cpu->ccsidr[1] = 0x201fe012; /* 48KB L1 icache */
@@ -149,7 +151,7 @@ static void aarch64_any_initfn(Object *obj)
     set_feature(&cpu->env, ARM_FEATURE_V8_SHA256);
     set_feature(&cpu->env, ARM_FEATURE_V8_PMULL);
     set_feature(&cpu->env, ARM_FEATURE_CRC);
-    cpu->ctr = 0x80030003; /* 32 byte I and D cacheline size, VIPT icache */
+    cpu->ctr = 0x80038003; /* 32 byte I and D cacheline size, VIPT icache */
     cpu->dcz_blocksize = 7; /*  512 bytes */
 }
 #endif
@@ -194,7 +196,10 @@ static void aarch64_cpu_class_init(ObjectClass *oc, void *data)
 {
     CPUClass *cc = CPU_CLASS(oc);
 
+#if !defined(CONFIG_USER_ONLY)
     cc->do_interrupt = aarch64_cpu_do_interrupt;
+#endif
+    cc->cpu_exec_interrupt = arm_cpu_exec_interrupt;
     cc->set_pc = aarch64_cpu_set_pc;
     cc->gdb_read_register = aarch64_cpu_gdb_read_register;
     cc->gdb_write_register = aarch64_cpu_gdb_write_register;
diff --git a/target-arm/helper-a64.c b/target-arm/helper-a64.c
index 2b4ce6ac6..81066ca93 100644
--- a/target-arm/helper-a64.c
+++ b/target-arm/helper-a64.c
@@ -438,15 +438,19 @@ uint64_t HELPER(crc32c_64)(uint64_t acc, uint64_t val, uint32_t bytes)
     return crc32c(acc, buf, bytes) ^ 0xffffffff;
 }
 
+#if !defined(CONFIG_USER_ONLY)
+
 /* Handle a CPU exception.  */
 void aarch64_cpu_do_interrupt(CPUState *cs)
 {
     ARMCPU *cpu = ARM_CPU(cs);
     CPUARMState *env = &cpu->env;
-    target_ulong addr = env->cp15.vbar_el[1];
+    unsigned int new_el = arm_excp_target_el(cs, cs->exception_index);
+    target_ulong addr = env->cp15.vbar_el[new_el];
+    unsigned int new_mode = aarch64_pstate_mode(new_el, true);
     int i;
 
-    if (arm_current_pl(env) == 0) {
+    if (arm_current_el(env) < new_el) {
         if (env->aarch64) {
             addr += 0x400;
         } else {
@@ -457,30 +461,40 @@ void aarch64_cpu_do_interrupt(CPUState *cs)
     }
 
     arm_log_exception(cs->exception_index);
-    qemu_log_mask(CPU_LOG_INT, "...from EL%d\n", arm_current_pl(env));
+    qemu_log_mask(CPU_LOG_INT, "...from EL%d\n", arm_current_el(env));
     if (qemu_loglevel_mask(CPU_LOG_INT)
         && !excp_is_internal(cs->exception_index)) {
         qemu_log_mask(CPU_LOG_INT, "...with ESR 0x%" PRIx32 "\n",
                       env->exception.syndrome);
     }
 
-    env->cp15.esr_el[1] = env->exception.syndrome;
-    env->cp15.far_el1 = env->exception.vaddress;
+    if (arm_is_psci_call(cpu, cs->exception_index)) {
+        arm_handle_psci_call(cpu);
+        qemu_log_mask(CPU_LOG_INT, "...handled as PSCI call\n");
+        return;
+    }
 
     switch (cs->exception_index) {
     case EXCP_PREFETCH_ABORT:
     case EXCP_DATA_ABORT:
+        env->cp15.far_el[new_el] = env->exception.vaddress;
         qemu_log_mask(CPU_LOG_INT, "...with FAR 0x%" PRIx64 "\n",
-                      env->cp15.far_el1);
-        break;
+                      env->cp15.far_el[new_el]);
+        /* fall through */
     case EXCP_BKPT:
     case EXCP_UDEF:
     case EXCP_SWI:
+    case EXCP_HVC:
+    case EXCP_HYP_TRAP:
+    case EXCP_SMC:
+        env->cp15.esr_el[new_el] = env->exception.syndrome;
         break;
     case EXCP_IRQ:
+    case EXCP_VIRQ:
         addr += 0x80;
         break;
     case EXCP_FIQ:
+    case EXCP_VFIQ:
         addr += 0x100;
         break;
     default:
@@ -488,16 +502,15 @@ void aarch64_cpu_do_interrupt(CPUState *cs)
     }
 
     if (is_a64(env)) {
-        env->banked_spsr[aarch64_banked_spsr_index(1)] = pstate_read(env);
-        env->sp_el[arm_current_pl(env)] = env->xregs[31];
-        env->xregs[31] = env->sp_el[1];
-        env->elr_el[1] = env->pc;
+        env->banked_spsr[aarch64_banked_spsr_index(new_el)] = pstate_read(env);
+        aarch64_save_sp(env, arm_current_el(env));
+        env->elr_el[new_el] = env->pc;
     } else {
         env->banked_spsr[0] = cpsr_read(env);
         if (!env->thumb) {
-            env->cp15.esr_el[1] |= 1 << 25;
+            env->cp15.esr_el[new_el] |= 1 << 25;
         }
-        env->elr_el[1] = env->regs[15];
+        env->elr_el[new_el] = env->regs[15];
 
         for (i = 0; i < 15; i++) {
             env->xregs[i] = env->regs[i];
@@ -506,9 +519,11 @@ void aarch64_cpu_do_interrupt(CPUState *cs)
         env->condexec_bits = 0;
     }
 
-    pstate_write(env, PSTATE_DAIF | PSTATE_MODE_EL1h);
+    pstate_write(env, PSTATE_DAIF | new_mode);
     env->aarch64 = 1;
+    aarch64_restore_sp(env, new_el);
 
     env->pc = addr;
     cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
 }
+#endif
diff --git a/target-arm/helper.c b/target-arm/helper.c
index d3438560e..b74d348a3 100644
--- a/target-arm/helper.c
+++ b/target-arm/helper.c
@@ -304,17 +304,6 @@ void init_cpreg_list(ARMCPU *cpu)
     g_list_free(keys);
 }
 
-/* Return true if extended addresses are enabled.
- * This is always the case if our translation regime is 64 bit,
- * but depends on TTBCR.EAE for 32 bit.
- */
-static inline bool extended_addresses_enabled(CPUARMState *env)
-{
-    return arm_el_is_aa64(env, 1)
-        || ((arm_feature(env, ARM_FEATURE_LPAE)
-             && (env->cp15.c2_control & TTBCR_EAE)));
-}
-
 static void dacr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 {
     ARMCPU *cpu = arm_env_get_cpu(env);
@@ -388,13 +377,48 @@ static void tlbimvaa_write(CPUARMState *env, const ARMCPRegInfo *ri,
     tlb_flush_page(CPU(cpu), value & TARGET_PAGE_MASK);
 }
 
+/* IS variants of TLB operations must affect all cores */
+static void tlbiall_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                             uint64_t value)
+{
+    CPUState *other_cs;
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush(other_cs, 1);
+    }
+}
+
+static void tlbiasid_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                             uint64_t value)
+{
+    CPUState *other_cs;
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush(other_cs, value == 0);
+    }
+}
+
+static void tlbimva_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                             uint64_t value)
+{
+    CPUState *other_cs;
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush_page(other_cs, value & TARGET_PAGE_MASK);
+    }
+}
+
+static void tlbimvaa_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                             uint64_t value)
+{
+    CPUState *other_cs;
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush_page(other_cs, value & TARGET_PAGE_MASK);
+    }
+}
+
 static const ARMCPRegInfo cp_reginfo[] = {
-    /* DBGDIDR: just RAZ. In particular this means the "debug architecture
-     * version" bits will read as a reserved value, which should cause
-     * Linux to not try to use the debug hardware.
-     */
-    { .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
-      .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
     { .name = "FCSEIDR", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c13_fcse),
       .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
@@ -420,21 +444,6 @@ static const ARMCPRegInfo not_v8_cp_reginfo[] = {
      */
     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = CP_ANY,
       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
-    /* MMU TLB control. Note that the wildcarding means we cover not just
-     * the unified TLB ops but also the dside/iside/inner-shareable variants.
-     */
-    { .name = "TLBIALL", .cp = 15, .crn = 8, .crm = CP_ANY,
-      .opc1 = CP_ANY, .opc2 = 0, .access = PL1_W, .writefn = tlbiall_write,
-      .type = ARM_CP_NO_MIGRATE },
-    { .name = "TLBIMVA", .cp = 15, .crn = 8, .crm = CP_ANY,
-      .opc1 = CP_ANY, .opc2 = 1, .access = PL1_W, .writefn = tlbimva_write,
-      .type = ARM_CP_NO_MIGRATE },
-    { .name = "TLBIASID", .cp = 15, .crn = 8, .crm = CP_ANY,
-      .opc1 = CP_ANY, .opc2 = 2, .access = PL1_W, .writefn = tlbiasid_write,
-      .type = ARM_CP_NO_MIGRATE },
-    { .name = "TLBIMVAA", .cp = 15, .crn = 8, .crm = CP_ANY,
-      .opc1 = CP_ANY, .opc2 = 3, .access = PL1_W, .writefn = tlbimvaa_write,
-      .type = ARM_CP_NO_MIGRATE },
     /* Cache maintenance ops; some of this space may be overridden later. */
     { .name = "CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
       .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W,
@@ -471,6 +480,28 @@ static const ARMCPRegInfo not_v7_cp_reginfo[] = {
     { .name = "DUMMY", .cp = 15, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = CP_ANY,
       .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE,
       .resetvalue = 0 },
+    /* We don't implement pre-v7 debug but most CPUs had at least a DBGDIDR;
+     * implementing it as RAZ means the "debug architecture version" bits
+     * will read as a reserved value, which should cause Linux to not try
+     * to use the debug hardware.
+     */
+    { .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
+      .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
+    /* MMU TLB control. Note that the wildcarding means we cover not just
+     * the unified TLB ops but also the dside/iside/inner-shareable variants.
+     */
+    { .name = "TLBIALL", .cp = 15, .crn = 8, .crm = CP_ANY,
+      .opc1 = CP_ANY, .opc2 = 0, .access = PL1_W, .writefn = tlbiall_write,
+      .type = ARM_CP_NO_MIGRATE },
+    { .name = "TLBIMVA", .cp = 15, .crn = 8, .crm = CP_ANY,
+      .opc1 = CP_ANY, .opc2 = 1, .access = PL1_W, .writefn = tlbimva_write,
+      .type = ARM_CP_NO_MIGRATE },
+    { .name = "TLBIASID", .cp = 15, .crn = 8, .crm = CP_ANY,
+      .opc1 = CP_ANY, .opc2 = 2, .access = PL1_W, .writefn = tlbiasid_write,
+      .type = ARM_CP_NO_MIGRATE },
+    { .name = "TLBIMVAA", .cp = 15, .crn = 8, .crm = CP_ANY,
+      .opc1 = CP_ANY, .opc2 = 3, .access = PL1_W, .writefn = tlbimvaa_write,
+      .type = ARM_CP_NO_MIGRATE },
     REGINFO_SENTINEL
 };
 
@@ -521,7 +552,7 @@ static const ARMCPRegInfo v6_cp_reginfo[] = {
       .access = PL0_W, .type = ARM_CP_NOP },
     { .name = "IFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 2,
       .access = PL1_RW,
-      .fieldoffset = offsetofhigh32(CPUARMState, cp15.far_el1),
+      .fieldoffset = offsetofhigh32(CPUARMState, cp15.far_el[1]),
       .resetvalue = 0, },
     /* Watchpoint Fault Address Register : should actually only be present
      * for 1136, 1176, 11MPCore.
@@ -540,32 +571,47 @@ static CPAccessResult pmreg_access(CPUARMState *env, const ARMCPRegInfo *ri)
     /* Performance monitor registers user accessibility is controlled
      * by PMUSERENR.
      */
-    if (arm_current_pl(env) == 0 && !env->cp15.c9_pmuserenr) {
+    if (arm_current_el(env) == 0 && !env->cp15.c9_pmuserenr) {
         return CP_ACCESS_TRAP;
     }
     return CP_ACCESS_OK;
 }
 
 #ifndef CONFIG_USER_ONLY
-static void pmcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                       uint64_t value)
+
+static inline bool arm_ccnt_enabled(CPUARMState *env)
 {
-    /* Don't computer the number of ticks in user mode */
-    uint32_t temp_ticks;
+    /* This does not support checking PMCCFILTR_EL0 register */
 
-    temp_ticks = qemu_clock_get_us(QEMU_CLOCK_VIRTUAL) *
-                  get_ticks_per_sec() / 1000000;
+    if (!(env->cp15.c9_pmcr & PMCRE)) {
+        return false;
+    }
 
-    if (env->cp15.c9_pmcr & PMCRE) {
-        /* If the counter is enabled */
-        if (env->cp15.c9_pmcr & PMCRD) {
-            /* Increment once every 64 processor clock cycles */
-            env->cp15.c15_ccnt = (temp_ticks/64) - env->cp15.c15_ccnt;
-        } else {
-            env->cp15.c15_ccnt = temp_ticks - env->cp15.c15_ccnt;
-        }
+    return true;
+}
+
+void pmccntr_sync(CPUARMState *env)
+{
+    uint64_t temp_ticks;
+
+    temp_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
+                          get_ticks_per_sec(), 1000000);
+
+    if (env->cp15.c9_pmcr & PMCRD) {
+        /* Increment once every 64 processor clock cycles */
+        temp_ticks /= 64;
     }
 
+    if (arm_ccnt_enabled(env)) {
+        env->cp15.c15_ccnt = temp_ticks - env->cp15.c15_ccnt;
+    }
+}
+
+static void pmcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                       uint64_t value)
+{
+    pmccntr_sync(env);
+
     if (value & PMCRC) {
         /* The counter has been reset */
         env->cp15.c15_ccnt = 0;
@@ -575,26 +621,20 @@ static void pmcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
     env->cp15.c9_pmcr &= ~0x39;
     env->cp15.c9_pmcr |= (value & 0x39);
 
-    if (env->cp15.c9_pmcr & PMCRE) {
-        if (env->cp15.c9_pmcr & PMCRD) {
-            /* Increment once every 64 processor clock cycles */
-            temp_ticks /= 64;
-        }
-        env->cp15.c15_ccnt = temp_ticks - env->cp15.c15_ccnt;
-    }
+    pmccntr_sync(env);
 }
 
 static uint64_t pmccntr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 {
-    uint32_t total_ticks;
+    uint64_t total_ticks;
 
-    if (!(env->cp15.c9_pmcr & PMCRE)) {
+    if (!arm_ccnt_enabled(env)) {
         /* Counter is disabled, do not change value */
         return env->cp15.c15_ccnt;
     }
 
-    total_ticks = qemu_clock_get_us(QEMU_CLOCK_VIRTUAL) *
-                  get_ticks_per_sec() / 1000000;
+    total_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
+                           get_ticks_per_sec(), 1000000);
 
     if (env->cp15.c9_pmcr & PMCRD) {
         /* Increment once every 64 processor clock cycles */
@@ -606,16 +646,16 @@ static uint64_t pmccntr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 static void pmccntr_write(CPUARMState *env, const ARMCPRegInfo *ri,
                         uint64_t value)
 {
-    uint32_t total_ticks;
+    uint64_t total_ticks;
 
-    if (!(env->cp15.c9_pmcr & PMCRE)) {
+    if (!arm_ccnt_enabled(env)) {
         /* Counter is disabled, set the absolute value */
         env->cp15.c15_ccnt = value;
         return;
     }
 
-    total_ticks = qemu_clock_get_us(QEMU_CLOCK_VIRTUAL) *
-                  get_ticks_per_sec() / 1000000;
+    total_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
+                           get_ticks_per_sec(), 1000000);
 
     if (env->cp15.c9_pmcr & PMCRD) {
         /* Increment once every 64 processor clock cycles */
@@ -623,8 +663,31 @@ static void pmccntr_write(CPUARMState *env, const ARMCPRegInfo *ri,
     }
     env->cp15.c15_ccnt = total_ticks - value;
 }
+
+static void pmccntr_write32(CPUARMState *env, const ARMCPRegInfo *ri,
+                            uint64_t value)
+{
+    uint64_t cur_val = pmccntr_read(env, NULL);
+
+    pmccntr_write(env, ri, deposit64(cur_val, 0, 32, value));
+}
+
+#else /* CONFIG_USER_ONLY */
+
+void pmccntr_sync(CPUARMState *env)
+{
+}
+
 #endif
 
+static void pmccfiltr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                            uint64_t value)
+{
+    pmccntr_sync(env);
+    env->cp15.pmccfiltr_el0 = value & 0x7E000000;
+    pmccntr_sync(env);
+}
+
 static void pmcntenset_write(CPUARMState *env, const ARMCPRegInfo *ri,
                             uint64_t value)
 {
@@ -684,6 +747,32 @@ static void vbar_write(CPUARMState *env, const ARMCPRegInfo *ri,
     raw_write(env, ri, value & ~0x1FULL);
 }
 
+static void scr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
+{
+    /* We only mask off bits that are RES0 both for AArch64 and AArch32.
+     * For bits that vary between AArch32/64, code needs to check the
+     * current execution mode before directly using the feature bit.
+     */
+    uint32_t valid_mask = SCR_AARCH64_MASK | SCR_AARCH32_MASK;
+
+    if (!arm_feature(env, ARM_FEATURE_EL2)) {
+        valid_mask &= ~SCR_HCE;
+
+        /* On ARMv7, SMD (or SCD as it is called in v7) is only
+         * supported if EL2 exists. The bit is UNK/SBZP when
+         * EL2 is unavailable. In QEMU ARMv7, we force it to always zero
+         * when EL2 is unavailable.
+         */
+        if (arm_feature(env, ARM_FEATURE_V7)) {
+            valid_mask &= ~SCR_SMD;
+        }
+    }
+
+    /* Clear all-context RES0 bits.  */
+    value &= valid_mask;
+    raw_write(env, ri, value);
+}
+
 static uint64_t ccsidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 {
     ARMCPU *cpu = arm_env_get_cpu(env);
@@ -712,13 +801,6 @@ static uint64_t isr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 }
 
 static const ARMCPRegInfo v7_cp_reginfo[] = {
-    /* DBGDRAR, DBGDSAR: always RAZ since we don't implement memory mapped
-     * debug components
-     */
-    { .name = "DBGDRAR", .cp = 14, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 0,
-      .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
-    { .name = "DBGDSAR", .cp = 14, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
-      .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
     /* the old v6 WFI, UNPREDICTABLE in v7 but we choose to NOP */
     { .name = "NOP", .cp = 15, .crn = 7, .crm = 0, .opc1 = 0, .opc2 = 4,
       .access = PL1_W, .type = ARM_CP_NOP },
@@ -734,16 +816,28 @@ static const ARMCPRegInfo v7_cp_reginfo[] = {
      * or PL0_RO as appropriate and then check PMUSERENR in the helper fn.
      */
     { .name = "PMCNTENSET", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 1,
-      .access = PL0_RW, .resetvalue = 0,
-      .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten),
+      .access = PL0_RW, .type = ARM_CP_NO_MIGRATE,
+      .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcnten),
       .writefn = pmcntenset_write,
       .accessfn = pmreg_access,
       .raw_writefn = raw_write },
+    { .name = "PMCNTENSET_EL0", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 1,
+      .access = PL0_RW, .accessfn = pmreg_access,
+      .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten), .resetvalue = 0,
+      .writefn = pmcntenset_write, .raw_writefn = raw_write },
     { .name = "PMCNTENCLR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 2,
-      .access = PL0_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten),
+      .access = PL0_RW,
+      .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcnten),
       .accessfn = pmreg_access,
       .writefn = pmcntenclr_write,
       .type = ARM_CP_NO_MIGRATE },
+    { .name = "PMCNTENCLR_EL0", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 2,
+      .access = PL0_RW, .accessfn = pmreg_access,
+      .type = ARM_CP_NO_MIGRATE,
+      .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten),
+      .writefn = pmcntenclr_write },
     { .name = "PMOVSR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 3,
       .access = PL0_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_pmovsr),
       .accessfn = pmreg_access,
@@ -761,9 +855,21 @@ static const ARMCPRegInfo v7_cp_reginfo[] = {
 #ifndef CONFIG_USER_ONLY
     { .name = "PMCCNTR", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 0,
       .access = PL0_RW, .resetvalue = 0, .type = ARM_CP_IO,
-      .readfn = pmccntr_read, .writefn = pmccntr_write,
+      .readfn = pmccntr_read, .writefn = pmccntr_write32,
       .accessfn = pmreg_access },
+    { .name = "PMCCNTR_EL0", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 13, .opc2 = 0,
+      .access = PL0_RW, .accessfn = pmreg_access,
+      .type = ARM_CP_IO,
+      .readfn = pmccntr_read, .writefn = pmccntr_write, },
 #endif
+    { .name = "PMCCFILTR_EL0", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 15, .opc2 = 7,
+      .writefn = pmccfiltr_write,
+      .access = PL0_RW, .accessfn = pmreg_access,
+      .type = ARM_CP_IO,
+      .fieldoffset = offsetof(CPUARMState, cp15.pmccfiltr_el0),
+      .resetvalue = 0, },
     { .name = "PMXEVTYPER", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 1,
       .access = PL0_RW,
       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmxevtyper),
@@ -793,8 +899,8 @@ static const ARMCPRegInfo v7_cp_reginfo[] = {
       .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[1]),
       .resetvalue = 0 },
     { .name = "SCR", .cp = 15, .crn = 1, .crm = 1, .opc1 = 0, .opc2 = 0,
-      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c1_scr),
-      .resetvalue = 0, },
+      .access = PL1_RW, .fieldoffset = offsetoflow32(CPUARMState, cp15.scr_el3),
+      .resetvalue = 0, .writefn = scr_write },
     { .name = "CCSIDR", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 0,
       .access = PL1_R, .readfn = ccsidr_read, .type = ARM_CP_NO_MIGRATE },
@@ -840,6 +946,44 @@ static const ARMCPRegInfo v7_cp_reginfo[] = {
     { .name = "ISR_EL1", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 1, .opc2 = 0,
       .type = ARM_CP_NO_MIGRATE, .access = PL1_R, .readfn = isr_read },
+    /* 32 bit ITLB invalidates */
+    { .name = "ITLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 0,
+      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_write },
+    { .name = "ITLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 1,
+      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
+    { .name = "ITLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 2,
+      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiasid_write },
+    /* 32 bit DTLB invalidates */
+    { .name = "DTLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 0,
+      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_write },
+    { .name = "DTLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 1,
+      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
+    { .name = "DTLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 2,
+      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiasid_write },
+    /* 32 bit TLB invalidates */
+    { .name = "TLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
+      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_write },
+    { .name = "TLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
+      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
+    { .name = "TLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
+      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiasid_write },
+    { .name = "TLBIMVAA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
+      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimvaa_write },
+    REGINFO_SENTINEL
+};
+
+static const ARMCPRegInfo v7mp_cp_reginfo[] = {
+    /* 32 bit TLB invalidates, Inner Shareable */
+    { .name = "TLBIALLIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
+      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_is_write },
+    { .name = "TLBIMVAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
+      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_is_write },
+    { .name = "TLBIASIDIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
+      .type = ARM_CP_NO_MIGRATE, .access = PL1_W,
+      .writefn = tlbiasid_is_write },
+    { .name = "TLBIMVAAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
+      .type = ARM_CP_NO_MIGRATE, .access = PL1_W,
+      .writefn = tlbimvaa_is_write },
     REGINFO_SENTINEL
 };
 
@@ -852,7 +996,7 @@ static void teecr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 
 static CPAccessResult teehbr_access(CPUARMState *env, const ARMCPRegInfo *ri)
 {
-    if (arm_current_pl(env) == 0 && (env->teecr & 1)) {
+    if (arm_current_el(env) == 0 && (env->teecr & 1)) {
         return CP_ACCESS_TRAP;
     }
     return CP_ACCESS_OK;
@@ -898,7 +1042,7 @@ static const ARMCPRegInfo v6k_cp_reginfo[] = {
 static CPAccessResult gt_cntfrq_access(CPUARMState *env, const ARMCPRegInfo *ri)
 {
     /* CNTFRQ: not visible from PL0 if both PL0PCTEN and PL0VCTEN are zero */
-    if (arm_current_pl(env) == 0 && !extract32(env->cp15.c14_cntkctl, 0, 2)) {
+    if (arm_current_el(env) == 0 && !extract32(env->cp15.c14_cntkctl, 0, 2)) {
         return CP_ACCESS_TRAP;
     }
     return CP_ACCESS_OK;
@@ -907,7 +1051,7 @@ static CPAccessResult gt_cntfrq_access(CPUARMState *env, const ARMCPRegInfo *ri)
 static CPAccessResult gt_counter_access(CPUARMState *env, int timeridx)
 {
     /* CNT[PV]CT: not visible from PL0 if ELO[PV]CTEN is zero */
-    if (arm_current_pl(env) == 0 &&
+    if (arm_current_el(env) == 0 &&
         !extract32(env->cp15.c14_cntkctl, timeridx, 1)) {
         return CP_ACCESS_TRAP;
     }
@@ -919,7 +1063,7 @@ static CPAccessResult gt_timer_access(CPUARMState *env, int timeridx)
     /* CNT[PV]_CVAL, CNT[PV]_CTL, CNT[PV]_TVAL: not visible from PL0 if
      * EL0[PV]TEN is zero.
      */
-    if (arm_current_pl(env) == 0 &&
+    if (arm_current_el(env) == 0 &&
         !extract32(env->cp15.c14_cntkctl, 9 - timeridx, 1)) {
         return CP_ACCESS_TRAP;
     }
@@ -1516,7 +1660,7 @@ static const ARMCPRegInfo vmsa_cp_reginfo[] = {
     /* 64-bit FAR; this entry also gives us the AArch32 DFAR */
     { .name = "FAR_EL1", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 0,
-      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el1),
+      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[1]),
       .resetvalue = 0, },
     REGINFO_SENTINEL
 };
@@ -1596,12 +1740,7 @@ static const ARMCPRegInfo omap_cp_reginfo[] = {
 static void xscale_cpar_write(CPUARMState *env, const ARMCPRegInfo *ri,
                               uint64_t value)
 {
-    value &= 0x3fff;
-    if (env->cp15.c15_cpar != value) {
-        /* Changes cp0 to cp13 behavior, so needs a TB flush.  */
-        tb_flush(env);
-        env->cp15.c15_cpar = value;
-    }
+    env->cp15.c15_cpar = value & 0x3fff;
 }
 
 static const ARMCPRegInfo xscale_cp_reginfo[] = {
@@ -1734,11 +1873,6 @@ static const ARMCPRegInfo lpae_cp_reginfo[] = {
     { .name = "AMAIR1", .cp = 15, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 1,
       .access = PL1_RW, .type = ARM_CP_CONST | ARM_CP_OVERRIDE,
       .resetvalue = 0 },
-    /* 64 bit access versions of the (dummy) debug registers */
-    { .name = "DBGDRAR", .cp = 14, .crm = 1, .opc1 = 0,
-      .access = PL0_R, .type = ARM_CP_CONST|ARM_CP_64BIT, .resetvalue = 0 },
-    { .name = "DBGDSAR", .cp = 14, .crm = 2, .opc1 = 0,
-      .access = PL0_R, .type = ARM_CP_CONST|ARM_CP_64BIT, .resetvalue = 0 },
     { .name = "PAR", .cp = 15, .crm = 7, .opc1 = 0,
       .access = PL1_RW, .type = ARM_CP_64BIT,
       .fieldoffset = offsetof(CPUARMState, cp15.par_el1), .resetvalue = 0 },
@@ -1777,7 +1911,7 @@ static void aa64_fpsr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 
 static CPAccessResult aa64_daif_access(CPUARMState *env, const ARMCPRegInfo *ri)
 {
-    if (arm_current_pl(env) == 0 && !(env->cp15.c1_sys & SCTLR_UMA)) {
+    if (arm_current_el(env) == 0 && !(env->cp15.c1_sys & SCTLR_UMA)) {
         return CP_ACCESS_TRAP;
     }
     return CP_ACCESS_OK;
@@ -1795,18 +1929,23 @@ static CPAccessResult aa64_cacheop_access(CPUARMState *env,
     /* Cache invalidate/clean: NOP, but EL0 must UNDEF unless
      * SCTLR_EL1.UCI is set.
      */
-    if (arm_current_pl(env) == 0 && !(env->cp15.c1_sys & SCTLR_UCI)) {
+    if (arm_current_el(env) == 0 && !(env->cp15.c1_sys & SCTLR_UCI)) {
         return CP_ACCESS_TRAP;
     }
     return CP_ACCESS_OK;
 }
 
+/* See: D4.7.2 TLB maintenance requirements and the TLB maintenance instructions
+ * Page D4-1736 (DDI0487A.b)
+ */
+
 static void tlbi_aa64_va_write(CPUARMState *env, const ARMCPRegInfo *ri,
                                uint64_t value)
 {
     /* Invalidate by VA (AArch64 version) */
     ARMCPU *cpu = arm_env_get_cpu(env);
-    uint64_t pageaddr = value << 12;
+    uint64_t pageaddr = sextract64(value << 12, 0, 56);
+
     tlb_flush_page(CPU(cpu), pageaddr);
 }
 
@@ -1815,7 +1954,8 @@ static void tlbi_aa64_vaa_write(CPUARMState *env, const ARMCPRegInfo *ri,
 {
     /* Invalidate by VA, all ASIDs (AArch64 version) */
     ARMCPU *cpu = arm_env_get_cpu(env);
-    uint64_t pageaddr = value << 12;
+    uint64_t pageaddr = sextract64(value << 12, 0, 56);
+
     tlb_flush_page(CPU(cpu), pageaddr);
 }
 
@@ -1828,12 +1968,45 @@ static void tlbi_aa64_asid_write(CPUARMState *env, const ARMCPRegInfo *ri,
     tlb_flush(CPU(cpu), asid == 0);
 }
 
+static void tlbi_aa64_va_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  uint64_t value)
+{
+    CPUState *other_cs;
+    uint64_t pageaddr = sextract64(value << 12, 0, 56);
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush_page(other_cs, pageaddr);
+    }
+}
+
+static void tlbi_aa64_vaa_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  uint64_t value)
+{
+    CPUState *other_cs;
+    uint64_t pageaddr = sextract64(value << 12, 0, 56);
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush_page(other_cs, pageaddr);
+    }
+}
+
+static void tlbi_aa64_asid_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  uint64_t value)
+{
+    CPUState *other_cs;
+    int asid = extract64(value, 48, 16);
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush(other_cs, asid == 0);
+    }
+}
+
 static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri)
 {
     /* We don't implement EL2, so the only control on DC ZVA is the
      * bit in the SCTLR which can prohibit access for EL0.
      */
-    if (arm_current_pl(env) == 0 && !(env->cp15.c1_sys & SCTLR_DZE)) {
+    if (arm_current_el(env) == 0 && !(env->cp15.c1_sys & SCTLR_DZE)) {
         return CP_ACCESS_TRAP;
     }
     return CP_ACCESS_OK;
@@ -1845,7 +2018,7 @@ static uint64_t aa64_dczid_read(CPUARMState *env, const ARMCPRegInfo *ri)
     int dzp_bit = 1 << 4;
 
     /* DZP indicates whether DC ZVA access is allowed */
-    if (aa64_zva_access(env, NULL) != CP_ACCESS_OK) {
+    if (aa64_zva_access(env, NULL) == CP_ACCESS_OK) {
         dzp_bit = 0;
     }
     return cpu->dcz_blocksize | dzp_bit;
@@ -1853,7 +2026,7 @@ static uint64_t aa64_dczid_read(CPUARMState *env, const ARMCPRegInfo *ri)
 
 static CPAccessResult sp_el0_access(CPUARMState *env, const ARMCPRegInfo *ri)
 {
-    if (!env->pstate & PSTATE_SP) {
+    if (!(env->pstate & PSTATE_SP)) {
         /* Access to SP_EL0 is undefined if it's being used as
          * the stack pointer.
          */
@@ -1945,27 +2118,27 @@ static const ARMCPRegInfo v8_cp_reginfo[] = {
     { .name = "TLBI_VMALLE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
-      .writefn = tlbiall_write },
+      .writefn = tlbiall_is_write },
     { .name = "TLBI_VAE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
-      .writefn = tlbi_aa64_va_write },
+      .writefn = tlbi_aa64_va_is_write },
     { .name = "TLBI_ASIDE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
-      .writefn = tlbi_aa64_asid_write },
+      .writefn = tlbi_aa64_asid_is_write },
     { .name = "TLBI_VAAE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
-      .writefn = tlbi_aa64_vaa_write },
+      .writefn = tlbi_aa64_vaa_is_write },
     { .name = "TLBI_VALE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
-      .writefn = tlbi_aa64_va_write },
+      .writefn = tlbi_aa64_va_is_write },
     { .name = "TLBI_VAALE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
-      .writefn = tlbi_aa64_vaa_write },
+      .writefn = tlbi_aa64_vaa_is_write },
     { .name = "TLBI_VMALLE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
@@ -2005,42 +2178,12 @@ static const ARMCPRegInfo v8_cp_reginfo[] = {
       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 3,
       .access = PL1_W, .type = ARM_CP_NO_MIGRATE, .writefn = ats_write },
 #endif
-    /* 32 bit TLB invalidates, Inner Shareable */
-    { .name = "TLBIALLIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_write },
-    { .name = "TLBIMVAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
-    { .name = "TLBIASIDIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiasid_write },
-    { .name = "TLBIMVAAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimvaa_write },
+    /* TLB invalidate last level of translation table walk */
     { .name = "TLBIMVALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
+      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_is_write },
     { .name = "TLBIMVAALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimvaa_write },
-    /* 32 bit ITLB invalidates */
-    { .name = "ITLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 0,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_write },
-    { .name = "ITLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 1,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
-    { .name = "ITLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 2,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiasid_write },
-    /* 32 bit DTLB invalidates */
-    { .name = "DTLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 0,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_write },
-    { .name = "DTLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 1,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
-    { .name = "DTLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 2,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiasid_write },
-    /* 32 bit TLB invalidates */
-    { .name = "TLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_write },
-    { .name = "TLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
-    { .name = "TLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiasid_write },
-    { .name = "TLBIMVAA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimvaa_write },
+      .type = ARM_CP_NO_MIGRATE, .access = PL1_W,
+      .writefn = tlbimvaa_is_write },
     { .name = "TLBIMVAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
       .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
     { .name = "TLBIMVAAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
@@ -2077,16 +2220,6 @@ static const ARMCPRegInfo v8_cp_reginfo[] = {
       .opc1 = 0, .crn = 3, .crm = 0, .opc2 = 0,
       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c3),
       .resetvalue = 0, .writefn = dacr_write, .raw_writefn = raw_write, },
-    /* Dummy implementation of monitor debug system control register:
-     * we don't support debug.
-     */
-    { .name = "MDSCR_EL1", .state = ARM_CP_STATE_AA64,
-      .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 2,
-      .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
-    /* We define a dummy WI OSLAR_EL1, because Linux writes to it. */
-    { .name = "OSLAR_EL1", .state = ARM_CP_STATE_AA64,
-      .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 4,
-      .access = PL1_W, .type = ARM_CP_NOP },
     { .name = "ELR_EL1", .state = ARM_CP_STATE_AA64,
       .type = ARM_CP_NO_MIGRATE,
       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 1,
@@ -2118,15 +2251,56 @@ static const ARMCPRegInfo v8_el3_no_el2_cp_reginfo[] = {
       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
       .access = PL2_RW,
       .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore },
+    { .name = "HCR_EL2", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_NO_MIGRATE,
+      .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
+      .access = PL2_RW,
+      .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore },
     REGINFO_SENTINEL
 };
 
+static void hcr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    uint64_t valid_mask = HCR_MASK;
+
+    if (arm_feature(env, ARM_FEATURE_EL3)) {
+        valid_mask &= ~HCR_HCD;
+    } else {
+        valid_mask &= ~HCR_TSC;
+    }
+
+    /* Clear RES0 bits.  */
+    value &= valid_mask;
+
+    /* These bits change the MMU setup:
+     * HCR_VM enables stage 2 translation
+     * HCR_PTW forbids certain page-table setups
+     * HCR_DC Disables stage1 and enables stage2 translation
+     */
+    if ((raw_read(env, ri) ^ value) & (HCR_VM | HCR_PTW | HCR_DC)) {
+        tlb_flush(CPU(cpu), 1);
+    }
+    raw_write(env, ri, value);
+}
+
 static const ARMCPRegInfo v8_el2_cp_reginfo[] = {
+    { .name = "HCR_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
+      .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.hcr_el2),
+      .writefn = hcr_write },
     { .name = "ELR_EL2", .state = ARM_CP_STATE_AA64,
       .type = ARM_CP_NO_MIGRATE,
       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 1,
       .access = PL2_RW,
       .fieldoffset = offsetof(CPUARMState, elr_el[2]) },
+    { .name = "ESR_EL2", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_NO_MIGRATE,
+      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 2, .opc2 = 0,
+      .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[2]) },
+    { .name = "FAR_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 0,
+      .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[2]) },
     { .name = "SPSR_EL2", .state = ARM_CP_STATE_AA64,
       .type = ARM_CP_NO_MIGRATE,
       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 0,
@@ -2145,6 +2319,13 @@ static const ARMCPRegInfo v8_el3_cp_reginfo[] = {
       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 1,
       .access = PL3_RW,
       .fieldoffset = offsetof(CPUARMState, elr_el[3]) },
+    { .name = "ESR_EL3", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_NO_MIGRATE,
+      .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 2, .opc2 = 0,
+      .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[3]) },
+    { .name = "FAR_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 6, .crm = 0, .opc2 = 0,
+      .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[3]) },
     { .name = "SPSR_EL3", .state = ARM_CP_STATE_AA64,
       .type = ARM_CP_NO_MIGRATE,
       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 0,
@@ -2154,6 +2335,11 @@ static const ARMCPRegInfo v8_el3_cp_reginfo[] = {
       .access = PL3_RW, .writefn = vbar_write,
       .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[3]),
       .resetvalue = 0 },
+    { .name = "SCR_EL3", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_NO_MIGRATE,
+      .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 0,
+      .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.scr_el3),
+      .writefn = scr_write },
     REGINFO_SENTINEL
 };
 
@@ -2180,38 +2366,380 @@ static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri)
     /* Only accessible in EL0 if SCTLR.UCT is set (and only in AArch64,
      * but the AArch32 CTR has its own reginfo struct)
      */
-    if (arm_current_pl(env) == 0 && !(env->cp15.c1_sys & SCTLR_UCT)) {
+    if (arm_current_el(env) == 0 && !(env->cp15.c1_sys & SCTLR_UCT)) {
         return CP_ACCESS_TRAP;
     }
     return CP_ACCESS_OK;
 }
 
-static void define_aarch64_debug_regs(ARMCPU *cpu)
+static const ARMCPRegInfo debug_cp_reginfo[] = {
+    /* DBGDRAR, DBGDSAR: always RAZ since we don't implement memory mapped
+     * debug components. The AArch64 version of DBGDRAR is named MDRAR_EL1;
+     * unlike DBGDRAR it is never accessible from EL0.
+     * DBGDSAR is deprecated and must RAZ from v8 anyway, so it has no AArch64
+     * accessor.
+     */
+    { .name = "DBGDRAR", .cp = 14, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 0,
+      .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "MDRAR_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 0,
+      .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "DBGDSAR", .cp = 14, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
+      .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
+    /* Monitor debug system control register; the 32-bit alias is DBGDSCRext. */
+    { .name = "MDSCR_EL1", .state = ARM_CP_STATE_BOTH,
+      .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 2,
+      .access = PL1_RW,
+      .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1),
+      .resetvalue = 0 },
+    /* MDCCSR_EL0, aka DBGDSCRint. This is a read-only mirror of MDSCR_EL1.
+     * We don't implement the configurable EL0 access.
+     */
+    { .name = "MDCCSR_EL0", .state = ARM_CP_STATE_BOTH,
+      .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 0,
+      .type = ARM_CP_NO_MIGRATE,
+      .access = PL1_R,
+      .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1),
+      .resetfn = arm_cp_reset_ignore },
+    /* We define a dummy WI OSLAR_EL1, because Linux writes to it. */
+    { .name = "OSLAR_EL1", .state = ARM_CP_STATE_BOTH,
+      .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 4,
+      .access = PL1_W, .type = ARM_CP_NOP },
+    /* Dummy OSDLR_EL1: 32-bit Linux will read this */
+    { .name = "OSDLR_EL1", .state = ARM_CP_STATE_BOTH,
+      .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 3, .opc2 = 4,
+      .access = PL1_RW, .type = ARM_CP_NOP },
+    /* Dummy DBGVCR: Linux wants to clear this on startup, but we don't
+     * implement vector catch debug events yet.
+     */
+    { .name = "DBGVCR",
+      .cp = 14, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
+      .access = PL1_RW, .type = ARM_CP_NOP },
+    REGINFO_SENTINEL
+};
+
+static const ARMCPRegInfo debug_lpae_cp_reginfo[] = {
+    /* 64 bit access versions of the (dummy) debug registers */
+    { .name = "DBGDRAR", .cp = 14, .crm = 1, .opc1 = 0,
+      .access = PL0_R, .type = ARM_CP_CONST|ARM_CP_64BIT, .resetvalue = 0 },
+    { .name = "DBGDSAR", .cp = 14, .crm = 2, .opc1 = 0,
+      .access = PL0_R, .type = ARM_CP_CONST|ARM_CP_64BIT, .resetvalue = 0 },
+    REGINFO_SENTINEL
+};
+
+void hw_watchpoint_update(ARMCPU *cpu, int n)
 {
-    /* Define breakpoint and watchpoint registers. These do nothing
-     * but read as written, for now.
+    CPUARMState *env = &cpu->env;
+    vaddr len = 0;
+    vaddr wvr = env->cp15.dbgwvr[n];
+    uint64_t wcr = env->cp15.dbgwcr[n];
+    int mask;
+    int flags = BP_CPU | BP_STOP_BEFORE_ACCESS;
+
+    if (env->cpu_watchpoint[n]) {
+        cpu_watchpoint_remove_by_ref(CPU(cpu), env->cpu_watchpoint[n]);
+        env->cpu_watchpoint[n] = NULL;
+    }
+
+    if (!extract64(wcr, 0, 1)) {
+        /* E bit clear : watchpoint disabled */
+        return;
+    }
+
+    switch (extract64(wcr, 3, 2)) {
+    case 0:
+        /* LSC 00 is reserved and must behave as if the wp is disabled */
+        return;
+    case 1:
+        flags |= BP_MEM_READ;
+        break;
+    case 2:
+        flags |= BP_MEM_WRITE;
+        break;
+    case 3:
+        flags |= BP_MEM_ACCESS;
+        break;
+    }
+
+    /* Attempts to use both MASK and BAS fields simultaneously are
+     * CONSTRAINED UNPREDICTABLE; we opt to ignore BAS in this case,
+     * thus generating a watchpoint for every byte in the masked region.
      */
+    mask = extract64(wcr, 24, 4);
+    if (mask == 1 || mask == 2) {
+        /* Reserved values of MASK; we must act as if the mask value was
+         * some non-reserved value, or as if the watchpoint were disabled.
+         * We choose the latter.
+         */
+        return;
+    } else if (mask) {
+        /* Watchpoint covers an aligned area up to 2GB in size */
+        len = 1ULL << mask;
+        /* If masked bits in WVR are not zero it's CONSTRAINED UNPREDICTABLE
+         * whether the watchpoint fires when the unmasked bits match; we opt
+         * to generate the exceptions.
+         */
+        wvr &= ~(len - 1);
+    } else {
+        /* Watchpoint covers bytes defined by the byte address select bits */
+        int bas = extract64(wcr, 5, 8);
+        int basstart;
+
+        if (bas == 0) {
+            /* This must act as if the watchpoint is disabled */
+            return;
+        }
+
+        if (extract64(wvr, 2, 1)) {
+            /* Deprecated case of an only 4-aligned address. BAS[7:4] are
+             * ignored, and BAS[3:0] define which bytes to watch.
+             */
+            bas &= 0xf;
+        }
+        /* The BAS bits are supposed to be programmed to indicate a contiguous
+         * range of bytes. Otherwise it is CONSTRAINED UNPREDICTABLE whether
+         * we fire for each byte in the word/doubleword addressed by the WVR.
+         * We choose to ignore any non-zero bits after the first range of 1s.
+         */
+        basstart = ctz32(bas);
+        len = cto32(bas >> basstart);
+        wvr += basstart;
+    }
+
+    cpu_watchpoint_insert(CPU(cpu), wvr, len, flags,
+                          &env->cpu_watchpoint[n]);
+}
+
+void hw_watchpoint_update_all(ARMCPU *cpu)
+{
     int i;
+    CPUARMState *env = &cpu->env;
 
-    for (i = 0; i < 16; i++) {
+    /* Completely clear out existing QEMU watchpoints and our array, to
+     * avoid possible stale entries following migration load.
+     */
+    cpu_watchpoint_remove_all(CPU(cpu), BP_CPU);
+    memset(env->cpu_watchpoint, 0, sizeof(env->cpu_watchpoint));
+
+    for (i = 0; i < ARRAY_SIZE(cpu->env.cpu_watchpoint); i++) {
+        hw_watchpoint_update(cpu, i);
+    }
+}
+
+static void dbgwvr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                         uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    int i = ri->crm;
+
+    /* Bits [63:49] are hardwired to the value of bit [48]; that is, the
+     * register reads and behaves as if values written are sign extended.
+     * Bits [1:0] are RES0.
+     */
+    value = sextract64(value, 0, 49) & ~3ULL;
+
+    raw_write(env, ri, value);
+    hw_watchpoint_update(cpu, i);
+}
+
+static void dbgwcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                         uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    int i = ri->crm;
+
+    raw_write(env, ri, value);
+    hw_watchpoint_update(cpu, i);
+}
+
+void hw_breakpoint_update(ARMCPU *cpu, int n)
+{
+    CPUARMState *env = &cpu->env;
+    uint64_t bvr = env->cp15.dbgbvr[n];
+    uint64_t bcr = env->cp15.dbgbcr[n];
+    vaddr addr;
+    int bt;
+    int flags = BP_CPU;
+
+    if (env->cpu_breakpoint[n]) {
+        cpu_breakpoint_remove_by_ref(CPU(cpu), env->cpu_breakpoint[n]);
+        env->cpu_breakpoint[n] = NULL;
+    }
+
+    if (!extract64(bcr, 0, 1)) {
+        /* E bit clear : watchpoint disabled */
+        return;
+    }
+
+    bt = extract64(bcr, 20, 4);
+
+    switch (bt) {
+    case 4: /* unlinked address mismatch (reserved if AArch64) */
+    case 5: /* linked address mismatch (reserved if AArch64) */
+        qemu_log_mask(LOG_UNIMP,
+                      "arm: address mismatch breakpoint types not implemented");
+        return;
+    case 0: /* unlinked address match */
+    case 1: /* linked address match */
+    {
+        /* Bits [63:49] are hardwired to the value of bit [48]; that is,
+         * we behave as if the register was sign extended. Bits [1:0] are
+         * RES0. The BAS field is used to allow setting breakpoints on 16
+         * bit wide instructions; it is CONSTRAINED UNPREDICTABLE whether
+         * a bp will fire if the addresses covered by the bp and the addresses
+         * covered by the insn overlap but the insn doesn't start at the
+         * start of the bp address range. We choose to require the insn and
+         * the bp to have the same address. The constraints on writing to
+         * BAS enforced in dbgbcr_write mean we have only four cases:
+         *  0b0000  => no breakpoint
+         *  0b0011  => breakpoint on addr
+         *  0b1100  => breakpoint on addr + 2
+         *  0b1111  => breakpoint on addr
+         * See also figure D2-3 in the v8 ARM ARM (DDI0487A.c).
+         */
+        int bas = extract64(bcr, 5, 4);
+        addr = sextract64(bvr, 0, 49) & ~3ULL;
+        if (bas == 0) {
+            return;
+        }
+        if (bas == 0xc) {
+            addr += 2;
+        }
+        break;
+    }
+    case 2: /* unlinked context ID match */
+    case 8: /* unlinked VMID match (reserved if no EL2) */
+    case 10: /* unlinked context ID and VMID match (reserved if no EL2) */
+        qemu_log_mask(LOG_UNIMP,
+                      "arm: unlinked context breakpoint types not implemented");
+        return;
+    case 9: /* linked VMID match (reserved if no EL2) */
+    case 11: /* linked context ID and VMID match (reserved if no EL2) */
+    case 3: /* linked context ID match */
+    default:
+        /* We must generate no events for Linked context matches (unless
+         * they are linked to by some other bp/wp, which is handled in
+         * updates for the linking bp/wp). We choose to also generate no events
+         * for reserved values.
+         */
+        return;
+    }
+
+    cpu_breakpoint_insert(CPU(cpu), addr, flags, &env->cpu_breakpoint[n]);
+}
+
+void hw_breakpoint_update_all(ARMCPU *cpu)
+{
+    int i;
+    CPUARMState *env = &cpu->env;
+
+    /* Completely clear out existing QEMU breakpoints and our array, to
+     * avoid possible stale entries following migration load.
+     */
+    cpu_breakpoint_remove_all(CPU(cpu), BP_CPU);
+    memset(env->cpu_breakpoint, 0, sizeof(env->cpu_breakpoint));
+
+    for (i = 0; i < ARRAY_SIZE(cpu->env.cpu_breakpoint); i++) {
+        hw_breakpoint_update(cpu, i);
+    }
+}
+
+static void dbgbvr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                         uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    int i = ri->crm;
+
+    raw_write(env, ri, value);
+    hw_breakpoint_update(cpu, i);
+}
+
+static void dbgbcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                         uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    int i = ri->crm;
+
+    /* BAS[3] is a read-only copy of BAS[2], and BAS[1] a read-only
+     * copy of BAS[0].
+     */
+    value = deposit64(value, 6, 1, extract64(value, 5, 1));
+    value = deposit64(value, 8, 1, extract64(value, 7, 1));
+
+    raw_write(env, ri, value);
+    hw_breakpoint_update(cpu, i);
+}
+
+static void define_debug_regs(ARMCPU *cpu)
+{
+    /* Define v7 and v8 architectural debug registers.
+     * These are just dummy implementations for now.
+     */
+    int i;
+    int wrps, brps, ctx_cmps;
+    ARMCPRegInfo dbgdidr = {
+        .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
+        .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = cpu->dbgdidr,
+    };
+
+    /* Note that all these register fields hold "number of Xs minus 1". */
+    brps = extract32(cpu->dbgdidr, 24, 4);
+    wrps = extract32(cpu->dbgdidr, 28, 4);
+    ctx_cmps = extract32(cpu->dbgdidr, 20, 4);
+
+    assert(ctx_cmps <= brps);
+
+    /* The DBGDIDR and ID_AA64DFR0_EL1 define various properties
+     * of the debug registers such as number of breakpoints;
+     * check that if they both exist then they agree.
+     */
+    if (arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
+        assert(extract32(cpu->id_aa64dfr0, 12, 4) == brps);
+        assert(extract32(cpu->id_aa64dfr0, 20, 4) == wrps);
+        assert(extract32(cpu->id_aa64dfr0, 28, 4) == ctx_cmps);
+    }
+
+    define_one_arm_cp_reg(cpu, &dbgdidr);
+    define_arm_cp_regs(cpu, debug_cp_reginfo);
+
+    if (arm_feature(&cpu->env, ARM_FEATURE_LPAE)) {
+        define_arm_cp_regs(cpu, debug_lpae_cp_reginfo);
+    }
+
+    for (i = 0; i < brps + 1; i++) {
         ARMCPRegInfo dbgregs[] = {
-            { .name = "DBGBVR", .state = ARM_CP_STATE_AA64,
-              .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 4,
+            { .name = "DBGBVR", .state = ARM_CP_STATE_BOTH,
+              .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 4,
               .access = PL1_RW,
-              .fieldoffset = offsetof(CPUARMState, cp15.dbgbvr[i]) },
-            { .name = "DBGBCR", .state = ARM_CP_STATE_AA64,
-              .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 5,
+              .fieldoffset = offsetof(CPUARMState, cp15.dbgbvr[i]),
+              .writefn = dbgbvr_write, .raw_writefn = raw_write
+            },
+            { .name = "DBGBCR", .state = ARM_CP_STATE_BOTH,
+              .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 5,
               .access = PL1_RW,
-              .fieldoffset = offsetof(CPUARMState, cp15.dbgbcr[i]) },
-            { .name = "DBGWVR", .state = ARM_CP_STATE_AA64,
-              .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 6,
+              .fieldoffset = offsetof(CPUARMState, cp15.dbgbcr[i]),
+              .writefn = dbgbcr_write, .raw_writefn = raw_write
+            },
+            REGINFO_SENTINEL
+        };
+        define_arm_cp_regs(cpu, dbgregs);
+    }
+
+    for (i = 0; i < wrps + 1; i++) {
+        ARMCPRegInfo dbgregs[] = {
+            { .name = "DBGWVR", .state = ARM_CP_STATE_BOTH,
+              .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 6,
               .access = PL1_RW,
-              .fieldoffset = offsetof(CPUARMState, cp15.dbgwvr[i]) },
-            { .name = "DBGWCR", .state = ARM_CP_STATE_AA64,
-              .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 7,
+              .fieldoffset = offsetof(CPUARMState, cp15.dbgwvr[i]),
+              .writefn = dbgwvr_write, .raw_writefn = raw_write
+            },
+            { .name = "DBGWCR", .state = ARM_CP_STATE_BOTH,
+              .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 7,
               .access = PL1_RW,
-              .fieldoffset = offsetof(CPUARMState, cp15.dbgwcr[i]) },
-               REGINFO_SENTINEL
+              .fieldoffset = offsetof(CPUARMState, cp15.dbgwcr[i]),
+              .writefn = dbgwcr_write, .raw_writefn = raw_write
+            },
+            REGINFO_SENTINEL
         };
         define_arm_cp_regs(cpu, dbgregs);
     }
@@ -2310,6 +2838,9 @@ void register_cp_regs_for_features(ARMCPU *cpu)
     if (arm_feature(env, ARM_FEATURE_V6K)) {
         define_arm_cp_regs(cpu, v6k_cp_reginfo);
     }
+    if (arm_feature(env, ARM_FEATURE_V7MP)) {
+        define_arm_cp_regs(cpu, v7mp_cp_reginfo);
+    }
     if (arm_feature(env, ARM_FEATURE_V7)) {
         /* v7 performance monitor control register: same implementor
          * field as main ID register, and we implement only the cycle
@@ -2318,13 +2849,23 @@ void register_cp_regs_for_features(ARMCPU *cpu)
 #ifndef CONFIG_USER_ONLY
         ARMCPRegInfo pmcr = {
             .name = "PMCR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 0,
-            .access = PL0_RW, .resetvalue = cpu->midr & 0xff000000,
-            .type = ARM_CP_IO,
-            .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcr),
+            .access = PL0_RW,
+            .type = ARM_CP_IO | ARM_CP_NO_MIGRATE,
+            .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcr),
             .accessfn = pmreg_access, .writefn = pmcr_write,
             .raw_writefn = raw_write,
         };
+        ARMCPRegInfo pmcr64 = {
+            .name = "PMCR_EL0", .state = ARM_CP_STATE_AA64,
+            .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 0,
+            .access = PL0_RW, .accessfn = pmreg_access,
+            .type = ARM_CP_IO,
+            .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcr),
+            .resetvalue = cpu->midr & 0xff000000,
+            .writefn = pmcr_write, .raw_writefn = raw_write,
+        };
         define_one_arm_cp_reg(cpu, &pmcr);
+        define_one_arm_cp_reg(cpu, &pmcr64);
 #endif
         ARMCPRegInfo clidr = {
             .name = "CLIDR", .state = ARM_CP_STATE_BOTH,
@@ -2333,6 +2874,7 @@ void register_cp_regs_for_features(ARMCPU *cpu)
         };
         define_one_arm_cp_reg(cpu, &clidr);
         define_arm_cp_regs(cpu, v7_cp_reginfo);
+        define_debug_regs(cpu);
     } else {
         define_arm_cp_regs(cpu, not_v7_cp_reginfo);
     }
@@ -2406,7 +2948,6 @@ void register_cp_regs_for_features(ARMCPU *cpu)
         define_one_arm_cp_reg(cpu, &rvbar);
         define_arm_cp_regs(cpu, v8_idregs);
         define_arm_cp_regs(cpu, v8_cp_reginfo);
-        define_aarch64_debug_regs(cpu);
     }
     if (arm_feature(env, ARM_FEATURE_EL2)) {
         define_arm_cp_regs(cpu, v8_el2_cp_reginfo);
@@ -2759,9 +3300,11 @@ static void add_cpreg_to_hashtable(ARMCPU *cpu, const ARMCPRegInfo *r,
         /* The AArch32 view of a shared register sees the lower 32 bits
          * of a 64 bit backing field. It is not migratable as the AArch64
          * view handles that. AArch64 also handles reset.
-         * We assume it is a cp15 register.
+         * We assume it is a cp15 register if the .cp field is left unset.
          */
-        r2->cp = 15;
+        if (r2->cp == 0) {
+            r2->cp = 15;
+        }
         r2->type |= ARM_CP_NO_MIGRATE;
         r2->resetfn = arm_cp_reset_ignore;
 #ifdef HOST_WORDS_BIGENDIAN
@@ -2774,8 +3317,11 @@ static void add_cpreg_to_hashtable(ARMCPU *cpu, const ARMCPRegInfo *r,
         /* To allow abbreviation of ARMCPRegInfo
          * definitions, we treat cp == 0 as equivalent to
          * the value for "standard guest-visible sysreg".
+         * STATE_BOTH definitions are also always "standard
+         * sysreg" in their AArch64 view (the .cp value may
+         * be non-zero for the benefit of the AArch32 view).
          */
-        if (r->cp == 0) {
+        if (r->cp == 0 || r->state == ARM_CP_STATE_BOTH) {
             r2->cp = CP_REG_ARM64_SYSREG_CP;
         }
         *key = ENCODE_AA64_CP_REG(r2->cp, r2->crn, crm,
@@ -2985,6 +3531,8 @@ static int bad_mode_switch(CPUARMState *env, int mode)
     case ARM_CPU_MODE_IRQ:
     case ARM_CPU_MODE_FIQ:
         return 0;
+    case ARM_CPU_MODE_MON:
+        return !arm_is_secure(env);
     default:
         return 1;
     }
@@ -3098,11 +3646,6 @@ uint32_t HELPER(rbit)(uint32_t x)
 
 #if defined(CONFIG_USER_ONLY)
 
-void arm_cpu_do_interrupt(CPUState *cs)
-{
-    cs->exception_index = -1;
-}
-
 int arm_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int rw,
                              int mmu_idx)
 {
@@ -3158,6 +3701,11 @@ uint32_t HELPER(get_r13_banked)(CPUARMState *env, uint32_t mode)
     return 0;
 }
 
+unsigned int arm_excp_target_el(CPUState *cs, unsigned int excp_idx)
+{
+    return 1;
+}
+
 #else
 
 /* Map CPU modes onto saved register banks.  */
@@ -3213,6 +3761,57 @@ void switch_mode(CPUARMState *env, int mode)
     env->spsr = env->banked_spsr[i];
 }
 
+/*
+ * Determine the target EL for a given exception type.
+ */
+unsigned int arm_excp_target_el(CPUState *cs, unsigned int excp_idx)
+{
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+    unsigned int cur_el = arm_current_el(env);
+    unsigned int target_el;
+    /* FIXME: Use actual secure state.  */
+    bool secure = false;
+
+    if (!env->aarch64) {
+        /* TODO: Add EL2 and 3 exception handling for AArch32.  */
+        return 1;
+    }
+
+    switch (excp_idx) {
+    case EXCP_HVC:
+    case EXCP_HYP_TRAP:
+        target_el = 2;
+        break;
+    case EXCP_SMC:
+        target_el = 3;
+        break;
+    case EXCP_FIQ:
+    case EXCP_IRQ:
+    {
+        const uint64_t hcr_mask = excp_idx == EXCP_FIQ ? HCR_FMO : HCR_IMO;
+        const uint32_t scr_mask = excp_idx == EXCP_FIQ ? SCR_FIQ : SCR_IRQ;
+
+        target_el = 1;
+        if (!secure && (env->cp15.hcr_el2 & hcr_mask)) {
+            target_el = 2;
+        }
+        if (env->cp15.scr_el3 & scr_mask) {
+            target_el = 3;
+        }
+        break;
+    }
+    case EXCP_VIRQ:
+    case EXCP_VFIQ:
+        target_el = 1;
+        break;
+    default:
+        target_el = MAX(cur_el, 1);
+        break;
+    }
+    return target_el;
+}
+
 static void v7m_push(CPUARMState *env, uint32_t val)
 {
     CPUState *cs = CPU(arm_env_get_cpu(env));
@@ -3367,11 +3966,43 @@ void arm_cpu_do_interrupt(CPUState *cs)
     uint32_t mask;
     int new_mode;
     uint32_t offset;
+    uint32_t moe;
 
     assert(!IS_M(env));
 
     arm_log_exception(cs->exception_index);
 
+    if (arm_is_psci_call(cpu, cs->exception_index)) {
+        arm_handle_psci_call(cpu);
+        qemu_log_mask(CPU_LOG_INT, "...handled as PSCI call\n");
+        return;
+    }
+
+    /* If this is a debug exception we must update the DBGDSCR.MOE bits */
+    switch (env->exception.syndrome >> ARM_EL_EC_SHIFT) {
+    case EC_BREAKPOINT:
+    case EC_BREAKPOINT_SAME_EL:
+        moe = 1;
+        break;
+    case EC_WATCHPOINT:
+    case EC_WATCHPOINT_SAME_EL:
+        moe = 10;
+        break;
+    case EC_AA32_BKPT:
+        moe = 3;
+        break;
+    case EC_VECTORCATCH:
+        moe = 5;
+        break;
+    default:
+        moe = 0;
+        break;
+    }
+
+    if (moe) {
+        env->cp15.mdscr_el1 = deposit64(env->cp15.mdscr_el1, 2, 4, moe);
+    }
+
     /* TODO: Vectored interrupt controller.  */
     switch (cs->exception_index) {
     case EXCP_UDEF:
@@ -3425,8 +4056,8 @@ void arm_cpu_do_interrupt(CPUState *cs)
         /* Fall through to prefetch abort.  */
     case EXCP_PREFETCH_ABORT:
         env->cp15.ifsr_el2 = env->exception.fsr;
-        env->cp15.far_el1 = deposit64(env->cp15.far_el1, 32, 32,
-                                      env->exception.vaddress);
+        env->cp15.far_el[1] = deposit64(env->cp15.far_el[1], 32, 32,
+                                        env->exception.vaddress);
         qemu_log_mask(CPU_LOG_INT, "...with IFSR 0x%x IFAR 0x%x\n",
                       env->cp15.ifsr_el2, (uint32_t)env->exception.vaddress);
         new_mode = ARM_CPU_MODE_ABT;
@@ -3436,8 +4067,8 @@ void arm_cpu_do_interrupt(CPUState *cs)
         break;
     case EXCP_DATA_ABORT:
         env->cp15.esr_el[1] = env->exception.fsr;
-        env->cp15.far_el1 = deposit64(env->cp15.far_el1, 0, 32,
-                                      env->exception.vaddress);
+        env->cp15.far_el[1] = deposit64(env->cp15.far_el[1], 0, 32,
+                                        env->exception.vaddress);
         qemu_log_mask(CPU_LOG_INT, "...with DFSR 0x%x DFAR 0x%x\n",
                       (uint32_t)env->cp15.esr_el[1],
                       (uint32_t)env->exception.vaddress);
@@ -3460,6 +4091,12 @@ void arm_cpu_do_interrupt(CPUState *cs)
         mask = CPSR_A | CPSR_I | CPSR_F;
         offset = 4;
         break;
+    case EXCP_SMC:
+        new_mode = ARM_CPU_MODE_MON;
+        addr = 0x08;
+        mask = CPSR_A | CPSR_I | CPSR_F;
+        offset = 0;
+        break;
     default:
         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
         return; /* Never happens.  Keep compiler happy.  */
@@ -3478,7 +4115,16 @@ void arm_cpu_do_interrupt(CPUState *cs)
          */
         addr += env->cp15.vbar_el[1];
     }
+
+    if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_MON) {
+        env->cp15.scr_el3 &= ~SCR_NS;
+    }
+
     switch_mode (env, new_mode);
+    /* For exceptions taken to AArch32 we must clear the SS bit in both
+     * PSTATE and in the old-state value we save to SPSR_<mode>, so zero it now.
+     */
+    env->uncached_cpsr &= ~PSTATE_SS;
     env->spsr = cpsr_read(env);
     /* Clear IT bits.  */
     env->condexec_bits = 0;
@@ -3899,16 +4545,18 @@ static int get_phys_addr_lpae(CPUARMState *env, target_ulong address,
         goto do_fault;
     }
 
-    /* The starting level depends on the virtual address size which can be
-     * up to 48-bits and the translation granule size.
+    /* The starting level depends on the virtual address size (which can be
+     * up to 48 bits) and the translation granule size. It indicates the number
+     * of strides (granule_sz bits at a time) needed to consume the bits
+     * of the input address. In the pseudocode this is:
+     *  level = 4 - RoundUp((inputsize - grainsize) / stride)
+     * where their 'inputsize' is our 'va_size - tsz', 'grainsize' is
+     * our 'granule_sz + 3' and 'stride' is our 'granule_sz'.
+     * Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying:
+     *     = 4 - (va_size - tsz - granule_sz - 3 + granule_sz - 1) / granule_sz
+     *     = 4 - (va_size - tsz - 4) / granule_sz;
      */
-    if ((va_size - tsz) > (granule_sz * 4 + 3)) {
-        level = 0;
-    } else if ((va_size - tsz) > (granule_sz * 3 + 3)) {
-        level = 1;
-    } else {
-        level = 2;
-    }
+    level = 4 - (va_size - tsz - 4) / granule_sz;
 
     /* Clear the vaddr bits which aren't part of the within-region address,
      * so that we don't have to special case things when calculating the
@@ -4135,15 +4783,15 @@ int arm_cpu_handle_mmu_fault(CPUState *cs, vaddr address,
     int prot;
     int ret, is_user;
     uint32_t syn;
-    bool same_el = (arm_current_pl(env) != 0);
+    bool same_el = (arm_current_el(env) != 0);
 
     is_user = mmu_idx == MMU_USER_IDX;
     ret = get_phys_addr(env, address, access_type, is_user, &phys_addr, &prot,
                         &page_size);
     if (ret == 0) {
         /* Map a single [sub]page.  */
-        phys_addr &= ~(hwaddr)0x3ff;
-        address &= ~(target_ulong)0x3ff;
+        phys_addr &= TARGET_PAGE_MASK;
+        address &= TARGET_PAGE_MASK;
         tlb_set_page(cs, address, phys_addr, prot, mmu_idx, page_size);
         return 0;
     }
diff --git a/target-arm/helper.h b/target-arm/helper.h
index facfcd2c1..dec372879 100644
--- a/target-arm/helper.h
+++ b/target-arm/helper.h
@@ -50,6 +50,8 @@ DEF_HELPER_2(exception_internal, void, env, i32)
 DEF_HELPER_3(exception_with_syndrome, void, env, i32, i32)
 DEF_HELPER_1(wfi, void, env)
 DEF_HELPER_1(wfe, void, env)
+DEF_HELPER_1(pre_hvc, void, env)
+DEF_HELPER_2(pre_smc, void, env, i32)
 
 DEF_HELPER_3(cpsr_write, void, env, i32, i32)
 DEF_HELPER_1(cpsr_read, i32, env)
@@ -64,6 +66,7 @@ DEF_HELPER_3(set_cp_reg64, void, env, ptr, i64)
 DEF_HELPER_2(get_cp_reg64, i64, env, ptr)
 
 DEF_HELPER_3(msr_i_pstate, void, env, i32, i32)
+DEF_HELPER_1(clear_pstate_ss, void, env)
 DEF_HELPER_1(exception_return, void, env)
 
 DEF_HELPER_2(get_r13_banked, i32, env, i32)
diff --git a/target-arm/internals.h b/target-arm/internals.h
index 564b5fa60..2dff4ffb1 100644
--- a/target-arm/internals.h
+++ b/target-arm/internals.h
@@ -53,6 +53,11 @@ static const char * const excnames[] = {
     [EXCP_EXCEPTION_EXIT] = "QEMU v7M exception exit",
     [EXCP_KERNEL_TRAP] = "QEMU intercept of kernel commpage",
     [EXCP_STREX] = "QEMU intercept of STREX",
+    [EXCP_HVC] = "Hypervisor Call",
+    [EXCP_HYP_TRAP] = "Hypervisor Trap",
+    [EXCP_SMC] = "Secure Monitor Call",
+    [EXCP_VIRQ] = "Virtual IRQ",
+    [EXCP_VFIQ] = "Virtual FIQ",
 };
 
 static inline void arm_log_exception(int idx)
@@ -105,30 +110,52 @@ enum arm_fprounding {
 
 int arm_rmode_to_sf(int rmode);
 
+static inline void aarch64_save_sp(CPUARMState *env, int el)
+{
+    if (env->pstate & PSTATE_SP) {
+        env->sp_el[el] = env->xregs[31];
+    } else {
+        env->sp_el[0] = env->xregs[31];
+    }
+}
+
+static inline void aarch64_restore_sp(CPUARMState *env, int el)
+{
+    if (env->pstate & PSTATE_SP) {
+        env->xregs[31] = env->sp_el[el];
+    } else {
+        env->xregs[31] = env->sp_el[0];
+    }
+}
+
 static inline void update_spsel(CPUARMState *env, uint32_t imm)
 {
-    unsigned int cur_el = arm_current_pl(env);
+    unsigned int cur_el = arm_current_el(env);
     /* Update PSTATE SPSel bit; this requires us to update the
      * working stack pointer in xregs[31].
      */
     if (!((imm ^ env->pstate) & PSTATE_SP)) {
         return;
     }
+    aarch64_save_sp(env, cur_el);
     env->pstate = deposit32(env->pstate, 0, 1, imm);
 
     /* We rely on illegal updates to SPsel from EL0 to get trapped
      * at translation time.
      */
     assert(cur_el >= 1 && cur_el <= 3);
-    if (env->pstate & PSTATE_SP) {
-        /* Switch from using SP_EL0 to using SP_ELx */
-        env->sp_el[0] = env->xregs[31];
-        env->xregs[31] = env->sp_el[cur_el];
-    } else {
-        /* Switch from SP_EL0 to SP_ELx */
-        env->sp_el[cur_el] = env->xregs[31];
-        env->xregs[31] = env->sp_el[0];
-    }
+    aarch64_restore_sp(env, cur_el);
+}
+
+/* Return true if extended addresses are enabled.
+ * This is always the case if our translation regime is 64 bit,
+ * but depends on TTBCR.EAE for 32 bit.
+ */
+static inline bool extended_addresses_enabled(CPUARMState *env)
+{
+    return arm_el_is_aa64(env, 1)
+        || ((arm_feature(env, ARM_FEATURE_LPAE)
+             && (env->cp15.c2_control & TTBCR_EAE)));
 }
 
 /* Valid Syndrome Register EC field values */
@@ -193,12 +220,32 @@ static inline uint32_t syn_aa64_svc(uint32_t imm16)
     return (EC_AA64_SVC << ARM_EL_EC_SHIFT) | ARM_EL_IL | (imm16 & 0xffff);
 }
 
+static inline uint32_t syn_aa64_hvc(uint32_t imm16)
+{
+    return (EC_AA64_HVC << ARM_EL_EC_SHIFT) | ARM_EL_IL | (imm16 & 0xffff);
+}
+
+static inline uint32_t syn_aa64_smc(uint32_t imm16)
+{
+    return (EC_AA64_SMC << ARM_EL_EC_SHIFT) | ARM_EL_IL | (imm16 & 0xffff);
+}
+
 static inline uint32_t syn_aa32_svc(uint32_t imm16, bool is_thumb)
 {
     return (EC_AA32_SVC << ARM_EL_EC_SHIFT) | (imm16 & 0xffff)
         | (is_thumb ? 0 : ARM_EL_IL);
 }
 
+static inline uint32_t syn_aa32_hvc(uint32_t imm16)
+{
+    return (EC_AA32_HVC << ARM_EL_EC_SHIFT) | ARM_EL_IL | (imm16 & 0xffff);
+}
+
+static inline uint32_t syn_aa32_smc(void)
+{
+    return (EC_AA32_SMC << ARM_EL_EC_SHIFT) | ARM_EL_IL;
+}
+
 static inline uint32_t syn_aa64_bkpt(uint32_t imm16)
 {
     return (EC_AA64_BKPT << ARM_EL_EC_SHIFT) | ARM_EL_IL | (imm16 & 0xffff);
@@ -279,4 +326,56 @@ static inline uint32_t syn_data_abort(int same_el, int ea, int cm, int s1ptw,
         | (ea << 9) | (cm << 8) | (s1ptw << 7) | (wnr << 6) | fsc;
 }
 
+static inline uint32_t syn_swstep(int same_el, int isv, int ex)
+{
+    return (EC_SOFTWARESTEP << ARM_EL_EC_SHIFT) | (same_el << ARM_EL_EC_SHIFT)
+        | (isv << 24) | (ex << 6) | 0x22;
+}
+
+static inline uint32_t syn_watchpoint(int same_el, int cm, int wnr)
+{
+    return (EC_WATCHPOINT << ARM_EL_EC_SHIFT) | (same_el << ARM_EL_EC_SHIFT)
+        | (cm << 8) | (wnr << 6) | 0x22;
+}
+
+static inline uint32_t syn_breakpoint(int same_el)
+{
+    return (EC_BREAKPOINT << ARM_EL_EC_SHIFT) | (same_el << ARM_EL_EC_SHIFT)
+        | ARM_EL_IL | 0x22;
+}
+
+/* Update a QEMU watchpoint based on the information the guest has set in the
+ * DBGWCR<n>_EL1 and DBGWVR<n>_EL1 registers.
+ */
+void hw_watchpoint_update(ARMCPU *cpu, int n);
+/* Update the QEMU watchpoints for every guest watchpoint. This does a
+ * complete delete-and-reinstate of the QEMU watchpoint list and so is
+ * suitable for use after migration or on reset.
+ */
+void hw_watchpoint_update_all(ARMCPU *cpu);
+/* Update a QEMU breakpoint based on the information the guest has set in the
+ * DBGBCR<n>_EL1 and DBGBVR<n>_EL1 registers.
+ */
+void hw_breakpoint_update(ARMCPU *cpu, int n);
+/* Update the QEMU breakpoints for every guest breakpoint. This does a
+ * complete delete-and-reinstate of the QEMU breakpoint list and so is
+ * suitable for use after migration or on reset.
+ */
+void hw_breakpoint_update_all(ARMCPU *cpu);
+
+/* Callback function for when a watchpoint or breakpoint triggers. */
+void arm_debug_excp_handler(CPUState *cs);
+
+#ifdef CONFIG_USER_ONLY
+static inline bool arm_is_psci_call(ARMCPU *cpu, int excp_type)
+{
+    return false;
+}
+#else
+/* Return true if the r0/x0 value indicates that this SMC/HVC is a PSCI call. */
+bool arm_is_psci_call(ARMCPU *cpu, int excp_type);
+/* Actually handle a PSCI call */
+void arm_handle_psci_call(ARMCPU *cpu);
+#endif
+
 #endif
diff --git a/target-arm/kvm-consts.h b/target-arm/kvm-consts.h
index 6009a33f1..aea12f1bc 100644
--- a/target-arm/kvm-consts.h
+++ b/target-arm/kvm-consts.h
@@ -17,6 +17,7 @@
 #ifdef CONFIG_KVM
 #include "qemu/compiler.h"
 #include <linux/kvm.h>
+#include <linux/psci.h>
 
 #define MISMATCH_CHECK(X, Y) QEMU_BUILD_BUG_ON(X != Y)
 
@@ -38,17 +39,83 @@ MISMATCH_CHECK(CP_REG_SIZE_U64, KVM_REG_SIZE_U64)
 MISMATCH_CHECK(CP_REG_ARM, KVM_REG_ARM)
 MISMATCH_CHECK(CP_REG_ARCH_MASK, KVM_REG_ARCH_MASK)
 
-#define PSCI_FN_BASE 0x95c1ba5e
-#define PSCI_FN(n) (PSCI_FN_BASE + (n))
-#define PSCI_FN_CPU_SUSPEND PSCI_FN(0)
-#define PSCI_FN_CPU_OFF PSCI_FN(1)
-#define PSCI_FN_CPU_ON PSCI_FN(2)
-#define PSCI_FN_MIGRATE PSCI_FN(3)
-
-MISMATCH_CHECK(PSCI_FN_CPU_SUSPEND, KVM_PSCI_FN_CPU_SUSPEND)
-MISMATCH_CHECK(PSCI_FN_CPU_OFF, KVM_PSCI_FN_CPU_OFF)
-MISMATCH_CHECK(PSCI_FN_CPU_ON, KVM_PSCI_FN_CPU_ON)
-MISMATCH_CHECK(PSCI_FN_MIGRATE, KVM_PSCI_FN_MIGRATE)
+#define QEMU_PSCI_0_1_FN_BASE 0x95c1ba5e
+#define QEMU_PSCI_0_1_FN(n) (QEMU_PSCI_0_1_FN_BASE + (n))
+#define QEMU_PSCI_0_1_FN_CPU_SUSPEND QEMU_PSCI_0_1_FN(0)
+#define QEMU_PSCI_0_1_FN_CPU_OFF QEMU_PSCI_0_1_FN(1)
+#define QEMU_PSCI_0_1_FN_CPU_ON QEMU_PSCI_0_1_FN(2)
+#define QEMU_PSCI_0_1_FN_MIGRATE QEMU_PSCI_0_1_FN(3)
+
+MISMATCH_CHECK(QEMU_PSCI_0_1_FN_CPU_SUSPEND, KVM_PSCI_FN_CPU_SUSPEND)
+MISMATCH_CHECK(QEMU_PSCI_0_1_FN_CPU_OFF, KVM_PSCI_FN_CPU_OFF)
+MISMATCH_CHECK(QEMU_PSCI_0_1_FN_CPU_ON, KVM_PSCI_FN_CPU_ON)
+MISMATCH_CHECK(QEMU_PSCI_0_1_FN_MIGRATE, KVM_PSCI_FN_MIGRATE)
+
+#define QEMU_PSCI_0_2_FN_BASE 0x84000000
+#define QEMU_PSCI_0_2_FN(n) (QEMU_PSCI_0_2_FN_BASE + (n))
+
+#define QEMU_PSCI_0_2_64BIT 0x40000000
+#define QEMU_PSCI_0_2_FN64_BASE \
+        (QEMU_PSCI_0_2_FN_BASE + QEMU_PSCI_0_2_64BIT)
+#define QEMU_PSCI_0_2_FN64(n) (QEMU_PSCI_0_2_FN64_BASE + (n))
+
+#define QEMU_PSCI_0_2_FN_PSCI_VERSION QEMU_PSCI_0_2_FN(0)
+#define QEMU_PSCI_0_2_FN_CPU_SUSPEND QEMU_PSCI_0_2_FN(1)
+#define QEMU_PSCI_0_2_FN_CPU_OFF QEMU_PSCI_0_2_FN(2)
+#define QEMU_PSCI_0_2_FN_CPU_ON QEMU_PSCI_0_2_FN(3)
+#define QEMU_PSCI_0_2_FN_AFFINITY_INFO QEMU_PSCI_0_2_FN(4)
+#define QEMU_PSCI_0_2_FN_MIGRATE QEMU_PSCI_0_2_FN(5)
+#define QEMU_PSCI_0_2_FN_MIGRATE_INFO_TYPE QEMU_PSCI_0_2_FN(6)
+#define QEMU_PSCI_0_2_FN_MIGRATE_INFO_UP_CPU QEMU_PSCI_0_2_FN(7)
+#define QEMU_PSCI_0_2_FN_SYSTEM_OFF QEMU_PSCI_0_2_FN(8)
+#define QEMU_PSCI_0_2_FN_SYSTEM_RESET QEMU_PSCI_0_2_FN(9)
+
+#define QEMU_PSCI_0_2_FN64_CPU_SUSPEND QEMU_PSCI_0_2_FN64(1)
+#define QEMU_PSCI_0_2_FN64_CPU_OFF QEMU_PSCI_0_2_FN64(2)
+#define QEMU_PSCI_0_2_FN64_CPU_ON QEMU_PSCI_0_2_FN64(3)
+#define QEMU_PSCI_0_2_FN64_AFFINITY_INFO QEMU_PSCI_0_2_FN64(4)
+#define QEMU_PSCI_0_2_FN64_MIGRATE QEMU_PSCI_0_2_FN64(5)
+
+MISMATCH_CHECK(QEMU_PSCI_0_2_FN_CPU_SUSPEND, PSCI_0_2_FN_CPU_SUSPEND)
+MISMATCH_CHECK(QEMU_PSCI_0_2_FN_CPU_OFF, PSCI_0_2_FN_CPU_OFF)
+MISMATCH_CHECK(QEMU_PSCI_0_2_FN_CPU_ON, PSCI_0_2_FN_CPU_ON)
+MISMATCH_CHECK(QEMU_PSCI_0_2_FN_MIGRATE, PSCI_0_2_FN_MIGRATE)
+MISMATCH_CHECK(QEMU_PSCI_0_2_FN64_CPU_SUSPEND, PSCI_0_2_FN64_CPU_SUSPEND)
+MISMATCH_CHECK(QEMU_PSCI_0_2_FN64_CPU_ON, PSCI_0_2_FN64_CPU_ON)
+MISMATCH_CHECK(QEMU_PSCI_0_2_FN64_MIGRATE, PSCI_0_2_FN64_MIGRATE)
+
+/* PSCI v0.2 return values used by TCG emulation of PSCI */
+
+/* No Trusted OS migration to worry about when offlining CPUs */
+#define QEMU_PSCI_0_2_RET_TOS_MIGRATION_NOT_REQUIRED        2
+
+/* We implement version 0.2 only */
+#define QEMU_PSCI_0_2_RET_VERSION_0_2                       2
+
+MISMATCH_CHECK(QEMU_PSCI_0_2_RET_TOS_MIGRATION_NOT_REQUIRED, PSCI_0_2_TOS_MP)
+MISMATCH_CHECK(QEMU_PSCI_0_2_RET_VERSION_0_2,
+               (PSCI_VERSION_MAJOR(0) | PSCI_VERSION_MINOR(2)))
+
+/* PSCI return values (inclusive of all PSCI versions) */
+#define QEMU_PSCI_RET_SUCCESS                     0
+#define QEMU_PSCI_RET_NOT_SUPPORTED               -1
+#define QEMU_PSCI_RET_INVALID_PARAMS              -2
+#define QEMU_PSCI_RET_DENIED                      -3
+#define QEMU_PSCI_RET_ALREADY_ON                  -4
+#define QEMU_PSCI_RET_ON_PENDING                  -5
+#define QEMU_PSCI_RET_INTERNAL_FAILURE            -6
+#define QEMU_PSCI_RET_NOT_PRESENT                 -7
+#define QEMU_PSCI_RET_DISABLED                    -8
+
+MISMATCH_CHECK(QEMU_PSCI_RET_SUCCESS, PSCI_RET_SUCCESS)
+MISMATCH_CHECK(QEMU_PSCI_RET_NOT_SUPPORTED, PSCI_RET_NOT_SUPPORTED)
+MISMATCH_CHECK(QEMU_PSCI_RET_INVALID_PARAMS, PSCI_RET_INVALID_PARAMS)
+MISMATCH_CHECK(QEMU_PSCI_RET_DENIED, PSCI_RET_DENIED)
+MISMATCH_CHECK(QEMU_PSCI_RET_ALREADY_ON, PSCI_RET_ALREADY_ON)
+MISMATCH_CHECK(QEMU_PSCI_RET_ON_PENDING, PSCI_RET_ON_PENDING)
+MISMATCH_CHECK(QEMU_PSCI_RET_INTERNAL_FAILURE, PSCI_RET_INTERNAL_FAILURE)
+MISMATCH_CHECK(QEMU_PSCI_RET_NOT_PRESENT, PSCI_RET_NOT_PRESENT)
+MISMATCH_CHECK(QEMU_PSCI_RET_DISABLED, PSCI_RET_DISABLED)
 
 /* Note that KVM uses overlapping values for AArch32 and AArch64
  * target CPU numbers. AArch32 targets:
diff --git a/target-arm/kvm64.c b/target-arm/kvm64.c
index 5d217ca2a..c61528615 100644
--- a/target-arm/kvm64.c
+++ b/target-arm/kvm64.c
@@ -21,6 +21,7 @@
 #include "sysemu/kvm.h"
 #include "kvm_arm.h"
 #include "cpu.h"
+#include "internals.h"
 #include "hw/arm/arm.h"
 
 static inline void set_feature(uint64_t *features, int feature)
@@ -132,11 +133,7 @@ int kvm_arch_put_registers(CPUState *cs, int level)
     /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the
      * QEMU side we keep the current SP in xregs[31] as well.
      */
-    if (env->pstate & PSTATE_SP) {
-        env->sp_el[1] = env->xregs[31];
-    } else {
-        env->sp_el[0] = env->xregs[31];
-    }
+    aarch64_save_sp(env, 1);
 
     reg.id = AARCH64_CORE_REG(regs.sp);
     reg.addr = (uintptr_t) &env->sp_el[0];
@@ -235,11 +232,7 @@ int kvm_arch_get_registers(CPUState *cs)
     /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the
      * QEMU side we keep the current SP in xregs[31] as well.
      */
-    if (env->pstate & PSTATE_SP) {
-        env->xregs[31] = env->sp_el[1];
-    } else {
-        env->xregs[31] = env->sp_el[0];
-    }
+    aarch64_restore_sp(env, 1);
 
     reg.id = AARCH64_CORE_REG(regs.pc);
     reg.addr = (uintptr_t) &env->pc;
diff --git a/target-arm/machine.c b/target-arm/machine.c
index 3bcc7cc83..6437690af 100644
--- a/target-arm/machine.c
+++ b/target-arm/machine.c
@@ -2,6 +2,7 @@
 #include "hw/boards.h"
 #include "sysemu/kvm.h"
 #include "kvm_arm.h"
+#include "internals.h"
 
 static bool vfp_needed(void *opaque)
 {
@@ -213,13 +214,16 @@ static int cpu_post_load(void *opaque, int version_id)
         }
     }
 
+    hw_breakpoint_update_all(cpu);
+    hw_watchpoint_update_all(cpu);
+
     return 0;
 }
 
 const VMStateDescription vmstate_arm_cpu = {
     .name = "cpu",
-    .version_id = 20,
-    .minimum_version_id = 20,
+    .version_id = 21,
+    .minimum_version_id = 21,
     .pre_save = cpu_pre_save,
     .post_load = cpu_post_load,
     .fields = (VMStateField[]) {
@@ -234,8 +238,8 @@ const VMStateDescription vmstate_arm_cpu = {
         },
         VMSTATE_UINT32(env.spsr, ARMCPU),
         VMSTATE_UINT64_ARRAY(env.banked_spsr, ARMCPU, 8),
-        VMSTATE_UINT32_ARRAY(env.banked_r13, ARMCPU, 6),
-        VMSTATE_UINT32_ARRAY(env.banked_r14, ARMCPU, 6),
+        VMSTATE_UINT32_ARRAY(env.banked_r13, ARMCPU, 8),
+        VMSTATE_UINT32_ARRAY(env.banked_r14, ARMCPU, 8),
         VMSTATE_UINT32_ARRAY(env.usr_regs, ARMCPU, 5),
         VMSTATE_UINT32_ARRAY(env.fiq_regs, ARMCPU, 5),
         VMSTATE_UINT64_ARRAY(env.elr_el, ARMCPU, 4),
@@ -259,6 +263,7 @@ const VMStateDescription vmstate_arm_cpu = {
         VMSTATE_UINT64(env.exception.vaddress, ARMCPU),
         VMSTATE_TIMER(gt_timer[GTIMER_PHYS], ARMCPU),
         VMSTATE_TIMER(gt_timer[GTIMER_VIRT], ARMCPU),
+        VMSTATE_BOOL(powered_off, ARMCPU),
         VMSTATE_END_OF_LIST()
     },
     .subsections = (VMStateSubsection[]) {
diff --git a/target-arm/op_helper.c b/target-arm/op_helper.c
index 9c1ef525a..62012c3a6 100644
--- a/target-arm/op_helper.c
+++ b/target-arm/op_helper.c
@@ -258,7 +258,7 @@ void HELPER(exception_with_syndrome)(CPUARMState *env, uint32_t excp,
 
 uint32_t HELPER(cpsr_read)(CPUARMState *env)
 {
-    return cpsr_read(env) & ~CPSR_EXEC;
+    return cpsr_read(env) & ~(CPSR_EXEC | CPSR_RESERVED);
 }
 
 void HELPER(cpsr_write)(CPUARMState *env, uint32_t val, uint32_t mask)
@@ -301,6 +301,17 @@ void HELPER(set_user_reg)(CPUARMState *env, uint32_t regno, uint32_t val)
 void HELPER(access_check_cp_reg)(CPUARMState *env, void *rip, uint32_t syndrome)
 {
     const ARMCPRegInfo *ri = rip;
+
+    if (arm_feature(env, ARM_FEATURE_XSCALE) && ri->cp < 14
+        && extract32(env->cp15.c15_cpar, ri->cp, 1) == 0) {
+        env->exception.syndrome = syndrome;
+        raise_exception(env, EXCP_UDEF);
+    }
+
+    if (!ri->accessfn) {
+        return;
+    }
+
     switch (ri->accessfn(env, ri)) {
     case CP_ACCESS_OK:
         return;
@@ -350,7 +361,7 @@ void HELPER(msr_i_pstate)(CPUARMState *env, uint32_t op, uint32_t imm)
      * Note that SPSel is never OK from EL0; we rely on handle_msr_i()
      * to catch that case at translate time.
      */
-    if (arm_current_pl(env) == 0 && !(env->cp15.c1_sys & SCTLR_UMA)) {
+    if (arm_current_el(env) == 0 && !(env->cp15.c1_sys & SCTLR_UMA)) {
         raise_exception(env, EXCP_UDEF);
     }
 
@@ -369,27 +380,117 @@ void HELPER(msr_i_pstate)(CPUARMState *env, uint32_t op, uint32_t imm)
     }
 }
 
+void HELPER(clear_pstate_ss)(CPUARMState *env)
+{
+    env->pstate &= ~PSTATE_SS;
+}
+
+void HELPER(pre_hvc)(CPUARMState *env)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    int cur_el = arm_current_el(env);
+    /* FIXME: Use actual secure state.  */
+    bool secure = false;
+    bool undef;
+
+    if (arm_is_psci_call(cpu, EXCP_HVC)) {
+        /* If PSCI is enabled and this looks like a valid PSCI call then
+         * that overrides the architecturally mandated HVC behaviour.
+         */
+        return;
+    }
+
+    if (!arm_feature(env, ARM_FEATURE_EL2)) {
+        /* If EL2 doesn't exist, HVC always UNDEFs */
+        undef = true;
+    } else if (arm_feature(env, ARM_FEATURE_EL3)) {
+        /* EL3.HCE has priority over EL2.HCD. */
+        undef = !(env->cp15.scr_el3 & SCR_HCE);
+    } else {
+        undef = env->cp15.hcr_el2 & HCR_HCD;
+    }
+
+    /* In ARMv7 and ARMv8/AArch32, HVC is undef in secure state.
+     * For ARMv8/AArch64, HVC is allowed in EL3.
+     * Note that we've already trapped HVC from EL0 at translation
+     * time.
+     */
+    if (secure && (!is_a64(env) || cur_el == 1)) {
+        undef = true;
+    }
+
+    if (undef) {
+        env->exception.syndrome = syn_uncategorized();
+        raise_exception(env, EXCP_UDEF);
+    }
+}
+
+void HELPER(pre_smc)(CPUARMState *env, uint32_t syndrome)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    int cur_el = arm_current_el(env);
+    bool secure = arm_is_secure(env);
+    bool smd = env->cp15.scr_el3 & SCR_SMD;
+    /* On ARMv8 AArch32, SMD only applies to NS state.
+     * On ARMv7 SMD only applies to NS state and only if EL2 is available.
+     * For ARMv7 non EL2, we force SMD to zero so we don't need to re-check
+     * the EL2 condition here.
+     */
+    bool undef = is_a64(env) ? smd : (!secure && smd);
+
+    if (arm_is_psci_call(cpu, EXCP_SMC)) {
+        /* If PSCI is enabled and this looks like a valid PSCI call then
+         * that overrides the architecturally mandated SMC behaviour.
+         */
+        return;
+    }
+
+    if (!arm_feature(env, ARM_FEATURE_EL3)) {
+        /* If we have no EL3 then SMC always UNDEFs */
+        undef = true;
+    } else if (!secure && cur_el == 1 && (env->cp15.hcr_el2 & HCR_TSC)) {
+        /* In NS EL1, HCR controlled routing to EL2 has priority over SMD. */
+        env->exception.syndrome = syndrome;
+        raise_exception(env, EXCP_HYP_TRAP);
+    }
+
+    if (undef) {
+        env->exception.syndrome = syn_uncategorized();
+        raise_exception(env, EXCP_UDEF);
+    }
+}
+
 void HELPER(exception_return)(CPUARMState *env)
 {
-    int cur_el = arm_current_pl(env);
+    int cur_el = arm_current_el(env);
     unsigned int spsr_idx = aarch64_banked_spsr_index(cur_el);
     uint32_t spsr = env->banked_spsr[spsr_idx];
     int new_el, i;
 
-    if (env->pstate & PSTATE_SP) {
-        env->sp_el[cur_el] = env->xregs[31];
-    } else {
-        env->sp_el[0] = env->xregs[31];
-    }
+    aarch64_save_sp(env, cur_el);
 
     env->exclusive_addr = -1;
 
+    /* We must squash the PSTATE.SS bit to zero unless both of the
+     * following hold:
+     *  1. debug exceptions are currently disabled
+     *  2. singlestep will be active in the EL we return to
+     * We check 1 here and 2 after we've done the pstate/cpsr write() to
+     * transition to the EL we're going to.
+     */
+    if (arm_generate_debug_exceptions(env)) {
+        spsr &= ~PSTATE_SS;
+    }
+
     if (spsr & PSTATE_nRW) {
         /* TODO: We currently assume EL1/2/3 are running in AArch64.  */
         env->aarch64 = 0;
         new_el = 0;
         env->uncached_cpsr = 0x10;
         cpsr_write(env, spsr, ~0);
+        if (!arm_singlestep_active(env)) {
+            env->uncached_cpsr &= ~PSTATE_SS;
+        }
         for (i = 0; i < 15; i++) {
             env->regs[i] = env->xregs[i];
         }
@@ -414,7 +515,10 @@ void HELPER(exception_return)(CPUARMState *env)
         }
         env->aarch64 = 1;
         pstate_write(env, spsr);
-        env->xregs[31] = env->sp_el[new_el];
+        if (!arm_singlestep_active(env)) {
+            env->pstate &= ~PSTATE_SS;
+        }
+        aarch64_restore_sp(env, new_el);
         env->pc = env->elr_el[cur_el];
     }
 
@@ -433,6 +537,242 @@ illegal_return:
     spsr &= PSTATE_NZCV | PSTATE_DAIF;
     spsr |= pstate_read(env) & ~(PSTATE_NZCV | PSTATE_DAIF);
     pstate_write(env, spsr);
+    if (!arm_singlestep_active(env)) {
+        env->pstate &= ~PSTATE_SS;
+    }
+}
+
+/* Return true if the linked breakpoint entry lbn passes its checks */
+static bool linked_bp_matches(ARMCPU *cpu, int lbn)
+{
+    CPUARMState *env = &cpu->env;
+    uint64_t bcr = env->cp15.dbgbcr[lbn];
+    int brps = extract32(cpu->dbgdidr, 24, 4);
+    int ctx_cmps = extract32(cpu->dbgdidr, 20, 4);
+    int bt;
+    uint32_t contextidr;
+
+    /* Links to unimplemented or non-context aware breakpoints are
+     * CONSTRAINED UNPREDICTABLE: either behave as if disabled, or
+     * as if linked to an UNKNOWN context-aware breakpoint (in which
+     * case DBGWCR<n>_EL1.LBN must indicate that breakpoint).
+     * We choose the former.
+     */
+    if (lbn > brps || lbn < (brps - ctx_cmps)) {
+        return false;
+    }
+
+    bcr = env->cp15.dbgbcr[lbn];
+
+    if (extract64(bcr, 0, 1) == 0) {
+        /* Linked breakpoint disabled : generate no events */
+        return false;
+    }
+
+    bt = extract64(bcr, 20, 4);
+
+    /* We match the whole register even if this is AArch32 using the
+     * short descriptor format (in which case it holds both PROCID and ASID),
+     * since we don't implement the optional v7 context ID masking.
+     */
+    contextidr = extract64(env->cp15.contextidr_el1, 0, 32);
+
+    switch (bt) {
+    case 3: /* linked context ID match */
+        if (arm_current_el(env) > 1) {
+            /* Context matches never fire in EL2 or (AArch64) EL3 */
+            return false;
+        }
+        return (contextidr == extract64(env->cp15.dbgbvr[lbn], 0, 32));
+    case 5: /* linked address mismatch (reserved in AArch64) */
+    case 9: /* linked VMID match (reserved if no EL2) */
+    case 11: /* linked context ID and VMID match (reserved if no EL2) */
+    default:
+        /* Links to Unlinked context breakpoints must generate no
+         * events; we choose to do the same for reserved values too.
+         */
+        return false;
+    }
+
+    return false;
+}
+
+static bool bp_wp_matches(ARMCPU *cpu, int n, bool is_wp)
+{
+    CPUARMState *env = &cpu->env;
+    uint64_t cr;
+    int pac, hmc, ssc, wt, lbn;
+    /* TODO: check against CPU security state when we implement TrustZone */
+    bool is_secure = false;
+
+    if (is_wp) {
+        if (!env->cpu_watchpoint[n]
+            || !(env->cpu_watchpoint[n]->flags & BP_WATCHPOINT_HIT)) {
+            return false;
+        }
+        cr = env->cp15.dbgwcr[n];
+    } else {
+        uint64_t pc = is_a64(env) ? env->pc : env->regs[15];
+
+        if (!env->cpu_breakpoint[n] || env->cpu_breakpoint[n]->pc != pc) {
+            return false;
+        }
+        cr = env->cp15.dbgbcr[n];
+    }
+    /* The WATCHPOINT_HIT flag guarantees us that the watchpoint is
+     * enabled and that the address and access type match; for breakpoints
+     * we know the address matched; check the remaining fields, including
+     * linked breakpoints. We rely on WCR and BCR having the same layout
+     * for the LBN, SSC, HMC, PAC/PMC and is-linked fields.
+     * Note that some combinations of {PAC, HMC, SSC} are reserved and
+     * must act either like some valid combination or as if the watchpoint
+     * were disabled. We choose the former, and use this together with
+     * the fact that EL3 must always be Secure and EL2 must always be
+     * Non-Secure to simplify the code slightly compared to the full
+     * table in the ARM ARM.
+     */
+    pac = extract64(cr, 1, 2);
+    hmc = extract64(cr, 13, 1);
+    ssc = extract64(cr, 14, 2);
+
+    switch (ssc) {
+    case 0:
+        break;
+    case 1:
+    case 3:
+        if (is_secure) {
+            return false;
+        }
+        break;
+    case 2:
+        if (!is_secure) {
+            return false;
+        }
+        break;
+    }
+
+    /* TODO: this is not strictly correct because the LDRT/STRT/LDT/STT
+     * "unprivileged access" instructions should match watchpoints as if
+     * they were accesses done at EL0, even if the CPU is at EL1 or higher.
+     * Implementing this would require reworking the core watchpoint code
+     * to plumb the mmu_idx through to this point. Luckily Linux does not
+     * rely on this behaviour currently.
+     * For breakpoints we do want to use the current CPU state.
+     */
+    switch (arm_current_el(env)) {
+    case 3:
+    case 2:
+        if (!hmc) {
+            return false;
+        }
+        break;
+    case 1:
+        if (extract32(pac, 0, 1) == 0) {
+            return false;
+        }
+        break;
+    case 0:
+        if (extract32(pac, 1, 1) == 0) {
+            return false;
+        }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    wt = extract64(cr, 20, 1);
+    lbn = extract64(cr, 16, 4);
+
+    if (wt && !linked_bp_matches(cpu, lbn)) {
+        return false;
+    }
+
+    return true;
+}
+
+static bool check_watchpoints(ARMCPU *cpu)
+{
+    CPUARMState *env = &cpu->env;
+    int n;
+
+    /* If watchpoints are disabled globally or we can't take debug
+     * exceptions here then watchpoint firings are ignored.
+     */
+    if (extract32(env->cp15.mdscr_el1, 15, 1) == 0
+        || !arm_generate_debug_exceptions(env)) {
+        return false;
+    }
+
+    for (n = 0; n < ARRAY_SIZE(env->cpu_watchpoint); n++) {
+        if (bp_wp_matches(cpu, n, true)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static bool check_breakpoints(ARMCPU *cpu)
+{
+    CPUARMState *env = &cpu->env;
+    int n;
+
+    /* If breakpoints are disabled globally or we can't take debug
+     * exceptions here then breakpoint firings are ignored.
+     */
+    if (extract32(env->cp15.mdscr_el1, 15, 1) == 0
+        || !arm_generate_debug_exceptions(env)) {
+        return false;
+    }
+
+    for (n = 0; n < ARRAY_SIZE(env->cpu_breakpoint); n++) {
+        if (bp_wp_matches(cpu, n, false)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+void arm_debug_excp_handler(CPUState *cs)
+{
+    /* Called by core code when a watchpoint or breakpoint fires;
+     * need to check which one and raise the appropriate exception.
+     */
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+    CPUWatchpoint *wp_hit = cs->watchpoint_hit;
+
+    if (wp_hit) {
+        if (wp_hit->flags & BP_CPU) {
+            cs->watchpoint_hit = NULL;
+            if (check_watchpoints(cpu)) {
+                bool wnr = (wp_hit->flags & BP_WATCHPOINT_HIT_WRITE) != 0;
+                bool same_el = arm_debug_target_el(env) == arm_current_el(env);
+
+                env->exception.syndrome = syn_watchpoint(same_el, 0, wnr);
+                if (extended_addresses_enabled(env)) {
+                    env->exception.fsr = (1 << 9) | 0x22;
+                } else {
+                    env->exception.fsr = 0x2;
+                }
+                env->exception.vaddress = wp_hit->hitaddr;
+                raise_exception(env, EXCP_DATA_ABORT);
+            } else {
+                cpu_resume_from_signal(cs, NULL);
+            }
+        }
+    } else {
+        if (check_breakpoints(cpu)) {
+            bool same_el = (arm_debug_target_el(env) == arm_current_el(env));
+            env->exception.syndrome = syn_breakpoint(same_el);
+            if (extended_addresses_enabled(env)) {
+                env->exception.fsr = (1 << 9) | 0x22;
+            } else {
+                env->exception.fsr = 0x2;
+            }
+            /* FAR is UNKNOWN, so doesn't need setting */
+            raise_exception(env, EXCP_PREFETCH_ABORT);
+        }
+    }
 }
 
 /* ??? Flag setting arithmetic is awkward because we need to do comparisons.
diff --git a/target-arm/psci.c b/target-arm/psci.c
new file mode 100644
index 000000000..d8fafab2f
--- /dev/null
+++ b/target-arm/psci.c
@@ -0,0 +1,242 @@
+/*
+ * Copyright (C) 2014 - Linaro
+ * Author: Rob Herring <rob.herring@linaro.org>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+#include <cpu.h>
+#include <cpu-qom.h>
+#include <exec/helper-proto.h>
+#include <kvm-consts.h>
+#include <sysemu/sysemu.h>
+#include "internals.h"
+
+bool arm_is_psci_call(ARMCPU *cpu, int excp_type)
+{
+    /* Return true if the r0/x0 value indicates a PSCI call and
+     * the exception type matches the configured PSCI conduit. This is
+     * called before the SMC/HVC instruction is executed, to decide whether
+     * we should treat it as a PSCI call or with the architecturally
+     * defined behaviour for an SMC or HVC (which might be UNDEF or trap
+     * to EL2 or to EL3).
+     */
+    CPUARMState *env = &cpu->env;
+    uint64_t param = is_a64(env) ? env->xregs[0] : env->regs[0];
+
+    switch (excp_type) {
+    case EXCP_HVC:
+        if (cpu->psci_conduit != QEMU_PSCI_CONDUIT_HVC) {
+            return false;
+        }
+        break;
+    case EXCP_SMC:
+        if (cpu->psci_conduit != QEMU_PSCI_CONDUIT_SMC) {
+            return false;
+        }
+        break;
+    default:
+        return false;
+    }
+
+    switch (param) {
+    case QEMU_PSCI_0_2_FN_PSCI_VERSION:
+    case QEMU_PSCI_0_2_FN_MIGRATE_INFO_TYPE:
+    case QEMU_PSCI_0_2_FN_AFFINITY_INFO:
+    case QEMU_PSCI_0_2_FN64_AFFINITY_INFO:
+    case QEMU_PSCI_0_2_FN_SYSTEM_RESET:
+    case QEMU_PSCI_0_2_FN_SYSTEM_OFF:
+    case QEMU_PSCI_0_1_FN_CPU_ON:
+    case QEMU_PSCI_0_2_FN_CPU_ON:
+    case QEMU_PSCI_0_2_FN64_CPU_ON:
+    case QEMU_PSCI_0_1_FN_CPU_OFF:
+    case QEMU_PSCI_0_2_FN_CPU_OFF:
+    case QEMU_PSCI_0_1_FN_CPU_SUSPEND:
+    case QEMU_PSCI_0_2_FN_CPU_SUSPEND:
+    case QEMU_PSCI_0_2_FN64_CPU_SUSPEND:
+    case QEMU_PSCI_0_1_FN_MIGRATE:
+    case QEMU_PSCI_0_2_FN_MIGRATE:
+        return true;
+    default:
+        return false;
+    }
+}
+
+void arm_handle_psci_call(ARMCPU *cpu)
+{
+    /*
+     * This function partially implements the logic for dispatching Power State
+     * Coordination Interface (PSCI) calls (as described in ARM DEN 0022B.b),
+     * to the extent required for bringing up and taking down secondary cores,
+     * and for handling reset and poweroff requests.
+     * Additional information about the calling convention used is available in
+     * the document 'SMC Calling Convention' (ARM DEN 0028)
+     */
+    CPUState *cs = CPU(cpu);
+    CPUARMState *env = &cpu->env;
+    uint64_t param[4];
+    uint64_t context_id, mpidr;
+    target_ulong entry;
+    int32_t ret = 0;
+    int i;
+
+    for (i = 0; i < 4; i++) {
+        /*
+         * All PSCI functions take explicit 32-bit or native int sized
+         * arguments so we can simply zero-extend all arguments regardless
+         * of which exact function we are about to call.
+         */
+        param[i] = is_a64(env) ? env->xregs[i] : env->regs[i];
+    }
+
+    if ((param[0] & QEMU_PSCI_0_2_64BIT) && !is_a64(env)) {
+        ret = QEMU_PSCI_RET_INVALID_PARAMS;
+        goto err;
+    }
+
+    switch (param[0]) {
+        CPUState *target_cpu_state;
+        ARMCPU *target_cpu;
+        CPUClass *target_cpu_class;
+
+    case QEMU_PSCI_0_2_FN_PSCI_VERSION:
+        ret = QEMU_PSCI_0_2_RET_VERSION_0_2;
+        break;
+    case QEMU_PSCI_0_2_FN_MIGRATE_INFO_TYPE:
+        ret = QEMU_PSCI_0_2_RET_TOS_MIGRATION_NOT_REQUIRED; /* No trusted OS */
+        break;
+    case QEMU_PSCI_0_2_FN_AFFINITY_INFO:
+    case QEMU_PSCI_0_2_FN64_AFFINITY_INFO:
+        mpidr = param[1];
+
+        switch (param[2]) {
+        case 0:
+            target_cpu_state = qemu_get_cpu(mpidr & 0xff);
+            if (!target_cpu_state) {
+                ret = QEMU_PSCI_RET_INVALID_PARAMS;
+                break;
+            }
+            target_cpu = ARM_CPU(target_cpu_state);
+            ret = target_cpu->powered_off ? 1 : 0;
+            break;
+        default:
+            /* Everything above affinity level 0 is always on. */
+            ret = 0;
+        }
+        break;
+    case QEMU_PSCI_0_2_FN_SYSTEM_RESET:
+        qemu_system_reset_request();
+        /* QEMU reset and shutdown are async requests, but PSCI
+         * mandates that we never return from the reset/shutdown
+         * call, so power the CPU off now so it doesn't execute
+         * anything further.
+         */
+        goto cpu_off;
+    case QEMU_PSCI_0_2_FN_SYSTEM_OFF:
+        qemu_system_shutdown_request();
+        goto cpu_off;
+    case QEMU_PSCI_0_1_FN_CPU_ON:
+    case QEMU_PSCI_0_2_FN_CPU_ON:
+    case QEMU_PSCI_0_2_FN64_CPU_ON:
+        mpidr = param[1];
+        entry = param[2];
+        context_id = param[3];
+
+        /* change to the cpu we are powering up */
+        target_cpu_state = qemu_get_cpu(mpidr & 0xff);
+        if (!target_cpu_state) {
+            ret = QEMU_PSCI_RET_INVALID_PARAMS;
+            break;
+        }
+        target_cpu = ARM_CPU(target_cpu_state);
+        if (!target_cpu->powered_off) {
+            ret = QEMU_PSCI_RET_ALREADY_ON;
+            break;
+        }
+        target_cpu_class = CPU_GET_CLASS(target_cpu);
+
+        /* Initialize the cpu we are turning on */
+        cpu_reset(target_cpu_state);
+        target_cpu->powered_off = false;
+        target_cpu_state->halted = 0;
+
+        /*
+         * The PSCI spec mandates that newly brought up CPUs enter the
+         * exception level of the caller in the same execution mode as
+         * the caller, with context_id in x0/r0, respectively.
+         *
+         * For now, it is sufficient to assert() that CPUs come out of
+         * reset in the same mode as the calling CPU, since we only
+         * implement EL1, which means that
+         * (a) there is no EL2 for the calling CPU to trap into to change
+         *     its state
+         * (b) the newly brought up CPU enters EL1 immediately after coming
+         *     out of reset in the default state
+         */
+        assert(is_a64(env) == is_a64(&target_cpu->env));
+        if (is_a64(env)) {
+            if (entry & 1) {
+                ret = QEMU_PSCI_RET_INVALID_PARAMS;
+                break;
+            }
+            target_cpu->env.xregs[0] = context_id;
+        } else {
+            target_cpu->env.regs[0] = context_id;
+            target_cpu->env.thumb = entry & 1;
+        }
+        target_cpu_class->set_pc(target_cpu_state, entry);
+
+        ret = 0;
+        break;
+    case QEMU_PSCI_0_1_FN_CPU_OFF:
+    case QEMU_PSCI_0_2_FN_CPU_OFF:
+        goto cpu_off;
+    case QEMU_PSCI_0_1_FN_CPU_SUSPEND:
+    case QEMU_PSCI_0_2_FN_CPU_SUSPEND:
+    case QEMU_PSCI_0_2_FN64_CPU_SUSPEND:
+        /* Affinity levels are not supported in QEMU */
+        if (param[1] & 0xfffe0000) {
+            ret = QEMU_PSCI_RET_INVALID_PARAMS;
+            break;
+        }
+        /* Powerdown is not supported, we always go into WFI */
+        if (is_a64(env)) {
+            env->xregs[0] = 0;
+        } else {
+            env->regs[0] = 0;
+        }
+        helper_wfi(env);
+        break;
+    case QEMU_PSCI_0_1_FN_MIGRATE:
+    case QEMU_PSCI_0_2_FN_MIGRATE:
+        ret = QEMU_PSCI_RET_NOT_SUPPORTED;
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+err:
+    if (is_a64(env)) {
+        env->xregs[0] = ret;
+    } else {
+        env->regs[0] = ret;
+    }
+    return;
+
+cpu_off:
+    cpu->powered_off = true;
+    cs->halted = 1;
+    cs->exception_index = EXCP_HLT;
+    cpu_loop_exit(cs);
+    /* notreached */
+}
diff --git a/target-arm/translate-a64.c b/target-arm/translate-a64.c
index 33b5025fe..80d2c07e8 100644
--- a/target-arm/translate-a64.c
+++ b/target-arm/translate-a64.c
@@ -35,6 +35,8 @@
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
 
+#include "trace-tcg.h"
+
 static TCGv_i64 cpu_X[32];
 static TCGv_i64 cpu_pc;
 static TCGv_i32 cpu_NF, cpu_ZF, cpu_CF, cpu_VF;
@@ -203,10 +205,39 @@ static void gen_exception_insn(DisasContext *s, int offset, int excp,
     s->is_jmp = DISAS_EXC;
 }
 
+static void gen_ss_advance(DisasContext *s)
+{
+    /* If the singlestep state is Active-not-pending, advance to
+     * Active-pending.
+     */
+    if (s->ss_active) {
+        s->pstate_ss = 0;
+        gen_helper_clear_pstate_ss(cpu_env);
+    }
+}
+
+static void gen_step_complete_exception(DisasContext *s)
+{
+    /* We just completed step of an insn. Move from Active-not-pending
+     * to Active-pending, and then also take the swstep exception.
+     * This corresponds to making the (IMPDEF) choice to prioritize
+     * swstep exceptions over asynchronous exceptions taken to an exception
+     * level where debug is disabled. This choice has the advantage that
+     * we do not need to maintain internal state corresponding to the
+     * ISV/EX syndrome bits between completion of the step and generation
+     * of the exception, and our syndrome information is always correct.
+     */
+    gen_ss_advance(s);
+    gen_exception(EXCP_UDEF, syn_swstep(s->ss_same_el, 1, s->is_ldex));
+    s->is_jmp = DISAS_EXC;
+}
+
 static inline bool use_goto_tb(DisasContext *s, int n, uint64_t dest)
 {
-    /* No direct tb linking with singlestep or deterministic io */
-    if (s->singlestep_enabled || (s->tb->cflags & CF_LAST_IO)) {
+    /* No direct tb linking with singlestep (either QEMU's or the ARM
+     * debug architecture kind) or deterministic io
+     */
+    if (s->singlestep_enabled || s->ss_active || (s->tb->cflags & CF_LAST_IO)) {
         return false;
     }
 
@@ -230,11 +261,14 @@ static inline void gen_goto_tb(DisasContext *s, int n, uint64_t dest)
         s->is_jmp = DISAS_TB_JUMP;
     } else {
         gen_a64_set_pc_im(dest);
-        if (s->singlestep_enabled) {
+        if (s->ss_active) {
+            gen_step_complete_exception(s);
+        } else if (s->singlestep_enabled) {
             gen_exception_internal(EXCP_DEBUG);
+        } else {
+            tcg_gen_exit_tb(0);
+            s->is_jmp = DISAS_TB_JUMP;
         }
-        tcg_gen_exit_tb(0);
-        s->is_jmp = DISAS_JUMP;
     }
 }
 
@@ -714,7 +748,6 @@ static void do_fp_st(DisasContext *s, int srcidx, TCGv_i64 tcg_addr, int size)
     } else {
         TCGv_i64 tcg_hiaddr = tcg_temp_new_i64();
         tcg_gen_qemu_st_i64(tmp, tcg_addr, get_mem_index(s), MO_TEQ);
-        tcg_gen_qemu_st64(tmp, tcg_addr, get_mem_index(s));
         tcg_gen_ld_i64(tmp, cpu_env, fp_reg_hi_offset(s, srcidx));
         tcg_gen_addi_i64(tcg_hiaddr, tcg_addr, 8);
         tcg_gen_qemu_st_i64(tmp, tcg_hiaddr, get_mem_index(s), MO_TEQ);
@@ -1192,7 +1225,7 @@ static void handle_msr_i(DisasContext *s, uint32_t insn,
     int op = op1 << 3 | op2;
     switch (op) {
     case 0x05: /* SPSel */
-        if (s->current_pl == 0) {
+        if (s->current_el == 0) {
             unallocated_encoding(s);
             return;
         }
@@ -1289,7 +1322,7 @@ static void handle_sys(DisasContext *s, uint32_t insn, bool isread,
     }
 
     /* Check access permissions */
-    if (!cp_access_ok(s->current_pl, ri, isread)) {
+    if (!cp_access_ok(s->current_el, ri, isread)) {
         unallocated_encoding(s);
         return;
     }
@@ -1328,7 +1361,7 @@ static void handle_sys(DisasContext *s, uint32_t insn, bool isread,
          * guaranteed to be constant by the tb flags.
          */
         tcg_rt = cpu_reg(s, rt);
-        tcg_gen_movi_i64(tcg_rt, s->current_pl << 2);
+        tcg_gen_movi_i64(tcg_rt, s->current_el << 2);
         return;
     case ARM_CP_DC_ZVA:
         /* Writes clear the aligned block of memory which rt points into. */
@@ -1436,17 +1469,49 @@ static void disas_exc(DisasContext *s, uint32_t insn)
     int opc = extract32(insn, 21, 3);
     int op2_ll = extract32(insn, 0, 5);
     int imm16 = extract32(insn, 5, 16);
+    TCGv_i32 tmp;
 
     switch (opc) {
     case 0:
-        /* SVC, HVC, SMC; since we don't support the Virtualization
-         * or TrustZone extensions these all UNDEF except SVC.
+        /* For SVC, HVC and SMC we advance the single-step state
+         * machine before taking the exception. This is architecturally
+         * mandated, to ensure that single-stepping a system call
+         * instruction works properly.
          */
-        if (op2_ll != 1) {
+        switch (op2_ll) {
+        case 1:
+            gen_ss_advance(s);
+            gen_exception_insn(s, 0, EXCP_SWI, syn_aa64_svc(imm16));
+            break;
+        case 2:
+            if (s->current_el == 0) {
+                unallocated_encoding(s);
+                break;
+            }
+            /* The pre HVC helper handles cases when HVC gets trapped
+             * as an undefined insn by runtime configuration.
+             */
+            gen_a64_set_pc_im(s->pc - 4);
+            gen_helper_pre_hvc(cpu_env);
+            gen_ss_advance(s);
+            gen_exception_insn(s, 0, EXCP_HVC, syn_aa64_hvc(imm16));
+            break;
+        case 3:
+            if (s->current_el == 0) {
+                unallocated_encoding(s);
+                break;
+            }
+            gen_a64_set_pc_im(s->pc - 4);
+            tmp = tcg_const_i32(syn_aa64_smc(imm16));
+            gen_helper_pre_smc(cpu_env, tmp);
+            tcg_temp_free_i32(tmp);
+            gen_ss_advance(s);
+            gen_exception_insn(s, 0, EXCP_SMC, syn_aa64_smc(imm16));
+            break;
+        default:
             unallocated_encoding(s);
             break;
         }
-        gen_exception_insn(s, 0, EXCP_SWI, syn_aa64_svc(imm16));
         break;
     case 1:
         if (op2_ll != 0) {
@@ -1454,7 +1519,7 @@ static void disas_exc(DisasContext *s, uint32_t insn)
             break;
         }
         /* BRK */
-        gen_exception_insn(s, 0, EXCP_BKPT, syn_aa64_bkpt(imm16));
+        gen_exception_insn(s, 4, EXCP_BKPT, syn_aa64_bkpt(imm16));
         break;
     case 2:
         if (op2_ll != 0) {
@@ -1509,7 +1574,7 @@ static void disas_uncond_b_reg(DisasContext *s, uint32_t insn)
         tcg_gen_movi_i64(cpu_reg(s, 30), s->pc);
         break;
     case 4: /* ERET */
-        if (s->current_pl == 0) {
+        if (s->current_el == 0) {
             unallocated_encoding(s);
             return;
         }
@@ -1726,6 +1791,7 @@ static void disas_ldst_excl(DisasContext *s, uint32_t insn)
 
     if (is_excl) {
         if (!is_store) {
+            s->is_ldex = true;
             gen_load_exclusive(s, rt, rt2, tcg_addr, size, is_pair);
         } else {
             gen_store_exclusive(s, rs, rt, rt2, tcg_addr, size, is_pair);
@@ -10863,9 +10929,29 @@ void gen_intermediate_code_internal_a64(ARMCPU *cpu,
     dc->vec_len = 0;
     dc->vec_stride = 0;
     dc->cp_regs = cpu->cp_regs;
-    dc->current_pl = arm_current_pl(env);
+    dc->current_el = arm_current_el(env);
     dc->features = env->features;
 
+    /* Single step state. The code-generation logic here is:
+     *  SS_ACTIVE == 0:
+     *   generate code with no special handling for single-stepping (except
+     *   that anything that can make us go to SS_ACTIVE == 1 must end the TB;
+     *   this happens anyway because those changes are all system register or
+     *   PSTATE writes).
+     *  SS_ACTIVE == 1, PSTATE.SS == 1: (active-not-pending)
+     *   emit code for one insn
+     *   emit code to clear PSTATE.SS
+     *   emit code to generate software step exception for completed step
+     *   end TB (as usual for having generated an exception)
+     *  SS_ACTIVE == 1, PSTATE.SS == 0: (active-pending)
+     *   emit code to generate a software step exception
+     *   end the TB
+     */
+    dc->ss_active = ARM_TBFLAG_AA64_SS_ACTIVE(tb->flags);
+    dc->pstate_ss = ARM_TBFLAG_AA64_PSTATE_SS(tb->flags);
+    dc->is_ldex = false;
+    dc->ss_same_el = (arm_debug_target_el(env) == dc->current_el);
+
     init_tmp_a64_array(dc);
 
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
@@ -10914,6 +11000,23 @@ void gen_intermediate_code_internal_a64(ARMCPU *cpu,
             tcg_gen_debug_insn_start(dc->pc);
         }
 
+        if (dc->ss_active && !dc->pstate_ss) {
+            /* Singlestep state is Active-pending.
+             * If we're in this state at the start of a TB then either
+             *  a) we just took an exception to an EL which is being debugged
+             *     and this is the first insn in the exception handler
+             *  b) debug exceptions were masked and we just unmasked them
+             *     without changing EL (eg by clearing PSTATE.D)
+             * In either case we're going to take a swstep exception in the
+             * "did not step an insn" case, and so the syndrome ISV and EX
+             * bits should be zero.
+             */
+            assert(num_insns == 0);
+            gen_exception(EXCP_UDEF, syn_swstep(dc->ss_same_el, 0, 0));
+            dc->is_jmp = DISAS_EXC;
+            break;
+        }
+
         disas_a64_insn(env, dc);
 
         if (tcg_check_temp_count()) {
@@ -10930,6 +11033,7 @@ void gen_intermediate_code_internal_a64(ARMCPU *cpu,
     } while (!dc->is_jmp && tcg_ctx.gen_opc_ptr < gen_opc_end &&
              !cs->singlestep_enabled &&
              !singlestep &&
+             !dc->ss_active &&
              dc->pc < next_page_start &&
              num_insns < max_insns);
 
@@ -10937,7 +11041,8 @@ void gen_intermediate_code_internal_a64(ARMCPU *cpu,
         gen_io_end();
     }
 
-    if (unlikely(cs->singlestep_enabled) && dc->is_jmp != DISAS_EXC) {
+    if (unlikely(cs->singlestep_enabled || dc->ss_active)
+        && dc->is_jmp != DISAS_EXC) {
         /* Note that this means single stepping WFI doesn't halt the CPU.
          * For conditional branch insns this is harmless unreachable code as
          * gen_goto_tb() has already handled emitting the debug exception
@@ -10947,7 +11052,11 @@ void gen_intermediate_code_internal_a64(ARMCPU *cpu,
         if (dc->is_jmp != DISAS_JUMP) {
             gen_a64_set_pc_im(dc->pc);
         }
-        gen_exception_internal(EXCP_DEBUG);
+        if (cs->singlestep_enabled) {
+            gen_exception_internal(EXCP_DEBUG);
+        } else {
+            gen_step_complete_exception(dc);
+        }
     } else {
         switch (dc->is_jmp) {
         case DISAS_NEXT:
diff --git a/target-arm/translate.c b/target-arm/translate.c
index cf4e767ff..af5156857 100644
--- a/target-arm/translate.c
+++ b/target-arm/translate.c
@@ -35,16 +35,19 @@
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
 
-#define ENABLE_ARCH_4T    arm_feature(env, ARM_FEATURE_V4T)
-#define ENABLE_ARCH_5     arm_feature(env, ARM_FEATURE_V5)
+#include "trace-tcg.h"
+
+
+#define ENABLE_ARCH_4T    arm_dc_feature(s, ARM_FEATURE_V4T)
+#define ENABLE_ARCH_5     arm_dc_feature(s, ARM_FEATURE_V5)
 /* currently all emulated v5 cores are also v5TE, so don't bother */
-#define ENABLE_ARCH_5TE   arm_feature(env, ARM_FEATURE_V5)
+#define ENABLE_ARCH_5TE   arm_dc_feature(s, ARM_FEATURE_V5)
 #define ENABLE_ARCH_5J    0
-#define ENABLE_ARCH_6     arm_feature(env, ARM_FEATURE_V6)
-#define ENABLE_ARCH_6K   arm_feature(env, ARM_FEATURE_V6K)
-#define ENABLE_ARCH_6T2   arm_feature(env, ARM_FEATURE_THUMB2)
-#define ENABLE_ARCH_7     arm_feature(env, ARM_FEATURE_V7)
-#define ENABLE_ARCH_8     arm_feature(env, ARM_FEATURE_V8)
+#define ENABLE_ARCH_6     arm_dc_feature(s, ARM_FEATURE_V6)
+#define ENABLE_ARCH_6K    arm_dc_feature(s, ARM_FEATURE_V6K)
+#define ENABLE_ARCH_6T2   arm_dc_feature(s, ARM_FEATURE_THUMB2)
+#define ENABLE_ARCH_7     arm_dc_feature(s, ARM_FEATURE_V7)
+#define ENABLE_ARCH_8     arm_dc_feature(s, ARM_FEATURE_V8)
 
 #define ARCH(x) do { if (!ENABLE_ARCH_##x) goto illegal_op; } while(0)
 
@@ -202,6 +205,33 @@ static void gen_exception(int excp, uint32_t syndrome)
     tcg_temp_free_i32(tcg_excp);
 }
 
+static void gen_ss_advance(DisasContext *s)
+{
+    /* If the singlestep state is Active-not-pending, advance to
+     * Active-pending.
+     */
+    if (s->ss_active) {
+        s->pstate_ss = 0;
+        gen_helper_clear_pstate_ss(cpu_env);
+    }
+}
+
+static void gen_step_complete_exception(DisasContext *s)
+{
+    /* We just completed step of an insn. Move from Active-not-pending
+     * to Active-pending, and then also take the swstep exception.
+     * This corresponds to making the (IMPDEF) choice to prioritize
+     * swstep exceptions over asynchronous exceptions taken to an exception
+     * level where debug is disabled. This choice has the advantage that
+     * we do not need to maintain internal state corresponding to the
+     * ISV/EX syndrome bits between completion of the step and generation
+     * of the exception, and our syndrome information is always correct.
+     */
+    gen_ss_advance(s);
+    gen_exception(EXCP_UDEF, syn_swstep(s->ss_same_el, 1, s->is_ldex));
+    s->is_jmp = DISAS_EXC;
+}
+
 static void gen_smul_dual(TCGv_i32 a, TCGv_i32 b)
 {
     TCGv_i32 tmp1 = tcg_temp_new_i32();
@@ -804,8 +834,7 @@ static inline void gen_bx(DisasContext *s, TCGv_i32 var)
 /* Variant of store_reg which uses branch&exchange logic when storing
    to r15 in ARM architecture v7 and above. The source must be a temporary
    and will be marked as dead. */
-static inline void store_reg_bx(CPUARMState *env, DisasContext *s,
-                                int reg, TCGv_i32 var)
+static inline void store_reg_bx(DisasContext *s, int reg, TCGv_i32 var)
 {
     if (reg == 15 && ENABLE_ARCH_7) {
         gen_bx(s, var);
@@ -818,8 +847,7 @@ static inline void store_reg_bx(CPUARMState *env, DisasContext *s,
  * to r15 in ARM architecture v5T and above. This is used for storing
  * the results of a LDR/LDM/POP into r15, and corresponds to the cases
  * in the ARM ARM which use the LoadWritePC() pseudocode function. */
-static inline void store_reg_from_load(CPUARMState *env, DisasContext *s,
-                                       int reg, TCGv_i32 var)
+static inline void store_reg_from_load(DisasContext *s, int reg, TCGv_i32 var)
 {
     if (reg == 15 && ENABLE_ARCH_5) {
         gen_bx(s, var);
@@ -911,6 +939,39 @@ static inline void gen_set_pc_im(DisasContext *s, target_ulong val)
     tcg_gen_movi_i32(cpu_R[15], val);
 }
 
+static inline void gen_hvc(DisasContext *s, int imm16)
+{
+    /* The pre HVC helper handles cases when HVC gets trapped
+     * as an undefined insn by runtime configuration (ie before
+     * the insn really executes).
+     */
+    gen_set_pc_im(s, s->pc - 4);
+    gen_helper_pre_hvc(cpu_env);
+    /* Otherwise we will treat this as a real exception which
+     * happens after execution of the insn. (The distinction matters
+     * for the PC value reported to the exception handler and also
+     * for single stepping.)
+     */
+    s->svc_imm = imm16;
+    gen_set_pc_im(s, s->pc);
+    s->is_jmp = DISAS_HVC;
+}
+
+static inline void gen_smc(DisasContext *s)
+{
+    /* As with HVC, we may take an exception either before or after
+     * the insn executes.
+     */
+    TCGv_i32 tmp;
+
+    gen_set_pc_im(s, s->pc - 4);
+    tmp = tcg_const_i32(syn_aa32_smc());
+    gen_helper_pre_smc(cpu_env, tmp);
+    tcg_temp_free_i32(tmp);
+    gen_set_pc_im(s, s->pc);
+    s->is_jmp = DISAS_SMC;
+}
+
 static inline void
 gen_set_condexec (DisasContext *s)
 {
@@ -1478,7 +1539,7 @@ static inline int gen_iwmmxt_shift(uint32_t insn, uint32_t mask, TCGv_i32 dest)
 
 /* Disassemble an iwMMXt instruction.  Returns nonzero if an error occurred
    (ie. an undefined instruction).  */
-static int disas_iwmmxt_insn(CPUARMState *env, DisasContext *s, uint32_t insn)
+static int disas_iwmmxt_insn(DisasContext *s, uint32_t insn)
 {
     int rd, wrd;
     int rdhi, rdlo, rd0, rd1, i;
@@ -2484,7 +2545,7 @@ static int disas_iwmmxt_insn(CPUARMState *env, DisasContext *s, uint32_t insn)
 
 /* Disassemble an XScale DSP instruction.  Returns nonzero if an error occurred
    (ie. an undefined instruction).  */
-static int disas_dsp_insn(CPUARMState *env, DisasContext *s, uint32_t insn)
+static int disas_dsp_insn(DisasContext *s, uint32_t insn)
 {
     int acc, rd0, rd1, rdhi, rdlo;
     TCGv_i32 tmp, tmp2;
@@ -2556,7 +2617,7 @@ static int disas_dsp_insn(CPUARMState *env, DisasContext *s, uint32_t insn)
 #define VFP_SREG(insn, bigbit, smallbit) \
   ((VFP_REG_SHR(insn, bigbit - 1) & 0x1e) | (((insn) >> (smallbit)) & 1))
 #define VFP_DREG(reg, insn, bigbit, smallbit) do { \
-    if (arm_feature(env, ARM_FEATURE_VFP3)) { \
+    if (arm_dc_feature(s, ARM_FEATURE_VFP3)) { \
         reg = (((insn) >> (bigbit)) & 0x0f) \
               | (((insn) >> ((smallbit) - 4)) & 0x10); \
     } else { \
@@ -2903,11 +2964,11 @@ static const uint8_t fp_decode_rm[] = {
     FPROUNDING_NEGINF,
 };
 
-static int disas_vfp_v8_insn(CPUARMState *env, DisasContext *s, uint32_t insn)
+static int disas_vfp_v8_insn(DisasContext *s, uint32_t insn)
 {
     uint32_t rd, rn, rm, dp = extract32(insn, 8, 1);
 
-    if (!arm_feature(env, ARM_FEATURE_V8)) {
+    if (!arm_dc_feature(s, ARM_FEATURE_V8)) {
         return 1;
     }
 
@@ -2939,7 +3000,7 @@ static int disas_vfp_v8_insn(CPUARMState *env, DisasContext *s, uint32_t insn)
 
 /* Disassemble a VFP instruction.  Returns nonzero if an error occurred
    (ie. an undefined instruction).  */
-static int disas_vfp_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
+static int disas_vfp_insn(DisasContext *s, uint32_t insn)
 {
     uint32_t rd, rn, rm, op, i, n, offset, delta_d, delta_m, bank_mask;
     int dp, veclen;
@@ -2947,8 +3008,9 @@ static int disas_vfp_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
     TCGv_i32 tmp;
     TCGv_i32 tmp2;
 
-    if (!arm_feature(env, ARM_FEATURE_VFP))
+    if (!arm_dc_feature(s, ARM_FEATURE_VFP)) {
         return 1;
+    }
 
     /* FIXME: this access check should not take precedence over UNDEF
      * for invalid encodings; we will generate incorrect syndrome information
@@ -2975,7 +3037,7 @@ static int disas_vfp_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
         /* Encodings with T=1 (Thumb) or unconditional (ARM):
          * only used in v8 and above.
          */
-        return disas_vfp_v8_insn(env, s, insn);
+        return disas_vfp_v8_insn(s, insn);
     }
 
     dp = ((insn & 0xf00) == 0xb00);
@@ -2992,8 +3054,9 @@ static int disas_vfp_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                 if (insn & 0xf)
                     return 1;
                 if (insn & 0x00c00060
-                    && !arm_feature(env, ARM_FEATURE_NEON))
+                    && !arm_dc_feature(s, ARM_FEATURE_NEON)) {
                     return 1;
+                }
 
                 pass = (insn >> 21) & 1;
                 if (insn & (1 << 22)) {
@@ -3088,8 +3151,9 @@ static int disas_vfp_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                                VFP3 restricts all id registers to privileged
                                accesses.  */
                             if (IS_USER(s)
-                                && arm_feature(env, ARM_FEATURE_VFP3))
+                                && arm_dc_feature(s, ARM_FEATURE_VFP3)) {
                                 return 1;
+                            }
                             tmp = load_cpu_field(vfp.xregs[rn]);
                             break;
                         case ARM_VFP_FPEXC:
@@ -3101,8 +3165,9 @@ static int disas_vfp_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                         case ARM_VFP_FPINST2:
                             /* Not present in VFP3.  */
                             if (IS_USER(s)
-                                || arm_feature(env, ARM_FEATURE_VFP3))
+                                || arm_dc_feature(s, ARM_FEATURE_VFP3)) {
                                 return 1;
+                            }
                             tmp = load_cpu_field(vfp.xregs[rn]);
                             break;
                         case ARM_VFP_FPSCR:
@@ -3115,15 +3180,16 @@ static int disas_vfp_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                             }
                             break;
                         case ARM_VFP_MVFR2:
-                            if (!arm_feature(env, ARM_FEATURE_V8)) {
+                            if (!arm_dc_feature(s, ARM_FEATURE_V8)) {
                                 return 1;
                             }
                             /* fall through */
                         case ARM_VFP_MVFR0:
                         case ARM_VFP_MVFR1:
                             if (IS_USER(s)
-                                || !arm_feature(env, ARM_FEATURE_MVFR))
+                                || !arm_dc_feature(s, ARM_FEATURE_MVFR)) {
                                 return 1;
+                            }
                             tmp = load_cpu_field(vfp.xregs[rn]);
                             break;
                         default:
@@ -3169,6 +3235,9 @@ static int disas_vfp_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                             break;
                         case ARM_VFP_FPINST:
                         case ARM_VFP_FPINST2:
+                            if (IS_USER(s)) {
+                                return 1;
+                            }
                             tmp = load_reg(s, rd);
                             store_cpu_field(tmp, vfp.xregs[rn]);
                             break;
@@ -3301,8 +3370,8 @@ static int disas_vfp_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                      * UNPREDICTABLE if bit 8 is set prior to ARMv8
                      * (we choose to UNDEF)
                      */
-                    if ((dp && !arm_feature(env, ARM_FEATURE_V8)) ||
-                        !arm_feature(env, ARM_FEATURE_VFP_FP16)) {
+                    if ((dp && !arm_dc_feature(s, ARM_FEATURE_V8)) ||
+                        !arm_dc_feature(s, ARM_FEATURE_VFP_FP16)) {
                         return 1;
                     }
                     if (!extract32(rn, 1, 1)) {
@@ -3381,7 +3450,7 @@ static int disas_vfp_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                      * correct : an input NaN should come out with its sign bit
                      * flipped if it is a negated-input.
                      */
-                    if (!arm_feature(env, ARM_FEATURE_VFP4)) {
+                    if (!arm_dc_feature(s, ARM_FEATURE_VFP4)) {
                         return 1;
                     }
                     if (dp) {
@@ -3422,8 +3491,9 @@ static int disas_vfp_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                     }
                     break;
                 case 14: /* fconst */
-                    if (!arm_feature(env, ARM_FEATURE_VFP3))
-                      return 1;
+                    if (!arm_dc_feature(s, ARM_FEATURE_VFP3)) {
+                        return 1;
+                    }
 
                     n = (insn << 12) & 0x80000000;
                     i = ((insn >> 12) & 0x70) | (insn & 0xf);
@@ -3578,23 +3648,27 @@ static int disas_vfp_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                         gen_vfp_sito(dp, 0);
                         break;
                     case 20: /* fshto */
-                        if (!arm_feature(env, ARM_FEATURE_VFP3))
-                          return 1;
+                        if (!arm_dc_feature(s, ARM_FEATURE_VFP3)) {
+                            return 1;
+                        }
                         gen_vfp_shto(dp, 16 - rm, 0);
                         break;
                     case 21: /* fslto */
-                        if (!arm_feature(env, ARM_FEATURE_VFP3))
-                          return 1;
+                        if (!arm_dc_feature(s, ARM_FEATURE_VFP3)) {
+                            return 1;
+                        }
                         gen_vfp_slto(dp, 32 - rm, 0);
                         break;
                     case 22: /* fuhto */
-                        if (!arm_feature(env, ARM_FEATURE_VFP3))
-                          return 1;
+                        if (!arm_dc_feature(s, ARM_FEATURE_VFP3)) {
+                            return 1;
+                        }
                         gen_vfp_uhto(dp, 16 - rm, 0);
                         break;
                     case 23: /* fulto */
-                        if (!arm_feature(env, ARM_FEATURE_VFP3))
-                          return 1;
+                        if (!arm_dc_feature(s, ARM_FEATURE_VFP3)) {
+                            return 1;
+                        }
                         gen_vfp_ulto(dp, 32 - rm, 0);
                         break;
                     case 24: /* ftoui */
@@ -3610,23 +3684,27 @@ static int disas_vfp_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                         gen_vfp_tosiz(dp, 0);
                         break;
                     case 28: /* ftosh */
-                        if (!arm_feature(env, ARM_FEATURE_VFP3))
-                          return 1;
+                        if (!arm_dc_feature(s, ARM_FEATURE_VFP3)) {
+                            return 1;
+                        }
                         gen_vfp_tosh(dp, 16 - rm, 0);
                         break;
                     case 29: /* ftosl */
-                        if (!arm_feature(env, ARM_FEATURE_VFP3))
-                          return 1;
+                        if (!arm_dc_feature(s, ARM_FEATURE_VFP3)) {
+                            return 1;
+                        }
                         gen_vfp_tosl(dp, 32 - rm, 0);
                         break;
                     case 30: /* ftouh */
-                        if (!arm_feature(env, ARM_FEATURE_VFP3))
-                          return 1;
+                        if (!arm_dc_feature(s, ARM_FEATURE_VFP3)) {
+                            return 1;
+                        }
                         gen_vfp_touh(dp, 16 - rm, 0);
                         break;
                     case 31: /* ftoul */
-                        if (!arm_feature(env, ARM_FEATURE_VFP3))
-                          return 1;
+                        if (!arm_dc_feature(s, ARM_FEATURE_VFP3)) {
+                            return 1;
+                        }
                         gen_vfp_toul(dp, 32 - rm, 0);
                         break;
                     default: /* undefined */
@@ -3857,7 +3935,7 @@ static inline void gen_goto_tb(DisasContext *s, int n, target_ulong dest)
 
 static inline void gen_jmp (DisasContext *s, uint32_t dest)
 {
-    if (unlikely(s->singlestep_enabled)) {
+    if (unlikely(s->singlestep_enabled || s->ss_active)) {
         /* An indirect jump so that we still trigger the debug exception.  */
         if (s->thumb)
             dest |= 1;
@@ -3882,7 +3960,8 @@ static inline void gen_mulxy(TCGv_i32 t0, TCGv_i32 t1, int x, int y)
 }
 
 /* Return the mask of PSR bits set by a MSR instruction.  */
-static uint32_t msr_mask(CPUARMState *env, DisasContext *s, int flags, int spsr) {
+static uint32_t msr_mask(DisasContext *s, int flags, int spsr)
+{
     uint32_t mask;
 
     mask = 0;
@@ -3897,17 +3976,22 @@ static uint32_t msr_mask(CPUARMState *env, DisasContext *s, int flags, int spsr)
 
     /* Mask out undefined bits.  */
     mask &= ~CPSR_RESERVED;
-    if (!arm_feature(env, ARM_FEATURE_V4T))
+    if (!arm_dc_feature(s, ARM_FEATURE_V4T)) {
         mask &= ~CPSR_T;
-    if (!arm_feature(env, ARM_FEATURE_V5))
+    }
+    if (!arm_dc_feature(s, ARM_FEATURE_V5)) {
         mask &= ~CPSR_Q; /* V5TE in reality*/
-    if (!arm_feature(env, ARM_FEATURE_V6))
+    }
+    if (!arm_dc_feature(s, ARM_FEATURE_V6)) {
         mask &= ~(CPSR_E | CPSR_GE);
-    if (!arm_feature(env, ARM_FEATURE_THUMB2))
+    }
+    if (!arm_dc_feature(s, ARM_FEATURE_THUMB2)) {
         mask &= ~CPSR_IT;
-    /* Mask out execution state bits.  */
-    if (!spsr)
-        mask &= ~CPSR_EXEC;
+    }
+    /* Mask out execution state and reserved bits.  */
+    if (!spsr) {
+        mask &= ~(CPSR_EXEC | CPSR_RESERVED);
+    }
     /* Mask out privileged bits.  */
     if (IS_USER(s))
         mask &= CPSR_USER;
@@ -3951,7 +4035,7 @@ static void gen_exception_return(DisasContext *s, TCGv_i32 pc)
     TCGv_i32 tmp;
     store_reg(s, 15, pc);
     tmp = load_cpu_field(spsr);
-    gen_set_cpsr(tmp, 0xffffffff);
+    gen_set_cpsr(tmp, CPSR_ERET_MASK);
     tcg_temp_free_i32(tmp);
     s->is_jmp = DISAS_UPDATE;
 }
@@ -3959,7 +4043,7 @@ static void gen_exception_return(DisasContext *s, TCGv_i32 pc)
 /* Generate a v6 exception return.  Marks both values as dead.  */
 static void gen_rfe(DisasContext *s, TCGv_i32 pc, TCGv_i32 cpsr)
 {
-    gen_set_cpsr(cpsr, 0xffffffff);
+    gen_set_cpsr(cpsr, CPSR_ERET_MASK);
     tcg_temp_free_i32(cpsr);
     store_reg(s, 15, pc);
     s->is_jmp = DISAS_UPDATE;
@@ -4227,7 +4311,7 @@ static struct {
 
 /* Translate a NEON load/store element instruction.  Return nonzero if the
    instruction is invalid.  */
-static int disas_neon_ls_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
+static int disas_neon_ls_insn(DisasContext *s, uint32_t insn)
 {
     int rd, rn, rm;
     int op;
@@ -4969,7 +5053,7 @@ static const uint8_t neon_2rm_sizes[] = {
    We process data in a mixture of 32-bit and 64-bit chunks.
    Mostly we use 32-bit chunks so we can use normal scalar instructions.  */
 
-static int disas_neon_data_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
+static int disas_neon_data_insn(DisasContext *s, uint32_t insn)
 {
     int op;
     int q;
@@ -5025,7 +5109,7 @@ static int disas_neon_data_insn(CPUARMState * env, DisasContext *s, uint32_t ins
                 return 1;
             }
             if (!u) { /* SHA-1 */
-                if (!arm_feature(env, ARM_FEATURE_V8_SHA1)) {
+                if (!arm_dc_feature(s, ARM_FEATURE_V8_SHA1)) {
                     return 1;
                 }
                 tmp = tcg_const_i32(rd);
@@ -5035,7 +5119,7 @@ static int disas_neon_data_insn(CPUARMState * env, DisasContext *s, uint32_t ins
                 gen_helper_crypto_sha1_3reg(cpu_env, tmp, tmp2, tmp3, tmp4);
                 tcg_temp_free_i32(tmp4);
             } else { /* SHA-256 */
-                if (!arm_feature(env, ARM_FEATURE_V8_SHA256) || size == 3) {
+                if (!arm_dc_feature(s, ARM_FEATURE_V8_SHA256) || size == 3) {
                     return 1;
                 }
                 tmp = tcg_const_i32(rd);
@@ -5170,7 +5254,7 @@ static int disas_neon_data_insn(CPUARMState * env, DisasContext *s, uint32_t ins
             break;
         case NEON_3R_FLOAT_MISC:
             /* VMAXNM/VMINNM in ARMv8 */
-            if (u && !arm_feature(env, ARM_FEATURE_V8)) {
+            if (u && !arm_dc_feature(s, ARM_FEATURE_V8)) {
                 return 1;
             }
             break;
@@ -5181,7 +5265,7 @@ static int disas_neon_data_insn(CPUARMState * env, DisasContext *s, uint32_t ins
             }
             break;
         case NEON_3R_VFM:
-            if (!arm_feature(env, ARM_FEATURE_VFP4) || u) {
+            if (!arm_dc_feature(s, ARM_FEATURE_VFP4) || u) {
                 return 1;
             }
             break;
@@ -6000,7 +6084,7 @@ static int disas_neon_data_insn(CPUARMState * env, DisasContext *s, uint32_t ins
                 if (op == 14 && size == 2) {
                     TCGv_i64 tcg_rn, tcg_rm, tcg_rd;
 
-                    if (!arm_feature(env, ARM_FEATURE_V8_PMULL)) {
+                    if (!arm_dc_feature(s, ARM_FEATURE_V8_PMULL)) {
                         return 1;
                     }
                     tcg_rn = tcg_temp_new_i64();
@@ -6488,7 +6572,7 @@ static int disas_neon_data_insn(CPUARMState * env, DisasContext *s, uint32_t ins
                     }
                     break;
                 case NEON_2RM_VCVT_F16_F32:
-                    if (!arm_feature(env, ARM_FEATURE_VFP_FP16) ||
+                    if (!arm_dc_feature(s, ARM_FEATURE_VFP_FP16) ||
                         q || (rm & 1)) {
                         return 1;
                     }
@@ -6512,7 +6596,7 @@ static int disas_neon_data_insn(CPUARMState * env, DisasContext *s, uint32_t ins
                     tcg_temp_free_i32(tmp);
                     break;
                 case NEON_2RM_VCVT_F32_F16:
-                    if (!arm_feature(env, ARM_FEATURE_VFP_FP16) ||
+                    if (!arm_dc_feature(s, ARM_FEATURE_VFP_FP16) ||
                         q || (rd & 1)) {
                         return 1;
                     }
@@ -6536,7 +6620,7 @@ static int disas_neon_data_insn(CPUARMState * env, DisasContext *s, uint32_t ins
                     tcg_temp_free_i32(tmp3);
                     break;
                 case NEON_2RM_AESE: case NEON_2RM_AESMC:
-                    if (!arm_feature(env, ARM_FEATURE_V8_AES)
+                    if (!arm_dc_feature(s, ARM_FEATURE_V8_AES)
                         || ((rm | rd) & 1)) {
                         return 1;
                     }
@@ -6558,7 +6642,7 @@ static int disas_neon_data_insn(CPUARMState * env, DisasContext *s, uint32_t ins
                     tcg_temp_free_i32(tmp3);
                     break;
                 case NEON_2RM_SHA1H:
-                    if (!arm_feature(env, ARM_FEATURE_V8_SHA1)
+                    if (!arm_dc_feature(s, ARM_FEATURE_V8_SHA1)
                         || ((rm | rd) & 1)) {
                         return 1;
                     }
@@ -6576,10 +6660,10 @@ static int disas_neon_data_insn(CPUARMState * env, DisasContext *s, uint32_t ins
                     }
                     /* bit 6 (q): set -> SHA256SU0, cleared -> SHA1SU1 */
                     if (q) {
-                        if (!arm_feature(env, ARM_FEATURE_V8_SHA256)) {
+                        if (!arm_dc_feature(s, ARM_FEATURE_V8_SHA256)) {
                             return 1;
                         }
-                    } else if (!arm_feature(env, ARM_FEATURE_V8_SHA1)) {
+                    } else if (!arm_dc_feature(s, ARM_FEATURE_V8_SHA1)) {
                         return 1;
                     }
                     tmp = tcg_const_i32(rd);
@@ -6964,28 +7048,24 @@ static int disas_neon_data_insn(CPUARMState * env, DisasContext *s, uint32_t ins
     return 0;
 }
 
-static int disas_coproc_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
+static int disas_coproc_insn(DisasContext *s, uint32_t insn)
 {
     int cpnum, is64, crn, crm, opc1, opc2, isread, rt, rt2;
     const ARMCPRegInfo *ri;
 
     cpnum = (insn >> 8) & 0xf;
-    if (arm_feature(env, ARM_FEATURE_XSCALE)
-	    && ((env->cp15.c15_cpar ^ 0x3fff) & (1 << cpnum)))
-	return 1;
-
-    /* First check for coprocessor space used for actual instructions */
-    switch (cpnum) {
-      case 0:
-      case 1:
-	if (arm_feature(env, ARM_FEATURE_IWMMXT)) {
-	    return disas_iwmmxt_insn(env, s, insn);
-	} else if (arm_feature(env, ARM_FEATURE_XSCALE)) {
-	    return disas_dsp_insn(env, s, insn);
-	}
-	return 1;
-    default:
-        break;
+
+    /* First check for coprocessor space used for XScale/iwMMXt insns */
+    if (arm_dc_feature(s, ARM_FEATURE_XSCALE) && (cpnum < 2)) {
+        if (extract32(s->c15_cpar, cpnum, 1) == 0) {
+            return 1;
+        }
+        if (arm_dc_feature(s, ARM_FEATURE_IWMMXT)) {
+            return disas_iwmmxt_insn(s, insn);
+        } else if (arm_dc_feature(s, ARM_FEATURE_XSCALE)) {
+            return disas_dsp_insn(s, insn);
+        }
+        return 1;
     }
 
     /* Otherwise treat as a generic register access */
@@ -7014,13 +7094,16 @@ static int disas_coproc_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                             ENCODE_CP_REG(cpnum, is64, crn, crm, opc1, opc2));
     if (ri) {
         /* Check access permissions */
-        if (!cp_access_ok(s->current_pl, ri, isread)) {
+        if (!cp_access_ok(s->current_el, ri, isread)) {
             return 1;
         }
 
-        if (ri->accessfn) {
+        if (ri->accessfn ||
+            (arm_dc_feature(s, ARM_FEATURE_XSCALE) && cpnum < 14)) {
             /* Emit code to perform further access permissions checks at
              * runtime; this may result in an exception.
+             * Note that on XScale all cp0..c13 registers do an access check
+             * call in order to handle c15_cpar.
              */
             TCGv_ptr tmpptr;
             TCGv_i32 tcg_syn;
@@ -7059,7 +7142,7 @@ static int disas_coproc_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                  * in which case the syndrome information won't actually be
                  * guest visible.
                  */
-                assert(!arm_feature(env, ARM_FEATURE_V8));
+                assert(!arm_dc_feature(s, ARM_FEATURE_V8));
                 syndrome = syn_uncategorized();
                 break;
             }
@@ -7277,6 +7360,8 @@ static void gen_load_exclusive(DisasContext *s, int rt, int rt2,
 {
     TCGv_i32 tmp = tcg_temp_new_i32();
 
+    s->is_ldex = true;
+
     switch (size) {
     case 0:
         gen_aa32_ld8u(tmp, addr, get_mem_index(s));
@@ -7475,21 +7560,19 @@ static void gen_srs(DisasContext *s,
     tcg_temp_free_i32(addr);
 }
 
-static void disas_arm_insn(CPUARMState * env, DisasContext *s)
+static void disas_arm_insn(DisasContext *s, unsigned int insn)
 {
-    unsigned int cond, insn, val, op1, i, shift, rm, rs, rn, rd, sh;
+    unsigned int cond, val, op1, i, shift, rm, rs, rn, rd, sh;
     TCGv_i32 tmp;
     TCGv_i32 tmp2;
     TCGv_i32 tmp3;
     TCGv_i32 addr;
     TCGv_i64 tmp64;
 
-    insn = arm_ldl_code(env, s->pc, s->bswap_code);
-    s->pc += 4;
-
     /* M variants do not implement ARM mode.  */
-    if (IS_M(env))
+    if (arm_dc_feature(s, ARM_FEATURE_M)) {
         goto illegal_op;
+    }
     cond = insn >> 28;
     if (cond == 0xf){
         /* In ARMv3 and v4 the NV condition is UNPREDICTABLE; we
@@ -7501,25 +7584,29 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
         /* Unconditional instructions.  */
         if (((insn >> 25) & 7) == 1) {
             /* NEON Data processing.  */
-            if (!arm_feature(env, ARM_FEATURE_NEON))
+            if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
                 goto illegal_op;
+            }
 
-            if (disas_neon_data_insn(env, s, insn))
+            if (disas_neon_data_insn(s, insn)) {
                 goto illegal_op;
+            }
             return;
         }
         if ((insn & 0x0f100000) == 0x04000000) {
             /* NEON load/store.  */
-            if (!arm_feature(env, ARM_FEATURE_NEON))
+            if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
                 goto illegal_op;
+            }
 
-            if (disas_neon_ls_insn(env, s, insn))
+            if (disas_neon_ls_insn(s, insn)) {
                 goto illegal_op;
+            }
             return;
         }
         if ((insn & 0x0f000e10) == 0x0e000a00) {
             /* VFP.  */
-            if (disas_vfp_insn(env, s, insn)) {
+            if (disas_vfp_insn(s, insn)) {
                 goto illegal_op;
             }
             return;
@@ -7528,7 +7615,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             ((insn & 0x0f30f010) == 0x0710f000)) {
             if ((insn & (1 << 22)) == 0) {
                 /* PLDW; v7MP */
-                if (!arm_feature(env, ARM_FEATURE_V7MP)) {
+                if (!arm_dc_feature(s, ARM_FEATURE_V7MP)) {
                     goto illegal_op;
                 }
             }
@@ -7543,7 +7630,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
         }
         if (((insn & 0x0f700000) == 0x04100000) ||
             ((insn & 0x0f700010) == 0x06100000)) {
-            if (!arm_feature(env, ARM_FEATURE_V7MP)) {
+            if (!arm_dc_feature(s, ARM_FEATURE_V7MP)) {
                 goto illegal_op;
             }
             return; /* v7MP: Unallocated memory hint: must NOP */
@@ -7640,11 +7727,13 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             gen_bx_im(s, val);
             return;
         } else if ((insn & 0x0e000f00) == 0x0c000100) {
-            if (arm_feature(env, ARM_FEATURE_IWMMXT)) {
+            if (arm_dc_feature(s, ARM_FEATURE_IWMMXT)) {
                 /* iWMMXt register transfer.  */
-                if (env->cp15.c15_cpar & (1 << 1))
-                    if (!disas_iwmmxt_insn(env, s, insn))
+                if (extract32(s->c15_cpar, 1, 1)) {
+                    if (!disas_iwmmxt_insn(s, insn)) {
                         return;
+                    }
+                }
             }
         } else if ((insn & 0x0fe00000) == 0x0c400000) {
             /* Coprocessor double register transfer.  */
@@ -7714,8 +7803,10 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                 if (shift)
                     val = (val >> shift) | (val << (32 - shift));
                 i = ((insn & (1 << 22)) != 0);
-                if (gen_set_psr_im(s, msr_mask(env, s, (insn >> 16) & 0xf, i), i, val))
+                if (gen_set_psr_im(s, msr_mask(s, (insn >> 16) & 0xf, i),
+                                   i, val)) {
                     goto illegal_op;
+                }
             }
         }
     } else if ((insn & 0x0f900000) == 0x01000000
@@ -7730,7 +7821,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                 /* PSR = reg */
                 tmp = load_reg(s, rm);
                 i = ((op1 & 2) != 0);
-                if (gen_set_psr(s, msr_mask(env, s, (insn >> 16) & 0xf, i), i, tmp))
+                if (gen_set_psr(s, msr_mask(s, (insn >> 16) & 0xf, i), i, tmp))
                     goto illegal_op;
             } else {
                 /* reg = PSR */
@@ -7794,7 +7885,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
              * op1 == 3 is UNPREDICTABLE but handle as UNDEFINED.
              * Bits 8, 10 and 11 should be zero.
              */
-            if (!arm_feature(env, ARM_FEATURE_CRC) || op1 == 0x3 ||
+            if (!arm_dc_feature(s, ARM_FEATURE_CRC) || op1 == 0x3 ||
                 (c & 0xd) != 0) {
                 goto illegal_op;
             }
@@ -7838,15 +7929,32 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
         case 7:
         {
             int imm16 = extract32(insn, 0, 4) | (extract32(insn, 8, 12) << 4);
-            /* SMC instruction (op1 == 3)
-               and undefined instructions (op1 == 0 || op1 == 2)
-               will trap */
-            if (op1 != 1) {
+            switch (op1) {
+            case 1:
+                /* bkpt */
+                ARCH(5);
+                gen_exception_insn(s, 4, EXCP_BKPT,
+                                   syn_aa32_bkpt(imm16, false));
+                break;
+            case 2:
+                /* Hypervisor call (v7) */
+                ARCH(7);
+                if (IS_USER(s)) {
+                    goto illegal_op;
+                }
+                gen_hvc(s, imm16);
+                break;
+            case 3:
+                /* Secure monitor call (v6+) */
+                ARCH(6K);
+                if (IS_USER(s)) {
+                    goto illegal_op;
+                }
+                gen_smc(s);
+                break;
+            default:
                 goto illegal_op;
             }
-            /* bkpt */
-            ARCH(5);
-            gen_exception_insn(s, 4, EXCP_BKPT, syn_aa32_bkpt(imm16, false));
             break;
         }
         case 0x8: /* signed multiply */
@@ -7951,14 +8059,14 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             if (logic_cc) {
                 gen_logic_CC(tmp);
             }
-            store_reg_bx(env, s, rd, tmp);
+            store_reg_bx(s, rd, tmp);
             break;
         case 0x01:
             tcg_gen_xor_i32(tmp, tmp, tmp2);
             if (logic_cc) {
                 gen_logic_CC(tmp);
             }
-            store_reg_bx(env, s, rd, tmp);
+            store_reg_bx(s, rd, tmp);
             break;
         case 0x02:
             if (set_cc && rd == 15) {
@@ -7974,7 +8082,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                 } else {
                     tcg_gen_sub_i32(tmp, tmp, tmp2);
                 }
-                store_reg_bx(env, s, rd, tmp);
+                store_reg_bx(s, rd, tmp);
             }
             break;
         case 0x03:
@@ -7983,7 +8091,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             } else {
                 tcg_gen_sub_i32(tmp, tmp2, tmp);
             }
-            store_reg_bx(env, s, rd, tmp);
+            store_reg_bx(s, rd, tmp);
             break;
         case 0x04:
             if (set_cc) {
@@ -7991,7 +8099,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             } else {
                 tcg_gen_add_i32(tmp, tmp, tmp2);
             }
-            store_reg_bx(env, s, rd, tmp);
+            store_reg_bx(s, rd, tmp);
             break;
         case 0x05:
             if (set_cc) {
@@ -7999,7 +8107,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             } else {
                 gen_add_carry(tmp, tmp, tmp2);
             }
-            store_reg_bx(env, s, rd, tmp);
+            store_reg_bx(s, rd, tmp);
             break;
         case 0x06:
             if (set_cc) {
@@ -8007,7 +8115,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             } else {
                 gen_sub_carry(tmp, tmp, tmp2);
             }
-            store_reg_bx(env, s, rd, tmp);
+            store_reg_bx(s, rd, tmp);
             break;
         case 0x07:
             if (set_cc) {
@@ -8015,7 +8123,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             } else {
                 gen_sub_carry(tmp, tmp2, tmp);
             }
-            store_reg_bx(env, s, rd, tmp);
+            store_reg_bx(s, rd, tmp);
             break;
         case 0x08:
             if (set_cc) {
@@ -8048,7 +8156,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             if (logic_cc) {
                 gen_logic_CC(tmp);
             }
-            store_reg_bx(env, s, rd, tmp);
+            store_reg_bx(s, rd, tmp);
             break;
         case 0x0d:
             if (logic_cc && rd == 15) {
@@ -8061,7 +8169,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                 if (logic_cc) {
                     gen_logic_CC(tmp2);
                 }
-                store_reg_bx(env, s, rd, tmp2);
+                store_reg_bx(s, rd, tmp2);
             }
             break;
         case 0x0e:
@@ -8069,7 +8177,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             if (logic_cc) {
                 gen_logic_CC(tmp);
             }
-            store_reg_bx(env, s, rd, tmp);
+            store_reg_bx(s, rd, tmp);
             break;
         default:
         case 0x0f:
@@ -8077,7 +8185,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             if (logic_cc) {
                 gen_logic_CC(tmp2);
             }
-            store_reg_bx(env, s, rd, tmp2);
+            store_reg_bx(s, rd, tmp2);
             break;
         }
         if (op1 != 0x0f && op1 != 0x0d) {
@@ -8586,7 +8694,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                     case 1:
                     case 3:
                         /* SDIV, UDIV */
-                        if (!arm_feature(env, ARM_FEATURE_ARM_DIV)) {
+                        if (!arm_dc_feature(s, ARM_FEATURE_ARM_DIV)) {
                             goto illegal_op;
                         }
                         if (((insn >> 5) & 7) || (rd != 15)) {
@@ -8715,7 +8823,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             }
             if (insn & (1 << 20)) {
                 /* Complete the load.  */
-                store_reg_from_load(env, s, rd, tmp);
+                store_reg_from_load(s, rd, tmp);
             }
             break;
         case 0x08:
@@ -8778,7 +8886,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                                 loaded_var = tmp;
                                 loaded_base = 1;
                             } else {
-                                store_reg_from_load(env, s, i, tmp);
+                                store_reg_from_load(s, i, tmp);
                             }
                         } else {
                             /* store */
@@ -8833,7 +8941,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                 if ((insn & (1 << 22)) && !user) {
                     /* Restore CPSR from SPSR.  */
                     tmp = load_cpu_field(spsr);
-                    gen_set_cpsr(tmp, 0xffffffff);
+                    gen_set_cpsr(tmp, CPSR_ERET_MASK);
                     tcg_temp_free_i32(tmp);
                     s->is_jmp = DISAS_UPDATE;
                 }
@@ -8861,10 +8969,10 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
         case 0xe:
             if (((insn >> 8) & 0xe) == 10) {
                 /* VFP.  */
-                if (disas_vfp_insn(env, s, insn)) {
+                if (disas_vfp_insn(s, insn)) {
                     goto illegal_op;
                 }
-            } else if (disas_coproc_insn(env, s, insn)) {
+            } else if (disas_coproc_insn(s, insn)) {
                 /* Coprocessor.  */
                 goto illegal_op;
             }
@@ -8982,8 +9090,8 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
     int conds;
     int logic_cc;
 
-    if (!(arm_feature(env, ARM_FEATURE_THUMB2)
-          || arm_feature (env, ARM_FEATURE_M))) {
+    if (!(arm_dc_feature(s, ARM_FEATURE_THUMB2)
+          || arm_dc_feature(s, ARM_FEATURE_M))) {
         /* Thumb-1 cores may need to treat bl and blx as a pair of
            16-bit instructions to get correct prefetch abort behavior.  */
         insn = insn_hw1;
@@ -9193,7 +9301,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
             /* Load/store multiple, RFE, SRS.  */
             if (((insn >> 23) & 1) == ((insn >> 24) & 1)) {
                 /* RFE, SRS: not available in user mode or on M profile */
-                if (IS_USER(s) || IS_M(env)) {
+                if (IS_USER(s) || arm_dc_feature(s, ARM_FEATURE_M)) {
                     goto illegal_op;
                 }
                 if (insn & (1 << 20)) {
@@ -9345,7 +9453,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
             gen_arm_shift_reg(tmp, op, tmp2, logic_cc);
             if (logic_cc)
                 gen_logic_CC(tmp);
-            store_reg_bx(env, s, rd, tmp);
+            store_reg_bx(s, rd, tmp);
             break;
         case 1: /* Sign/zero extend.  */
             tmp = load_reg(s, rm);
@@ -9436,7 +9544,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                     uint32_t sz = op & 0x3;
                     uint32_t c = op & 0x8;
 
-                    if (!arm_feature(env, ARM_FEATURE_CRC)) {
+                    if (!arm_dc_feature(s, ARM_FEATURE_CRC)) {
                         goto illegal_op;
                     }
 
@@ -9564,7 +9672,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
             tmp2 = load_reg(s, rm);
             if ((op & 0x50) == 0x10) {
                 /* sdiv, udiv */
-                if (!arm_feature(env, ARM_FEATURE_THUMB_DIV)) {
+                if (!arm_dc_feature(s, ARM_FEATURE_THUMB_DIV)) {
                     goto illegal_op;
                 }
                 if (op & 0x20)
@@ -9627,17 +9735,19 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
         if (((insn >> 24) & 3) == 3) {
             /* Translate into the equivalent ARM encoding.  */
             insn = (insn & 0xe2ffffff) | ((insn & (1 << 28)) >> 4) | (1 << 28);
-            if (disas_neon_data_insn(env, s, insn))
+            if (disas_neon_data_insn(s, insn)) {
                 goto illegal_op;
+            }
         } else if (((insn >> 8) & 0xe) == 10) {
-            if (disas_vfp_insn(env, s, insn)) {
+            if (disas_vfp_insn(s, insn)) {
                 goto illegal_op;
             }
         } else {
             if (insn & (1 << 28))
                 goto illegal_op;
-            if (disas_coproc_insn (env, s, insn))
+            if (disas_coproc_insn(s, insn)) {
                 goto illegal_op;
+            }
         }
         break;
     case 8: case 9: case 10: case 11:
@@ -9676,15 +9786,28 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                     goto illegal_op;
 
                 if (insn & (1 << 26)) {
-                    /* Secure monitor call (v6Z) */
-                    qemu_log_mask(LOG_UNIMP,
-                                  "arm: unimplemented secure monitor call\n");
-                    goto illegal_op; /* not implemented.  */
+                    if (!(insn & (1 << 20))) {
+                        /* Hypervisor call (v7) */
+                        int imm16 = extract32(insn, 16, 4) << 12
+                            | extract32(insn, 0, 12);
+                        ARCH(7);
+                        if (IS_USER(s)) {
+                            goto illegal_op;
+                        }
+                        gen_hvc(s, imm16);
+                    } else {
+                        /* Secure monitor call (v6+) */
+                        ARCH(6K);
+                        if (IS_USER(s)) {
+                            goto illegal_op;
+                        }
+                        gen_smc(s);
+                    }
                 } else {
                     op = (insn >> 20) & 7;
                     switch (op) {
                     case 0: /* msr cpsr.  */
-                        if (IS_M(env)) {
+                        if (arm_dc_feature(s, ARM_FEATURE_M)) {
                             tmp = load_reg(s, rn);
                             addr = tcg_const_i32(insn & 0xff);
                             gen_helper_v7m_msr(cpu_env, addr, tmp);
@@ -9695,11 +9818,12 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                         }
                         /* fall through */
                     case 1: /* msr spsr.  */
-                        if (IS_M(env))
+                        if (arm_dc_feature(s, ARM_FEATURE_M)) {
                             goto illegal_op;
+                        }
                         tmp = load_reg(s, rn);
                         if (gen_set_psr(s,
-                              msr_mask(env, s, (insn >> 8) & 0xf, op == 1),
+                              msr_mask(s, (insn >> 8) & 0xf, op == 1),
                               op == 1, tmp))
                             goto illegal_op;
                         break;
@@ -9764,7 +9888,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                         break;
                     case 6: /* mrs cpsr.  */
                         tmp = tcg_temp_new_i32();
-                        if (IS_M(env)) {
+                        if (arm_dc_feature(s, ARM_FEATURE_M)) {
                             addr = tcg_const_i32(insn & 0xff);
                             gen_helper_v7m_mrs(tmp, cpu_env, addr);
                             tcg_temp_free_i32(addr);
@@ -9775,8 +9899,9 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                         break;
                     case 7: /* mrs spsr.  */
                         /* Not accessible in user mode.  */
-                        if (IS_USER(s) || IS_M(env))
+                        if (IS_USER(s) || arm_dc_feature(s, ARM_FEATURE_M)) {
                             goto illegal_op;
+                        }
                         tmp = load_cpu_field(spsr);
                         store_reg(s, rd, tmp);
                         break;
@@ -9964,8 +10089,9 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
         int writeback = 0;
         int memidx;
         if ((insn & 0x01100000) == 0x01000000) {
-            if (disas_neon_ls_insn(env, s, insn))
+            if (disas_neon_ls_insn(s, insn)) {
                 goto illegal_op;
+            }
             break;
         }
         op = ((insn >> 21) & 3) | ((insn >> 22) & 4);
@@ -10661,7 +10787,7 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
             store_reg(s, 13, addr);
             /* set the new PC value */
             if ((insn & 0x0900) == 0x0900) {
-                store_reg_from_load(env, s, 15, tmp);
+                store_reg_from_load(s, 15, tmp);
             }
             break;
 
@@ -10731,7 +10857,7 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
                 if (IS_USER(s)) {
                     break;
                 }
-                if (IS_M(env)) {
+                if (arm_dc_feature(s, ARM_FEATURE_M)) {
                     tmp = tcg_const_i32((insn & (1 << 4)) != 0);
                     /* FAULTMASK */
                     if (insn & 1) {
@@ -10909,10 +11035,31 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
     dc->vfp_enabled = ARM_TBFLAG_VFPEN(tb->flags);
     dc->vec_len = ARM_TBFLAG_VECLEN(tb->flags);
     dc->vec_stride = ARM_TBFLAG_VECSTRIDE(tb->flags);
+    dc->c15_cpar = ARM_TBFLAG_XSCALE_CPAR(tb->flags);
     dc->cp_regs = cpu->cp_regs;
-    dc->current_pl = arm_current_pl(env);
+    dc->current_el = arm_current_el(env);
     dc->features = env->features;
 
+    /* Single step state. The code-generation logic here is:
+     *  SS_ACTIVE == 0:
+     *   generate code with no special handling for single-stepping (except
+     *   that anything that can make us go to SS_ACTIVE == 1 must end the TB;
+     *   this happens anyway because those changes are all system register or
+     *   PSTATE writes).
+     *  SS_ACTIVE == 1, PSTATE.SS == 1: (active-not-pending)
+     *   emit code for one insn
+     *   emit code to clear PSTATE.SS
+     *   emit code to generate software step exception for completed step
+     *   end TB (as usual for having generated an exception)
+     *  SS_ACTIVE == 1, PSTATE.SS == 0: (active-pending)
+     *   emit code to generate a software step exception
+     *   end the TB
+     */
+    dc->ss_active = ARM_TBFLAG_SS_ACTIVE(tb->flags);
+    dc->pstate_ss = ARM_TBFLAG_PSTATE_SS(tb->flags);
+    dc->is_ldex = false;
+    dc->ss_same_el = false; /* Can't be true since EL_d must be AArch64 */
+
     cpu_F0s = tcg_temp_new_i32();
     cpu_F1s = tcg_temp_new_i32();
     cpu_F0d = tcg_temp_new_i64();
@@ -10982,7 +11129,7 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
             break;
         }
 #else
-        if (dc->pc >= 0xfffffff0 && IS_M(env)) {
+        if (dc->pc >= 0xfffffff0 && arm_dc_feature(dc, ARM_FEATURE_M)) {
             /* We always get here via a jump, so know we are not in a
                conditional execution block.  */
             gen_exception_internal(EXCP_EXCEPTION_EXIT);
@@ -11022,6 +11169,22 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
             tcg_gen_debug_insn_start(dc->pc);
         }
 
+        if (dc->ss_active && !dc->pstate_ss) {
+            /* Singlestep state is Active-pending.
+             * If we're in this state at the start of a TB then either
+             *  a) we just took an exception to an EL which is being debugged
+             *     and this is the first insn in the exception handler
+             *  b) debug exceptions were masked and we just unmasked them
+             *     without changing EL (eg by clearing PSTATE.D)
+             * In either case we're going to take a swstep exception in the
+             * "did not step an insn" case, and so the syndrome ISV and EX
+             * bits should be zero.
+             */
+            assert(num_insns == 0);
+            gen_exception(EXCP_UDEF, syn_swstep(dc->ss_same_el, 0, 0));
+            goto done_generating;
+        }
+
         if (dc->thumb) {
             disas_thumb_insn(env, dc);
             if (dc->condexec_mask) {
@@ -11033,7 +11196,9 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
                 }
             }
         } else {
-            disas_arm_insn(env, dc);
+            unsigned int insn = arm_ldl_code(env, dc->pc, dc->bswap_code);
+            dc->pc += 4;
+            disas_arm_insn(dc, insn);
         }
 
         if (dc->condjmp && !dc->is_jmp) {
@@ -11054,6 +11219,7 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
     } while (!dc->is_jmp && tcg_ctx.gen_opc_ptr < gen_opc_end &&
              !cs->singlestep_enabled &&
              !singlestep &&
+             !dc->ss_active &&
              dc->pc < next_page_start &&
              num_insns < max_insns);
 
@@ -11069,12 +11235,21 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
     /* At this stage dc->condjmp will only be set when the skipped
        instruction was a conditional branch or trap, and the PC has
        already been written.  */
-    if (unlikely(cs->singlestep_enabled)) {
+    if (unlikely(cs->singlestep_enabled || dc->ss_active)) {
         /* Make sure the pc is updated, and raise a debug exception.  */
         if (dc->condjmp) {
             gen_set_condexec(dc);
             if (dc->is_jmp == DISAS_SWI) {
+                gen_ss_advance(dc);
                 gen_exception(EXCP_SWI, syn_aa32_svc(dc->svc_imm, dc->thumb));
+            } else if (dc->is_jmp == DISAS_HVC) {
+                gen_ss_advance(dc);
+                gen_exception(EXCP_HVC, syn_aa32_hvc(dc->svc_imm));
+            } else if (dc->is_jmp == DISAS_SMC) {
+                gen_ss_advance(dc);
+                gen_exception(EXCP_SMC, syn_aa32_smc());
+            } else if (dc->ss_active) {
+                gen_step_complete_exception(dc);
             } else {
                 gen_exception_internal(EXCP_DEBUG);
             }
@@ -11086,7 +11261,16 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
         }
         gen_set_condexec(dc);
         if (dc->is_jmp == DISAS_SWI && !dc->condjmp) {
+            gen_ss_advance(dc);
             gen_exception(EXCP_SWI, syn_aa32_svc(dc->svc_imm, dc->thumb));
+        } else if (dc->is_jmp == DISAS_HVC && !dc->condjmp) {
+            gen_ss_advance(dc);
+            gen_exception(EXCP_HVC, syn_aa32_hvc(dc->svc_imm));
+        } else if (dc->is_jmp == DISAS_SMC && !dc->condjmp) {
+            gen_ss_advance(dc);
+            gen_exception(EXCP_SMC, syn_aa32_smc());
+        } else if (dc->ss_active) {
+            gen_step_complete_exception(dc);
         } else {
             /* FIXME: Single stepping a WFI insn will not halt
                the CPU.  */
@@ -11124,6 +11308,12 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
         case DISAS_SWI:
             gen_exception(EXCP_SWI, syn_aa32_svc(dc->svc_imm, dc->thumb));
             break;
+        case DISAS_HVC:
+            gen_exception(EXCP_HVC, syn_aa32_hvc(dc->svc_imm));
+            break;
+        case DISAS_SMC:
+            gen_exception(EXCP_SMC, syn_aa32_smc());
+            break;
         }
         if (dc->condjmp) {
             gen_set_label(dc->condlabel);
diff --git a/target-arm/translate.h b/target-arm/translate.h
index 31a0104b5..41a907157 100644
--- a/target-arm/translate.h
+++ b/target-arm/translate.h
@@ -29,7 +29,7 @@ typedef struct DisasContext {
      */
     uint32_t svc_imm;
     int aarch64;
-    int current_pl;
+    int current_el;
     GHashTable *cp_regs;
     uint64_t features; /* CPU features bits */
     /* Because unallocated encodings generate different exception syndrome
@@ -40,6 +40,20 @@ typedef struct DisasContext {
      * that it is set at the point where we actually touch the FP regs.
      */
     bool fp_access_checked;
+    /* ARMv8 single-step state (this is distinct from the QEMU gdbstub
+     * single-step support).
+     */
+    bool ss_active;
+    bool pstate_ss;
+    /* True if the insn just emitted was a load-exclusive instruction
+     * (necessary for syndrome information for single step exceptions),
+     * ie A64 LDX*, LDAX*, A32/T32 LDREX*, LDAEX*.
+     */
+    bool is_ldex;
+    /* True if a single-step exception will be taken to the current EL */
+    bool ss_same_el;
+    /* Bottom two bits of XScale c15_cpar coprocessor access control reg */
+    int c15_cpar;
 #define TMP_A64_MAX 16
     int tmp_a64_count;
     TCGv_i64 tmp_a64[TMP_A64_MAX];
@@ -54,7 +68,7 @@ static inline int arm_dc_feature(DisasContext *dc, int feature)
 
 static inline int get_mem_index(DisasContext *s)
 {
-    return s->current_pl;
+    return s->current_el;
 }
 
 /* target-specific extra values for is_jmp */
@@ -70,6 +84,8 @@ static inline int get_mem_index(DisasContext *s)
 #define DISAS_EXC 6
 /* WFE */
 #define DISAS_WFE 7
+#define DISAS_HVC 8
+#define DISAS_SMC 9
 
 #ifdef TARGET_AARCH64
 void a64_translate_init(void);
diff --git a/target-cris/cpu-qom.h b/target-cris/cpu-qom.h
index 75593667d..6fc30c208 100644
--- a/target-cris/cpu-qom.h
+++ b/target-cris/cpu-qom.h
@@ -75,6 +75,7 @@ static inline CRISCPU *cris_env_get_cpu(CPUCRISState *env)
 
 void cris_cpu_do_interrupt(CPUState *cpu);
 void crisv10_cpu_do_interrupt(CPUState *cpu);
+bool cris_cpu_exec_interrupt(CPUState *cpu, int int_req);
 
 void cris_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                          int flags);
diff --git a/target-cris/cpu.c b/target-cris/cpu.c
index 20d880969..16cfba95f 100644
--- a/target-cris/cpu.c
+++ b/target-cris/cpu.c
@@ -279,6 +279,7 @@ static void cris_cpu_class_init(ObjectClass *oc, void *data)
     cc->class_by_name = cris_cpu_class_by_name;
     cc->has_work = cris_cpu_has_work;
     cc->do_interrupt = cris_cpu_do_interrupt;
+    cc->cpu_exec_interrupt = cris_cpu_exec_interrupt;
     cc->dump_state = cris_cpu_dump_state;
     cc->set_pc = cris_cpu_set_pc;
     cc->gdb_read_register = cris_cpu_gdb_read_register;
@@ -290,6 +291,7 @@ static void cris_cpu_class_init(ObjectClass *oc, void *data)
 #endif
 
     cc->gdb_num_core_regs = 49;
+    cc->gdb_stop_before_watchpoint = true;
 }
 
 static const TypeInfo cris_cpu_type_info = {
diff --git a/target-cris/helper.c b/target-cris/helper.c
index e8b8261fe..e901c3a00 100644
--- a/target-cris/helper.c
+++ b/target-cris/helper.c
@@ -283,3 +283,34 @@ hwaddr cris_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
     return phy;
 }
 #endif
+
+bool cris_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    CPUClass *cc = CPU_GET_CLASS(cs);
+    CRISCPU *cpu = CRIS_CPU(cs);
+    CPUCRISState *env = &cpu->env;
+    bool ret = false;
+
+    if (interrupt_request & CPU_INTERRUPT_HARD
+        && (env->pregs[PR_CCS] & I_FLAG)
+        && !env->locked_irq) {
+        cs->exception_index = EXCP_IRQ;
+        cc->do_interrupt(cs);
+        ret = true;
+    }
+    if (interrupt_request & CPU_INTERRUPT_NMI) {
+        unsigned int m_flag_archval;
+        if (env->pregs[PR_VR] < 32) {
+            m_flag_archval = M_FLAG_V10;
+        } else {
+            m_flag_archval = M_FLAG_V32;
+        }
+        if ((env->pregs[PR_CCS] & m_flag_archval)) {
+            cs->exception_index = EXCP_NMI;
+            cc->do_interrupt(cs);
+            ret = true;
+        }
+    }
+
+    return ret;
+}
diff --git a/target-cris/translate.c b/target-cris/translate.c
index ab0e47962..76406af98 100644
--- a/target-cris/translate.c
+++ b/target-cris/translate.c
@@ -33,6 +33,9 @@
 
 #include "exec/helper-gen.h"
 
+#include "trace-tcg.h"
+
+
 #define DISAS_CRIS 0
 #if DISAS_CRIS
 #  define LOG_DIS(...) qemu_log_mask(CPU_LOG_TB_IN_ASM, ## __VA_ARGS__)
@@ -166,9 +169,7 @@ static int preg_sizes[] = {
 
 static inline void t_gen_mov_TN_preg(TCGv tn, int r)
 {
-    if (r < 0 || r > 15) {
-        fprintf(stderr, "wrong register read $p%d\n", r);
-    }
+    assert(r >= 0 && r <= 15);
     if (r == PR_BZ || r == PR_WZ || r == PR_DZ) {
         tcg_gen_mov_tl(tn, tcg_const_tl(0));
     } else if (r == PR_VR) {
@@ -179,9 +180,7 @@ static inline void t_gen_mov_TN_preg(TCGv tn, int r)
 }
 static inline void t_gen_mov_preg_TN(DisasContext *dc, int r, TCGv tn)
 {
-    if (r < 0 || r > 15) {
-        fprintf(stderr, "wrong register write $p%d\n", r);
-    }
+    assert(r >= 0 && r <= 15);
     if (r == PR_BZ || r == PR_WZ || r == PR_DZ) {
         return;
     } else if (r == PR_SRS) {
diff --git a/target-i386/cpu-qom.h b/target-i386/cpu-qom.h
index 71a1b97cf..b557b619c 100644
--- a/target-i386/cpu-qom.h
+++ b/target-i386/cpu-qom.h
@@ -92,6 +92,7 @@ typedef struct X86CPU {
     bool enforce_cpuid;
     bool expose_kvm;
     bool migratable;
+    bool host_features;
 
     /* if true the CPUID code directly forward host cache leaves to the guest */
     bool cache_info_passthrough;
@@ -129,6 +130,7 @@ extern struct VMStateDescription vmstate_x86_cpu;
  * @cpu: vCPU the interrupt is to be handled by.
  */
 void x86_cpu_do_interrupt(CPUState *cpu);
+bool x86_cpu_exec_interrupt(CPUState *cpu, int int_req);
 
 int x86_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cpu,
                              int cpuid, void *opaque);
@@ -150,4 +152,7 @@ hwaddr x86_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
 int x86_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
 int x86_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
 
+void x86_cpu_exec_enter(CPUState *cpu);
+void x86_cpu_exec_exit(CPUState *cpu);
+
 #endif
diff --git a/target-i386/cpu.c b/target-i386/cpu.c
index 6d008ab5e..e9df33e5c 100644
--- a/target-i386/cpu.c
+++ b/target-i386/cpu.c
@@ -257,10 +257,10 @@ static const char *svm_feature_name[] = {
 };
 
 static const char *cpuid_7_0_ebx_feature_name[] = {
-    "fsgsbase", NULL, NULL, "bmi1", "hle", "avx2", NULL, "smep",
-    "bmi2", "erms", "invpcid", "rtm", NULL, NULL, NULL, NULL,
-    NULL, NULL, "rdseed", "adx", "smap", NULL, NULL, NULL,
-    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+    "fsgsbase", "tsc_adjust", NULL, "bmi1", "hle", "avx2", NULL, "smep",
+    "bmi2", "erms", "invpcid", "rtm", NULL, NULL, "mpx", NULL,
+    "avx512f", NULL, "rdseed", "adx", "smap", NULL, NULL, NULL,
+    NULL, NULL, "avx512pf", "avx512er", "avx512cd", NULL, NULL, NULL,
 };
 
 static const char *cpuid_apm_edx_feature_name[] = {
@@ -426,6 +426,12 @@ static const ExtSaveArea ext_save_areas[] = {
             .offset = 0x3c0, .size = 0x40  },
     [4] = { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_MPX,
             .offset = 0x400, .size = 0x40  },
+    [5] = { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_AVX512F,
+            .offset = 0x440, .size = 0x40 },
+    [6] = { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_AVX512F,
+            .offset = 0x480, .size = 0x200 },
+    [7] = { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_AVX512F,
+            .offset = 0x680, .size = 0x400 },
 };
 
 const char *get_register_name_32(unsigned int reg)
@@ -436,14 +442,6 @@ const char *get_register_name_32(unsigned int reg)
     return x86_reg_info_32[reg].name;
 }
 
-/* collects per-function cpuid data
- */
-typedef struct model_features_t {
-    uint32_t *guest_feat;
-    uint32_t *host_feat;
-    FeatureWord feat_word;
-} model_features_t;
-
 /* KVM-specific features that are automatically added to all CPU models
  * when KVM is enabled.
  */
@@ -461,14 +459,21 @@ static uint32_t kvm_default_features[FEATURE_WORDS] = {
 /* Features that are not added by default to any CPU model when KVM is enabled.
  */
 static uint32_t kvm_default_unset_features[FEATURE_WORDS] = {
+    [FEAT_1_EDX] = CPUID_ACPI,
     [FEAT_1_ECX] = CPUID_EXT_MONITOR,
+    [FEAT_8000_0001_ECX] = CPUID_EXT3_SVM,
 };
 
-void x86_cpu_compat_disable_kvm_features(FeatureWord w, uint32_t features)
+void x86_cpu_compat_kvm_no_autoenable(FeatureWord w, uint32_t features)
 {
     kvm_default_features[w] &= ~features;
 }
 
+void x86_cpu_compat_kvm_no_autodisable(FeatureWord w, uint32_t features)
+{
+    kvm_default_unset_features[w] &= ~features;
+}
+
 /*
  * Returns the set of feature flags that are supported and migratable by
  * QEMU, for a given FeatureWord.
@@ -535,8 +540,8 @@ void host_cpuid(uint32_t function, uint32_t count,
  * otherwise the string is assumed to sized by a terminating nul.
  * Return lexical ordering of *s1:*s2.
  */
-static int sstrcmp(const char *s1, const char *e1, const char *s2,
-    const char *e2)
+static int sstrcmp(const char *s1, const char *e1,
+                   const char *s2, const char *e2)
 {
     for (;;) {
         if (!*s1 || !*s2 || *s1 != *s2)
@@ -592,7 +597,8 @@ static bool lookup_feature(uint32_t *pval, const char *s, const char *e,
 }
 
 static void add_flagname_to_bitmaps(const char *flagname,
-                                    FeatureWordArray words)
+                                    FeatureWordArray words,
+                                    Error **errp)
 {
     FeatureWord w;
     for (w = 0; w < FEATURE_WORDS; w++) {
@@ -603,7 +609,7 @@ static void add_flagname_to_bitmaps(const char *flagname,
         }
     }
     if (w == FEATURE_WORDS) {
-        fprintf(stderr, "CPU feature %s not found\n", flagname);
+        error_setg(errp, "CPU feature %s not found", flagname);
     }
 }
 
@@ -679,10 +685,11 @@ static X86CPUDefinition builtin_x86_defs[] = {
         .family = 16,
         .model = 2,
         .stepping = 3,
+        /* Missing: CPUID_HT */
         .features[FEAT_1_EDX] =
             PPRO_FEATURES |
             CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA |
-            CPUID_PSE36 | CPUID_VME | CPUID_HT,
+            CPUID_PSE36 | CPUID_VME,
         .features[FEAT_1_ECX] =
             CPUID_EXT_SSE3 | CPUID_EXT_MONITOR | CPUID_EXT_CX16 |
             CPUID_EXT_POPCNT,
@@ -698,8 +705,9 @@ static X86CPUDefinition builtin_x86_defs[] = {
         .features[FEAT_8000_0001_ECX] =
             CPUID_EXT3_LAHF_LM | CPUID_EXT3_SVM |
             CPUID_EXT3_ABM | CPUID_EXT3_SSE4A,
+        /* Missing: CPUID_SVM_LBRV */
         .features[FEAT_SVM] =
-            CPUID_SVM_NPT | CPUID_SVM_LBRV,
+            CPUID_SVM_NPT,
         .xlevel = 0x8000001A,
         .model_id = "AMD Phenom(tm) 9550 Quad-Core Processor"
     },
@@ -710,15 +718,16 @@ static X86CPUDefinition builtin_x86_defs[] = {
         .family = 6,
         .model = 15,
         .stepping = 11,
+        /* Missing: CPUID_DTS, CPUID_HT, CPUID_TM, CPUID_PBE */
         .features[FEAT_1_EDX] =
             PPRO_FEATURES |
             CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA |
-            CPUID_PSE36 | CPUID_VME | CPUID_DTS | CPUID_ACPI | CPUID_SS |
-            CPUID_HT | CPUID_TM | CPUID_PBE,
+            CPUID_PSE36 | CPUID_VME | CPUID_ACPI | CPUID_SS,
+        /* Missing: CPUID_EXT_DTES64, CPUID_EXT_DSCPL, CPUID_EXT_EST,
+         * CPUID_EXT_TM2, CPUID_EXT_XTPR, CPUID_EXT_PDCM, CPUID_EXT_VMX */
         .features[FEAT_1_ECX] =
             CPUID_EXT_SSE3 | CPUID_EXT_MONITOR | CPUID_EXT_SSSE3 |
-            CPUID_EXT_DTES64 | CPUID_EXT_DSCPL | CPUID_EXT_VMX | CPUID_EXT_EST |
-            CPUID_EXT_TM2 | CPUID_EXT_CX16 | CPUID_EXT_XTPR | CPUID_EXT_PDCM,
+            CPUID_EXT_CX16,
         .features[FEAT_8000_0001_EDX] =
             CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX,
         .features[FEAT_8000_0001_ECX] =
@@ -793,13 +802,15 @@ static X86CPUDefinition builtin_x86_defs[] = {
         .family = 6,
         .model = 14,
         .stepping = 8,
+        /* Missing: CPUID_DTS, CPUID_HT, CPUID_TM, CPUID_PBE */
         .features[FEAT_1_EDX] =
             PPRO_FEATURES | CPUID_VME |
-            CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA | CPUID_DTS | CPUID_ACPI |
-            CPUID_SS | CPUID_HT | CPUID_TM | CPUID_PBE,
+            CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA | CPUID_ACPI |
+            CPUID_SS,
+        /* Missing: CPUID_EXT_EST, CPUID_EXT_TM2 , CPUID_EXT_XTPR,
+         * CPUID_EXT_PDCM, CPUID_EXT_VMX */
         .features[FEAT_1_ECX] =
-            CPUID_EXT_SSE3 | CPUID_EXT_MONITOR | CPUID_EXT_VMX |
-            CPUID_EXT_EST | CPUID_EXT_TM2 | CPUID_EXT_XTPR | CPUID_EXT_PDCM,
+            CPUID_EXT_SSE3 | CPUID_EXT_MONITOR,
         .features[FEAT_8000_0001_EDX] =
             CPUID_EXT2_NX,
         .xlevel = 0x80000008,
@@ -872,14 +883,16 @@ static X86CPUDefinition builtin_x86_defs[] = {
         .family = 6,
         .model = 28,
         .stepping = 2,
+        /* Missing: CPUID_DTS, CPUID_HT, CPUID_TM, CPUID_PBE */
         .features[FEAT_1_EDX] =
             PPRO_FEATURES |
-            CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA | CPUID_VME | CPUID_DTS |
-            CPUID_ACPI | CPUID_SS | CPUID_HT | CPUID_TM | CPUID_PBE,
+            CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA | CPUID_VME |
+            CPUID_ACPI | CPUID_SS,
             /* Some CPUs got no CPUID_SEP */
+        /* Missing: CPUID_EXT_DSCPL, CPUID_EXT_EST, CPUID_EXT_TM2,
+         * CPUID_EXT_XTPR */
         .features[FEAT_1_ECX] =
             CPUID_EXT_SSE3 | CPUID_EXT_MONITOR | CPUID_EXT_SSSE3 |
-            CPUID_EXT_DSCPL | CPUID_EXT_EST | CPUID_EXT_TM2 | CPUID_EXT_XTPR |
             CPUID_EXT_MOVBE,
         .features[FEAT_8000_0001_EDX] =
             (PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES) |
@@ -1254,6 +1267,9 @@ void x86_cpu_compat_set_features(const char *cpu_model, FeatureWord w,
     }
 }
 
+static uint32_t x86_cpu_get_supported_feature_word(FeatureWord w,
+                                                   bool migratable_only);
+
 #ifdef CONFIG_KVM
 
 static int cpu_x86_fill_model_id(char *str)
@@ -1310,26 +1326,23 @@ static void host_x86_cpu_class_init(ObjectClass *oc, void *data)
     dc->props = host_x86_cpu_properties;
 }
 
-static uint32_t x86_cpu_get_supported_feature_word(FeatureWord w,
-                                                   bool migratable_only);
-
 static void host_x86_cpu_initfn(Object *obj)
 {
     X86CPU *cpu = X86_CPU(obj);
     CPUX86State *env = &cpu->env;
     KVMState *s = kvm_state;
-    FeatureWord w;
 
     assert(kvm_enabled());
 
+    /* We can't fill the features array here because we don't know yet if
+     * "migratable" is true or false.
+     */
+    cpu->host_features = true;
+
     env->cpuid_level = kvm_arch_get_supported_cpuid(s, 0x0, 0, R_EAX);
     env->cpuid_xlevel = kvm_arch_get_supported_cpuid(s, 0x80000000, 0, R_EAX);
     env->cpuid_xlevel2 = kvm_arch_get_supported_cpuid(s, 0xC0000000, 0, R_EAX);
 
-    for (w = 0; w < FEATURE_WORDS; w++) {
-        env->features[w] =
-            x86_cpu_get_supported_feature_word(w, cpu->migratable);
-    }
     object_property_set_bool(OBJECT(cpu), true, "pmu", &error_abort);
 }
 
@@ -1716,9 +1729,9 @@ static void x86_set_hv_spinlocks(Object *obj, Visitor *v, void *opaque,
 
     if (value < min || value > max) {
         error_setg(errp, "Property %s.%s doesn't take value %" PRId64
-                  " (minimum: %" PRId64 ", maximum: %" PRId64 ")",
-                  object_get_typename(obj), name ? name : "null",
-                  value, min, max);
+                   " (minimum: %" PRId64 ", maximum: %" PRId64 ")",
+                   object_get_typename(obj), name ? name : "null",
+                   value, min, max);
         return;
     }
     cpu->hyperv_spinlock_attempts = value;
@@ -1761,9 +1774,9 @@ static void x86_cpu_parse_featurestr(CPUState *cs, char *features,
     while (featurestr) {
         char *val;
         if (featurestr[0] == '+') {
-            add_flagname_to_bitmaps(featurestr + 1, plus_features);
+            add_flagname_to_bitmaps(featurestr + 1, plus_features, &local_err);
         } else if (featurestr[0] == '-') {
-            add_flagname_to_bitmaps(featurestr + 1, minus_features);
+            add_flagname_to_bitmaps(featurestr + 1, minus_features, &local_err);
         } else if ((val = strchr(featurestr, '='))) {
             *val = 0; val++;
             feat2prop(featurestr);
@@ -1808,8 +1821,8 @@ static void x86_cpu_parse_featurestr(CPUState *cs, char *features,
                 }
                 if (numvalue < min) {
                     error_report("hv-spinlocks value shall always be >= 0x%x"
-                            ", fixup will be removed in future versions",
-                            min);
+                                 ", fixup will be removed in future versions",
+                                 min);
                     numvalue = min;
                 }
                 snprintf(num, sizeof(num), "%" PRId32, numvalue);
@@ -1828,6 +1841,13 @@ static void x86_cpu_parse_featurestr(CPUState *cs, char *features,
         featurestr = strtok(NULL, ",");
     }
 
+    if (cpu->host_features) {
+        for (w = 0; w < FEATURE_WORDS; w++) {
+            env->features[w] =
+                x86_cpu_get_supported_feature_word(w, cpu->migratable);
+        }
+    }
+
     for (w = 0; w < FEATURE_WORDS; w++) {
         env->features[w] |= plus_features[w];
         env->features[w] &= ~minus_features[w];
@@ -1839,7 +1859,7 @@ static void x86_cpu_parse_featurestr(CPUState *cs, char *features,
  * if flags, suppress names undefined in featureset.
  */
 static void listflags(char *buf, int bufsize, uint32_t fbits,
-    const char **featureset, uint32_t flags)
+                      const char **featureset, uint32_t flags)
 {
     const char **p = &featureset[31];
     char *q, *b, bit;
@@ -2572,7 +2592,7 @@ static void x86_cpu_reset(CPUState *s)
     for (i = 0; i < 8; i++) {
         env->fptags[i] = 1;
     }
-    env->fpuc = 0x37f;
+    cpu_set_fpuc(env, 0x37f);
 
     env->mxcsr = 0x1f80;
     env->xstate_bv = XSTATE_FP | XSTATE_SSE;
@@ -2588,6 +2608,16 @@ static void x86_cpu_reset(CPUState *s)
 
     env->xcr0 = 1;
 
+    /*
+     * SDM 11.11.5 requires:
+     *  - IA32_MTRR_DEF_TYPE MSR.E = 0
+     *  - IA32_MTRR_PHYSMASKn.V = 0
+     * All other bits are undefined.  For simplification, zero it all.
+     */
+    env->mtrr_deftype = 0;
+    memset(env->mtrr_var, 0, sizeof(env->mtrr_var));
+    memset(env->mtrr_fixed, 0, sizeof(env->mtrr_fixed));
+
 #if !defined(CONFIG_USER_ONLY)
     /* We hard-wire the BSP to the first CPU. */
     if (s->cpu_index == 0) {
@@ -2678,6 +2708,13 @@ static void x86_cpu_apic_realize(X86CPU *cpu, Error **errp)
 }
 #endif
 
+
+#define IS_INTEL_CPU(env) ((env)->cpuid_vendor1 == CPUID_VENDOR_INTEL_1 && \
+                           (env)->cpuid_vendor2 == CPUID_VENDOR_INTEL_2 && \
+                           (env)->cpuid_vendor3 == CPUID_VENDOR_INTEL_3)
+#define IS_AMD_CPU(env) ((env)->cpuid_vendor1 == CPUID_VENDOR_AMD_1 && \
+                         (env)->cpuid_vendor2 == CPUID_VENDOR_AMD_2 && \
+                         (env)->cpuid_vendor3 == CPUID_VENDOR_AMD_3)
 static void x86_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
@@ -2685,6 +2722,7 @@ static void x86_cpu_realizefn(DeviceState *dev, Error **errp)
     X86CPUClass *xcc = X86_CPU_GET_CLASS(dev);
     CPUX86State *env = &cpu->env;
     Error *local_err = NULL;
+    static bool ht_warned;
 
     if (env->features[FEAT_7_0_EBX] && env->cpuid_level < 7) {
         env->cpuid_level = 7;
@@ -2693,9 +2731,7 @@ static void x86_cpu_realizefn(DeviceState *dev, Error **errp)
     /* On AMD CPUs, some CPUID[8000_0001].EDX bits must match the bits on
      * CPUID[1].EDX.
      */
-    if (env->cpuid_vendor1 == CPUID_VENDOR_AMD_1 &&
-        env->cpuid_vendor2 == CPUID_VENDOR_AMD_2 &&
-        env->cpuid_vendor3 == CPUID_VENDOR_AMD_3) {
+    if (IS_AMD_CPU(env)) {
         env->features[FEAT_8000_0001_EDX] &= ~CPUID_EXT2_AMD_ALIASES;
         env->features[FEAT_8000_0001_EDX] |= (env->features[FEAT_1_EDX]
            & CPUID_EXT2_AMD_ALIASES);
@@ -2724,6 +2760,20 @@ static void x86_cpu_realizefn(DeviceState *dev, Error **errp)
     mce_init(cpu);
     qemu_init_vcpu(cs);
 
+    /* Only Intel CPUs support hyperthreading. Even though QEMU fixes this
+     * issue by adjusting CPUID_0000_0001_EBX and CPUID_8000_0008_ECX
+     * based on inputs (sockets,cores,threads), it is still better to gives
+     * users a warning.
+     *
+     * NOTE: the following code has to follow qemu_init_vcpu(). Otherwise
+     * cs->nr_threads hasn't be populated yet and the checking is incorrect.
+     */
+    if (!IS_INTEL_CPU(env) && cs->nr_threads > 1 && !ht_warned) {
+        error_report("AMD CPU doesn't support hyperthreading. Please configure"
+                     " -smp options properly.");
+        ht_warned = true;
+    }
+
     x86_cpu_apic_realize(cpu, &local_err);
     if (local_err != NULL) {
         goto out;
@@ -2825,9 +2875,6 @@ static void x86_cpu_initfn(Object *obj)
     if (tcg_enabled() && !inited) {
         inited = 1;
         optimize_flags_init();
-#ifndef CONFIG_USER_ONLY
-        cpu_set_debug_excp_handler(breakpoint_handler);
-#endif
     }
 }
 
@@ -2865,8 +2912,14 @@ static bool x86_cpu_has_work(CPUState *cs)
     X86CPU *cpu = X86_CPU(cs);
     CPUX86State *env = &cpu->env;
 
-    return ((cs->interrupt_request & (CPU_INTERRUPT_HARD |
-                                      CPU_INTERRUPT_POLL)) &&
+#if !defined(CONFIG_USER_ONLY)
+    if (cs->interrupt_request & CPU_INTERRUPT_POLL) {
+        apic_poll_irq(cpu->apic_state);
+        cpu_reset_interrupt(cs, CPU_INTERRUPT_POLL);
+    }
+#endif
+
+    return ((cs->interrupt_request & CPU_INTERRUPT_HARD) &&
             (env->eflags & IF_MASK)) ||
            (cs->interrupt_request & (CPU_INTERRUPT_NMI |
                                      CPU_INTERRUPT_INIT |
@@ -2905,6 +2958,7 @@ static void x86_cpu_common_class_init(ObjectClass *oc, void *data)
     cc->parse_features = x86_cpu_parse_featurestr;
     cc->has_work = x86_cpu_has_work;
     cc->do_interrupt = x86_cpu_do_interrupt;
+    cc->cpu_exec_interrupt = x86_cpu_exec_interrupt;
     cc->dump_state = x86_cpu_dump_state;
     cc->set_pc = x86_cpu_set_pc;
     cc->synchronize_from_tb = x86_cpu_synchronize_from_tb;
@@ -2924,6 +2978,11 @@ static void x86_cpu_common_class_init(ObjectClass *oc, void *data)
     cc->vmsd = &vmstate_x86_cpu;
 #endif
     cc->gdb_num_core_regs = CPU_NB_REGS * 2 + 25;
+#ifndef CONFIG_USER_ONLY
+    cc->debug_excp_handler = breakpoint_handler;
+#endif
+    cc->cpu_exec_enter = x86_cpu_exec_enter;
+    cc->cpu_exec_exit = x86_cpu_exec_exit;
 }
 
 static const TypeInfo x86_cpu_type_info = {
diff --git a/target-i386/cpu.h b/target-i386/cpu.h
index e634d83e8..015f5b527 100644
--- a/target-i386/cpu.h
+++ b/target-i386/cpu.h
@@ -337,6 +337,8 @@
 #define MSR_MTRRphysBase(reg)           (0x200 + 2 * (reg))
 #define MSR_MTRRphysMask(reg)           (0x200 + 2 * (reg) + 1)
 
+#define MSR_MTRRphysIndex(addr)         ((((addr) & ~1u) - 0x200) / 2)
+
 #define MSR_MTRRfix64K_00000            0x250
 #define MSR_MTRRfix16K_80000            0x258
 #define MSR_MTRRfix16K_A0000            0x259
@@ -393,6 +395,9 @@
 #define XSTATE_YMM                      (1ULL << 2)
 #define XSTATE_BNDREGS                  (1ULL << 3)
 #define XSTATE_BNDCSR                   (1ULL << 4)
+#define XSTATE_OPMASK                   (1ULL << 5)
+#define XSTATE_ZMM_Hi256                (1ULL << 6)
+#define XSTATE_Hi16_ZMM                 (1ULL << 7)
 
 
 /* CPUID feature words */
@@ -558,9 +563,13 @@ typedef uint32_t FeatureWordArray[FEATURE_WORDS];
 #define CPUID_7_0_EBX_INVPCID  (1U << 10)
 #define CPUID_7_0_EBX_RTM      (1U << 11)
 #define CPUID_7_0_EBX_MPX      (1U << 14)
+#define CPUID_7_0_EBX_AVX512F  (1U << 16) /* AVX-512 Foundation */
 #define CPUID_7_0_EBX_RDSEED   (1U << 18)
 #define CPUID_7_0_EBX_ADX      (1U << 19)
 #define CPUID_7_0_EBX_SMAP     (1U << 20)
+#define CPUID_7_0_EBX_AVX512PF (1U << 26) /* AVX-512 Prefetch */
+#define CPUID_7_0_EBX_AVX512ER (1U << 27) /* AVX-512 Exponential and Reciprocal */
+#define CPUID_7_0_EBX_AVX512CD (1U << 28) /* AVX-512 Conflict Detection */
 
 /* CPUID[0x80000007].EDX flags: */
 #define CPUID_APM_INVTSC       (1U << 8)
@@ -705,6 +714,24 @@ typedef union {
 } XMMReg;
 
 typedef union {
+    uint8_t _b[32];
+    uint16_t _w[16];
+    uint32_t _l[8];
+    uint64_t _q[4];
+    float32 _s[8];
+    float64 _d[4];
+} YMMReg;
+
+typedef union {
+    uint8_t _b[64];
+    uint16_t _w[32];
+    uint32_t _l[16];
+    uint64_t _q[8];
+    float32 _s[16];
+    float64 _d[8];
+} ZMMReg;
+
+typedef union {
     uint8_t _b[8];
     uint16_t _w[4];
     uint32_t _l[2];
@@ -723,6 +750,20 @@ typedef struct BNDCSReg {
 } BNDCSReg;
 
 #ifdef HOST_WORDS_BIGENDIAN
+#define ZMM_B(n) _b[63 - (n)]
+#define ZMM_W(n) _w[31 - (n)]
+#define ZMM_L(n) _l[15 - (n)]
+#define ZMM_S(n) _s[15 - (n)]
+#define ZMM_Q(n) _q[7 - (n)]
+#define ZMM_D(n) _d[7 - (n)]
+
+#define YMM_B(n) _b[31 - (n)]
+#define YMM_W(n) _w[15 - (n)]
+#define YMM_L(n) _l[7 - (n)]
+#define YMM_S(n) _s[7 - (n)]
+#define YMM_Q(n) _q[3 - (n)]
+#define YMM_D(n) _d[3 - (n)]
+
 #define XMM_B(n) _b[15 - (n)]
 #define XMM_W(n) _w[7 - (n)]
 #define XMM_L(n) _l[3 - (n)]
@@ -735,6 +776,20 @@ typedef struct BNDCSReg {
 #define MMX_L(n) _l[1 - (n)]
 #define MMX_S(n) _s[1 - (n)]
 #else
+#define ZMM_B(n) _b[n]
+#define ZMM_W(n) _w[n]
+#define ZMM_L(n) _l[n]
+#define ZMM_S(n) _s[n]
+#define ZMM_Q(n) _q[n]
+#define ZMM_D(n) _d[n]
+
+#define YMM_B(n) _b[n]
+#define YMM_W(n) _w[n]
+#define YMM_L(n) _l[n]
+#define YMM_S(n) _s[n]
+#define YMM_Q(n) _q[n]
+#define YMM_D(n) _d[n]
+
 #define XMM_B(n) _b[n]
 #define XMM_W(n) _w[n]
 #define XMM_L(n) _l[n]
@@ -773,6 +828,8 @@ typedef struct {
 
 #define NB_MMU_MODES 3
 
+#define NB_OPMASK_REGS 8
+
 typedef enum TPRAccess {
     TPR_ACCESS_READ,
     TPR_ACCESS_WRITE,
@@ -837,6 +894,12 @@ typedef struct CPUX86State {
 
     XMMReg ymmh_regs[CPU_NB_REGS];
 
+    uint64_t opmask_regs[NB_OPMASK_REGS];
+    YMMReg zmmh_regs[CPU_NB_REGS];
+#ifdef TARGET_X86_64
+    ZMMReg hi16_zmm_regs[CPU_NB_REGS];
+#endif
+
     /* sysenter registers */
     uint32_t sysenter_cs;
     target_ulong sysenter_esp;
@@ -930,7 +993,7 @@ typedef struct CPUX86State {
     /* MTRRs */
     uint64_t mtrr_fixed[11];
     uint64_t mtrr_deftype;
-    MTRRVar mtrr_var[8];
+    MTRRVar mtrr_var[MSR_MTRRcap_VCNT];
 
     /* For KVM */
     uint32_t mp_state;
@@ -1041,7 +1104,7 @@ static inline void cpu_x86_load_seg_cache(CPUX86State *env,
 }
 
 static inline void cpu_x86_load_seg_cache_sipi(X86CPU *cpu,
-                                               int sipi_vector)
+                                               uint8_t sipi_vector)
 {
     CPUState *cs = CPU(cpu);
     CPUX86State *env = &cpu->env;
@@ -1119,7 +1182,7 @@ static inline int hw_breakpoint_len(unsigned long dr7, int index)
 void hw_breakpoint_insert(CPUX86State *env, int index);
 void hw_breakpoint_remove(CPUX86State *env, int index);
 bool check_hw_breakpoints(CPUX86State *env, bool force_dr6_update);
-void breakpoint_handler(CPUX86State *env);
+void breakpoint_handler(CPUState *cs);
 
 /* will be suppressed */
 void cpu_x86_update_cr0(CPUX86State *env, uint32_t new_cr0);
@@ -1249,6 +1312,7 @@ void QEMU_NORETURN raise_interrupt(CPUX86State *nenv, int intno, int is_int,
 /* cc_helper.c */
 extern const uint8_t parity_table[256];
 uint32_t cpu_cc_compute_all(CPUX86State *env1, int op);
+void update_fp_status(CPUX86State *env);
 
 static inline uint32_t cpu_compute_eflags(CPUX86State *env)
 {
@@ -1284,6 +1348,7 @@ static inline void cpu_load_efer(CPUX86State *env, uint64_t val)
 
 /* fpu_helper.c */
 void cpu_set_mxcsr(CPUX86State *env, uint32_t val);
+void cpu_set_fpuc(CPUX86State *env, uint16_t val);
 
 /* svm_helper.c */
 void cpu_svm_check_intercept_param(CPUX86State *env1, uint32_t type,
@@ -1300,7 +1365,8 @@ void cpu_report_tpr_access(CPUX86State *env, TPRAccess access);
 void x86_cpu_compat_set_features(const char *cpu_model, FeatureWord w,
                                  uint32_t feat_add, uint32_t feat_remove);
 
-void x86_cpu_compat_disable_kvm_features(FeatureWord w, uint32_t features);
+void x86_cpu_compat_kvm_no_autoenable(FeatureWord w, uint32_t features);
+void x86_cpu_compat_kvm_no_autodisable(FeatureWord w, uint32_t features);
 
 
 /* Return name of 32-bit register, from a R_* constant */
diff --git a/target-i386/fpu_helper.c b/target-i386/fpu_helper.c
index 1b2900d5d..1d4eee397 100644
--- a/target-i386/fpu_helper.c
+++ b/target-i386/fpu_helper.c
@@ -537,7 +537,7 @@ uint32_t helper_fnstcw(CPUX86State *env)
     return env->fpuc;
 }
 
-static void update_fp_status(CPUX86State *env)
+void update_fp_status(CPUX86State *env)
 {
     int rnd_type;
 
@@ -575,8 +575,7 @@ static void update_fp_status(CPUX86State *env)
 
 void helper_fldcw(CPUX86State *env, uint32_t val)
 {
-    env->fpuc = val;
-    update_fp_status(env);
+    cpu_set_fpuc(env, val);
 }
 
 void helper_fclex(CPUX86State *env)
@@ -595,7 +594,7 @@ void helper_fninit(CPUX86State *env)
 {
     env->fpus = 0;
     env->fpstt = 0;
-    env->fpuc = 0x37f;
+    cpu_set_fpuc(env, 0x37f);
     env->fptags[0] = 1;
     env->fptags[1] = 1;
     env->fptags[2] = 1;
@@ -1013,11 +1012,11 @@ void helper_fldenv(CPUX86State *env, target_ulong ptr, int data32)
     int i, fpus, fptag;
 
     if (data32) {
-        env->fpuc = cpu_lduw_data(env, ptr);
+        cpu_set_fpuc(env, cpu_lduw_data(env, ptr));
         fpus = cpu_lduw_data(env, ptr + 4);
         fptag = cpu_lduw_data(env, ptr + 8);
     } else {
-        env->fpuc = cpu_lduw_data(env, ptr);
+        cpu_set_fpuc(env, cpu_lduw_data(env, ptr));
         fpus = cpu_lduw_data(env, ptr + 2);
         fptag = cpu_lduw_data(env, ptr + 4);
     }
@@ -1046,7 +1045,7 @@ void helper_fsave(CPUX86State *env, target_ulong ptr, int data32)
     /* fninit */
     env->fpus = 0;
     env->fpstt = 0;
-    env->fpuc = 0x37f;
+    cpu_set_fpuc(env, 0x37f);
     env->fptags[0] = 1;
     env->fptags[1] = 1;
     env->fptags[2] = 1;
@@ -1157,7 +1156,7 @@ void helper_fxrstor(CPUX86State *env, target_ulong ptr, int data64)
         raise_exception(env, EXCP0D_GPF);
     }
 
-    env->fpuc = cpu_lduw_data(env, ptr);
+    cpu_set_fpuc(env, cpu_lduw_data(env, ptr));
     fpus = cpu_lduw_data(env, ptr + 2);
     fptag = cpu_lduw_data(env, ptr + 4);
     env->fpstt = (fpus >> 11) & 7;
@@ -1257,6 +1256,12 @@ void cpu_set_mxcsr(CPUX86State *env, uint32_t mxcsr)
     set_flush_to_zero((mxcsr & SSE_FZ) ? 1 : 0, &env->fp_status);
 }
 
+void cpu_set_fpuc(CPUX86State *env, uint16_t val)
+{
+    env->fpuc = val;
+    update_fp_status(env);
+}
+
 void helper_ldmxcsr(CPUX86State *env, uint32_t val)
 {
     cpu_set_mxcsr(env, val);
diff --git a/target-i386/gdbstub.c b/target-i386/gdbstub.c
index 19fe9adc3..ff99cfb00 100644
--- a/target-i386/gdbstub.c
+++ b/target-i386/gdbstub.c
@@ -203,7 +203,7 @@ int x86_cpu_gdb_write_register(CPUState *cs, uint8_t *mem_buf, int n)
             return x86_cpu_gdb_load_seg(cpu, R_GS, mem_buf);
 
         case IDX_FP_REGS + 8:
-            env->fpuc = ldl_p(mem_buf);
+            cpu_set_fpuc(env, ldl_p(mem_buf));
             return 4;
         case IDX_FP_REGS + 9:
             tmp = ldl_p(mem_buf);
diff --git a/target-i386/helper.c b/target-i386/helper.c
index 47b982b43..345bda188 100644
--- a/target-i386/helper.c
+++ b/target-i386/helper.c
@@ -615,8 +615,8 @@ int x86_cpu_handle_mmu_fault(CPUState *cs, vaddr addr,
             if (!(pdpe & PG_PRESENT_MASK)) {
                 goto do_fault;
             }
-            rsvd_mask |= PG_HI_USER_MASK | PG_NX_MASK;
-            if (pdpe & rsvd_mask) {
+            rsvd_mask |= PG_HI_USER_MASK;
+            if (pdpe & (rsvd_mask | PG_NX_MASK)) {
                 goto do_fault_rsvd;
             }
             ptep = PG_NX_MASK | PG_USER_MASK | PG_RW_MASK;
@@ -1011,9 +1011,10 @@ bool check_hw_breakpoints(CPUX86State *env, bool force_dr6_update)
     return hit_enabled;
 }
 
-void breakpoint_handler(CPUX86State *env)
+void breakpoint_handler(CPUState *cs)
 {
-    CPUState *cs = CPU(x86_env_get_cpu(env));
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
     CPUBreakpoint *bp;
 
     if (cs->watchpoint_hit) {
@@ -1261,3 +1262,24 @@ void do_cpu_sipi(X86CPU *cpu)
 {
 }
 #endif
+
+/* Frob eflags into and out of the CPU temporary format.  */
+
+void x86_cpu_exec_enter(CPUState *cs)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+
+    CC_SRC = env->eflags & (CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C);
+    env->df = 1 - (2 * ((env->eflags >> 10) & 1));
+    CC_OP = CC_OP_EFLAGS;
+    env->eflags &= ~(DF_MASK | CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C);
+}
+
+void x86_cpu_exec_exit(CPUState *cs)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+
+    env->eflags = cpu_compute_eflags(env);
+}
diff --git a/target-i386/kvm.c b/target-i386/kvm.c
index 097fe1188..ccf36e871 100644
--- a/target-i386/kvm.c
+++ b/target-i386/kvm.c
@@ -79,6 +79,7 @@ static int lm_capable_kernel;
 static bool has_msr_hv_hypercall;
 static bool has_msr_hv_vapic;
 static bool has_msr_hv_tsc;
+static bool has_msr_mtrr;
 
 static bool has_msr_architectural_pmu;
 static uint32_t num_architectural_pmu_counters;
@@ -739,6 +740,10 @@ int kvm_arch_init_vcpu(CPUState *cs)
         env->kvm_xsave_buf = qemu_memalign(4096, sizeof(struct kvm_xsave));
     }
 
+    if (env->features[FEAT_1_EDX] & CPUID_MTRR) {
+        has_msr_mtrr = true;
+    }
+
     return 0;
 }
 
@@ -1026,6 +1031,9 @@ static int kvm_put_fpu(X86CPU *cpu)
 #define XSAVE_YMMH_SPACE  144
 #define XSAVE_BNDREGS     240
 #define XSAVE_BNDCSR      256
+#define XSAVE_OPMASK      272
+#define XSAVE_ZMM_Hi256   288
+#define XSAVE_Hi16_ZMM    416
 
 static int kvm_put_xsave(X86CPU *cpu)
 {
@@ -1062,6 +1070,14 @@ static int kvm_put_xsave(X86CPU *cpu)
             sizeof env->bnd_regs);
     memcpy(&xsave->region[XSAVE_BNDCSR], &env->bndcs_regs,
             sizeof(env->bndcs_regs));
+    memcpy(&xsave->region[XSAVE_OPMASK], env->opmask_regs,
+            sizeof env->opmask_regs);
+    memcpy(&xsave->region[XSAVE_ZMM_Hi256], env->zmmh_regs,
+            sizeof env->zmmh_regs);
+#ifdef TARGET_X86_64
+    memcpy(&xsave->region[XSAVE_Hi16_ZMM], env->hi16_zmm_regs,
+            sizeof env->hi16_zmm_regs);
+#endif
     r = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_XSAVE, xsave);
     return r;
 }
@@ -1183,7 +1199,7 @@ static int kvm_put_msrs(X86CPU *cpu, int level)
     CPUX86State *env = &cpu->env;
     struct {
         struct kvm_msrs info;
-        struct kvm_msr_entry entries[100];
+        struct kvm_msr_entry entries[150];
     } msr_data;
     struct kvm_msr_entry *msrs = msr_data.entries;
     int n = 0, i;
@@ -1278,6 +1294,37 @@ static int kvm_put_msrs(X86CPU *cpu, int level)
             kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_REFERENCE_TSC,
                               env->msr_hv_tsc);
         }
+        if (has_msr_mtrr) {
+            kvm_msr_entry_set(&msrs[n++], MSR_MTRRdefType, env->mtrr_deftype);
+            kvm_msr_entry_set(&msrs[n++],
+                              MSR_MTRRfix64K_00000, env->mtrr_fixed[0]);
+            kvm_msr_entry_set(&msrs[n++],
+                              MSR_MTRRfix16K_80000, env->mtrr_fixed[1]);
+            kvm_msr_entry_set(&msrs[n++],
+                              MSR_MTRRfix16K_A0000, env->mtrr_fixed[2]);
+            kvm_msr_entry_set(&msrs[n++],
+                              MSR_MTRRfix4K_C0000, env->mtrr_fixed[3]);
+            kvm_msr_entry_set(&msrs[n++],
+                              MSR_MTRRfix4K_C8000, env->mtrr_fixed[4]);
+            kvm_msr_entry_set(&msrs[n++],
+                              MSR_MTRRfix4K_D0000, env->mtrr_fixed[5]);
+            kvm_msr_entry_set(&msrs[n++],
+                              MSR_MTRRfix4K_D8000, env->mtrr_fixed[6]);
+            kvm_msr_entry_set(&msrs[n++],
+                              MSR_MTRRfix4K_E0000, env->mtrr_fixed[7]);
+            kvm_msr_entry_set(&msrs[n++],
+                              MSR_MTRRfix4K_E8000, env->mtrr_fixed[8]);
+            kvm_msr_entry_set(&msrs[n++],
+                              MSR_MTRRfix4K_F0000, env->mtrr_fixed[9]);
+            kvm_msr_entry_set(&msrs[n++],
+                              MSR_MTRRfix4K_F8000, env->mtrr_fixed[10]);
+            for (i = 0; i < MSR_MTRRcap_VCNT; i++) {
+                kvm_msr_entry_set(&msrs[n++],
+                                  MSR_MTRRphysBase(i), env->mtrr_var[i].base);
+                kvm_msr_entry_set(&msrs[n++],
+                                  MSR_MTRRphysMask(i), env->mtrr_var[i].mask);
+            }
+        }
 
         /* Note: MSR_IA32_FEATURE_CONTROL is written separately, see
          *       kvm_put_msr_feature_control. */
@@ -1366,6 +1413,14 @@ static int kvm_get_xsave(X86CPU *cpu)
             sizeof env->bnd_regs);
     memcpy(&env->bndcs_regs, &xsave->region[XSAVE_BNDCSR],
             sizeof(env->bndcs_regs));
+    memcpy(env->opmask_regs, &xsave->region[XSAVE_OPMASK],
+            sizeof env->opmask_regs);
+    memcpy(env->zmmh_regs, &xsave->region[XSAVE_ZMM_Hi256],
+            sizeof env->zmmh_regs);
+#ifdef TARGET_X86_64
+    memcpy(env->hi16_zmm_regs, &xsave->region[XSAVE_Hi16_ZMM],
+            sizeof env->hi16_zmm_regs);
+#endif
     return 0;
 }
 
@@ -1484,7 +1539,7 @@ static int kvm_get_msrs(X86CPU *cpu)
     CPUX86State *env = &cpu->env;
     struct {
         struct kvm_msrs info;
-        struct kvm_msr_entry entries[100];
+        struct kvm_msr_entry entries[150];
     } msr_data;
     struct kvm_msr_entry *msrs = msr_data.entries;
     int ret, i, n;
@@ -1572,6 +1627,24 @@ static int kvm_get_msrs(X86CPU *cpu)
     if (has_msr_hv_tsc) {
         msrs[n++].index = HV_X64_MSR_REFERENCE_TSC;
     }
+    if (has_msr_mtrr) {
+        msrs[n++].index = MSR_MTRRdefType;
+        msrs[n++].index = MSR_MTRRfix64K_00000;
+        msrs[n++].index = MSR_MTRRfix16K_80000;
+        msrs[n++].index = MSR_MTRRfix16K_A0000;
+        msrs[n++].index = MSR_MTRRfix4K_C0000;
+        msrs[n++].index = MSR_MTRRfix4K_C8000;
+        msrs[n++].index = MSR_MTRRfix4K_D0000;
+        msrs[n++].index = MSR_MTRRfix4K_D8000;
+        msrs[n++].index = MSR_MTRRfix4K_E0000;
+        msrs[n++].index = MSR_MTRRfix4K_E8000;
+        msrs[n++].index = MSR_MTRRfix4K_F0000;
+        msrs[n++].index = MSR_MTRRfix4K_F8000;
+        for (i = 0; i < MSR_MTRRcap_VCNT; i++) {
+            msrs[n++].index = MSR_MTRRphysBase(i);
+            msrs[n++].index = MSR_MTRRphysMask(i);
+        }
+    }
 
     msr_data.info.nmsrs = n;
     ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, &msr_data);
@@ -1692,6 +1765,49 @@ static int kvm_get_msrs(X86CPU *cpu)
         case HV_X64_MSR_REFERENCE_TSC:
             env->msr_hv_tsc = msrs[i].data;
             break;
+        case MSR_MTRRdefType:
+            env->mtrr_deftype = msrs[i].data;
+            break;
+        case MSR_MTRRfix64K_00000:
+            env->mtrr_fixed[0] = msrs[i].data;
+            break;
+        case MSR_MTRRfix16K_80000:
+            env->mtrr_fixed[1] = msrs[i].data;
+            break;
+        case MSR_MTRRfix16K_A0000:
+            env->mtrr_fixed[2] = msrs[i].data;
+            break;
+        case MSR_MTRRfix4K_C0000:
+            env->mtrr_fixed[3] = msrs[i].data;
+            break;
+        case MSR_MTRRfix4K_C8000:
+            env->mtrr_fixed[4] = msrs[i].data;
+            break;
+        case MSR_MTRRfix4K_D0000:
+            env->mtrr_fixed[5] = msrs[i].data;
+            break;
+        case MSR_MTRRfix4K_D8000:
+            env->mtrr_fixed[6] = msrs[i].data;
+            break;
+        case MSR_MTRRfix4K_E0000:
+            env->mtrr_fixed[7] = msrs[i].data;
+            break;
+        case MSR_MTRRfix4K_E8000:
+            env->mtrr_fixed[8] = msrs[i].data;
+            break;
+        case MSR_MTRRfix4K_F0000:
+            env->mtrr_fixed[9] = msrs[i].data;
+            break;
+        case MSR_MTRRfix4K_F8000:
+            env->mtrr_fixed[10] = msrs[i].data;
+            break;
+        case MSR_MTRRphysBase(0) ... MSR_MTRRphysMask(MSR_MTRRcap_VCNT - 1):
+            if (index & 1) {
+                env->mtrr_var[MSR_MTRRphysIndex(index)].mask = msrs[i].data;
+            } else {
+                env->mtrr_var[MSR_MTRRphysIndex(index)].base = msrs[i].data;
+            }
+            break;
         }
     }
 
diff --git a/target-i386/machine.c b/target-i386/machine.c
index 16d2f6a80..1c13b1435 100644
--- a/target-i386/machine.c
+++ b/target-i386/machine.c
@@ -60,6 +60,44 @@ static const VMStateDescription vmstate_ymmh_reg = {
 #define VMSTATE_YMMH_REGS_VARS(_field, _state, _n, _v)                         \
     VMSTATE_STRUCT_ARRAY(_field, _state, _n, _v, vmstate_ymmh_reg, XMMReg)
 
+static const VMStateDescription vmstate_zmmh_reg = {
+    .name = "zmmh_reg",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(YMM_Q(0), YMMReg),
+        VMSTATE_UINT64(YMM_Q(1), YMMReg),
+        VMSTATE_UINT64(YMM_Q(2), YMMReg),
+        VMSTATE_UINT64(YMM_Q(3), YMMReg),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+#define VMSTATE_ZMMH_REGS_VARS(_field, _state, _n)                             \
+    VMSTATE_STRUCT_ARRAY(_field, _state, _n, 0, vmstate_zmmh_reg, YMMReg)
+
+#ifdef TARGET_X86_64
+static const VMStateDescription vmstate_hi16_zmm_reg = {
+    .name = "hi16_zmm_reg",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(ZMM_Q(0), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(1), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(2), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(3), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(4), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(5), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(6), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(7), ZMMReg),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+#define VMSTATE_Hi16_ZMM_REGS_VARS(_field, _state, _n)                         \
+    VMSTATE_STRUCT_ARRAY(_field, _state, _n, 0, vmstate_hi16_zmm_reg, ZMMReg)
+#endif
+
 static const VMStateDescription vmstate_bnd_regs = {
     .name = "bnd_regs",
     .version_id = 1,
@@ -315,13 +353,13 @@ static int cpu_post_load(void *opaque, int version_id)
     env->hflags &= ~HF_CPL_MASK;
     env->hflags |= (env->segs[R_SS].flags >> DESC_DPL_SHIFT) & HF_CPL_MASK;
 
-    /* XXX: restore FPU round state */
     env->fpstt = (env->fpus_vmstate >> 11) & 7;
     env->fpus = env->fpus_vmstate & ~0x3800;
     env->fptag_vmstate ^= 0xff;
     for(i = 0; i < 8; i++) {
         env->fptags[i] = (env->fptag_vmstate >> i) & 1;
     }
+    update_fp_status(env);
 
     cpu_breakpoint_remove_all(cs, BP_CPU);
     cpu_watchpoint_remove_all(cs, BP_CPU);
@@ -603,6 +641,52 @@ static const VMStateDescription vmstate_msr_hyperv_time = {
     }
 };
 
+static bool avx512_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+    unsigned int i;
+
+    for (i = 0; i < NB_OPMASK_REGS; i++) {
+        if (env->opmask_regs[i]) {
+            return true;
+        }
+    }
+
+    for (i = 0; i < CPU_NB_REGS; i++) {
+#define ENV_ZMMH(reg, field) (env->zmmh_regs[reg].YMM_Q(field))
+        if (ENV_ZMMH(i, 0) || ENV_ZMMH(i, 1) ||
+            ENV_ZMMH(i, 2) || ENV_ZMMH(i, 3)) {
+            return true;
+        }
+#ifdef TARGET_X86_64
+#define ENV_Hi16_ZMM(reg, field) (env->hi16_zmm_regs[reg].ZMM_Q(field))
+        if (ENV_Hi16_ZMM(i, 0) || ENV_Hi16_ZMM(i, 1) ||
+            ENV_Hi16_ZMM(i, 2) || ENV_Hi16_ZMM(i, 3) ||
+            ENV_Hi16_ZMM(i, 4) || ENV_Hi16_ZMM(i, 5) ||
+            ENV_Hi16_ZMM(i, 6) || ENV_Hi16_ZMM(i, 7)) {
+            return true;
+        }
+#endif
+    }
+
+    return false;
+}
+
+static const VMStateDescription vmstate_avx512 = {
+    .name = "cpu/avx512",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64_ARRAY(env.opmask_regs, X86CPU, NB_OPMASK_REGS),
+        VMSTATE_ZMMH_REGS_VARS(env.zmmh_regs, X86CPU, CPU_NB_REGS),
+#ifdef TARGET_X86_64
+        VMSTATE_Hi16_ZMM_REGS_VARS(env.hi16_zmm_regs, X86CPU, CPU_NB_REGS),
+#endif
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 VMStateDescription vmstate_x86_cpu = {
     .name = "cpu",
     .version_id = 12,
@@ -677,7 +761,7 @@ VMStateDescription vmstate_x86_cpu = {
         /* MTRRs */
         VMSTATE_UINT64_ARRAY_V(env.mtrr_fixed, X86CPU, 11, 8),
         VMSTATE_UINT64_V(env.mtrr_deftype, X86CPU, 8),
-        VMSTATE_MTRR_VARS(env.mtrr_var, X86CPU, 8, 8),
+        VMSTATE_MTRR_VARS(env.mtrr_var, X86CPU, MSR_MTRRcap_VCNT, 8),
         /* KVM-related states */
         VMSTATE_INT32_V(env.interrupt_injected, X86CPU, 9),
         VMSTATE_UINT32_V(env.mp_state, X86CPU, 9),
@@ -745,6 +829,9 @@ VMStateDescription vmstate_x86_cpu = {
         }, {
             .vmsd = &vmstate_msr_hyperv_time,
             .needed = hyperv_time_enable_needed,
+        }, {
+            .vmsd = &vmstate_avx512,
+            .needed = avx512_needed,
         } , {
             /* empty */
         }
diff --git a/target-i386/seg_helper.c b/target-i386/seg_helper.c
index 2d970d0cb..c98eeb435 100644
--- a/target-i386/seg_helper.c
+++ b/target-i386/seg_helper.c
@@ -883,32 +883,23 @@ static void do_interrupt64(CPUX86State *env, int intno, int is_int,
     }
     if ((!(e2 & DESC_C_MASK) && dpl < cpl) || ist != 0) {
         /* to inner privilege */
-        if (ist != 0) {
-            esp = get_rsp_from_tss(env, ist + 3);
-        } else {
-            esp = get_rsp_from_tss(env, dpl);
-        }
-        esp &= ~0xfLL; /* align stack */
-        ss = 0;
         new_stack = 1;
+        esp = get_rsp_from_tss(env, ist != 0 ? ist + 3 : dpl);
+        ss = 0;
     } else if ((e2 & DESC_C_MASK) || dpl == cpl) {
         /* to same privilege */
         if (env->eflags & VM_MASK) {
             raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
         }
         new_stack = 0;
-        if (ist != 0) {
-            esp = get_rsp_from_tss(env, ist + 3);
-        } else {
-            esp = env->regs[R_ESP];
-        }
-        esp &= ~0xfLL; /* align stack */
+        esp = env->regs[R_ESP];
         dpl = cpl;
     } else {
         raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
         new_stack = 0; /* avoid warning */
         esp = 0; /* avoid warning */
     }
+    esp &= ~0xfLL; /* align stack */
 
     PUSHQ(esp, env->segs[R_SS].selector);
     PUSHQ(esp, env->regs[R_ESP]);
@@ -1127,8 +1118,8 @@ static void do_interrupt_user(CPUX86State *env, int intno, int is_int,
 
     /* Since we emulate only user space, we cannot do more than
        exiting the emulation with the suitable exception and error
-       code */
-    if (is_int) {
+       code. So update EIP for INT 0x80 and EXCP_SYSCALL. */
+    if (is_int || intno == EXCP_SYSCALL) {
         env->eip = next_eip;
     }
 }
@@ -1279,6 +1270,75 @@ void do_interrupt_x86_hardirq(CPUX86State *env, int intno, int is_hw)
     do_interrupt_all(x86_env_get_cpu(env), intno, 0, 0, 0, is_hw);
 }
 
+bool x86_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+    bool ret = false;
+
+#if !defined(CONFIG_USER_ONLY)
+    if (interrupt_request & CPU_INTERRUPT_POLL) {
+        cs->interrupt_request &= ~CPU_INTERRUPT_POLL;
+        apic_poll_irq(cpu->apic_state);
+    }
+#endif
+    if (interrupt_request & CPU_INTERRUPT_SIPI) {
+        do_cpu_sipi(cpu);
+    } else if (env->hflags2 & HF2_GIF_MASK) {
+        if ((interrupt_request & CPU_INTERRUPT_SMI) &&
+            !(env->hflags & HF_SMM_MASK)) {
+            cpu_svm_check_intercept_param(env, SVM_EXIT_SMI, 0);
+            cs->interrupt_request &= ~CPU_INTERRUPT_SMI;
+            do_smm_enter(cpu);
+            ret = true;
+        } else if ((interrupt_request & CPU_INTERRUPT_NMI) &&
+                   !(env->hflags2 & HF2_NMI_MASK)) {
+            cs->interrupt_request &= ~CPU_INTERRUPT_NMI;
+            env->hflags2 |= HF2_NMI_MASK;
+            do_interrupt_x86_hardirq(env, EXCP02_NMI, 1);
+            ret = true;
+        } else if (interrupt_request & CPU_INTERRUPT_MCE) {
+            cs->interrupt_request &= ~CPU_INTERRUPT_MCE;
+            do_interrupt_x86_hardirq(env, EXCP12_MCHK, 0);
+            ret = true;
+        } else if ((interrupt_request & CPU_INTERRUPT_HARD) &&
+                   (((env->hflags2 & HF2_VINTR_MASK) &&
+                     (env->hflags2 & HF2_HIF_MASK)) ||
+                    (!(env->hflags2 & HF2_VINTR_MASK) &&
+                     (env->eflags & IF_MASK &&
+                      !(env->hflags & HF_INHIBIT_IRQ_MASK))))) {
+            int intno;
+            cpu_svm_check_intercept_param(env, SVM_EXIT_INTR, 0);
+            cs->interrupt_request &= ~(CPU_INTERRUPT_HARD |
+                                       CPU_INTERRUPT_VIRQ);
+            intno = cpu_get_pic_interrupt(env);
+            qemu_log_mask(CPU_LOG_TB_IN_ASM,
+                          "Servicing hardware INT=0x%02x\n", intno);
+            do_interrupt_x86_hardirq(env, intno, 1);
+            /* ensure that no TB jump will be modified as
+               the program flow was changed */
+            ret = true;
+#if !defined(CONFIG_USER_ONLY)
+        } else if ((interrupt_request & CPU_INTERRUPT_VIRQ) &&
+                   (env->eflags & IF_MASK) &&
+                   !(env->hflags & HF_INHIBIT_IRQ_MASK)) {
+            int intno;
+            /* FIXME: this should respect TPR */
+            cpu_svm_check_intercept_param(env, SVM_EXIT_VINTR, 0);
+            intno = ldl_phys(cs->as, env->vm_vmcb
+                             + offsetof(struct vmcb, control.int_vector));
+            qemu_log_mask(CPU_LOG_TB_IN_ASM,
+                          "Servicing virtual hardware INT=0x%02x\n", intno);
+            do_interrupt_x86_hardirq(env, intno, 1);
+            cs->interrupt_request &= ~CPU_INTERRUPT_VIRQ;
+            ret = true;
+#endif
+        }
+    }
+
+    return ret;
+}
+
 void helper_enter_level(CPUX86State *env, int level, int data32,
                         target_ulong t1)
 {
diff --git a/target-i386/translate.c b/target-i386/translate.c
index 6fcd8245d..782f7d266 100644
--- a/target-i386/translate.c
+++ b/target-i386/translate.c
@@ -32,6 +32,9 @@
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
 
+#include "trace-tcg.h"
+
+
 #define PREFIX_REPZ   0x01
 #define PREFIX_REPNZ  0x02
 #define PREFIX_LOCK   0x04
@@ -7984,7 +7987,7 @@ static inline void gen_intermediate_code_internal(X86CPU *cpu,
                 if (bp->pc == pc_ptr &&
                     !((bp->flags & BP_CPU) && (tb->flags & HF_RF_MASK))) {
                     gen_debug(dc, pc_ptr - dc->cs_base);
-                    break;
+                    goto done_generating;
                 }
             }
         }
@@ -8035,6 +8038,7 @@ static inline void gen_intermediate_code_internal(X86CPU *cpu,
     }
     if (tb->cflags & CF_LAST_IO)
         gen_io_end();
+done_generating:
     gen_tb_end(tb, num_insns);
     *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     /* we don't forget to fill the last values */
diff --git a/target-lm32/cpu-qom.h b/target-lm32/cpu-qom.h
index 9f1509387..77bc7b268 100644
--- a/target-lm32/cpu-qom.h
+++ b/target-lm32/cpu-qom.h
@@ -82,6 +82,7 @@ extern const struct VMStateDescription vmstate_lm32_cpu;
 #endif
 
 void lm32_cpu_do_interrupt(CPUState *cpu);
+bool lm32_cpu_exec_interrupt(CPUState *cs, int int_req);
 void lm32_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
                          int flags);
 hwaddr lm32_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
diff --git a/target-lm32/cpu.c b/target-lm32/cpu.c
index c5c20d74c..f8081f52c 100644
--- a/target-lm32/cpu.c
+++ b/target-lm32/cpu.c
@@ -158,7 +158,6 @@ static void lm32_cpu_initfn(Object *obj)
     if (tcg_enabled() && !tcg_initialized) {
         tcg_initialized = true;
         lm32_translate_init();
-        cpu_set_debug_excp_handler(lm32_debug_excp_handler);
     }
 }
 
@@ -262,6 +261,7 @@ static void lm32_cpu_class_init(ObjectClass *oc, void *data)
     cc->class_by_name = lm32_cpu_class_by_name;
     cc->has_work = lm32_cpu_has_work;
     cc->do_interrupt = lm32_cpu_do_interrupt;
+    cc->cpu_exec_interrupt = lm32_cpu_exec_interrupt;
     cc->dump_state = lm32_cpu_dump_state;
     cc->set_pc = lm32_cpu_set_pc;
     cc->gdb_read_register = lm32_cpu_gdb_read_register;
@@ -273,6 +273,8 @@ static void lm32_cpu_class_init(ObjectClass *oc, void *data)
     cc->vmsd = &vmstate_lm32_cpu;
 #endif
     cc->gdb_num_core_regs = 32 + 7;
+    cc->gdb_stop_before_watchpoint = true;
+    cc->debug_excp_handler = lm32_debug_excp_handler;
 }
 
 static void lm32_register_cpu_type(const LM32CPUInfo *info)
diff --git a/target-lm32/cpu.h b/target-lm32/cpu.h
index 70600aa47..0dab6e89a 100644
--- a/target-lm32/cpu.h
+++ b/target-lm32/cpu.h
@@ -211,7 +211,7 @@ void lm32_cpu_list(FILE *f, fprintf_function cpu_fprintf);
 void lm32_translate_init(void);
 void cpu_lm32_set_phys_msb_ignore(CPULM32State *env, int value);
 void QEMU_NORETURN raise_exception(CPULM32State *env, int index);
-void lm32_debug_excp_handler(CPULM32State *env);
+void lm32_debug_excp_handler(CPUState *cs);
 void lm32_breakpoint_insert(CPULM32State *env, int index, target_ulong address);
 void lm32_breakpoint_remove(CPULM32State *env, int index);
 void lm32_watchpoint_insert(CPULM32State *env, int index, target_ulong address,
diff --git a/target-lm32/helper.c b/target-lm32/helper.c
index 1bca1961a..7a41f2973 100644
--- a/target-lm32/helper.c
+++ b/target-lm32/helper.c
@@ -125,9 +125,10 @@ static bool check_watchpoints(CPULM32State *env)
     return false;
 }
 
-void lm32_debug_excp_handler(CPULM32State *env)
+void lm32_debug_excp_handler(CPUState *cs)
 {
-    CPUState *cs = CPU(lm32_env_get_cpu(env));
+    LM32CPU *cpu = LM32_CPU(cs);
+    CPULM32State *env = &cpu->env;
     CPUBreakpoint *bp;
 
     if (cs->watchpoint_hit) {
@@ -201,6 +202,19 @@ void lm32_cpu_do_interrupt(CPUState *cs)
     }
 }
 
+bool lm32_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    LM32CPU *cpu = LM32_CPU(cs);
+    CPULM32State *env = &cpu->env;
+
+    if ((interrupt_request & CPU_INTERRUPT_HARD) && (env->ie & IE_IE)) {
+        cs->exception_index = EXCP_IRQ;
+        lm32_cpu_do_interrupt(cs);
+        return true;
+    }
+    return false;
+}
+
 LM32CPU *cpu_lm32_init(const char *cpu_model)
 {
     return LM32_CPU(cpu_generic_init(TYPE_LM32_CPU, cpu_model));
diff --git a/target-lm32/translate.c b/target-lm32/translate.c
index a51ade9a1..8454e8b51 100644
--- a/target-lm32/translate.c
+++ b/target-lm32/translate.c
@@ -27,6 +27,9 @@
 
 #include "exec/helper-gen.h"
 
+#include "trace-tcg.h"
+
+
 #define DISAS_LM32 1
 #if DISAS_LM32
 #  define LOG_DIS(...) qemu_log_mask(CPU_LOG_TB_IN_ASM, ## __VA_ARGS__)
diff --git a/target-m68k/cpu-qom.h b/target-m68k/cpu-qom.h
index 7f388eda6..c28e55d6b 100644
--- a/target-m68k/cpu-qom.h
+++ b/target-m68k/cpu-qom.h
@@ -71,10 +71,14 @@ static inline M68kCPU *m68k_env_get_cpu(CPUM68KState *env)
 #define ENV_OFFSET offsetof(M68kCPU, env)
 
 void m68k_cpu_do_interrupt(CPUState *cpu);
+bool m68k_cpu_exec_interrupt(CPUState *cpu, int int_req);
 void m68k_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
                          int flags);
 hwaddr m68k_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
 int m68k_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
 int m68k_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
 
+void m68k_cpu_exec_enter(CPUState *cs);
+void m68k_cpu_exec_exit(CPUState *cs);
+
 #endif
diff --git a/target-m68k/cpu.c b/target-m68k/cpu.c
index c9cff19ef..4cfb7256c 100644
--- a/target-m68k/cpu.c
+++ b/target-m68k/cpu.c
@@ -196,6 +196,7 @@ static void m68k_cpu_class_init(ObjectClass *c, void *data)
     cc->class_by_name = m68k_cpu_class_by_name;
     cc->has_work = m68k_cpu_has_work;
     cc->do_interrupt = m68k_cpu_do_interrupt;
+    cc->cpu_exec_interrupt = m68k_cpu_exec_interrupt;
     cc->dump_state = m68k_cpu_dump_state;
     cc->set_pc = m68k_cpu_set_pc;
     cc->gdb_read_register = m68k_cpu_gdb_read_register;
@@ -205,6 +206,9 @@ static void m68k_cpu_class_init(ObjectClass *c, void *data)
 #else
     cc->get_phys_page_debug = m68k_cpu_get_phys_page_debug;
 #endif
+    cc->cpu_exec_enter = m68k_cpu_exec_enter;
+    cc->cpu_exec_exit = m68k_cpu_exec_exit;
+
     dc->vmsd = &vmstate_m68k_cpu;
     cc->gdb_num_core_regs = 18;
     cc->gdb_core_xml_file = "cf-core.xml";
diff --git a/target-m68k/cpu.h b/target-m68k/cpu.h
index 6e4001d52..f67bbcc64 100644
--- a/target-m68k/cpu.h
+++ b/target-m68k/cpu.h
@@ -120,7 +120,6 @@ void m68k_tcg_init(void);
 void m68k_cpu_init_gdb(M68kCPU *cpu);
 M68kCPU *cpu_m68k_init(const char *cpu_model);
 int cpu_m68k_exec(CPUM68KState *s);
-void do_interrupt_m68k_hardirq(CPUM68KState *env1);
 /* you can call this signal handler from your SIGBUS and SIGSEGV
    signal handlers to inform the virtual CPU of exceptions. non zero
    is returned if the signal was handled by the virtual CPU.  */
diff --git a/target-m68k/helper.c b/target-m68k/helper.c
index 8be974569..77225a200 100644
--- a/target-m68k/helper.c
+++ b/target-m68k/helper.c
@@ -864,3 +864,23 @@ void HELPER(set_mac_extu)(CPUM68KState *env, uint32_t val, uint32_t acc)
     res |= (uint64_t)(val & 0xffff0000) << 16;
     env->macc[acc + 1] = res;
 }
+
+void m68k_cpu_exec_enter(CPUState *cs)
+{
+    M68kCPU *cpu = M68K_CPU(cs);
+    CPUM68KState *env = &cpu->env;
+
+    env->cc_op = CC_OP_FLAGS;
+    env->cc_dest = env->sr & 0xf;
+    env->cc_x = (env->sr >> 4) & 1;
+}
+
+void m68k_cpu_exec_exit(CPUState *cs)
+{
+    M68kCPU *cpu = M68K_CPU(cs);
+    CPUM68KState *env = &cpu->env;
+
+    cpu_m68k_flush_flags(env, env->cc_op);
+    env->cc_op = CC_OP_FLAGS;
+    env->sr = (env->sr & 0xffe0) | env->cc_dest | (env->cc_x << 4);
+}
diff --git a/target-m68k/op_helper.c b/target-m68k/op_helper.c
index 9dd3e74ab..06661f58c 100644
--- a/target-m68k/op_helper.c
+++ b/target-m68k/op_helper.c
@@ -27,7 +27,7 @@ void m68k_cpu_do_interrupt(CPUState *cs)
     cs->exception_index = -1;
 }
 
-void do_interrupt_m68k_hardirq(CPUM68KState *env)
+static inline void do_interrupt_m68k_hardirq(CPUM68KState *env)
 {
 }
 
@@ -141,12 +141,30 @@ void m68k_cpu_do_interrupt(CPUState *cs)
     do_interrupt_all(env, 0);
 }
 
-void do_interrupt_m68k_hardirq(CPUM68KState *env)
+static inline void do_interrupt_m68k_hardirq(CPUM68KState *env)
 {
     do_interrupt_all(env, 1);
 }
 #endif
 
+bool m68k_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    M68kCPU *cpu = M68K_CPU(cs);
+    CPUM68KState *env = &cpu->env;
+
+    if (interrupt_request & CPU_INTERRUPT_HARD
+        && ((env->sr & SR_I) >> SR_I_SHIFT) < env->pending_level) {
+        /* Real hardware gets the interrupt vector via an IACK cycle
+           at this point.  Current emulated hardware doesn't rely on
+           this, so we provide/save the vector when the interrupt is
+           first signalled.  */
+        cs->exception_index = env->pending_vector;
+        do_interrupt_m68k_hardirq(env);
+        return true;
+    }
+    return false;
+}
+
 static void raise_exception(CPUM68KState *env, int tt)
 {
     CPUState *cs = CPU(m68k_env_get_cpu(env));
diff --git a/target-m68k/translate.c b/target-m68k/translate.c
index 50df4d384..efd4cfc3c 100644
--- a/target-m68k/translate.c
+++ b/target-m68k/translate.c
@@ -27,6 +27,9 @@
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
 
+#include "trace-tcg.h"
+
+
 //#define DEBUG_DISPATCH 1
 
 /* Fake floating point.  */
diff --git a/target-microblaze/cpu-qom.h b/target-microblaze/cpu-qom.h
index 35a12b42a..e3e070159 100644
--- a/target-microblaze/cpu-qom.h
+++ b/target-microblaze/cpu-qom.h
@@ -72,6 +72,7 @@ static inline MicroBlazeCPU *mb_env_get_cpu(CPUMBState *env)
 #define ENV_OFFSET offsetof(MicroBlazeCPU, env)
 
 void mb_cpu_do_interrupt(CPUState *cs);
+bool mb_cpu_exec_interrupt(CPUState *cs, int int_req);
 void mb_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
                        int flags);
 hwaddr mb_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
diff --git a/target-microblaze/cpu.c b/target-microblaze/cpu.c
index 0379f2be2..67e3182f7 100644
--- a/target-microblaze/cpu.c
+++ b/target-microblaze/cpu.c
@@ -169,6 +169,7 @@ static void mb_cpu_class_init(ObjectClass *oc, void *data)
 
     cc->has_work = mb_cpu_has_work;
     cc->do_interrupt = mb_cpu_do_interrupt;
+    cc->cpu_exec_interrupt = mb_cpu_exec_interrupt;
     cc->dump_state = mb_cpu_dump_state;
     cc->set_pc = mb_cpu_set_pc;
     cc->gdb_read_register = mb_cpu_gdb_read_register;
diff --git a/target-microblaze/helper.c b/target-microblaze/helper.c
index 59c9ad5ae..59466c974 100644
--- a/target-microblaze/helper.c
+++ b/target-microblaze/helper.c
@@ -286,3 +286,19 @@ hwaddr mb_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
     return paddr;
 }
 #endif
+
+bool mb_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    MicroBlazeCPU *cpu = MICROBLAZE_CPU(cs);
+    CPUMBState *env = &cpu->env;
+
+    if ((interrupt_request & CPU_INTERRUPT_HARD)
+        && (env->sregs[SR_MSR] & MSR_IE)
+        && !(env->sregs[SR_MSR] & (MSR_EIP | MSR_BIP))
+        && !(env->iflags & (D_FLAG | IMM_FLAG))) {
+        cs->exception_index = EXCP_IRQ;
+        mb_cpu_do_interrupt(cs);
+        return true;
+    }
+    return false;
+}
diff --git a/target-microblaze/translate.c b/target-microblaze/translate.c
index 03ea15803..fd2b77164 100644
--- a/target-microblaze/translate.c
+++ b/target-microblaze/translate.c
@@ -26,6 +26,9 @@
 #include "exec/cpu_ldst.h"
 #include "exec/helper-gen.h"
 
+#include "trace-tcg.h"
+
+
 #define SIM_COMPAT 0
 #define DISAS_GNU 1
 #define DISAS_MB 1
diff --git a/target-mips/Makefile.objs b/target-mips/Makefile.objs
index 716244f3d..108fd9b50 100644
--- a/target-mips/Makefile.objs
+++ b/target-mips/Makefile.objs
@@ -1,4 +1,4 @@
 obj-y += translate.o dsp_helper.o op_helper.o lmi_helper.o helper.o cpu.o
-obj-y += gdbstub.o
+obj-y += gdbstub.o msa_helper.o
 obj-$(CONFIG_SOFTMMU) += machine.o
 obj-$(CONFIG_KVM) += kvm.o
diff --git a/target-mips/cpu-qom.h b/target-mips/cpu-qom.h
index 2cff15a27..2ffc1bf3f 100644
--- a/target-mips/cpu-qom.h
+++ b/target-mips/cpu-qom.h
@@ -75,6 +75,7 @@ static inline MIPSCPU *mips_env_get_cpu(CPUMIPSState *env)
 #define ENV_OFFSET offsetof(MIPSCPU, env)
 
 void mips_cpu_do_interrupt(CPUState *cpu);
+bool mips_cpu_exec_interrupt(CPUState *cpu, int int_req);
 void mips_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
                          int flags);
 hwaddr mips_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
diff --git a/target-mips/cpu.c b/target-mips/cpu.c
index b3e0e6cce..98dc94e74 100644
--- a/target-mips/cpu.c
+++ b/target-mips/cpu.c
@@ -136,6 +136,7 @@ static void mips_cpu_class_init(ObjectClass *c, void *data)
 
     cc->has_work = mips_cpu_has_work;
     cc->do_interrupt = mips_cpu_do_interrupt;
+    cc->cpu_exec_interrupt = mips_cpu_exec_interrupt;
     cc->dump_state = mips_cpu_dump_state;
     cc->set_pc = mips_cpu_set_pc;
     cc->synchronize_from_tb = mips_cpu_synchronize_from_tb;
@@ -150,6 +151,7 @@ static void mips_cpu_class_init(ObjectClass *c, void *data)
 #endif
 
     cc->gdb_num_core_regs = 73;
+    cc->gdb_stop_before_watchpoint = true;
 }
 
 static const TypeInfo mips_cpu_type_info = {
diff --git a/target-mips/cpu.h b/target-mips/cpu.h
index 8b9a92ebd..c01bbdac2 100644
--- a/target-mips/cpu.h
+++ b/target-mips/cpu.h
@@ -30,6 +30,11 @@ struct r4k_tlb_t {
     uint_fast16_t V1:1;
     uint_fast16_t D0:1;
     uint_fast16_t D1:1;
+    uint_fast16_t XI0:1;
+    uint_fast16_t XI1:1;
+    uint_fast16_t RI0:1;
+    uint_fast16_t RI1:1;
+    uint_fast16_t EHINV:1;
     target_ulong PFN[2];
 };
 
@@ -43,6 +48,8 @@ struct CPUMIPSTLBContext {
     void (*helper_tlbwr)(struct CPUMIPSState *env);
     void (*helper_tlbp)(struct CPUMIPSState *env);
     void (*helper_tlbr)(struct CPUMIPSState *env);
+    void (*helper_tlbinv)(struct CPUMIPSState *env);
+    void (*helper_tlbinvf)(struct CPUMIPSState *env);
     union {
         struct {
             r4k_tlb_t tlb[MIPS_TLB_MAX];
@@ -51,12 +58,32 @@ struct CPUMIPSTLBContext {
 };
 #endif
 
+/* MSA Context */
+#define MSA_WRLEN (128)
+
+enum CPUMIPSMSADataFormat {
+    DF_BYTE = 0,
+    DF_HALF,
+    DF_WORD,
+    DF_DOUBLE
+};
+
+typedef union wr_t wr_t;
+union wr_t {
+    int8_t  b[MSA_WRLEN/8];
+    int16_t h[MSA_WRLEN/16];
+    int32_t w[MSA_WRLEN/32];
+    int64_t d[MSA_WRLEN/64];
+};
+
 typedef union fpr_t fpr_t;
 union fpr_t {
     float64  fd;   /* ieee double precision */
     float32  fs[2];/* ieee single precision */
     uint64_t d;    /* binary double fixed-point */
     uint32_t w[2]; /* binary single fixed-point */
+/* FPU/MSA register mapping is not tested on big-endian hosts. */
+    wr_t     wr;   /* vector data */
 };
 /* define FP_ENDIAN_IDX to access the same location
  * in the fpr_t union regardless of the host endianness
@@ -136,6 +163,7 @@ typedef struct mips_def_t mips_def_t;
 #define MIPS_TC_MAX 5
 #define MIPS_FPU_MAX 1
 #define MIPS_DSP_ACC 4
+#define MIPS_KSCRATCH_NUM 6
 
 typedef struct TCState TCState;
 struct TCState {
@@ -169,6 +197,21 @@ struct TCState {
     target_ulong CP0_TCScheFBack;
     int32_t CP0_Debug_tcstatus;
     target_ulong CP0_UserLocal;
+
+    int32_t msacsr;
+
+#define MSACSR_FS       24
+#define MSACSR_FS_MASK  (1 << MSACSR_FS)
+#define MSACSR_NX       18
+#define MSACSR_NX_MASK  (1 << MSACSR_NX)
+#define MSACSR_CEF      2
+#define MSACSR_CEF_MASK (0xffff << MSACSR_CEF)
+#define MSACSR_RM       0
+#define MSACSR_RM_MASK  (0x3 << MSACSR_RM)
+#define MSACSR_MASK     (MSACSR_RM_MASK | MSACSR_CEF_MASK | MSACSR_NX_MASK | \
+        MSACSR_FS_MASK)
+
+    float_status msa_fp_status;
 };
 
 typedef struct CPUMIPSState CPUMIPSState;
@@ -184,6 +227,10 @@ struct CPUMIPSState {
     target_ulong SEGMask;
     target_ulong PAMask;
 
+    int32_t msair;
+#define MSAIR_ProcID    8
+#define MSAIR_Rev       0
+
     int32_t CP0_Index;
     /* CP0_MVP* are per MVP registers. */
     int32_t CP0_Random;
@@ -228,9 +275,21 @@ struct CPUMIPSState {
 #define CP0VPEOpt_DWX0	0
     target_ulong CP0_EntryLo0;
     target_ulong CP0_EntryLo1;
+#if defined(TARGET_MIPS64)
+# define CP0EnLo_RI 63
+# define CP0EnLo_XI 62
+#else
+# define CP0EnLo_RI 31
+# define CP0EnLo_XI 30
+#endif
     target_ulong CP0_Context;
+    target_ulong CP0_KScratch[MIPS_KSCRATCH_NUM];
     int32_t CP0_PageMask;
+    int32_t CP0_PageGrain_rw_bitmask;
     int32_t CP0_PageGrain;
+#define CP0PG_RIE 31
+#define CP0PG_XIE 30
+#define CP0PG_IEC 27
     int32_t CP0_Wired;
     int32_t CP0_SRSConf0_rw_bitmask;
     int32_t CP0_SRSConf0;
@@ -263,8 +322,11 @@ struct CPUMIPSState {
 #define CP0SRSC4_SRS13	0
     int32_t CP0_HWREna;
     target_ulong CP0_BadVAddr;
+    uint32_t CP0_BadInstr;
+    uint32_t CP0_BadInstrP;
     int32_t CP0_Count;
     target_ulong CP0_EntryHi;
+#define CP0EnHi_EHINV 10
     int32_t CP0_Compare;
     int32_t CP0_Status;
 #define CP0St_CU3   31
@@ -362,19 +424,38 @@ struct CPUMIPSState {
 #define CP0C2_SA   0
     int32_t CP0_Config3;
 #define CP0C3_M    31
+#define CP0C3_BPG  30
+#define CP0C3_CMCGR 29
+#define CP0C3_MSAP  28
+#define CP0C3_BP 27
+#define CP0C3_BI 26
+#define CP0C3_IPLW 21
+#define CP0C3_MMAR 18
+#define CP0C3_MCU  17
 #define CP0C3_ISA_ON_EXC 16
+#define CP0C3_ISA  14
 #define CP0C3_ULRI 13
+#define CP0C3_RXI  12
+#define CP0C3_DSP2P 11
 #define CP0C3_DSPP 10
 #define CP0C3_LPA  7
 #define CP0C3_VEIC 6
 #define CP0C3_VInt 5
 #define CP0C3_SP   4
+#define CP0C3_CDMM 3
 #define CP0C3_MT   2
 #define CP0C3_SM   1
 #define CP0C3_TL   0
     uint32_t CP0_Config4;
     uint32_t CP0_Config4_rw_bitmask;
 #define CP0C4_M    31
+#define CP0C4_IE   29
+#define CP0C4_KScrExist 16
+#define CP0C4_MMUExtDef 14
+#define CP0C4_FTLBPageSize 8
+#define CP0C4_FTLBWays 4
+#define CP0C4_FTLBSets 0
+#define CP0C4_MMUSizeExt 0
     uint32_t CP0_Config5;
     uint32_t CP0_Config5_rw_bitmask;
 #define CP0C5_M          31
@@ -382,6 +463,7 @@ struct CPUMIPSState {
 #define CP0C5_CV         29
 #define CP0C5_EVA        28
 #define CP0C5_MSAEn      27
+#define CP0C5_SBRI       6
 #define CP0C5_UFR        2
 #define CP0C5_NFExists   0
     int32_t CP0_Config6;
@@ -429,9 +511,11 @@ struct CPUMIPSState {
     CPUMIPSFPUContext fpus[MIPS_FPU_MAX];
     /* QEMU */
     int error_code;
+#define EXCP_TLB_NOMATCH   0x1
+#define EXCP_INST_NOTAVAIL 0x2 /* No valid instruction word for BadInstr */
     uint32_t hflags;    /* CPU State */
     /* TMASK defines different execution modes */
-#define MIPS_HFLAG_TMASK  0xC07FF
+#define MIPS_HFLAG_TMASK  0x15807FF
 #define MIPS_HFLAG_MODE   0x00007 /* execution modes                    */
     /* The KSU flags must be the lowest bits in hflags. The flag order
        must be the same as defined for CP0 Status. This allows to use
@@ -450,30 +534,34 @@ struct CPUMIPSState {
        and RSQRT.D.  */
 #define MIPS_HFLAG_COP1X  0x00080 /* COP1X instructions enabled         */
 #define MIPS_HFLAG_RE     0x00100 /* Reversed endianness                */
-#define MIPS_HFLAG_UX     0x00200 /* 64-bit user mode                   */
+#define MIPS_HFLAG_AWRAP  0x00200 /* 32-bit compatibility address wrapping */
 #define MIPS_HFLAG_M16    0x00400 /* MIPS16 mode flag                   */
 #define MIPS_HFLAG_M16_SHIFT 10
     /* If translation is interrupted between the branch instruction and
      * the delay slot, record what type of branch it is so that we can
      * resume translation properly.  It might be possible to reduce
      * this from three bits to two.  */
-#define MIPS_HFLAG_BMASK_BASE  0x03800
+#define MIPS_HFLAG_BMASK_BASE  0x803800
 #define MIPS_HFLAG_B      0x00800 /* Unconditional branch               */
 #define MIPS_HFLAG_BC     0x01000 /* Conditional branch                 */
 #define MIPS_HFLAG_BL     0x01800 /* Likely branch                      */
 #define MIPS_HFLAG_BR     0x02000 /* branch to register (can't link TB) */
     /* Extra flags about the current pending branch.  */
-#define MIPS_HFLAG_BMASK_EXT 0x3C000
+#define MIPS_HFLAG_BMASK_EXT 0x7C000
 #define MIPS_HFLAG_B16    0x04000 /* branch instruction was 16 bits     */
 #define MIPS_HFLAG_BDS16  0x08000 /* branch requires 16-bit delay slot  */
 #define MIPS_HFLAG_BDS32  0x10000 /* branch requires 32-bit delay slot  */
-#define MIPS_HFLAG_BX     0x20000 /* branch exchanges execution mode    */
+#define MIPS_HFLAG_BDS_STRICT  0x20000 /* Strict delay slot size */
+#define MIPS_HFLAG_BX     0x40000 /* branch exchanges execution mode    */
 #define MIPS_HFLAG_BMASK  (MIPS_HFLAG_BMASK_BASE | MIPS_HFLAG_BMASK_EXT)
     /* MIPS DSP resources access. */
-#define MIPS_HFLAG_DSP   0x40000  /* Enable access to MIPS DSP resources. */
-#define MIPS_HFLAG_DSPR2 0x80000  /* Enable access to MIPS DSPR2 resources. */
+#define MIPS_HFLAG_DSP   0x080000  /* Enable access to MIPS DSP resources. */
+#define MIPS_HFLAG_DSPR2 0x100000  /* Enable access to MIPS DSPR2 resources. */
     /* Extra flag about HWREna register. */
-#define MIPS_HFLAG_HWRENA_ULR 0x100000 /* ULR bit from HWREna is set. */
+#define MIPS_HFLAG_HWRENA_ULR 0x200000 /* ULR bit from HWREna is set. */
+#define MIPS_HFLAG_SBRI  0x400000 /* R6 SDBBP causes RI excpt. in user mode */
+#define MIPS_HFLAG_FBNSLOT 0x800000 /* Forbidden slot                   */
+#define MIPS_HFLAG_MSA   0x1000000
     target_ulong btarget;        /* Jump / branch target               */
     target_ulong bcond;          /* Branch condition (if needed)       */
 
@@ -509,6 +597,8 @@ void r4k_helper_tlbwi(CPUMIPSState *env);
 void r4k_helper_tlbwr(CPUMIPSState *env);
 void r4k_helper_tlbp(CPUMIPSState *env);
 void r4k_helper_tlbr(CPUMIPSState *env);
+void r4k_helper_tlbinv(CPUMIPSState *env);
+void r4k_helper_tlbinvf(CPUMIPSState *env);
 
 void mips_cpu_unassigned_access(CPUState *cpu, hwaddr addr,
                                 bool is_write, bool is_exec, int unused,
@@ -525,7 +615,7 @@ void mips_cpu_list (FILE *f, fprintf_function cpu_fprintf);
 extern void cpu_wrdsp(uint32_t rs, uint32_t mask_num, CPUMIPSState *env);
 extern uint32_t cpu_rddsp(uint32_t mask_num, CPUMIPSState *env);
 
-#define CPU_SAVE_VERSION 4
+#define CPU_SAVE_VERSION 5
 
 /* MMU modes definitions. We carefully match the indices with our
    hflags layout. */
@@ -627,8 +717,12 @@ enum {
     EXCP_C2E,
     EXCP_CACHE, /* 32 */
     EXCP_DSPDIS,
+    EXCP_MSADIS,
+    EXCP_MSAFPE,
+    EXCP_TLBXI,
+    EXCP_TLBRI,
 
-    EXCP_LAST = EXCP_DSPDIS,
+    EXCP_LAST = EXCP_TLBRI,
 };
 /* Dummy exception for conditional stores.  */
 #define EXCP_SC 0x100
@@ -679,6 +773,10 @@ hwaddr cpu_mips_translate_address (CPUMIPSState *env, target_ulong address,
 #endif
 target_ulong exception_resume_pc (CPUMIPSState *env);
 
+/* op_helper.c */
+extern unsigned int ieee_rm[];
+int ieee_ex_to_mips(int xcpt);
+
 static inline void cpu_get_tb_cpu_state(CPUMIPSState *env, target_ulong *pc,
                                         target_ulong *cs_base, int *flags)
 {
@@ -725,7 +823,8 @@ static inline void compute_hflags(CPUMIPSState *env)
 {
     env->hflags &= ~(MIPS_HFLAG_COP1X | MIPS_HFLAG_64 | MIPS_HFLAG_CP0 |
                      MIPS_HFLAG_F64 | MIPS_HFLAG_FPU | MIPS_HFLAG_KSU |
-                     MIPS_HFLAG_UX | MIPS_HFLAG_DSP | MIPS_HFLAG_DSPR2);
+                     MIPS_HFLAG_AWRAP | MIPS_HFLAG_DSP | MIPS_HFLAG_DSPR2 |
+                     MIPS_HFLAG_SBRI | MIPS_HFLAG_MSA);
     if (!(env->CP0_Status & (1 << CP0St_EXL)) &&
         !(env->CP0_Status & (1 << CP0St_ERL)) &&
         !(env->hflags & MIPS_HFLAG_DM)) {
@@ -737,11 +836,22 @@ static inline void compute_hflags(CPUMIPSState *env)
         (env->CP0_Status & (1 << CP0St_UX))) {
         env->hflags |= MIPS_HFLAG_64;
     }
-    if (env->CP0_Status & (1 << CP0St_UX)) {
-        env->hflags |= MIPS_HFLAG_UX;
+
+    if (((env->hflags & MIPS_HFLAG_KSU) == MIPS_HFLAG_UM) &&
+        !(env->CP0_Status & (1 << CP0St_UX))) {
+        env->hflags |= MIPS_HFLAG_AWRAP;
+    } else if (env->insn_flags & ISA_MIPS32R6) {
+        /* Address wrapping for Supervisor and Kernel is specified in R6 */
+        if ((((env->hflags & MIPS_HFLAG_KSU) == MIPS_HFLAG_SM) &&
+             !(env->CP0_Status & (1 << CP0St_SX))) ||
+            (((env->hflags & MIPS_HFLAG_KSU) == MIPS_HFLAG_KM) &&
+             !(env->CP0_Status & (1 << CP0St_KX)))) {
+            env->hflags |= MIPS_HFLAG_AWRAP;
+        }
     }
 #endif
-    if ((env->CP0_Status & (1 << CP0St_CU0)) ||
+    if (((env->CP0_Status & (1 << CP0St_CU0)) &&
+         !(env->insn_flags & ISA_MIPS32R6)) ||
         !(env->hflags & MIPS_HFLAG_KSU)) {
         env->hflags |= MIPS_HFLAG_CP0;
     }
@@ -751,6 +861,10 @@ static inline void compute_hflags(CPUMIPSState *env)
     if (env->CP0_Status & (1 << CP0St_FR)) {
         env->hflags |= MIPS_HFLAG_F64;
     }
+    if (((env->hflags & MIPS_HFLAG_KSU) != MIPS_HFLAG_KM) &&
+        (env->CP0_Config5 & (1 << CP0C5_SBRI))) {
+        env->hflags |= MIPS_HFLAG_SBRI;
+    }
     if (env->insn_flags & ASE_DSPR2) {
         /* Enables access MIPS DSP resources, now our cpu is DSP ASER2,
            so enable to access DSPR2 resources. */
@@ -783,6 +897,11 @@ static inline void compute_hflags(CPUMIPSState *env)
             env->hflags |= MIPS_HFLAG_COP1X;
         }
     }
+    if (env->insn_flags & ASE_MSA) {
+        if (env->CP0_Config5 & (1 << CP0C5_MSAEn)) {
+            env->hflags |= MIPS_HFLAG_MSA;
+        }
+    }
 }
 
 #endif /* !defined (__MIPS_CPU_H__) */
diff --git a/target-mips/dsp_helper.c b/target-mips/dsp_helper.c
index 94083fb42..349f2a003 100644
--- a/target-mips/dsp_helper.c
+++ b/target-mips/dsp_helper.c
@@ -76,15 +76,6 @@ static inline void set_DSPControl_24(uint32_t flag, int len, CPUMIPSState *env)
   env->active_tc.DSPControl |= (target_ulong)flag << 24;
 }
 
-static inline uint32_t get_DSPControl_24(int len, CPUMIPSState *env)
-{
-  uint32_t filter;
-
-  filter = (0x01 << len) - 1;
-
-  return (env->active_tc.DSPControl >> 24) & filter;
-}
-
 static inline void set_DSPControl_pos(uint32_t pos, CPUMIPSState *env)
 {
     target_ulong dspc;
@@ -283,6 +274,7 @@ static inline int32_t mipsdsp_sat32_acc_q31(int32_t acc, int32_t a,
     return result;
 }
 
+#ifdef TARGET_MIPS64
 /* a[0] is LO, a[1] is HI. */
 static inline void mipsdsp_sat64_acc_add_q63(int64_t *ret,
                                              int32_t ac,
@@ -336,6 +328,7 @@ static inline void mipsdsp_sat64_acc_sub_q63(int64_t *ret,
         set_DSPControl_overflow_flag(1, 16 + ac, env);
     }
 }
+#endif
 
 static inline int32_t mipsdsp_mul_i16_i16(int16_t a, int16_t b,
                                           CPUMIPSState *env)
@@ -357,10 +350,12 @@ static inline int32_t mipsdsp_mul_u16_u16(int32_t a, int32_t b)
     return a * b;
 }
 
+#ifdef TARGET_MIPS64
 static inline int32_t mipsdsp_mul_i32_i32(int32_t a, int32_t b)
 {
     return a * b;
 }
+#endif
 
 static inline int32_t mipsdsp_sat16_mul_i16_i16(int16_t a, int16_t b,
                                                 CPUMIPSState *env)
@@ -417,10 +412,12 @@ static inline int16_t mipsdsp_rashift16(int16_t a, target_ulong mov)
     return a >> mov;
 }
 
+#ifdef TARGET_MIPS64
 static inline int32_t mipsdsp_rashift32(int32_t a, target_ulong mov)
 {
     return a >> mov;
 }
+#endif
 
 static inline int16_t mipsdsp_rshift1_add_q16(int16_t a, int16_t b)
 {
@@ -479,6 +476,7 @@ static inline uint8_t mipsdsp_rrshift1_add_u8(uint8_t a, uint8_t b)
     return (temp >> 1) & 0x00FF;
 }
 
+#ifdef TARGET_MIPS64
 static inline uint8_t mipsdsp_rshift1_sub_u8(uint8_t a, uint8_t b)
 {
     uint16_t temp;
@@ -496,6 +494,7 @@ static inline uint8_t mipsdsp_rrshift1_sub_u8(uint8_t a, uint8_t b)
 
     return (temp >> 1) & 0x00FF;
 }
+#endif
 
 /*  128 bits long. p[0] is LO, p[1] is HI. */
 static inline void mipsdsp_rndrashift_short_acc(int64_t *p,
@@ -511,6 +510,7 @@ static inline void mipsdsp_rndrashift_short_acc(int64_t *p,
     p[1] = (acc >> 63) & 0x01;
 }
 
+#ifdef TARGET_MIPS64
 /* 128 bits long. p[0] is LO, p[1] is HI */
 static inline void mipsdsp_rashift_acc(uint64_t *p,
                                        uint32_t ac,
@@ -558,6 +558,7 @@ static inline void mipsdsp_rndrashift_acc(uint64_t *p,
         }
     }
 }
+#endif
 
 static inline int32_t mipsdsp_mul_q15_q15(int32_t ac, uint16_t a, uint16_t b,
                                           CPUMIPSState *env)
@@ -608,10 +609,12 @@ static inline uint16_t mipsdsp_mul_u8_u16(uint8_t a, uint16_t b,
     return tempI & 0x0000FFFF;
 }
 
+#ifdef TARGET_MIPS64
 static inline uint64_t mipsdsp_mul_u32_u32(uint32_t a, uint32_t b)
 {
     return (uint64_t)a * (uint64_t)b;
 }
+#endif
 
 static inline int16_t mipsdsp_rndq15_mul_q15_q15(uint16_t a, uint16_t b,
                                                  CPUMIPSState *env)
@@ -717,7 +720,7 @@ static inline uint16_t mipsdsp_lshift16(uint16_t a, uint8_t s,
     return a << s;
 }
 
-
+#ifdef TARGET_MIPS64
 static inline uint32_t mipsdsp_lshift32(uint32_t a, uint8_t s,
                                         CPUMIPSState *env)
 {
@@ -734,6 +737,7 @@ static inline uint32_t mipsdsp_lshift32(uint32_t a, uint8_t s,
         return a << s;
     }
 }
+#endif
 
 static inline uint16_t mipsdsp_sat16_lshift(uint16_t a, uint8_t s,
                                             CPUMIPSState *env)
@@ -973,6 +977,7 @@ static inline uint8_t mipsdsp_satu8_sub(uint8_t a, uint8_t b, CPUMIPSState *env)
     return temp & 0x00FF;
 }
 
+#ifdef TARGET_MIPS64
 static inline uint32_t mipsdsp_sub32(int32_t a, int32_t b, CPUMIPSState *env)
 {
     int32_t temp;
@@ -997,6 +1002,7 @@ static inline int32_t mipsdsp_add_i32(int32_t a, int32_t b, CPUMIPSState *env)
 
     return temp;
 }
+#endif
 
 static inline int32_t mipsdsp_cmp_eq(int32_t a, int32_t b)
 {
diff --git a/target-mips/gdbstub.c b/target-mips/gdbstub.c
index 5b72d58a4..f65fec23c 100644
--- a/target-mips/gdbstub.c
+++ b/target-mips/gdbstub.c
@@ -73,13 +73,6 @@ int mips_cpu_gdb_read_register(CPUState *cs, uint8_t *mem_buf, int n)
     return 0;
 }
 
-/* convert MIPS rounding mode in FCR31 to IEEE library */
-static unsigned int ieee_rm[] = {
-    float_round_nearest_even,
-    float_round_to_zero,
-    float_round_up,
-    float_round_down
-};
 #define RESTORE_ROUNDING_MODE \
     set_float_rounding_mode(ieee_rm[env->active_fpu.fcr31 & 3], \
                             &env->active_fpu.fp_status)
diff --git a/target-mips/helper.c b/target-mips/helper.c
index 8a997e44e..3a93c206e 100644
--- a/target-mips/helper.c
+++ b/target-mips/helper.c
@@ -25,8 +25,11 @@
 
 #include "cpu.h"
 #include "sysemu/kvm.h"
+#include "exec/cpu_ldst.h"
 
 enum {
+    TLBRET_XI = -6,
+    TLBRET_RI = -5,
     TLBRET_DIRTY = -4,
     TLBRET_INVALID = -3,
     TLBRET_NOMATCH = -2,
@@ -81,13 +84,20 @@ int r4k_map_address (CPUMIPSState *env, hwaddr *physical, int *prot,
 #endif
 
         /* Check ASID, virtual page number & size */
-        if ((tlb->G == 1 || tlb->ASID == ASID) && VPN == tag) {
+        if ((tlb->G == 1 || tlb->ASID == ASID) && VPN == tag && !tlb->EHINV) {
             /* TLB match */
             int n = !!(address & mask & ~(mask >> 1));
             /* Check access rights */
-            if (!(n ? tlb->V1 : tlb->V0))
+            if (!(n ? tlb->V1 : tlb->V0)) {
                 return TLBRET_INVALID;
-            if (rw == 0 || (n ? tlb->D1 : tlb->D0)) {
+            }
+            if (rw == MMU_INST_FETCH && (n ? tlb->XI1 : tlb->XI0)) {
+                return TLBRET_XI;
+            }
+            if (rw == MMU_DATA_LOAD && (n ? tlb->RI1 : tlb->RI0)) {
+                return TLBRET_RI;
+            }
+            if (rw != MMU_DATA_STORE || (n ? tlb->D1 : tlb->D0)) {
                 *physical = tlb->PFN[n] | (address & (mask >> 1));
                 *prot = PAGE_READ;
                 if (n ? tlb->D1 : tlb->D0)
@@ -232,36 +242,58 @@ static void raise_mmu_exception(CPUMIPSState *env, target_ulong address,
     CPUState *cs = CPU(mips_env_get_cpu(env));
     int exception = 0, error_code = 0;
 
+    if (rw == MMU_INST_FETCH) {
+        error_code |= EXCP_INST_NOTAVAIL;
+    }
+
     switch (tlb_error) {
     default:
     case TLBRET_BADADDR:
         /* Reference to kernel address from user mode or supervisor mode */
         /* Reference to supervisor address from user mode */
-        if (rw)
+        if (rw == MMU_DATA_STORE) {
             exception = EXCP_AdES;
-        else
+        } else {
             exception = EXCP_AdEL;
+        }
         break;
     case TLBRET_NOMATCH:
         /* No TLB match for a mapped address */
-        if (rw)
+        if (rw == MMU_DATA_STORE) {
             exception = EXCP_TLBS;
-        else
+        } else {
             exception = EXCP_TLBL;
-        error_code = 1;
+        }
+        error_code |= EXCP_TLB_NOMATCH;
         break;
     case TLBRET_INVALID:
         /* TLB match with no valid bit */
-        if (rw)
+        if (rw == MMU_DATA_STORE) {
             exception = EXCP_TLBS;
-        else
+        } else {
             exception = EXCP_TLBL;
+        }
         break;
     case TLBRET_DIRTY:
         /* TLB match but 'D' bit is cleared */
         exception = EXCP_LTLBL;
         break;
-
+    case TLBRET_XI:
+        /* Execute-Inhibit Exception */
+        if (env->CP0_PageGrain & (1 << CP0PG_IEC)) {
+            exception = EXCP_TLBXI;
+        } else {
+            exception = EXCP_TLBL;
+        }
+        break;
+    case TLBRET_RI:
+        /* Read-Inhibit Exception */
+        if (env->CP0_PageGrain & (1 << CP0PG_IEC)) {
+            exception = EXCP_TLBRI;
+        } else {
+            exception = EXCP_TLBL;
+        }
+        break;
     }
     /* Raise exception */
     env->CP0_BadVAddr = address;
@@ -312,8 +344,6 @@ int mips_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int rw,
     qemu_log("%s pc " TARGET_FMT_lx " ad %" VADDR_PRIx " rw %d mmu_idx %d\n",
               __func__, env->active_tc.PC, address, rw, mmu_idx);
 
-    rw &= 1;
-
     /* data access */
 #if !defined(CONFIG_USER_ONLY)
     /* XXX: put correct access by using cpu_restore_state()
@@ -347,8 +377,6 @@ hwaddr cpu_mips_translate_address(CPUMIPSState *env, target_ulong address, int r
     int access_type;
     int ret = 0;
 
-    rw &= 1;
-
     /* data access */
     access_type = ACCESS_INT;
     ret = get_physical_address(env, &physical, &prot,
@@ -396,6 +424,10 @@ static const char * const excp_names[EXCP_LAST + 1] = {
     [EXCP_MDMX] = "MDMX",
     [EXCP_C2E] = "precise coprocessor 2",
     [EXCP_CACHE] = "cache error",
+    [EXCP_TLBXI] = "TLB execute-inhibit",
+    [EXCP_TLBRI] = "TLB read-inhibit",
+    [EXCP_MSADIS] = "MSA disabled",
+    [EXCP_MSAFPE] = "MSA floating point",
 };
 
 target_ulong exception_resume_pc (CPUMIPSState *env)
@@ -426,6 +458,21 @@ static void set_hflags_for_handler (CPUMIPSState *env)
                         << MIPS_HFLAG_M16_SHIFT);
     }
 }
+
+static inline void set_badinstr_registers(CPUMIPSState *env)
+{
+    if (env->hflags & MIPS_HFLAG_M16) {
+        /* TODO: add BadInstr support for microMIPS */
+        return;
+    }
+    if (env->CP0_Config3 & (1 << CP0C3_BI)) {
+        env->CP0_BadInstr = cpu_ldl_code(env, env->active_tc.PC);
+    }
+    if ((env->CP0_Config3 & (1 << CP0C3_BP)) &&
+        (env->hflags & MIPS_HFLAG_BMASK)) {
+        env->CP0_BadInstrP = cpu_ldl_code(env, env->active_tc.PC - 4);
+    }
+}
 #endif
 
 void mips_cpu_do_interrupt(CPUState *cs)
@@ -433,6 +480,7 @@ void mips_cpu_do_interrupt(CPUState *cs)
 #if !defined(CONFIG_USER_ONLY)
     MIPSCPU *cpu = MIPS_CPU(cs);
     CPUMIPSState *env = &cpu->env;
+    bool update_badinstr = 0;
     target_ulong offset;
     int cause = -1;
     const char *name;
@@ -541,10 +589,13 @@ void mips_cpu_do_interrupt(CPUState *cs)
         goto set_EPC;
     case EXCP_LTLBL:
         cause = 1;
+        update_badinstr = !(env->error_code & EXCP_INST_NOTAVAIL);
         goto set_EPC;
     case EXCP_TLBL:
         cause = 2;
-        if (env->error_code == 1 && !(env->CP0_Status & (1 << CP0St_EXL))) {
+        update_badinstr = !(env->error_code & EXCP_INST_NOTAVAIL);
+        if ((env->error_code & EXCP_TLB_NOMATCH) &&
+            !(env->CP0_Status & (1 << CP0St_EXL))) {
 #if defined(TARGET_MIPS64)
             int R = env->CP0_BadVAddr >> 62;
             int UX = (env->CP0_Status & (1 << CP0St_UX)) != 0;
@@ -561,7 +612,9 @@ void mips_cpu_do_interrupt(CPUState *cs)
         goto set_EPC;
     case EXCP_TLBS:
         cause = 3;
-        if (env->error_code == 1 && !(env->CP0_Status & (1 << CP0St_EXL))) {
+        update_badinstr = 1;
+        if ((env->error_code & EXCP_TLB_NOMATCH) &&
+            !(env->CP0_Status & (1 << CP0St_EXL))) {
 #if defined(TARGET_MIPS64)
             int R = env->CP0_BadVAddr >> 62;
             int UX = (env->CP0_Status & (1 << CP0St_UX)) != 0;
@@ -578,9 +631,11 @@ void mips_cpu_do_interrupt(CPUState *cs)
         goto set_EPC;
     case EXCP_AdEL:
         cause = 4;
+        update_badinstr = !(env->error_code & EXCP_INST_NOTAVAIL);
         goto set_EPC;
     case EXCP_AdES:
         cause = 5;
+        update_badinstr = 1;
         goto set_EPC;
     case EXCP_IBE:
         cause = 6;
@@ -590,30 +645,52 @@ void mips_cpu_do_interrupt(CPUState *cs)
         goto set_EPC;
     case EXCP_SYSCALL:
         cause = 8;
+        update_badinstr = 1;
         goto set_EPC;
     case EXCP_BREAK:
         cause = 9;
+        update_badinstr = 1;
         goto set_EPC;
     case EXCP_RI:
         cause = 10;
+        update_badinstr = 1;
         goto set_EPC;
     case EXCP_CpU:
         cause = 11;
+        update_badinstr = 1;
         env->CP0_Cause = (env->CP0_Cause & ~(0x3 << CP0Ca_CE)) |
                          (env->error_code << CP0Ca_CE);
         goto set_EPC;
     case EXCP_OVERFLOW:
         cause = 12;
+        update_badinstr = 1;
         goto set_EPC;
     case EXCP_TRAP:
         cause = 13;
+        update_badinstr = 1;
+        goto set_EPC;
+    case EXCP_MSAFPE:
+        cause = 14;
+        update_badinstr = 1;
         goto set_EPC;
     case EXCP_FPE:
         cause = 15;
+        update_badinstr = 1;
         goto set_EPC;
     case EXCP_C2E:
         cause = 18;
         goto set_EPC;
+    case EXCP_TLBRI:
+        cause = 19;
+        update_badinstr = 1;
+        goto set_EPC;
+    case EXCP_TLBXI:
+        cause = 20;
+        goto set_EPC;
+    case EXCP_MSADIS:
+        cause = 21;
+        update_badinstr = 1;
+        goto set_EPC;
     case EXCP_MDMX:
         cause = 22;
         goto set_EPC;
@@ -640,6 +717,9 @@ void mips_cpu_do_interrupt(CPUState *cs)
  set_EPC:
         if (!(env->CP0_Status & (1 << CP0St_EXL))) {
             env->CP0_EPC = exception_resume_pc(env);
+            if (update_badinstr) {
+                set_badinstr_registers(env);
+            }
             if (env->hflags & MIPS_HFLAG_BMASK) {
                 env->CP0_Cause |= (1U << CP0Ca_BD);
             } else {
@@ -675,6 +755,23 @@ void mips_cpu_do_interrupt(CPUState *cs)
     cs->exception_index = EXCP_NONE;
 }
 
+bool mips_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    if (interrupt_request & CPU_INTERRUPT_HARD) {
+        MIPSCPU *cpu = MIPS_CPU(cs);
+        CPUMIPSState *env = &cpu->env;
+
+        if (cpu_mips_hw_interrupts_pending(env)) {
+            /* Raise it */
+            cs->exception_index = EXCP_EXT_INTERRUPT;
+            env->error_code = 0;
+            mips_cpu_do_interrupt(cs);
+            return true;
+        }
+    }
+    return false;
+}
+
 #if !defined(CONFIG_USER_ONLY)
 void r4k_invalidate_tlb (CPUMIPSState *env, int idx, int use_extra)
 {
diff --git a/target-mips/helper.h b/target-mips/helper.h
index 74ef09485..9d0275891 100644
--- a/target-mips/helper.h
+++ b/target-mips/helper.h
@@ -39,6 +39,11 @@ DEF_HELPER_3(macchiu, tl, env, tl, tl)
 DEF_HELPER_3(msachi, tl, env, tl, tl)
 DEF_HELPER_3(msachiu, tl, env, tl, tl)
 
+DEF_HELPER_FLAGS_1(bitswap, TCG_CALL_NO_RWG_SE, tl, tl)
+#ifdef TARGET_MIPS64
+DEF_HELPER_FLAGS_1(dbitswap, TCG_CALL_NO_RWG_SE, tl, tl)
+#endif
+
 #ifndef CONFIG_USER_ONLY
 /* CP0 helpers */
 DEF_HELPER_1(mfc0_mvpcontrol, tl, env)
@@ -147,6 +152,11 @@ DEF_HELPER_2(mtc0_datalo, void, env, tl)
 DEF_HELPER_2(mtc0_taghi, void, env, tl)
 DEF_HELPER_2(mtc0_datahi, void, env, tl)
 
+#if defined(TARGET_MIPS64)
+DEF_HELPER_2(dmtc0_entrylo0, void, env, i64)
+DEF_HELPER_2(dmtc0_entrylo1, void, env, i64)
+#endif
+
 /* MIPS MT functions */
 DEF_HELPER_2(mftgpr, tl, env, i32)
 DEF_HELPER_2(mftlo, tl, env, i32)
@@ -197,6 +207,25 @@ DEF_HELPER_2(float_cvtw_d, i32, env, i64)
 DEF_HELPER_3(float_addr_ps, i64, env, i64, i64)
 DEF_HELPER_3(float_mulr_ps, i64, env, i64, i64)
 
+DEF_HELPER_FLAGS_1(float_class_s, TCG_CALL_NO_RWG_SE, i32, i32)
+DEF_HELPER_FLAGS_1(float_class_d, TCG_CALL_NO_RWG_SE, i64, i64)
+
+#define FOP_PROTO(op)                                     \
+DEF_HELPER_4(float_ ## op ## _s, i32, env, i32, i32, i32) \
+DEF_HELPER_4(float_ ## op ## _d, i64, env, i64, i64, i64)
+FOP_PROTO(maddf)
+FOP_PROTO(msubf)
+#undef FOP_PROTO
+
+#define FOP_PROTO(op)                                \
+DEF_HELPER_3(float_ ## op ## _s, i32, env, i32, i32) \
+DEF_HELPER_3(float_ ## op ## _d, i64, env, i64, i64)
+FOP_PROTO(max)
+FOP_PROTO(maxa)
+FOP_PROTO(min)
+FOP_PROTO(mina)
+#undef FOP_PROTO
+
 #define FOP_PROTO(op)                            \
 DEF_HELPER_2(float_ ## op ## l_s, i64, env, i32) \
 DEF_HELPER_2(float_ ## op ## l_d, i64, env, i64) \
@@ -214,6 +243,7 @@ DEF_HELPER_2(float_ ## op ## _d, i64, env, i64)
 FOP_PROTO(sqrt)
 FOP_PROTO(rsqrt)
 FOP_PROTO(recip)
+FOP_PROTO(rint)
 #undef FOP_PROTO
 
 #define FOP_PROTO(op)                       \
@@ -279,12 +309,41 @@ FOP_PROTO(le)
 FOP_PROTO(ngt)
 #undef FOP_PROTO
 
+#define FOP_PROTO(op) \
+DEF_HELPER_3(r6_cmp_d_ ## op, i64, env, i64, i64) \
+DEF_HELPER_3(r6_cmp_s_ ## op, i32, env, i32, i32)
+FOP_PROTO(af)
+FOP_PROTO(un)
+FOP_PROTO(eq)
+FOP_PROTO(ueq)
+FOP_PROTO(lt)
+FOP_PROTO(ult)
+FOP_PROTO(le)
+FOP_PROTO(ule)
+FOP_PROTO(saf)
+FOP_PROTO(sun)
+FOP_PROTO(seq)
+FOP_PROTO(sueq)
+FOP_PROTO(slt)
+FOP_PROTO(sult)
+FOP_PROTO(sle)
+FOP_PROTO(sule)
+FOP_PROTO(or)
+FOP_PROTO(une)
+FOP_PROTO(ne)
+FOP_PROTO(sor)
+FOP_PROTO(sune)
+FOP_PROTO(sne)
+#undef FOP_PROTO
+
 /* Special functions */
 #ifndef CONFIG_USER_ONLY
 DEF_HELPER_1(tlbwi, void, env)
 DEF_HELPER_1(tlbwr, void, env)
 DEF_HELPER_1(tlbp, void, env)
 DEF_HELPER_1(tlbr, void, env)
+DEF_HELPER_1(tlbinv, void, env)
+DEF_HELPER_1(tlbinvf, void, env)
 DEF_HELPER_1(di, tl, env)
 DEF_HELPER_1(ei, tl, env)
 DEF_HELPER_1(eret, void, env)
@@ -689,3 +748,187 @@ DEF_HELPER_FLAGS_3(dmthlip, 0, void, tl, tl, env)
 #endif
 DEF_HELPER_FLAGS_3(wrdsp, 0, void, tl, tl, env)
 DEF_HELPER_FLAGS_2(rddsp, 0, tl, tl, env)
+
+/* MIPS SIMD Architecture */
+DEF_HELPER_4(msa_andi_b, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_ori_b, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_nori_b, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_xori_b, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_bmnzi_b, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_bmzi_b, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_bseli_b, void, env, i32, i32, i32)
+DEF_HELPER_5(msa_shf_df, void, env, i32, i32, i32, i32)
+
+DEF_HELPER_5(msa_addvi_df, void, env, i32, i32, i32, s32)
+DEF_HELPER_5(msa_subvi_df, void, env, i32, i32, i32, s32)
+DEF_HELPER_5(msa_maxi_s_df, void, env, i32, i32, i32, s32)
+DEF_HELPER_5(msa_maxi_u_df, void, env, i32, i32, i32, s32)
+DEF_HELPER_5(msa_mini_s_df, void, env, i32, i32, i32, s32)
+DEF_HELPER_5(msa_mini_u_df, void, env, i32, i32, i32, s32)
+DEF_HELPER_5(msa_ceqi_df, void, env, i32, i32, i32, s32)
+DEF_HELPER_5(msa_clti_s_df, void, env, i32, i32, i32, s32)
+DEF_HELPER_5(msa_clti_u_df, void, env, i32, i32, i32, s32)
+DEF_HELPER_5(msa_clei_s_df, void, env, i32, i32, i32, s32)
+DEF_HELPER_5(msa_clei_u_df, void, env, i32, i32, i32, s32)
+DEF_HELPER_4(msa_ldi_df, void, env, i32, i32, s32)
+
+DEF_HELPER_5(msa_slli_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_srai_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_srli_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_bclri_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_bseti_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_bnegi_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_binsli_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_binsri_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_sat_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_sat_u_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_srari_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_srlri_df, void, env, i32, i32, i32, i32)
+
+DEF_HELPER_5(msa_sll_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_sra_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_srl_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_bclr_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_bset_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_bneg_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_binsl_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_binsr_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_addv_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_subv_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_max_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_max_u_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_min_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_min_u_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_max_a_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_min_a_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_ceq_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_clt_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_clt_u_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_cle_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_cle_u_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_add_a_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_adds_a_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_adds_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_adds_u_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_ave_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_ave_u_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_aver_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_aver_u_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_subs_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_subs_u_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_subsus_u_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_subsuu_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_asub_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_asub_u_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_mulv_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_maddv_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_msubv_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_div_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_div_u_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_mod_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_mod_u_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_dotp_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_dotp_u_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_dpadd_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_dpadd_u_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_dpsub_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_dpsub_u_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_sld_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_splat_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_pckev_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_pckod_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_ilvl_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_ilvr_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_ilvev_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_ilvod_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_vshf_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_srar_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_srlr_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_hadd_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_hadd_u_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_hsub_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_hsub_u_df, void, env, i32, i32, i32, i32)
+
+DEF_HELPER_5(msa_sldi_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_splati_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_copy_s_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_copy_u_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_insert_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_insve_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_3(msa_ctcmsa, void, env, tl, i32)
+DEF_HELPER_2(msa_cfcmsa, tl, env, i32)
+DEF_HELPER_3(msa_move_v, void, env, i32, i32)
+
+DEF_HELPER_5(msa_fcaf_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fcun_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fceq_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fcueq_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fclt_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fcult_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fcle_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fcule_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fsaf_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fsun_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fseq_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fsueq_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fslt_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fsult_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fsle_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fsule_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fadd_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fsub_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fmul_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fdiv_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fmadd_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fmsub_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fexp2_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fexdo_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_ftq_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fmin_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fmin_a_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fmax_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fmax_a_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fcor_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fcune_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fcne_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_mul_q_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_madd_q_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_msub_q_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fsor_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fsune_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_fsne_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_mulr_q_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_maddr_q_df, void, env, i32, i32, i32, i32)
+DEF_HELPER_5(msa_msubr_q_df, void, env, i32, i32, i32, i32)
+
+DEF_HELPER_4(msa_and_v, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_or_v, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_nor_v, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_xor_v, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_bmnz_v, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_bmz_v, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_bsel_v, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_fill_df, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_pcnt_df, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_nloc_df, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_nlzc_df, void, env, i32, i32, i32)
+
+DEF_HELPER_4(msa_fclass_df, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_ftrunc_s_df, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_ftrunc_u_df, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_fsqrt_df, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_frsqrt_df, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_frcp_df, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_frint_df, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_flog2_df, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_fexupl_df, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_fexupr_df, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_ffql_df, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_ffqr_df, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_ftint_s_df, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_ftint_u_df, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_ffint_s_df, void, env, i32, i32, i32)
+DEF_HELPER_4(msa_ffint_u_df, void, env, i32, i32, i32)
+
+DEF_HELPER_5(msa_ld_df, void, env, i32, i32, i32, s32)
+DEF_HELPER_5(msa_st_df, void, env, i32, i32, i32, s32)
diff --git a/target-mips/machine.c b/target-mips/machine.c
index 0496faa91..0ba7d736d 100644
--- a/target-mips/machine.c
+++ b/target-mips/machine.c
@@ -61,7 +61,12 @@ void cpu_save(QEMUFile *f, void *opaque)
     qemu_put_be32s(f, &env->tlb->nb_tlb);
     qemu_put_be32s(f, &env->tlb->tlb_in_use);
     for(i = 0; i < MIPS_TLB_MAX; i++) {
-        uint16_t flags = ((env->tlb->mmu.r4k.tlb[i].G << 10) |
+        uint16_t flags = ((env->tlb->mmu.r4k.tlb[i].EHINV << 15) |
+                          (env->tlb->mmu.r4k.tlb[i].RI1 << 14) |
+                          (env->tlb->mmu.r4k.tlb[i].RI0 << 13) |
+                          (env->tlb->mmu.r4k.tlb[i].XI1 << 12) |
+                          (env->tlb->mmu.r4k.tlb[i].XI0 << 11) |
+                          (env->tlb->mmu.r4k.tlb[i].G << 10) |
                           (env->tlb->mmu.r4k.tlb[i].C0 << 7) |
                           (env->tlb->mmu.r4k.tlb[i].C1 << 4) |
                           (env->tlb->mmu.r4k.tlb[i].V0 << 3) |
@@ -111,6 +116,8 @@ void cpu_save(QEMUFile *f, void *opaque)
     qemu_put_sbe32s(f, &env->CP0_SRSConf4);
     qemu_put_sbe32s(f, &env->CP0_HWREna);
     qemu_put_betls(f, &env->CP0_BadVAddr);
+    qemu_put_be32s(f, &env->CP0_BadInstr);
+    qemu_put_be32s(f, &env->CP0_BadInstrP);
     qemu_put_sbe32s(f, &env->CP0_Count);
     qemu_put_betls(f, &env->CP0_EntryHi);
     qemu_put_sbe32s(f, &env->CP0_Compare);
@@ -144,6 +151,9 @@ void cpu_save(QEMUFile *f, void *opaque)
     qemu_put_sbe32s(f, &env->CP0_DataHi);
     qemu_put_betls(f, &env->CP0_ErrorEPC);
     qemu_put_sbe32s(f, &env->CP0_DESAVE);
+    for (i = 0; i < MIPS_KSCRATCH_NUM; i++) {
+        qemu_put_betls(f, &env->CP0_KScratch[i]);
+    }
 
     /* Save inactive TC state */
     for (i = 0; i < MIPS_SHADOW_SET_MAX; i++)
@@ -232,6 +242,13 @@ int cpu_load(QEMUFile *f, void *opaque, int version_id)
         env->tlb->mmu.r4k.tlb[i].V1 = (flags >> 2) & 1;
         env->tlb->mmu.r4k.tlb[i].D0 = (flags >> 1) & 1;
         env->tlb->mmu.r4k.tlb[i].D1 = (flags >> 0) & 1;
+        if (version_id >= 5) {
+            env->tlb->mmu.r4k.tlb[i].EHINV = (flags >> 15) & 1;
+            env->tlb->mmu.r4k.tlb[i].RI1 = (flags >> 14) & 1;
+            env->tlb->mmu.r4k.tlb[i].RI0 = (flags >> 13) & 1;
+            env->tlb->mmu.r4k.tlb[i].XI1 = (flags >> 12) & 1;
+            env->tlb->mmu.r4k.tlb[i].XI0 = (flags >> 11) & 1;
+        }
         qemu_get_betls(f, &env->tlb->mmu.r4k.tlb[i].PFN[0]);
         qemu_get_betls(f, &env->tlb->mmu.r4k.tlb[i].PFN[1]);
     }
@@ -301,6 +318,13 @@ int cpu_load(QEMUFile *f, void *opaque, int version_id)
     qemu_get_sbe32s(f, &env->CP0_DataHi);
     qemu_get_betls(f, &env->CP0_ErrorEPC);
     qemu_get_sbe32s(f, &env->CP0_DESAVE);
+    if (version_id >= 5) {
+        qemu_get_be32s(f, &env->CP0_BadInstr);
+        qemu_get_be32s(f, &env->CP0_BadInstrP);
+        for (i = 0; i < MIPS_KSCRATCH_NUM; i++) {
+            qemu_get_betls(f, &env->CP0_KScratch[i]);
+        }
+    }
 
     /* Load inactive TC state */
     for (i = 0; i < MIPS_SHADOW_SET_MAX; i++) {
diff --git a/target-mips/mips-defs.h b/target-mips/mips-defs.h
index 9dfa5168d..178422749 100644
--- a/target-mips/mips-defs.h
+++ b/target-mips/mips-defs.h
@@ -30,17 +30,22 @@
 #define		ISA_MIPS64	0x00000080
 #define		ISA_MIPS64R2	0x00000100
 #define   ISA_MIPS32R3  0x00000200
-#define   ISA_MIPS32R5  0x00000400
+#define   ISA_MIPS64R3  0x00000400
+#define   ISA_MIPS32R5  0x00000800
+#define   ISA_MIPS64R5  0x00001000
+#define   ISA_MIPS32R6  0x00002000
+#define   ISA_MIPS64R6  0x00004000
 
 /* MIPS ASEs. */
-#define		ASE_MIPS16	0x00001000
-#define		ASE_MIPS3D	0x00002000
-#define		ASE_MDMX	0x00004000
-#define		ASE_DSP		0x00008000
-#define		ASE_DSPR2	0x00010000
-#define		ASE_MT		0x00020000
-#define		ASE_SMARTMIPS	0x00040000
-#define 	ASE_MICROMIPS	0x00080000
+#define   ASE_MIPS16    0x00010000
+#define   ASE_MIPS3D    0x00020000
+#define   ASE_MDMX      0x00040000
+#define   ASE_DSP       0x00080000
+#define   ASE_DSPR2     0x00100000
+#define   ASE_MT        0x00200000
+#define   ASE_SMARTMIPS 0x00400000
+#define   ASE_MICROMIPS 0x00800000
+#define   ASE_MSA       0x01000000
 
 /* Chip specific instructions. */
 #define		INSN_LOONGSON2E  0x20000000
@@ -68,9 +73,15 @@
 
 /* MIPS Technologies "Release 3" */
 #define CPU_MIPS32R3 (CPU_MIPS32R2 | ISA_MIPS32R3)
+#define CPU_MIPS64R3 (CPU_MIPS64R2 | CPU_MIPS32R3 | ISA_MIPS64R3)
 
 /* MIPS Technologies "Release 5" */
 #define CPU_MIPS32R5 (CPU_MIPS32R3 | ISA_MIPS32R5)
+#define CPU_MIPS64R5 (CPU_MIPS64R3 | CPU_MIPS32R5 | ISA_MIPS64R5)
+
+/* MIPS Technologies "Release 6" */
+#define CPU_MIPS32R6 (CPU_MIPS32R5 | ISA_MIPS32R6)
+#define CPU_MIPS64R6 (CPU_MIPS64R5 | CPU_MIPS32R6 | ISA_MIPS64R6)
 
 /* Strictly follow the architecture standard:
    - Disallow "special" instruction handling for PMON/SPIM.
diff --git a/target-mips/msa_helper.c b/target-mips/msa_helper.c
new file mode 100644
index 000000000..b08f37f78
--- /dev/null
+++ b/target-mips/msa_helper.c
@@ -0,0 +1,3436 @@
+/*
+ * MIPS SIMD Architecture Module Instruction emulation helpers for QEMU.
+ *
+ * Copyright (c) 2014 Imagination Technologies
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cpu.h"
+#include "exec/helper-proto.h"
+
+/* Data format min and max values */
+#define DF_BITS(df) (1 << ((df) + 3))
+
+#define DF_MAX_INT(df)  (int64_t)((1LL << (DF_BITS(df) - 1)) - 1)
+#define M_MAX_INT(m)    (int64_t)((1LL << ((m)         - 1)) - 1)
+
+#define DF_MIN_INT(df)  (int64_t)(-(1LL << (DF_BITS(df) - 1)))
+#define M_MIN_INT(m)    (int64_t)(-(1LL << ((m)         - 1)))
+
+#define DF_MAX_UINT(df) (uint64_t)(-1ULL >> (64 - DF_BITS(df)))
+#define M_MAX_UINT(m)   (uint64_t)(-1ULL >> (64 - (m)))
+
+#define UNSIGNED(x, df) ((x) & DF_MAX_UINT(df))
+#define SIGNED(x, df)                                                   \
+    ((((int64_t)x) << (64 - DF_BITS(df))) >> (64 - DF_BITS(df)))
+
+/* Element-by-element access macros */
+#define DF_ELEMENTS(df) (MSA_WRLEN / DF_BITS(df))
+
+static inline void msa_move_v(wr_t *pwd, wr_t *pws)
+{
+    uint32_t i;
+
+    for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+        pwd->d[i] = pws->d[i];
+    }
+}
+
+#define MSA_FN_IMM8(FUNC, DEST, OPERATION)                              \
+void helper_msa_ ## FUNC(CPUMIPSState *env, uint32_t wd, uint32_t ws,   \
+        uint32_t i8)                                                    \
+{                                                                       \
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);                          \
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);                          \
+    uint32_t i;                                                         \
+    for (i = 0; i < DF_ELEMENTS(DF_BYTE); i++) {                        \
+        DEST = OPERATION;                                               \
+    }                                                                   \
+}
+
+MSA_FN_IMM8(andi_b, pwd->b[i], pws->b[i] & i8)
+MSA_FN_IMM8(ori_b, pwd->b[i], pws->b[i] | i8)
+MSA_FN_IMM8(nori_b, pwd->b[i], ~(pws->b[i] | i8))
+MSA_FN_IMM8(xori_b, pwd->b[i], pws->b[i] ^ i8)
+
+#define BIT_MOVE_IF_NOT_ZERO(dest, arg1, arg2, df) \
+            UNSIGNED(((dest & (~arg2)) | (arg1 & arg2)), df)
+MSA_FN_IMM8(bmnzi_b, pwd->b[i],
+        BIT_MOVE_IF_NOT_ZERO(pwd->b[i], pws->b[i], i8, DF_BYTE))
+
+#define BIT_MOVE_IF_ZERO(dest, arg1, arg2, df) \
+            UNSIGNED((dest & arg2) | (arg1 & (~arg2)), df)
+MSA_FN_IMM8(bmzi_b, pwd->b[i],
+        BIT_MOVE_IF_ZERO(pwd->b[i], pws->b[i], i8, DF_BYTE))
+
+#define BIT_SELECT(dest, arg1, arg2, df) \
+            UNSIGNED((arg1 & (~dest)) | (arg2 & dest), df)
+MSA_FN_IMM8(bseli_b, pwd->b[i],
+        BIT_SELECT(pwd->b[i], pws->b[i], i8, DF_BYTE))
+
+#undef MSA_FN_IMM8
+
+#define SHF_POS(i, imm) (((i) & 0xfc) + (((imm) >> (2 * ((i) & 0x03))) & 0x03))
+
+void helper_msa_shf_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                       uint32_t ws, uint32_t imm)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t wx, *pwx = &wx;
+    uint32_t i;
+
+    switch (df) {
+    case DF_BYTE:
+        for (i = 0; i < DF_ELEMENTS(DF_BYTE); i++) {
+            pwx->b[i] = pws->b[SHF_POS(i, imm)];
+        }
+        break;
+    case DF_HALF:
+        for (i = 0; i < DF_ELEMENTS(DF_HALF); i++) {
+            pwx->h[i] = pws->h[SHF_POS(i, imm)];
+        }
+        break;
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            pwx->w[i] = pws->w[SHF_POS(i, imm)];
+        }
+        break;
+    default:
+        assert(0);
+    }
+    msa_move_v(pwd, pwx);
+}
+
+#define MSA_FN_VECTOR(FUNC, DEST, OPERATION)                            \
+void helper_msa_ ## FUNC(CPUMIPSState *env, uint32_t wd, uint32_t ws,   \
+        uint32_t wt)                                                    \
+{                                                                       \
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);                          \
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);                          \
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);                          \
+    uint32_t i;                                                         \
+    for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {                      \
+        DEST = OPERATION;                                               \
+    }                                                                   \
+}
+
+MSA_FN_VECTOR(and_v, pwd->d[i], pws->d[i] & pwt->d[i])
+MSA_FN_VECTOR(or_v, pwd->d[i], pws->d[i] | pwt->d[i])
+MSA_FN_VECTOR(nor_v, pwd->d[i], ~(pws->d[i] | pwt->d[i]))
+MSA_FN_VECTOR(xor_v, pwd->d[i], pws->d[i] ^ pwt->d[i])
+MSA_FN_VECTOR(bmnz_v, pwd->d[i],
+        BIT_MOVE_IF_NOT_ZERO(pwd->d[i], pws->d[i], pwt->d[i], DF_DOUBLE))
+MSA_FN_VECTOR(bmz_v, pwd->d[i],
+        BIT_MOVE_IF_ZERO(pwd->d[i], pws->d[i], pwt->d[i], DF_DOUBLE))
+MSA_FN_VECTOR(bsel_v, pwd->d[i],
+        BIT_SELECT(pwd->d[i], pws->d[i], pwt->d[i], DF_DOUBLE))
+#undef BIT_MOVE_IF_NOT_ZERO
+#undef BIT_MOVE_IF_ZERO
+#undef BIT_SELECT
+#undef MSA_FN_VECTOR
+
+static inline int64_t msa_addv_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    return arg1 + arg2;
+}
+
+static inline int64_t msa_subv_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    return arg1 - arg2;
+}
+
+static inline int64_t msa_ceq_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    return arg1 == arg2 ? -1 : 0;
+}
+
+static inline int64_t msa_cle_s_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    return arg1 <= arg2 ? -1 : 0;
+}
+
+static inline int64_t msa_cle_u_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    uint64_t u_arg1 = UNSIGNED(arg1, df);
+    uint64_t u_arg2 = UNSIGNED(arg2, df);
+    return u_arg1 <= u_arg2 ? -1 : 0;
+}
+
+static inline int64_t msa_clt_s_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    return arg1 < arg2 ? -1 : 0;
+}
+
+static inline int64_t msa_clt_u_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    uint64_t u_arg1 = UNSIGNED(arg1, df);
+    uint64_t u_arg2 = UNSIGNED(arg2, df);
+    return u_arg1 < u_arg2 ? -1 : 0;
+}
+
+static inline int64_t msa_max_s_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    return arg1 > arg2 ? arg1 : arg2;
+}
+
+static inline int64_t msa_max_u_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    uint64_t u_arg1 = UNSIGNED(arg1, df);
+    uint64_t u_arg2 = UNSIGNED(arg2, df);
+    return u_arg1 > u_arg2 ? arg1 : arg2;
+}
+
+static inline int64_t msa_min_s_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    return arg1 < arg2 ? arg1 : arg2;
+}
+
+static inline int64_t msa_min_u_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    uint64_t u_arg1 = UNSIGNED(arg1, df);
+    uint64_t u_arg2 = UNSIGNED(arg2, df);
+    return u_arg1 < u_arg2 ? arg1 : arg2;
+}
+
+#define MSA_BINOP_IMM_DF(helper, func)                                  \
+void helper_msa_ ## helper ## _df(CPUMIPSState *env, uint32_t df,       \
+                        uint32_t wd, uint32_t ws, int32_t u5)           \
+{                                                                       \
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);                          \
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);                          \
+    uint32_t i;                                                         \
+                                                                        \
+    switch (df) {                                                       \
+    case DF_BYTE:                                                       \
+        for (i = 0; i < DF_ELEMENTS(DF_BYTE); i++) {                    \
+            pwd->b[i] = msa_ ## func ## _df(df, pws->b[i], u5);         \
+        }                                                               \
+        break;                                                          \
+    case DF_HALF:                                                       \
+        for (i = 0; i < DF_ELEMENTS(DF_HALF); i++) {                    \
+            pwd->h[i] = msa_ ## func ## _df(df, pws->h[i], u5);         \
+        }                                                               \
+        break;                                                          \
+    case DF_WORD:                                                       \
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {                    \
+            pwd->w[i] = msa_ ## func ## _df(df, pws->w[i], u5);         \
+        }                                                               \
+        break;                                                          \
+    case DF_DOUBLE:                                                     \
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {                  \
+            pwd->d[i] = msa_ ## func ## _df(df, pws->d[i], u5);         \
+        }                                                               \
+        break;                                                          \
+    default:                                                            \
+        assert(0);                                                      \
+    }                                                                   \
+}
+
+MSA_BINOP_IMM_DF(addvi, addv)
+MSA_BINOP_IMM_DF(subvi, subv)
+MSA_BINOP_IMM_DF(ceqi, ceq)
+MSA_BINOP_IMM_DF(clei_s, cle_s)
+MSA_BINOP_IMM_DF(clei_u, cle_u)
+MSA_BINOP_IMM_DF(clti_s, clt_s)
+MSA_BINOP_IMM_DF(clti_u, clt_u)
+MSA_BINOP_IMM_DF(maxi_s, max_s)
+MSA_BINOP_IMM_DF(maxi_u, max_u)
+MSA_BINOP_IMM_DF(mini_s, min_s)
+MSA_BINOP_IMM_DF(mini_u, min_u)
+#undef MSA_BINOP_IMM_DF
+
+void helper_msa_ldi_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                       int32_t s10)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    uint32_t i;
+
+    switch (df) {
+    case DF_BYTE:
+        for (i = 0; i < DF_ELEMENTS(DF_BYTE); i++) {
+            pwd->b[i] = (int8_t)s10;
+        }
+        break;
+    case DF_HALF:
+        for (i = 0; i < DF_ELEMENTS(DF_HALF); i++) {
+            pwd->h[i] = (int16_t)s10;
+        }
+        break;
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            pwd->w[i] = (int32_t)s10;
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            pwd->d[i] = (int64_t)s10;
+        }
+       break;
+    default:
+        assert(0);
+    }
+}
+
+/* Data format bit position and unsigned values */
+#define BIT_POSITION(x, df) ((uint64_t)(x) % DF_BITS(df))
+
+static inline int64_t msa_sll_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    int32_t b_arg2 = BIT_POSITION(arg2, df);
+    return arg1 << b_arg2;
+}
+
+static inline int64_t msa_sra_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    int32_t b_arg2 = BIT_POSITION(arg2, df);
+    return arg1 >> b_arg2;
+}
+
+static inline int64_t msa_srl_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    uint64_t u_arg1 = UNSIGNED(arg1, df);
+    int32_t b_arg2 = BIT_POSITION(arg2, df);
+    return u_arg1 >> b_arg2;
+}
+
+static inline int64_t msa_bclr_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    int32_t b_arg2 = BIT_POSITION(arg2, df);
+    return UNSIGNED(arg1 & (~(1LL << b_arg2)), df);
+}
+
+static inline int64_t msa_bset_df(uint32_t df, int64_t arg1,
+        int64_t arg2)
+{
+    int32_t b_arg2 = BIT_POSITION(arg2, df);
+    return UNSIGNED(arg1 | (1LL << b_arg2), df);
+}
+
+static inline int64_t msa_bneg_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    int32_t b_arg2 = BIT_POSITION(arg2, df);
+    return UNSIGNED(arg1 ^ (1LL << b_arg2), df);
+}
+
+static inline int64_t msa_binsl_df(uint32_t df, int64_t dest, int64_t arg1,
+                                   int64_t arg2)
+{
+    uint64_t u_arg1 = UNSIGNED(arg1, df);
+    uint64_t u_dest = UNSIGNED(dest, df);
+    int32_t sh_d = BIT_POSITION(arg2, df) + 1;
+    int32_t sh_a = DF_BITS(df) - sh_d;
+    if (sh_d == DF_BITS(df)) {
+        return u_arg1;
+    } else {
+        return UNSIGNED(UNSIGNED(u_dest << sh_d, df) >> sh_d, df) |
+               UNSIGNED(UNSIGNED(u_arg1 >> sh_a, df) << sh_a, df);
+    }
+}
+
+static inline int64_t msa_binsr_df(uint32_t df, int64_t dest, int64_t arg1,
+                                   int64_t arg2)
+{
+    uint64_t u_arg1 = UNSIGNED(arg1, df);
+    uint64_t u_dest = UNSIGNED(dest, df);
+    int32_t sh_d = BIT_POSITION(arg2, df) + 1;
+    int32_t sh_a = DF_BITS(df) - sh_d;
+    if (sh_d == DF_BITS(df)) {
+        return u_arg1;
+    } else {
+        return UNSIGNED(UNSIGNED(u_dest >> sh_d, df) << sh_d, df) |
+               UNSIGNED(UNSIGNED(u_arg1 << sh_a, df) >> sh_a, df);
+    }
+}
+
+static inline int64_t msa_sat_s_df(uint32_t df, int64_t arg, uint32_t m)
+{
+    return arg < M_MIN_INT(m+1) ? M_MIN_INT(m+1) :
+                                  arg > M_MAX_INT(m+1) ? M_MAX_INT(m+1) :
+                                                         arg;
+}
+
+static inline int64_t msa_sat_u_df(uint32_t df, int64_t arg, uint32_t m)
+{
+    uint64_t u_arg = UNSIGNED(arg, df);
+    return  u_arg < M_MAX_UINT(m+1) ? u_arg :
+                                      M_MAX_UINT(m+1);
+}
+
+static inline int64_t msa_srar_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    int32_t b_arg2 = BIT_POSITION(arg2, df);
+    if (b_arg2 == 0) {
+        return arg1;
+    } else {
+        int64_t r_bit = (arg1 >> (b_arg2 - 1)) & 1;
+        return (arg1 >> b_arg2) + r_bit;
+    }
+}
+
+static inline int64_t msa_srlr_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    uint64_t u_arg1 = UNSIGNED(arg1, df);
+    int32_t b_arg2 = BIT_POSITION(arg2, df);
+    if (b_arg2 == 0) {
+        return u_arg1;
+    } else {
+        uint64_t r_bit = (u_arg1 >> (b_arg2 - 1)) & 1;
+        return (u_arg1 >> b_arg2) + r_bit;
+    }
+}
+
+#define MSA_BINOP_IMMU_DF(helper, func)                                  \
+void helper_msa_ ## helper ## _df(CPUMIPSState *env, uint32_t df, uint32_t wd, \
+                       uint32_t ws, uint32_t u5)                        \
+{                                                                       \
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);                          \
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);                          \
+    uint32_t i;                                                         \
+                                                                        \
+    switch (df) {                                                       \
+    case DF_BYTE:                                                       \
+        for (i = 0; i < DF_ELEMENTS(DF_BYTE); i++) {                    \
+            pwd->b[i] = msa_ ## func ## _df(df, pws->b[i], u5);         \
+        }                                                               \
+        break;                                                          \
+    case DF_HALF:                                                       \
+        for (i = 0; i < DF_ELEMENTS(DF_HALF); i++) {                    \
+            pwd->h[i] = msa_ ## func ## _df(df, pws->h[i], u5);         \
+        }                                                               \
+        break;                                                          \
+    case DF_WORD:                                                       \
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {                    \
+            pwd->w[i] = msa_ ## func ## _df(df, pws->w[i], u5);         \
+        }                                                               \
+        break;                                                          \
+    case DF_DOUBLE:                                                     \
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {                  \
+            pwd->d[i] = msa_ ## func ## _df(df, pws->d[i], u5);         \
+        }                                                               \
+        break;                                                          \
+    default:                                                            \
+        assert(0);                                                      \
+    }                                                                   \
+}
+
+MSA_BINOP_IMMU_DF(slli, sll)
+MSA_BINOP_IMMU_DF(srai, sra)
+MSA_BINOP_IMMU_DF(srli, srl)
+MSA_BINOP_IMMU_DF(bclri, bclr)
+MSA_BINOP_IMMU_DF(bseti, bset)
+MSA_BINOP_IMMU_DF(bnegi, bneg)
+MSA_BINOP_IMMU_DF(sat_s, sat_s)
+MSA_BINOP_IMMU_DF(sat_u, sat_u)
+MSA_BINOP_IMMU_DF(srari, srar)
+MSA_BINOP_IMMU_DF(srlri, srlr)
+#undef MSA_BINOP_IMMU_DF
+
+#define MSA_TEROP_IMMU_DF(helper, func)                                  \
+void helper_msa_ ## helper ## _df(CPUMIPSState *env, uint32_t df,       \
+                                  uint32_t wd, uint32_t ws, uint32_t u5) \
+{                                                                       \
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);                          \
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);                          \
+    uint32_t i;                                                         \
+                                                                        \
+    switch (df) {                                                       \
+    case DF_BYTE:                                                       \
+        for (i = 0; i < DF_ELEMENTS(DF_BYTE); i++) {                    \
+            pwd->b[i] = msa_ ## func ## _df(df, pwd->b[i], pws->b[i],   \
+                                            u5);                        \
+        }                                                               \
+        break;                                                          \
+    case DF_HALF:                                                       \
+        for (i = 0; i < DF_ELEMENTS(DF_HALF); i++) {                    \
+            pwd->h[i] = msa_ ## func ## _df(df, pwd->h[i], pws->h[i],   \
+                                            u5);                        \
+        }                                                               \
+        break;                                                          \
+    case DF_WORD:                                                       \
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {                    \
+            pwd->w[i] = msa_ ## func ## _df(df, pwd->w[i], pws->w[i],   \
+                                            u5);                        \
+        }                                                               \
+        break;                                                          \
+    case DF_DOUBLE:                                                     \
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {                  \
+            pwd->d[i] = msa_ ## func ## _df(df, pwd->d[i], pws->d[i],   \
+                                            u5);                        \
+        }                                                               \
+        break;                                                          \
+    default:                                                            \
+        assert(0);                                                      \
+    }                                                                   \
+}
+
+MSA_TEROP_IMMU_DF(binsli, binsl)
+MSA_TEROP_IMMU_DF(binsri, binsr)
+#undef MSA_TEROP_IMMU_DF
+
+static inline int64_t msa_max_a_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    uint64_t abs_arg1 = arg1 >= 0 ? arg1 : -arg1;
+    uint64_t abs_arg2 = arg2 >= 0 ? arg2 : -arg2;
+    return abs_arg1 > abs_arg2 ? arg1 : arg2;
+}
+
+static inline int64_t msa_min_a_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    uint64_t abs_arg1 = arg1 >= 0 ? arg1 : -arg1;
+    uint64_t abs_arg2 = arg2 >= 0 ? arg2 : -arg2;
+    return abs_arg1 < abs_arg2 ? arg1 : arg2;
+}
+
+static inline int64_t msa_add_a_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    uint64_t abs_arg1 = arg1 >= 0 ? arg1 : -arg1;
+    uint64_t abs_arg2 = arg2 >= 0 ? arg2 : -arg2;
+    return abs_arg1 + abs_arg2;
+}
+
+static inline int64_t msa_adds_a_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    uint64_t max_int = (uint64_t)DF_MAX_INT(df);
+    uint64_t abs_arg1 = arg1 >= 0 ? arg1 : -arg1;
+    uint64_t abs_arg2 = arg2 >= 0 ? arg2 : -arg2;
+    if (abs_arg1 > max_int || abs_arg2 > max_int) {
+        return (int64_t)max_int;
+    } else {
+        return (abs_arg1 < max_int - abs_arg2) ? abs_arg1 + abs_arg2 : max_int;
+    }
+}
+
+static inline int64_t msa_adds_s_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    int64_t max_int = DF_MAX_INT(df);
+    int64_t min_int = DF_MIN_INT(df);
+    if (arg1 < 0) {
+        return (min_int - arg1 < arg2) ? arg1 + arg2 : min_int;
+    } else {
+        return (arg2 < max_int - arg1) ? arg1 + arg2 : max_int;
+    }
+}
+
+static inline uint64_t msa_adds_u_df(uint32_t df, uint64_t arg1, uint64_t arg2)
+{
+    uint64_t max_uint = DF_MAX_UINT(df);
+    uint64_t u_arg1 = UNSIGNED(arg1, df);
+    uint64_t u_arg2 = UNSIGNED(arg2, df);
+    return (u_arg1 < max_uint - u_arg2) ? u_arg1 + u_arg2 : max_uint;
+}
+
+static inline int64_t msa_ave_s_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    /* signed shift */
+    return (arg1 >> 1) + (arg2 >> 1) + (arg1 & arg2 & 1);
+}
+
+static inline uint64_t msa_ave_u_df(uint32_t df, uint64_t arg1, uint64_t arg2)
+{
+    uint64_t u_arg1 = UNSIGNED(arg1, df);
+    uint64_t u_arg2 = UNSIGNED(arg2, df);
+    /* unsigned shift */
+    return (u_arg1 >> 1) + (u_arg2 >> 1) + (u_arg1 & u_arg2 & 1);
+}
+
+static inline int64_t msa_aver_s_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    /* signed shift */
+    return (arg1 >> 1) + (arg2 >> 1) + ((arg1 | arg2) & 1);
+}
+
+static inline uint64_t msa_aver_u_df(uint32_t df, uint64_t arg1, uint64_t arg2)
+{
+    uint64_t u_arg1 = UNSIGNED(arg1, df);
+    uint64_t u_arg2 = UNSIGNED(arg2, df);
+    /* unsigned shift */
+    return (u_arg1 >> 1) + (u_arg2 >> 1) + ((u_arg1 | u_arg2) & 1);
+}
+
+static inline int64_t msa_subs_s_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    int64_t max_int = DF_MAX_INT(df);
+    int64_t min_int = DF_MIN_INT(df);
+    if (arg2 > 0) {
+        return (min_int + arg2 < arg1) ? arg1 - arg2 : min_int;
+    } else {
+        return (arg1 < max_int + arg2) ? arg1 - arg2 : max_int;
+    }
+}
+
+static inline int64_t msa_subs_u_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    uint64_t u_arg1 = UNSIGNED(arg1, df);
+    uint64_t u_arg2 = UNSIGNED(arg2, df);
+    return (u_arg1 > u_arg2) ? u_arg1 - u_arg2 : 0;
+}
+
+static inline int64_t msa_subsus_u_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    uint64_t u_arg1 = UNSIGNED(arg1, df);
+    uint64_t max_uint = DF_MAX_UINT(df);
+    if (arg2 >= 0) {
+        uint64_t u_arg2 = (uint64_t)arg2;
+        return (u_arg1 > u_arg2) ?
+            (int64_t)(u_arg1 - u_arg2) :
+            0;
+    } else {
+        uint64_t u_arg2 = (uint64_t)(-arg2);
+        return (u_arg1 < max_uint - u_arg2) ?
+            (int64_t)(u_arg1 + u_arg2) :
+            (int64_t)max_uint;
+    }
+}
+
+static inline int64_t msa_subsuu_s_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    uint64_t u_arg1 = UNSIGNED(arg1, df);
+    uint64_t u_arg2 = UNSIGNED(arg2, df);
+    int64_t max_int = DF_MAX_INT(df);
+    int64_t min_int = DF_MIN_INT(df);
+    if (u_arg1 > u_arg2) {
+        return u_arg1 - u_arg2 < (uint64_t)max_int ?
+            (int64_t)(u_arg1 - u_arg2) :
+            max_int;
+    } else {
+        return u_arg2 - u_arg1 < (uint64_t)(-min_int) ?
+            (int64_t)(u_arg1 - u_arg2) :
+            min_int;
+    }
+}
+
+static inline int64_t msa_asub_s_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    /* signed compare */
+    return (arg1 < arg2) ?
+        (uint64_t)(arg2 - arg1) : (uint64_t)(arg1 - arg2);
+}
+
+static inline uint64_t msa_asub_u_df(uint32_t df, uint64_t arg1, uint64_t arg2)
+{
+    uint64_t u_arg1 = UNSIGNED(arg1, df);
+    uint64_t u_arg2 = UNSIGNED(arg2, df);
+    /* unsigned compare */
+    return (u_arg1 < u_arg2) ?
+        (uint64_t)(u_arg2 - u_arg1) : (uint64_t)(u_arg1 - u_arg2);
+}
+
+static inline int64_t msa_mulv_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    return arg1 * arg2;
+}
+
+static inline int64_t msa_div_s_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    if (arg1 == DF_MIN_INT(df) && arg2 == -1) {
+        return DF_MIN_INT(df);
+    }
+    return arg2 ? arg1 / arg2 : 0;
+}
+
+static inline int64_t msa_div_u_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    uint64_t u_arg1 = UNSIGNED(arg1, df);
+    uint64_t u_arg2 = UNSIGNED(arg2, df);
+    return u_arg2 ? u_arg1 / u_arg2 : 0;
+}
+
+static inline int64_t msa_mod_s_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    if (arg1 == DF_MIN_INT(df) && arg2 == -1) {
+        return 0;
+    }
+    return arg2 ? arg1 % arg2 : 0;
+}
+
+static inline int64_t msa_mod_u_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    uint64_t u_arg1 = UNSIGNED(arg1, df);
+    uint64_t u_arg2 = UNSIGNED(arg2, df);
+    return u_arg2 ? u_arg1 % u_arg2 : 0;
+}
+
+#define SIGNED_EVEN(a, df) \
+        ((((int64_t)(a)) << (64 - DF_BITS(df)/2)) >> (64 - DF_BITS(df)/2))
+
+#define UNSIGNED_EVEN(a, df) \
+        ((((uint64_t)(a)) << (64 - DF_BITS(df)/2)) >> (64 - DF_BITS(df)/2))
+
+#define SIGNED_ODD(a, df) \
+        ((((int64_t)(a)) << (64 - DF_BITS(df))) >> (64 - DF_BITS(df)/2))
+
+#define UNSIGNED_ODD(a, df) \
+        ((((uint64_t)(a)) << (64 - DF_BITS(df))) >> (64 - DF_BITS(df)/2))
+
+#define SIGNED_EXTRACT(e, o, a, df)     \
+    do {                                \
+        e = SIGNED_EVEN(a, df);         \
+        o = SIGNED_ODD(a, df);          \
+    } while (0);
+
+#define UNSIGNED_EXTRACT(e, o, a, df)   \
+    do {                                \
+        e = UNSIGNED_EVEN(a, df);       \
+        o = UNSIGNED_ODD(a, df);        \
+    } while (0);
+
+static inline int64_t msa_dotp_s_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    int64_t even_arg1;
+    int64_t even_arg2;
+    int64_t odd_arg1;
+    int64_t odd_arg2;
+    SIGNED_EXTRACT(even_arg1, odd_arg1, arg1, df);
+    SIGNED_EXTRACT(even_arg2, odd_arg2, arg2, df);
+    return (even_arg1 * even_arg2) + (odd_arg1 * odd_arg2);
+}
+
+static inline int64_t msa_dotp_u_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    int64_t even_arg1;
+    int64_t even_arg2;
+    int64_t odd_arg1;
+    int64_t odd_arg2;
+    UNSIGNED_EXTRACT(even_arg1, odd_arg1, arg1, df);
+    UNSIGNED_EXTRACT(even_arg2, odd_arg2, arg2, df);
+    return (even_arg1 * even_arg2) + (odd_arg1 * odd_arg2);
+}
+
+#define CONCATENATE_AND_SLIDE(s, k)             \
+    do {                                        \
+        for (i = 0; i < s; i++) {               \
+            v[i]     = pws->b[s * k + i];       \
+            v[i + s] = pwd->b[s * k + i];       \
+        }                                       \
+        for (i = 0; i < s; i++) {               \
+            pwd->b[s * k + i] = v[i + n];       \
+        }                                       \
+    } while (0)
+
+static inline void msa_sld_df(uint32_t df, wr_t *pwd,
+                              wr_t *pws, target_ulong rt)
+{
+    uint32_t n = rt % DF_ELEMENTS(df);
+    uint8_t v[64];
+    uint32_t i, k;
+
+    switch (df) {
+    case DF_BYTE:
+        CONCATENATE_AND_SLIDE(DF_ELEMENTS(DF_BYTE), 0);
+        break;
+    case DF_HALF:
+        for (k = 0; k < 2; k++) {
+            CONCATENATE_AND_SLIDE(DF_ELEMENTS(DF_HALF), k);
+        }
+        break;
+    case DF_WORD:
+        for (k = 0; k < 4; k++) {
+            CONCATENATE_AND_SLIDE(DF_ELEMENTS(DF_WORD), k);
+        }
+        break;
+    case DF_DOUBLE:
+        for (k = 0; k < 8; k++) {
+            CONCATENATE_AND_SLIDE(DF_ELEMENTS(DF_DOUBLE), k);
+        }
+        break;
+    default:
+        assert(0);
+    }
+}
+
+static inline int64_t msa_hadd_s_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    return SIGNED_ODD(arg1, df) + SIGNED_EVEN(arg2, df);
+}
+
+static inline int64_t msa_hadd_u_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    return UNSIGNED_ODD(arg1, df) + UNSIGNED_EVEN(arg2, df);
+}
+
+static inline int64_t msa_hsub_s_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    return SIGNED_ODD(arg1, df) - SIGNED_EVEN(arg2, df);
+}
+
+static inline int64_t msa_hsub_u_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    return UNSIGNED_ODD(arg1, df) - UNSIGNED_EVEN(arg2, df);
+}
+
+static inline int64_t msa_mul_q_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    int64_t q_min = DF_MIN_INT(df);
+    int64_t q_max = DF_MAX_INT(df);
+
+    if (arg1 == q_min && arg2 == q_min) {
+        return q_max;
+    }
+    return (arg1 * arg2) >> (DF_BITS(df) - 1);
+}
+
+static inline int64_t msa_mulr_q_df(uint32_t df, int64_t arg1, int64_t arg2)
+{
+    int64_t q_min = DF_MIN_INT(df);
+    int64_t q_max = DF_MAX_INT(df);
+    int64_t r_bit = 1 << (DF_BITS(df) - 2);
+
+    if (arg1 == q_min && arg2 == q_min) {
+        return q_max;
+    }
+    return (arg1 * arg2 + r_bit) >> (DF_BITS(df) - 1);
+}
+
+#define MSA_BINOP_DF(func) \
+void helper_msa_ ## func ## _df(CPUMIPSState *env, uint32_t df,         \
+                                uint32_t wd, uint32_t ws, uint32_t wt)  \
+{                                                                       \
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);                          \
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);                          \
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);                          \
+    uint32_t i;                                                         \
+                                                                        \
+    switch (df) {                                                       \
+    case DF_BYTE:                                                       \
+        for (i = 0; i < DF_ELEMENTS(DF_BYTE); i++) {                    \
+            pwd->b[i] = msa_ ## func ## _df(df, pws->b[i], pwt->b[i]);  \
+        }                                                               \
+        break;                                                          \
+    case DF_HALF:                                                       \
+        for (i = 0; i < DF_ELEMENTS(DF_HALF); i++) {                    \
+            pwd->h[i] = msa_ ## func ## _df(df, pws->h[i], pwt->h[i]);  \
+        }                                                               \
+        break;                                                          \
+    case DF_WORD:                                                       \
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {                    \
+            pwd->w[i] = msa_ ## func ## _df(df, pws->w[i], pwt->w[i]);  \
+        }                                                               \
+        break;                                                          \
+    case DF_DOUBLE:                                                     \
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {                  \
+            pwd->d[i] = msa_ ## func ## _df(df, pws->d[i], pwt->d[i]);  \
+        }                                                               \
+        break;                                                          \
+    default:                                                            \
+        assert(0);                                                      \
+    }                                                                   \
+}
+
+MSA_BINOP_DF(sll)
+MSA_BINOP_DF(sra)
+MSA_BINOP_DF(srl)
+MSA_BINOP_DF(bclr)
+MSA_BINOP_DF(bset)
+MSA_BINOP_DF(bneg)
+MSA_BINOP_DF(addv)
+MSA_BINOP_DF(subv)
+MSA_BINOP_DF(max_s)
+MSA_BINOP_DF(max_u)
+MSA_BINOP_DF(min_s)
+MSA_BINOP_DF(min_u)
+MSA_BINOP_DF(max_a)
+MSA_BINOP_DF(min_a)
+MSA_BINOP_DF(ceq)
+MSA_BINOP_DF(clt_s)
+MSA_BINOP_DF(clt_u)
+MSA_BINOP_DF(cle_s)
+MSA_BINOP_DF(cle_u)
+MSA_BINOP_DF(add_a)
+MSA_BINOP_DF(adds_a)
+MSA_BINOP_DF(adds_s)
+MSA_BINOP_DF(adds_u)
+MSA_BINOP_DF(ave_s)
+MSA_BINOP_DF(ave_u)
+MSA_BINOP_DF(aver_s)
+MSA_BINOP_DF(aver_u)
+MSA_BINOP_DF(subs_s)
+MSA_BINOP_DF(subs_u)
+MSA_BINOP_DF(subsus_u)
+MSA_BINOP_DF(subsuu_s)
+MSA_BINOP_DF(asub_s)
+MSA_BINOP_DF(asub_u)
+MSA_BINOP_DF(mulv)
+MSA_BINOP_DF(div_s)
+MSA_BINOP_DF(div_u)
+MSA_BINOP_DF(mod_s)
+MSA_BINOP_DF(mod_u)
+MSA_BINOP_DF(dotp_s)
+MSA_BINOP_DF(dotp_u)
+MSA_BINOP_DF(srar)
+MSA_BINOP_DF(srlr)
+MSA_BINOP_DF(hadd_s)
+MSA_BINOP_DF(hadd_u)
+MSA_BINOP_DF(hsub_s)
+MSA_BINOP_DF(hsub_u)
+
+MSA_BINOP_DF(mul_q)
+MSA_BINOP_DF(mulr_q)
+#undef MSA_BINOP_DF
+
+void helper_msa_sld_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                       uint32_t ws, uint32_t rt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+
+    msa_sld_df(df, pwd, pws, env->active_tc.gpr[rt]);
+}
+
+static inline int64_t msa_maddv_df(uint32_t df, int64_t dest, int64_t arg1,
+                                   int64_t arg2)
+{
+    return dest + arg1 * arg2;
+}
+
+static inline int64_t msa_msubv_df(uint32_t df, int64_t dest, int64_t arg1,
+                                   int64_t arg2)
+{
+    return dest - arg1 * arg2;
+}
+
+static inline int64_t msa_dpadd_s_df(uint32_t df, int64_t dest, int64_t arg1,
+                                     int64_t arg2)
+{
+    int64_t even_arg1;
+    int64_t even_arg2;
+    int64_t odd_arg1;
+    int64_t odd_arg2;
+    SIGNED_EXTRACT(even_arg1, odd_arg1, arg1, df);
+    SIGNED_EXTRACT(even_arg2, odd_arg2, arg2, df);
+    return dest + (even_arg1 * even_arg2) + (odd_arg1 * odd_arg2);
+}
+
+static inline int64_t msa_dpadd_u_df(uint32_t df, int64_t dest, int64_t arg1,
+                                     int64_t arg2)
+{
+    int64_t even_arg1;
+    int64_t even_arg2;
+    int64_t odd_arg1;
+    int64_t odd_arg2;
+    UNSIGNED_EXTRACT(even_arg1, odd_arg1, arg1, df);
+    UNSIGNED_EXTRACT(even_arg2, odd_arg2, arg2, df);
+    return dest + (even_arg1 * even_arg2) + (odd_arg1 * odd_arg2);
+}
+
+static inline int64_t msa_dpsub_s_df(uint32_t df, int64_t dest, int64_t arg1,
+                                     int64_t arg2)
+{
+    int64_t even_arg1;
+    int64_t even_arg2;
+    int64_t odd_arg1;
+    int64_t odd_arg2;
+    SIGNED_EXTRACT(even_arg1, odd_arg1, arg1, df);
+    SIGNED_EXTRACT(even_arg2, odd_arg2, arg2, df);
+    return dest - ((even_arg1 * even_arg2) + (odd_arg1 * odd_arg2));
+}
+
+static inline int64_t msa_dpsub_u_df(uint32_t df, int64_t dest, int64_t arg1,
+                                     int64_t arg2)
+{
+    int64_t even_arg1;
+    int64_t even_arg2;
+    int64_t odd_arg1;
+    int64_t odd_arg2;
+    UNSIGNED_EXTRACT(even_arg1, odd_arg1, arg1, df);
+    UNSIGNED_EXTRACT(even_arg2, odd_arg2, arg2, df);
+    return dest - ((even_arg1 * even_arg2) + (odd_arg1 * odd_arg2));
+}
+
+static inline int64_t msa_madd_q_df(uint32_t df, int64_t dest, int64_t arg1,
+                                    int64_t arg2)
+{
+    int64_t q_prod, q_ret;
+
+    int64_t q_max = DF_MAX_INT(df);
+    int64_t q_min = DF_MIN_INT(df);
+
+    q_prod = arg1 * arg2;
+    q_ret = ((dest << (DF_BITS(df) - 1)) + q_prod) >> (DF_BITS(df) - 1);
+
+    return (q_ret < q_min) ? q_min : (q_max < q_ret) ? q_max : q_ret;
+}
+
+static inline int64_t msa_msub_q_df(uint32_t df, int64_t dest, int64_t arg1,
+                                    int64_t arg2)
+{
+    int64_t q_prod, q_ret;
+
+    int64_t q_max = DF_MAX_INT(df);
+    int64_t q_min = DF_MIN_INT(df);
+
+    q_prod = arg1 * arg2;
+    q_ret = ((dest << (DF_BITS(df) - 1)) - q_prod) >> (DF_BITS(df) - 1);
+
+    return (q_ret < q_min) ? q_min : (q_max < q_ret) ? q_max : q_ret;
+}
+
+static inline int64_t msa_maddr_q_df(uint32_t df, int64_t dest, int64_t arg1,
+                                     int64_t arg2)
+{
+    int64_t q_prod, q_ret;
+
+    int64_t q_max = DF_MAX_INT(df);
+    int64_t q_min = DF_MIN_INT(df);
+    int64_t r_bit = 1 << (DF_BITS(df) - 2);
+
+    q_prod = arg1 * arg2;
+    q_ret = ((dest << (DF_BITS(df) - 1)) + q_prod + r_bit) >> (DF_BITS(df) - 1);
+
+    return (q_ret < q_min) ? q_min : (q_max < q_ret) ? q_max : q_ret;
+}
+
+static inline int64_t msa_msubr_q_df(uint32_t df, int64_t dest, int64_t arg1,
+                                     int64_t arg2)
+{
+    int64_t q_prod, q_ret;
+
+    int64_t q_max = DF_MAX_INT(df);
+    int64_t q_min = DF_MIN_INT(df);
+    int64_t r_bit = 1 << (DF_BITS(df) - 2);
+
+    q_prod = arg1 * arg2;
+    q_ret = ((dest << (DF_BITS(df) - 1)) - q_prod + r_bit) >> (DF_BITS(df) - 1);
+
+    return (q_ret < q_min) ? q_min : (q_max < q_ret) ? q_max : q_ret;
+}
+
+#define MSA_TEROP_DF(func) \
+void helper_msa_ ## func ## _df(CPUMIPSState *env, uint32_t df, uint32_t wd,   \
+                          uint32_t ws, uint32_t wt)                     \
+{                                                                       \
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);                          \
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);                          \
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);                          \
+    uint32_t i;                                                         \
+                                                                        \
+    switch (df) {                                                       \
+    case DF_BYTE:                                                       \
+        for (i = 0; i < DF_ELEMENTS(DF_BYTE); i++) {                    \
+            pwd->b[i] = msa_ ## func ## _df(df, pwd->b[i], pws->b[i],   \
+                                            pwt->b[i]);                 \
+        }                                                               \
+        break;                                                          \
+    case DF_HALF:                                                       \
+        for (i = 0; i < DF_ELEMENTS(DF_HALF); i++) {                    \
+            pwd->h[i] = msa_ ## func ## _df(df, pwd->h[i], pws->h[i],   \
+                                            pwt->h[i]);                 \
+        }                                                               \
+        break;                                                          \
+    case DF_WORD:                                                       \
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {                    \
+            pwd->w[i] = msa_ ## func ## _df(df, pwd->w[i], pws->w[i],   \
+                                            pwt->w[i]);                 \
+        }                                                               \
+        break;                                                          \
+    case DF_DOUBLE:                                                     \
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {                  \
+            pwd->d[i] = msa_ ## func ## _df(df, pwd->d[i], pws->d[i],   \
+                                            pwt->d[i]);                 \
+        }                                                               \
+        break;                                                          \
+    default:                                                            \
+        assert(0);                                                      \
+    }                                                                   \
+}
+
+MSA_TEROP_DF(maddv)
+MSA_TEROP_DF(msubv)
+MSA_TEROP_DF(dpadd_s)
+MSA_TEROP_DF(dpadd_u)
+MSA_TEROP_DF(dpsub_s)
+MSA_TEROP_DF(dpsub_u)
+MSA_TEROP_DF(binsl)
+MSA_TEROP_DF(binsr)
+MSA_TEROP_DF(madd_q)
+MSA_TEROP_DF(msub_q)
+MSA_TEROP_DF(maddr_q)
+MSA_TEROP_DF(msubr_q)
+#undef MSA_TEROP_DF
+
+static inline void msa_splat_df(uint32_t df, wr_t *pwd,
+                                wr_t *pws, target_ulong rt)
+{
+    uint32_t n = rt % DF_ELEMENTS(df);
+    uint32_t i;
+
+    switch (df) {
+    case DF_BYTE:
+        for (i = 0; i < DF_ELEMENTS(DF_BYTE); i++) {
+            pwd->b[i] = pws->b[n];
+        }
+        break;
+    case DF_HALF:
+        for (i = 0; i < DF_ELEMENTS(DF_HALF); i++) {
+            pwd->h[i] = pws->h[n];
+        }
+        break;
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            pwd->w[i] = pws->w[n];
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            pwd->d[i] = pws->d[n];
+        }
+       break;
+    default:
+        assert(0);
+    }
+}
+
+void helper_msa_splat_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                         uint32_t ws, uint32_t rt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+
+    msa_splat_df(df, pwd, pws, env->active_tc.gpr[rt]);
+}
+
+#define MSA_DO_B MSA_DO(b)
+#define MSA_DO_H MSA_DO(h)
+#define MSA_DO_W MSA_DO(w)
+#define MSA_DO_D MSA_DO(d)
+
+#define MSA_LOOP_B MSA_LOOP(B)
+#define MSA_LOOP_H MSA_LOOP(H)
+#define MSA_LOOP_W MSA_LOOP(W)
+#define MSA_LOOP_D MSA_LOOP(D)
+
+#define MSA_LOOP_COND_B MSA_LOOP_COND(DF_BYTE)
+#define MSA_LOOP_COND_H MSA_LOOP_COND(DF_HALF)
+#define MSA_LOOP_COND_W MSA_LOOP_COND(DF_WORD)
+#define MSA_LOOP_COND_D MSA_LOOP_COND(DF_DOUBLE)
+
+#define MSA_LOOP(DF) \
+        for (i = 0; i < (MSA_LOOP_COND_ ## DF) ; i++) { \
+            MSA_DO_ ## DF \
+        }
+
+#define MSA_FN_DF(FUNC)                                             \
+void helper_msa_##FUNC(CPUMIPSState *env, uint32_t df, uint32_t wd, \
+        uint32_t ws, uint32_t wt)                                   \
+{                                                                   \
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);                      \
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);                      \
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);                      \
+    wr_t wx, *pwx = &wx;                                            \
+    uint32_t i;                                                     \
+    switch (df) {                                                   \
+    case DF_BYTE:                                                   \
+        MSA_LOOP_B                                                  \
+        break;                                                      \
+    case DF_HALF:                                                   \
+        MSA_LOOP_H                                                  \
+        break;                                                      \
+    case DF_WORD:                                                   \
+        MSA_LOOP_W                                                  \
+        break;                                                      \
+    case DF_DOUBLE:                                                 \
+        MSA_LOOP_D                                                  \
+       break;                                                       \
+    default:                                                        \
+        assert(0);                                                  \
+    }                                                               \
+    msa_move_v(pwd, pwx);                                           \
+}
+
+#define MSA_LOOP_COND(DF) \
+            (DF_ELEMENTS(DF) / 2)
+
+#define Rb(pwr, i) (pwr->b[i])
+#define Lb(pwr, i) (pwr->b[i + DF_ELEMENTS(DF_BYTE)/2])
+#define Rh(pwr, i) (pwr->h[i])
+#define Lh(pwr, i) (pwr->h[i + DF_ELEMENTS(DF_HALF)/2])
+#define Rw(pwr, i) (pwr->w[i])
+#define Lw(pwr, i) (pwr->w[i + DF_ELEMENTS(DF_WORD)/2])
+#define Rd(pwr, i) (pwr->d[i])
+#define Ld(pwr, i) (pwr->d[i + DF_ELEMENTS(DF_DOUBLE)/2])
+
+#define MSA_DO(DF)                      \
+    do {                                \
+        R##DF(pwx, i) = pwt->DF[2*i];   \
+        L##DF(pwx, i) = pws->DF[2*i];   \
+    } while (0);
+MSA_FN_DF(pckev_df)
+#undef MSA_DO
+
+#define MSA_DO(DF)                      \
+    do {                                \
+        R##DF(pwx, i) = pwt->DF[2*i+1]; \
+        L##DF(pwx, i) = pws->DF[2*i+1]; \
+    } while (0);
+MSA_FN_DF(pckod_df)
+#undef MSA_DO
+
+#define MSA_DO(DF)                      \
+    do {                                \
+        pwx->DF[2*i]   = L##DF(pwt, i); \
+        pwx->DF[2*i+1] = L##DF(pws, i); \
+    } while (0);
+MSA_FN_DF(ilvl_df)
+#undef MSA_DO
+
+#define MSA_DO(DF)                      \
+    do {                                \
+        pwx->DF[2*i]   = R##DF(pwt, i); \
+        pwx->DF[2*i+1] = R##DF(pws, i); \
+    } while (0);
+MSA_FN_DF(ilvr_df)
+#undef MSA_DO
+
+#define MSA_DO(DF)                      \
+    do {                                \
+        pwx->DF[2*i]   = pwt->DF[2*i];  \
+        pwx->DF[2*i+1] = pws->DF[2*i];  \
+    } while (0);
+MSA_FN_DF(ilvev_df)
+#undef MSA_DO
+
+#define MSA_DO(DF)                          \
+    do {                                    \
+        pwx->DF[2*i]   = pwt->DF[2*i+1];    \
+        pwx->DF[2*i+1] = pws->DF[2*i+1];    \
+    } while (0);
+MSA_FN_DF(ilvod_df)
+#undef MSA_DO
+#undef MSA_LOOP_COND
+
+#define MSA_LOOP_COND(DF) \
+            (DF_ELEMENTS(DF))
+
+#define MSA_DO(DF)                                                          \
+    do {                                                                    \
+        uint32_t n = DF_ELEMENTS(df);                                       \
+        uint32_t k = (pwd->DF[i] & 0x3f) % (2 * n);                         \
+        pwx->DF[i] =                                                        \
+            (pwd->DF[i] & 0xc0) ? 0 : k < n ? pwt->DF[k] : pws->DF[k - n];  \
+    } while (0);
+MSA_FN_DF(vshf_df)
+#undef MSA_DO
+#undef MSA_LOOP_COND
+#undef MSA_FN_DF
+
+void helper_msa_sldi_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t ws, uint32_t n)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+
+    msa_sld_df(df, pwd, pws, n);
+}
+
+void helper_msa_splati_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                          uint32_t ws, uint32_t n)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+
+    msa_splat_df(df, pwd, pws, n);
+}
+
+void helper_msa_copy_s_df(CPUMIPSState *env, uint32_t df, uint32_t rd,
+                          uint32_t ws, uint32_t n)
+{
+    n %= DF_ELEMENTS(df);
+
+    switch (df) {
+    case DF_BYTE:
+        env->active_tc.gpr[rd] = (int8_t)env->active_fpu.fpr[ws].wr.b[n];
+        break;
+    case DF_HALF:
+        env->active_tc.gpr[rd] = (int16_t)env->active_fpu.fpr[ws].wr.h[n];
+        break;
+    case DF_WORD:
+        env->active_tc.gpr[rd] = (int32_t)env->active_fpu.fpr[ws].wr.w[n];
+        break;
+#ifdef TARGET_MIPS64
+    case DF_DOUBLE:
+        env->active_tc.gpr[rd] = (int64_t)env->active_fpu.fpr[ws].wr.d[n];
+        break;
+#endif
+    default:
+        assert(0);
+    }
+}
+
+void helper_msa_copy_u_df(CPUMIPSState *env, uint32_t df, uint32_t rd,
+                          uint32_t ws, uint32_t n)
+{
+    n %= DF_ELEMENTS(df);
+
+    switch (df) {
+    case DF_BYTE:
+        env->active_tc.gpr[rd] = (uint8_t)env->active_fpu.fpr[ws].wr.b[n];
+        break;
+    case DF_HALF:
+        env->active_tc.gpr[rd] = (uint16_t)env->active_fpu.fpr[ws].wr.h[n];
+        break;
+    case DF_WORD:
+        env->active_tc.gpr[rd] = (uint32_t)env->active_fpu.fpr[ws].wr.w[n];
+        break;
+#ifdef TARGET_MIPS64
+    case DF_DOUBLE:
+        env->active_tc.gpr[rd] = (uint64_t)env->active_fpu.fpr[ws].wr.d[n];
+        break;
+#endif
+    default:
+        assert(0);
+    }
+}
+
+void helper_msa_insert_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                          uint32_t rs_num, uint32_t n)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    target_ulong rs = env->active_tc.gpr[rs_num];
+
+    switch (df) {
+    case DF_BYTE:
+        pwd->b[n] = (int8_t)rs;
+        break;
+    case DF_HALF:
+        pwd->h[n] = (int16_t)rs;
+        break;
+    case DF_WORD:
+        pwd->w[n] = (int32_t)rs;
+        break;
+    case DF_DOUBLE:
+        pwd->d[n] = (int64_t)rs;
+        break;
+    default:
+        assert(0);
+    }
+}
+
+void helper_msa_insve_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                         uint32_t ws, uint32_t n)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+
+    switch (df) {
+    case DF_BYTE:
+        pwd->b[n] = (int8_t)pws->b[0];
+        break;
+    case DF_HALF:
+        pwd->h[n] = (int16_t)pws->h[0];
+        break;
+    case DF_WORD:
+        pwd->w[n] = (int32_t)pws->w[0];
+        break;
+    case DF_DOUBLE:
+        pwd->d[n] = (int64_t)pws->d[0];
+        break;
+    default:
+        assert(0);
+    }
+}
+
+void helper_msa_ctcmsa(CPUMIPSState *env, target_ulong elm, uint32_t cd)
+{
+    switch (cd) {
+    case 0:
+        break;
+    case 1:
+        env->active_tc.msacsr = (int32_t)elm & MSACSR_MASK;
+        /* set float_status rounding mode */
+        set_float_rounding_mode(
+            ieee_rm[(env->active_tc.msacsr & MSACSR_RM_MASK) >> MSACSR_RM],
+            &env->active_tc.msa_fp_status);
+        /* set float_status flush modes */
+        set_flush_to_zero(
+          (env->active_tc.msacsr & MSACSR_FS_MASK) != 0 ? 1 : 0,
+          &env->active_tc.msa_fp_status);
+        set_flush_inputs_to_zero(
+          (env->active_tc.msacsr & MSACSR_FS_MASK) != 0 ? 1 : 0,
+          &env->active_tc.msa_fp_status);
+        /* check exception */
+        if ((GET_FP_ENABLE(env->active_tc.msacsr) | FP_UNIMPLEMENTED)
+            & GET_FP_CAUSE(env->active_tc.msacsr)) {
+            helper_raise_exception(env, EXCP_MSAFPE);
+        }
+        break;
+    }
+}
+
+target_ulong helper_msa_cfcmsa(CPUMIPSState *env, uint32_t cs)
+{
+    switch (cs) {
+    case 0:
+        return env->msair;
+    case 1:
+        return env->active_tc.msacsr & MSACSR_MASK;
+    }
+    return 0;
+}
+
+void helper_msa_move_v(CPUMIPSState *env, uint32_t wd, uint32_t ws)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+
+    msa_move_v(pwd, pws);
+}
+
+static inline int64_t msa_pcnt_df(uint32_t df, int64_t arg)
+{
+    uint64_t x;
+
+    x = UNSIGNED(arg, df);
+
+    x = (x & 0x5555555555555555ULL) + ((x >>  1) & 0x5555555555555555ULL);
+    x = (x & 0x3333333333333333ULL) + ((x >>  2) & 0x3333333333333333ULL);
+    x = (x & 0x0F0F0F0F0F0F0F0FULL) + ((x >>  4) & 0x0F0F0F0F0F0F0F0FULL);
+    x = (x & 0x00FF00FF00FF00FFULL) + ((x >>  8) & 0x00FF00FF00FF00FFULL);
+    x = (x & 0x0000FFFF0000FFFFULL) + ((x >> 16) & 0x0000FFFF0000FFFFULL);
+    x = (x & 0x00000000FFFFFFFFULL) + ((x >> 32));
+
+    return x;
+}
+
+static inline int64_t msa_nlzc_df(uint32_t df, int64_t arg)
+{
+    uint64_t x, y;
+    int n, c;
+
+    x = UNSIGNED(arg, df);
+    n = DF_BITS(df);
+    c = DF_BITS(df) / 2;
+
+    do {
+        y = x >> c;
+        if (y != 0) {
+            n = n - c;
+            x = y;
+        }
+        c = c >> 1;
+    } while (c != 0);
+
+    return n - x;
+}
+
+static inline int64_t msa_nloc_df(uint32_t df, int64_t arg)
+{
+    return msa_nlzc_df(df, UNSIGNED((~arg), df));
+}
+
+void helper_msa_fill_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t rs)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    uint32_t i;
+
+    switch (df) {
+    case DF_BYTE:
+        for (i = 0; i < DF_ELEMENTS(DF_BYTE); i++) {
+            pwd->b[i] = (int8_t)env->active_tc.gpr[rs];
+        }
+        break;
+    case DF_HALF:
+        for (i = 0; i < DF_ELEMENTS(DF_HALF); i++) {
+            pwd->h[i] = (int16_t)env->active_tc.gpr[rs];
+        }
+        break;
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            pwd->w[i] = (int32_t)env->active_tc.gpr[rs];
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            pwd->d[i] = (int64_t)env->active_tc.gpr[rs];
+        }
+       break;
+    default:
+        assert(0);
+    }
+}
+
+#define MSA_UNOP_DF(func) \
+void helper_msa_ ## func ## _df(CPUMIPSState *env, uint32_t df,         \
+                              uint32_t wd, uint32_t ws)                 \
+{                                                                       \
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);                          \
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);                          \
+    uint32_t i;                                                         \
+                                                                        \
+    switch (df) {                                                       \
+    case DF_BYTE:                                                       \
+        for (i = 0; i < DF_ELEMENTS(DF_BYTE); i++) {                    \
+            pwd->b[i] = msa_ ## func ## _df(df, pws->b[i]);             \
+        }                                                               \
+        break;                                                          \
+    case DF_HALF:                                                       \
+        for (i = 0; i < DF_ELEMENTS(DF_HALF); i++) {                    \
+            pwd->h[i] = msa_ ## func ## _df(df, pws->h[i]);             \
+        }                                                               \
+        break;                                                          \
+    case DF_WORD:                                                       \
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {                    \
+            pwd->w[i] = msa_ ## func ## _df(df, pws->w[i]);             \
+        }                                                               \
+        break;                                                          \
+    case DF_DOUBLE:                                                     \
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {                  \
+            pwd->d[i] = msa_ ## func ## _df(df, pws->d[i]);             \
+        }                                                               \
+        break;                                                          \
+    default:                                                            \
+        assert(0);                                                      \
+    }                                                                   \
+}
+
+MSA_UNOP_DF(nlzc)
+MSA_UNOP_DF(nloc)
+MSA_UNOP_DF(pcnt)
+#undef MSA_UNOP_DF
+
+#define FLOAT_ONE32 make_float32(0x3f8 << 20)
+#define FLOAT_ONE64 make_float64(0x3ffULL << 52)
+
+#define FLOAT_SNAN16 (float16_default_nan ^ 0x0220)
+        /* 0x7c20 */
+#define FLOAT_SNAN32 (float32_default_nan ^ 0x00400020)
+        /* 0x7f800020 */
+#define FLOAT_SNAN64 (float64_default_nan ^ 0x0008000000000020ULL)
+        /* 0x7ff0000000000020 */
+
+static inline void clear_msacsr_cause(CPUMIPSState *env)
+{
+    SET_FP_CAUSE(env->active_tc.msacsr, 0);
+}
+
+static inline void check_msacsr_cause(CPUMIPSState *env)
+{
+    if ((GET_FP_CAUSE(env->active_tc.msacsr) &
+            (GET_FP_ENABLE(env->active_tc.msacsr) | FP_UNIMPLEMENTED)) == 0) {
+        UPDATE_FP_FLAGS(env->active_tc.msacsr,
+                GET_FP_CAUSE(env->active_tc.msacsr));
+    } else {
+        helper_raise_exception(env, EXCP_MSAFPE);
+    }
+}
+
+/* Flush-to-zero use cases for update_msacsr() */
+#define CLEAR_FS_UNDERFLOW 1
+#define CLEAR_IS_INEXACT   2
+#define RECIPROCAL_INEXACT 4
+
+static inline int update_msacsr(CPUMIPSState *env, int action, int denormal)
+{
+    int ieee_ex;
+
+    int c;
+    int cause;
+    int enable;
+
+    ieee_ex = get_float_exception_flags(&env->active_tc.msa_fp_status);
+
+    /* QEMU softfloat does not signal all underflow cases */
+    if (denormal) {
+        ieee_ex |= float_flag_underflow;
+    }
+
+    c = ieee_ex_to_mips(ieee_ex);
+    enable = GET_FP_ENABLE(env->active_tc.msacsr) | FP_UNIMPLEMENTED;
+
+    /* Set Inexact (I) when flushing inputs to zero */
+    if ((ieee_ex & float_flag_input_denormal) &&
+            (env->active_tc.msacsr & MSACSR_FS_MASK) != 0) {
+        if (action & CLEAR_IS_INEXACT) {
+            c &= ~FP_INEXACT;
+        } else {
+            c |=  FP_INEXACT;
+        }
+    }
+
+    /* Set Inexact (I) and Underflow (U) when flushing outputs to zero */
+    if ((ieee_ex & float_flag_output_denormal) &&
+            (env->active_tc.msacsr & MSACSR_FS_MASK) != 0) {
+        c |= FP_INEXACT;
+        if (action & CLEAR_FS_UNDERFLOW) {
+            c &= ~FP_UNDERFLOW;
+        } else {
+            c |=  FP_UNDERFLOW;
+        }
+    }
+
+    /* Set Inexact (I) when Overflow (O) is not enabled */
+    if ((c & FP_OVERFLOW) != 0 && (enable & FP_OVERFLOW) == 0) {
+        c |= FP_INEXACT;
+    }
+
+    /* Clear Exact Underflow when Underflow (U) is not enabled */
+    if ((c & FP_UNDERFLOW) != 0 && (enable & FP_UNDERFLOW) == 0 &&
+            (c & FP_INEXACT) == 0) {
+        c &= ~FP_UNDERFLOW;
+    }
+
+    /* Reciprocal operations set only Inexact when valid and not
+       divide by zero */
+    if ((action & RECIPROCAL_INEXACT) &&
+            (c & (FP_INVALID | FP_DIV0)) == 0) {
+        c = FP_INEXACT;
+    }
+
+    cause = c & enable;    /* all current enabled exceptions */
+
+    if (cause == 0) {
+        /* No enabled exception, update the MSACSR Cause
+         with all current exceptions */
+        SET_FP_CAUSE(env->active_tc.msacsr,
+                (GET_FP_CAUSE(env->active_tc.msacsr) | c));
+    } else {
+        /* Current exceptions are enabled */
+        if ((env->active_tc.msacsr & MSACSR_NX_MASK) == 0) {
+            /* Exception(s) will trap, update MSACSR Cause
+           with all enabled exceptions */
+            SET_FP_CAUSE(env->active_tc.msacsr,
+                    (GET_FP_CAUSE(env->active_tc.msacsr) | c));
+        }
+    }
+
+    return c;
+}
+
+static inline int get_enabled_exceptions(const CPUMIPSState *env, int c)
+{
+    int enable = GET_FP_ENABLE(env->active_tc.msacsr) | FP_UNIMPLEMENTED;
+    return c & enable;
+}
+
+static inline float16 float16_from_float32(int32 a, flag ieee STATUS_PARAM)
+{
+      float16 f_val;
+
+      f_val = float32_to_float16((float32)a, ieee  STATUS_VAR);
+      f_val = float16_maybe_silence_nan(f_val);
+
+      return a < 0 ? (f_val | (1 << 15)) : f_val;
+}
+
+static inline float32 float32_from_float64(int64 a STATUS_PARAM)
+{
+      float32 f_val;
+
+      f_val = float64_to_float32((float64)a STATUS_VAR);
+      f_val = float32_maybe_silence_nan(f_val);
+
+      return a < 0 ? (f_val | (1 << 31)) : f_val;
+}
+
+static inline float32 float32_from_float16(int16_t a, flag ieee STATUS_PARAM)
+{
+      float32 f_val;
+
+      f_val = float16_to_float32((float16)a, ieee STATUS_VAR);
+      f_val = float32_maybe_silence_nan(f_val);
+
+      return a < 0 ? (f_val | (1 << 31)) : f_val;
+}
+
+static inline float64 float64_from_float32(int32 a STATUS_PARAM)
+{
+      float64 f_val;
+
+      f_val = float32_to_float64((float64)a STATUS_VAR);
+      f_val = float64_maybe_silence_nan(f_val);
+
+      return a < 0 ? (f_val | (1ULL << 63)) : f_val;
+}
+
+static inline float32 float32_from_q16(int16_t a STATUS_PARAM)
+{
+    float32 f_val;
+
+    /* conversion as integer and scaling */
+    f_val = int32_to_float32(a STATUS_VAR);
+    f_val = float32_scalbn(f_val, -15 STATUS_VAR);
+
+    return f_val;
+}
+
+static inline float64 float64_from_q32(int32 a STATUS_PARAM)
+{
+    float64 f_val;
+
+    /* conversion as integer and scaling */
+    f_val = int32_to_float64(a STATUS_VAR);
+    f_val = float64_scalbn(f_val, -31 STATUS_VAR);
+
+    return f_val;
+}
+
+static inline int16_t float32_to_q16(float32 a STATUS_PARAM)
+{
+    int32 q_val;
+    int32 q_min = 0xffff8000;
+    int32 q_max = 0x00007fff;
+
+    int ieee_ex;
+
+    if (float32_is_any_nan(a)) {
+        float_raise(float_flag_invalid STATUS_VAR);
+        return 0;
+    }
+
+    /* scaling */
+    a = float32_scalbn(a, 15 STATUS_VAR);
+
+    ieee_ex = get_float_exception_flags(status);
+    set_float_exception_flags(ieee_ex & (~float_flag_underflow)
+                              STATUS_VAR);
+
+    if (ieee_ex & float_flag_overflow) {
+        float_raise(float_flag_inexact STATUS_VAR);
+        return (int32)a < 0 ? q_min : q_max;
+    }
+
+    /* conversion to int */
+    q_val = float32_to_int32(a STATUS_VAR);
+
+    ieee_ex = get_float_exception_flags(status);
+    set_float_exception_flags(ieee_ex & (~float_flag_underflow)
+                              STATUS_VAR);
+
+    if (ieee_ex & float_flag_invalid) {
+        set_float_exception_flags(ieee_ex & (~float_flag_invalid)
+                                STATUS_VAR);
+        float_raise(float_flag_overflow | float_flag_inexact STATUS_VAR);
+        return (int32)a < 0 ? q_min : q_max;
+    }
+
+    if (q_val < q_min) {
+        float_raise(float_flag_overflow | float_flag_inexact STATUS_VAR);
+        return (int16_t)q_min;
+    }
+
+    if (q_max < q_val) {
+        float_raise(float_flag_overflow | float_flag_inexact STATUS_VAR);
+        return (int16_t)q_max;
+    }
+
+    return (int16_t)q_val;
+}
+
+static inline int32 float64_to_q32(float64 a STATUS_PARAM)
+{
+    int64 q_val;
+    int64 q_min = 0xffffffff80000000LL;
+    int64 q_max = 0x000000007fffffffLL;
+
+    int ieee_ex;
+
+    if (float64_is_any_nan(a)) {
+        float_raise(float_flag_invalid STATUS_VAR);
+        return 0;
+    }
+
+    /* scaling */
+    a = float64_scalbn(a, 31 STATUS_VAR);
+
+    ieee_ex = get_float_exception_flags(status);
+    set_float_exception_flags(ieee_ex & (~float_flag_underflow)
+            STATUS_VAR);
+
+    if (ieee_ex & float_flag_overflow) {
+        float_raise(float_flag_inexact STATUS_VAR);
+        return (int64)a < 0 ? q_min : q_max;
+    }
+
+    /* conversion to integer */
+    q_val = float64_to_int64(a STATUS_VAR);
+
+    ieee_ex = get_float_exception_flags(status);
+    set_float_exception_flags(ieee_ex & (~float_flag_underflow)
+            STATUS_VAR);
+
+    if (ieee_ex & float_flag_invalid) {
+        set_float_exception_flags(ieee_ex & (~float_flag_invalid)
+                STATUS_VAR);
+        float_raise(float_flag_overflow | float_flag_inexact STATUS_VAR);
+        return (int64)a < 0 ? q_min : q_max;
+    }
+
+    if (q_val < q_min) {
+        float_raise(float_flag_overflow | float_flag_inexact STATUS_VAR);
+        return (int32)q_min;
+    }
+
+    if (q_max < q_val) {
+        float_raise(float_flag_overflow | float_flag_inexact STATUS_VAR);
+        return (int32)q_max;
+    }
+
+    return (int32)q_val;
+}
+
+#define MSA_FLOAT_COND(DEST, OP, ARG1, ARG2, BITS, QUIET)                   \
+    do {                                                                    \
+        int c;                                                              \
+        int64_t cond;                                                       \
+        set_float_exception_flags(0, &env->active_tc.msa_fp_status);        \
+        if (!QUIET) {                                                       \
+            cond = float ## BITS ## _ ## OP(ARG1, ARG2,                     \
+                                          &env->active_tc.msa_fp_status);   \
+        } else {                                                            \
+            cond = float ## BITS ## _ ## OP ## _quiet(ARG1, ARG2,           \
+                                          &env->active_tc.msa_fp_status);   \
+        }                                                                   \
+        DEST = cond ? M_MAX_UINT(BITS) : 0;                                 \
+        c = update_msacsr(env, CLEAR_IS_INEXACT, 0);                        \
+                                                                            \
+        if (get_enabled_exceptions(env, c)) {                               \
+            DEST = ((FLOAT_SNAN ## BITS >> 6) << 6) | c;                    \
+        }                                                                   \
+    } while (0)
+
+#define MSA_FLOAT_AF(DEST, ARG1, ARG2, BITS, QUIET)                 \
+    do {                                                            \
+        MSA_FLOAT_COND(DEST, eq, ARG1, ARG2, BITS, QUIET);          \
+        if ((DEST & M_MAX_UINT(BITS)) == M_MAX_UINT(BITS)) {        \
+            DEST = 0;                                               \
+        }                                                           \
+    } while (0)
+
+#define MSA_FLOAT_UEQ(DEST, ARG1, ARG2, BITS, QUIET)                \
+    do {                                                            \
+        MSA_FLOAT_COND(DEST, unordered, ARG1, ARG2, BITS, QUIET);   \
+        if (DEST == 0) {                                            \
+            MSA_FLOAT_COND(DEST, eq, ARG1, ARG2, BITS, QUIET);      \
+        }                                                           \
+    } while (0)
+
+#define MSA_FLOAT_NE(DEST, ARG1, ARG2, BITS, QUIET)                 \
+    do {                                                            \
+        MSA_FLOAT_COND(DEST, lt, ARG1, ARG2, BITS, QUIET);          \
+        if (DEST == 0) {                                            \
+            MSA_FLOAT_COND(DEST, lt, ARG2, ARG1, BITS, QUIET);      \
+        }                                                           \
+    } while (0)
+
+#define MSA_FLOAT_UNE(DEST, ARG1, ARG2, BITS, QUIET)                \
+    do {                                                            \
+        MSA_FLOAT_COND(DEST, unordered, ARG1, ARG2, BITS, QUIET);   \
+        if (DEST == 0) {                                            \
+            MSA_FLOAT_COND(DEST, lt, ARG1, ARG2, BITS, QUIET);      \
+            if (DEST == 0) {                                        \
+                MSA_FLOAT_COND(DEST, lt, ARG2, ARG1, BITS, QUIET);  \
+            }                                                       \
+        }                                                           \
+    } while (0)
+
+#define MSA_FLOAT_ULE(DEST, ARG1, ARG2, BITS, QUIET)                \
+    do {                                                            \
+        MSA_FLOAT_COND(DEST, unordered, ARG1, ARG2, BITS, QUIET);   \
+        if (DEST == 0) {                                            \
+            MSA_FLOAT_COND(DEST, le, ARG1, ARG2, BITS, QUIET);      \
+        }                                                           \
+    } while (0)
+
+#define MSA_FLOAT_ULT(DEST, ARG1, ARG2, BITS, QUIET)                \
+    do {                                                            \
+        MSA_FLOAT_COND(DEST, unordered, ARG1, ARG2, BITS, QUIET);   \
+        if (DEST == 0) {                                            \
+            MSA_FLOAT_COND(DEST, lt, ARG1, ARG2, BITS, QUIET);      \
+        }                                                           \
+    } while (0)
+
+#define MSA_FLOAT_OR(DEST, ARG1, ARG2, BITS, QUIET)                 \
+    do {                                                            \
+        MSA_FLOAT_COND(DEST, le, ARG1, ARG2, BITS, QUIET);          \
+        if (DEST == 0) {                                            \
+            MSA_FLOAT_COND(DEST, le, ARG2, ARG1, BITS, QUIET);      \
+        }                                                           \
+    } while (0)
+
+static inline void compare_af(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
+                              wr_t *pwt, uint32_t df, int quiet)
+{
+    wr_t wx, *pwx = &wx;
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_AF(pwx->w[i], pws->w[i], pwt->w[i], 32, quiet);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_AF(pwx->d[i], pws->d[i], pwt->d[i], 64, quiet);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+static inline void compare_un(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
+                              wr_t *pwt, uint32_t df, int quiet)
+{
+    wr_t wx, *pwx = &wx;
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_COND(pwx->w[i], unordered, pws->w[i], pwt->w[i], 32,
+                    quiet);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_COND(pwx->d[i], unordered, pws->d[i], pwt->d[i], 64,
+                    quiet);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+static inline void compare_eq(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
+                              wr_t *pwt, uint32_t df, int quiet)
+{
+    wr_t wx, *pwx = &wx;
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_COND(pwx->w[i], eq, pws->w[i], pwt->w[i], 32, quiet);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_COND(pwx->d[i], eq, pws->d[i], pwt->d[i], 64, quiet);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+static inline void compare_ueq(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
+                               wr_t *pwt, uint32_t df, int quiet)
+{
+    wr_t wx, *pwx = &wx;
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_UEQ(pwx->w[i], pws->w[i], pwt->w[i], 32, quiet);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_UEQ(pwx->d[i], pws->d[i], pwt->d[i], 64, quiet);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+static inline void compare_lt(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
+                              wr_t *pwt, uint32_t df, int quiet)
+{
+    wr_t wx, *pwx = &wx;
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_COND(pwx->w[i], lt, pws->w[i], pwt->w[i], 32, quiet);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_COND(pwx->d[i], lt, pws->d[i], pwt->d[i], 64, quiet);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+static inline void compare_ult(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
+                               wr_t *pwt, uint32_t df, int quiet)
+{
+    wr_t wx, *pwx = &wx;
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_ULT(pwx->w[i], pws->w[i], pwt->w[i], 32, quiet);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_ULT(pwx->d[i], pws->d[i], pwt->d[i], 64, quiet);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+static inline void compare_le(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
+                              wr_t *pwt, uint32_t df, int quiet)
+{
+    wr_t wx, *pwx = &wx;
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_COND(pwx->w[i], le, pws->w[i], pwt->w[i], 32, quiet);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_COND(pwx->d[i], le, pws->d[i], pwt->d[i], 64, quiet);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+static inline void compare_ule(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
+                               wr_t *pwt, uint32_t df, int quiet)
+{
+    wr_t wx, *pwx = &wx;
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_ULE(pwx->w[i], pws->w[i], pwt->w[i], 32, quiet);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_ULE(pwx->d[i], pws->d[i], pwt->d[i], 64, quiet);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+static inline void compare_or(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
+                              wr_t *pwt, uint32_t df, int quiet)
+{
+    wr_t wx, *pwx = &wx;
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_OR(pwx->w[i], pws->w[i], pwt->w[i], 32, quiet);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_OR(pwx->d[i], pws->d[i], pwt->d[i], 64, quiet);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+static inline void compare_une(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
+                               wr_t *pwt, uint32_t df, int quiet)
+{
+    wr_t wx, *pwx = &wx;
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_UNE(pwx->w[i], pws->w[i], pwt->w[i], 32, quiet);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_UNE(pwx->d[i], pws->d[i], pwt->d[i], 64, quiet);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+static inline void compare_ne(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
+                              wr_t *pwt, uint32_t df, int quiet) {
+    wr_t wx, *pwx = &wx;
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_NE(pwx->w[i], pws->w[i], pwt->w[i], 32, quiet);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_NE(pwx->d[i], pws->d[i], pwt->d[i], 64, quiet);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_fcaf_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_af(env, pwd, pws, pwt, df, 1);
+}
+
+void helper_msa_fcun_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_un(env, pwd, pws, pwt, df, 1);
+}
+
+void helper_msa_fceq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_eq(env, pwd, pws, pwt, df, 1);
+}
+
+void helper_msa_fcueq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                         uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_ueq(env, pwd, pws, pwt, df, 1);
+}
+
+void helper_msa_fclt_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_lt(env, pwd, pws, pwt, df, 1);
+}
+
+void helper_msa_fcult_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                         uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_ult(env, pwd, pws, pwt, df, 1);
+}
+
+void helper_msa_fcle_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_le(env, pwd, pws, pwt, df, 1);
+}
+
+void helper_msa_fcule_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                         uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_ule(env, pwd, pws, pwt, df, 1);
+}
+
+void helper_msa_fsaf_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_af(env, pwd, pws, pwt, df, 0);
+}
+
+void helper_msa_fsun_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_un(env, pwd, pws, pwt, df, 0);
+}
+
+void helper_msa_fseq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_eq(env, pwd, pws, pwt, df, 0);
+}
+
+void helper_msa_fsueq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                         uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_ueq(env, pwd, pws, pwt, df, 0);
+}
+
+void helper_msa_fslt_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_lt(env, pwd, pws, pwt, df, 0);
+}
+
+void helper_msa_fsult_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                         uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_ult(env, pwd, pws, pwt, df, 0);
+}
+
+void helper_msa_fsle_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_le(env, pwd, pws, pwt, df, 0);
+}
+
+void helper_msa_fsule_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                         uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_ule(env, pwd, pws, pwt, df, 0);
+}
+
+void helper_msa_fcor_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_or(env, pwd, pws, pwt, df, 1);
+}
+
+void helper_msa_fcune_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                         uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_une(env, pwd, pws, pwt, df, 1);
+}
+
+void helper_msa_fcne_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_ne(env, pwd, pws, pwt, df, 1);
+}
+
+void helper_msa_fsor_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_or(env, pwd, pws, pwt, df, 0);
+}
+
+void helper_msa_fsune_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                         uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_une(env, pwd, pws, pwt, df, 0);
+}
+
+void helper_msa_fsne_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t ws, uint32_t wt)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    compare_ne(env, pwd, pws, pwt, df, 0);
+}
+
+#define float16_is_zero(ARG) 0
+#define float16_is_zero_or_denormal(ARG) 0
+
+#define IS_DENORMAL(ARG, BITS)                      \
+    (!float ## BITS ## _is_zero(ARG)                \
+    && float ## BITS ## _is_zero_or_denormal(ARG))
+
+#define MSA_FLOAT_BINOP(DEST, OP, ARG1, ARG2, BITS)                         \
+    do {                                                                    \
+        int c;                                                              \
+                                                                            \
+        set_float_exception_flags(0, &env->active_tc.msa_fp_status);        \
+        DEST = float ## BITS ## _ ## OP(ARG1, ARG2,                         \
+                                        &env->active_tc.msa_fp_status);     \
+        c = update_msacsr(env, 0, IS_DENORMAL(DEST, BITS));                 \
+                                                                            \
+        if (get_enabled_exceptions(env, c)) {                               \
+            DEST = ((FLOAT_SNAN ## BITS >> 6) << 6) | c;                    \
+        }                                                                   \
+    } while (0)
+
+void helper_msa_fadd_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+        uint32_t ws, uint32_t wt)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_BINOP(pwx->w[i], add, pws->w[i], pwt->w[i], 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_BINOP(pwx->d[i], add, pws->d[i], pwt->d[i], 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_fsub_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+        uint32_t ws, uint32_t wt)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_BINOP(pwx->w[i], sub, pws->w[i], pwt->w[i], 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_BINOP(pwx->d[i], sub, pws->d[i], pwt->d[i], 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_fmul_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+        uint32_t ws, uint32_t wt)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_BINOP(pwx->w[i], mul, pws->w[i], pwt->w[i], 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_BINOP(pwx->d[i], mul, pws->d[i], pwt->d[i], 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_fdiv_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+        uint32_t ws, uint32_t wt)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_BINOP(pwx->w[i], div, pws->w[i], pwt->w[i], 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_BINOP(pwx->d[i], div, pws->d[i], pwt->d[i], 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+#define MSA_FLOAT_MULADD(DEST, ARG1, ARG2, ARG3, NEGATE, BITS)              \
+    do {                                                                    \
+        int c;                                                              \
+                                                                            \
+        set_float_exception_flags(0, &env->active_tc.msa_fp_status);        \
+        DEST = float ## BITS ## _muladd(ARG2, ARG3, ARG1, NEGATE,           \
+                                        &env->active_tc.msa_fp_status);     \
+        c = update_msacsr(env, 0, IS_DENORMAL(DEST, BITS));                 \
+                                                                            \
+        if (get_enabled_exceptions(env, c)) {                               \
+            DEST = ((FLOAT_SNAN ## BITS >> 6) << 6) | c;                    \
+        }                                                                   \
+    } while (0)
+
+void helper_msa_fmadd_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+        uint32_t ws, uint32_t wt)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_MULADD(pwx->w[i], pwd->w[i],
+                           pws->w[i], pwt->w[i], 0, 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_MULADD(pwx->d[i], pwd->d[i],
+                           pws->d[i], pwt->d[i], 0, 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_fmsub_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+        uint32_t ws, uint32_t wt)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_MULADD(pwx->w[i], pwd->w[i],
+                           pws->w[i], pwt->w[i],
+                           float_muladd_negate_product, 32);
+      }
+      break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_MULADD(pwx->d[i], pwd->d[i],
+                           pws->d[i], pwt->d[i],
+                           float_muladd_negate_product, 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_fexp2_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+        uint32_t ws, uint32_t wt)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_BINOP(pwx->w[i], scalbn, pws->w[i],
+                            pwt->w[i] >  0x200 ?  0x200 :
+                            pwt->w[i] < -0x200 ? -0x200 : pwt->w[i],
+                            32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_BINOP(pwx->d[i], scalbn, pws->d[i],
+                            pwt->d[i] >  0x1000 ?  0x1000 :
+                            pwt->d[i] < -0x1000 ? -0x1000 : pwt->d[i],
+                            64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+#define MSA_FLOAT_UNOP(DEST, OP, ARG, BITS)                                 \
+    do {                                                                    \
+        int c;                                                              \
+                                                                            \
+        set_float_exception_flags(0, &env->active_tc.msa_fp_status);        \
+        DEST = float ## BITS ## _ ## OP(ARG, &env->active_tc.msa_fp_status);\
+        c = update_msacsr(env, 0, IS_DENORMAL(DEST, BITS));                 \
+                                                                            \
+        if (get_enabled_exceptions(env, c)) {                               \
+            DEST = ((FLOAT_SNAN ## BITS >> 6) << 6) | c;                    \
+        }                                                                   \
+    } while (0)
+
+void helper_msa_fexdo_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                         uint32_t ws, uint32_t wt)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    uint32_t i;
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            /* Half precision floats come in two formats: standard
+               IEEE and "ARM" format.  The latter gains extra exponent
+               range by omitting the NaN/Inf encodings.  */
+            flag ieee = 1;
+
+            MSA_FLOAT_BINOP(Lh(pwx, i), from_float32, pws->w[i], ieee, 16);
+            MSA_FLOAT_BINOP(Rh(pwx, i), from_float32, pwt->w[i], ieee, 16);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_UNOP(Lw(pwx, i), from_float64, pws->d[i], 32);
+            MSA_FLOAT_UNOP(Rw(pwx, i), from_float64, pwt->d[i], 32);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+    msa_move_v(pwd, pwx);
+}
+
+#define MSA_FLOAT_UNOP_XD(DEST, OP, ARG, BITS, XBITS)                       \
+    do {                                                                    \
+        int c;                                                              \
+                                                                            \
+        set_float_exception_flags(0, &env->active_tc.msa_fp_status);        \
+        DEST = float ## BITS ## _ ## OP(ARG, &env->active_tc.msa_fp_status);\
+        c = update_msacsr(env, CLEAR_FS_UNDERFLOW, 0);                      \
+                                                                            \
+        if (get_enabled_exceptions(env, c)) {                               \
+            DEST = ((FLOAT_SNAN ## XBITS >> 6) << 6) | c;                   \
+        }                                                                   \
+    } while (0)
+
+void helper_msa_ftq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                       uint32_t ws, uint32_t wt)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_UNOP_XD(Lh(pwx, i), to_q16, pws->w[i], 32, 16);
+            MSA_FLOAT_UNOP_XD(Rh(pwx, i), to_q16, pwt->w[i], 32, 16);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_UNOP_XD(Lw(pwx, i), to_q32, pws->d[i], 64, 32);
+            MSA_FLOAT_UNOP_XD(Rw(pwx, i), to_q32, pwt->d[i], 64, 32);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+#define NUMBER_QNAN_PAIR(ARG1, ARG2, BITS)      \
+    !float ## BITS ## _is_any_nan(ARG1)         \
+    && float ## BITS ## _is_quiet_nan(ARG2)
+
+#define MSA_FLOAT_MAXOP(DEST, OP, ARG1, ARG2, BITS)                         \
+    do {                                                                    \
+        int c;                                                              \
+                                                                            \
+        set_float_exception_flags(0, &env->active_tc.msa_fp_status);        \
+        DEST = float ## BITS ## _ ## OP(ARG1, ARG2,                         \
+                                        &env->active_tc.msa_fp_status);     \
+        c = update_msacsr(env, 0, 0);                                       \
+                                                                            \
+        if (get_enabled_exceptions(env, c)) {                               \
+            DEST = ((FLOAT_SNAN ## BITS >> 6) << 6) | c;                    \
+        }                                                                   \
+    } while (0)
+
+#define FMAXMIN_A(F, G, X, _S, _T, BITS)                            \
+    do {                                                            \
+        uint## BITS ##_t S = _S, T = _T;                            \
+        uint## BITS ##_t as, at, xs, xt, xd;                        \
+        if (NUMBER_QNAN_PAIR(S, T, BITS)) {                         \
+            T = S;                                                  \
+        }                                                           \
+        else if (NUMBER_QNAN_PAIR(T, S, BITS)) {                    \
+            S = T;                                                  \
+        }                                                           \
+        as = float## BITS ##_abs(S);                                \
+        at = float## BITS ##_abs(T);                                \
+        MSA_FLOAT_MAXOP(xs, F,  S,  T, BITS);                       \
+        MSA_FLOAT_MAXOP(xt, G,  S,  T, BITS);                       \
+        MSA_FLOAT_MAXOP(xd, F, as, at, BITS);                       \
+        X = (as == at || xd == float## BITS ##_abs(xs)) ? xs : xt;  \
+    } while (0)
+
+void helper_msa_fmin_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+        uint32_t ws, uint32_t wt)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            if (NUMBER_QNAN_PAIR(pws->w[i], pwt->w[i], 32)) {
+                MSA_FLOAT_MAXOP(pwx->w[i], min, pws->w[i], pws->w[i], 32);
+            } else if (NUMBER_QNAN_PAIR(pwt->w[i], pws->w[i], 32)) {
+                MSA_FLOAT_MAXOP(pwx->w[i], min, pwt->w[i], pwt->w[i], 32);
+            } else {
+                MSA_FLOAT_MAXOP(pwx->w[i], min, pws->w[i], pwt->w[i], 32);
+            }
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            if (NUMBER_QNAN_PAIR(pws->d[i], pwt->d[i], 64)) {
+                MSA_FLOAT_MAXOP(pwx->d[i], min, pws->d[i], pws->d[i], 64);
+            } else if (NUMBER_QNAN_PAIR(pwt->d[i], pws->d[i], 64)) {
+                MSA_FLOAT_MAXOP(pwx->d[i], min, pwt->d[i], pwt->d[i], 64);
+            } else {
+                MSA_FLOAT_MAXOP(pwx->d[i], min, pws->d[i], pwt->d[i], 64);
+            }
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_fmin_a_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+        uint32_t ws, uint32_t wt)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            FMAXMIN_A(min, max, pwx->w[i], pws->w[i], pwt->w[i], 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            FMAXMIN_A(min, max, pwx->d[i], pws->d[i], pwt->d[i], 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_fmax_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+        uint32_t ws, uint32_t wt)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            if (NUMBER_QNAN_PAIR(pws->w[i], pwt->w[i], 32)) {
+                MSA_FLOAT_MAXOP(pwx->w[i], max, pws->w[i], pws->w[i], 32);
+            } else if (NUMBER_QNAN_PAIR(pwt->w[i], pws->w[i], 32)) {
+                MSA_FLOAT_MAXOP(pwx->w[i], max, pwt->w[i], pwt->w[i], 32);
+            } else {
+                MSA_FLOAT_MAXOP(pwx->w[i], max, pws->w[i], pwt->w[i], 32);
+            }
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            if (NUMBER_QNAN_PAIR(pws->d[i], pwt->d[i], 64)) {
+                MSA_FLOAT_MAXOP(pwx->d[i], max, pws->d[i], pws->d[i], 64);
+            } else if (NUMBER_QNAN_PAIR(pwt->d[i], pws->d[i], 64)) {
+                MSA_FLOAT_MAXOP(pwx->d[i], max, pwt->d[i], pwt->d[i], 64);
+            } else {
+                MSA_FLOAT_MAXOP(pwx->d[i], max, pws->d[i], pwt->d[i], 64);
+            }
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_fmax_a_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+        uint32_t ws, uint32_t wt)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            FMAXMIN_A(max, min, pwx->w[i], pws->w[i], pwt->w[i], 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            FMAXMIN_A(max, min, pwx->d[i], pws->d[i], pwt->d[i], 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_fclass_df(CPUMIPSState *env, uint32_t df,
+        uint32_t wd, uint32_t ws)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    if (df == DF_WORD) {
+        pwd->w[0] = helper_float_class_s(pws->w[0]);
+        pwd->w[1] = helper_float_class_s(pws->w[1]);
+        pwd->w[2] = helper_float_class_s(pws->w[2]);
+        pwd->w[3] = helper_float_class_s(pws->w[3]);
+    } else {
+        pwd->d[0] = helper_float_class_d(pws->d[0]);
+        pwd->d[1] = helper_float_class_d(pws->d[1]);
+    }
+}
+
+#define MSA_FLOAT_UNOP0(DEST, OP, ARG, BITS)                                \
+    do {                                                                    \
+        int c;                                                              \
+                                                                            \
+        set_float_exception_flags(0, &env->active_tc.msa_fp_status);        \
+        DEST = float ## BITS ## _ ## OP(ARG, &env->active_tc.msa_fp_status);\
+        c = update_msacsr(env, CLEAR_FS_UNDERFLOW, 0);                      \
+                                                                            \
+        if (get_enabled_exceptions(env, c)) {                               \
+            DEST = ((FLOAT_SNAN ## BITS >> 6) << 6) | c;                    \
+        } else if (float ## BITS ## _is_any_nan(ARG)) {                     \
+            DEST = 0;                                                       \
+        }                                                                   \
+    } while (0)
+
+void helper_msa_ftrunc_s_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                            uint32_t ws)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_UNOP0(pwx->w[i], to_int32_round_to_zero, pws->w[i], 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_UNOP0(pwx->d[i], to_int64_round_to_zero, pws->d[i], 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_ftrunc_u_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                            uint32_t ws)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_UNOP0(pwx->w[i], to_uint32_round_to_zero, pws->w[i], 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_UNOP0(pwx->d[i], to_uint64_round_to_zero, pws->d[i], 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_fsqrt_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                         uint32_t ws)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_UNOP(pwx->w[i], sqrt, pws->w[i], 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_UNOP(pwx->d[i], sqrt, pws->d[i], 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+#define MSA_FLOAT_RECIPROCAL(DEST, ARG, BITS)                               \
+    do {                                                                    \
+        int c;                                                              \
+                                                                            \
+        set_float_exception_flags(0, &env->active_tc.msa_fp_status);        \
+        DEST = float ## BITS ## _ ## div(FLOAT_ONE ## BITS, ARG,            \
+                                         &env->active_tc.msa_fp_status);    \
+        c = update_msacsr(env, float ## BITS ## _is_infinity(ARG) ||        \
+                          float ## BITS ## _is_quiet_nan(DEST) ?            \
+                          0 : RECIPROCAL_INEXACT,                           \
+                          IS_DENORMAL(DEST, BITS));                         \
+                                                                            \
+        if (get_enabled_exceptions(env, c)) {                               \
+            DEST = ((FLOAT_SNAN ## BITS >> 6) << 6) | c;                    \
+        }                                                                   \
+    } while (0)
+
+void helper_msa_frsqrt_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                          uint32_t ws)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_RECIPROCAL(pwx->w[i], float32_sqrt(pws->w[i],
+                    &env->active_tc.msa_fp_status), 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_RECIPROCAL(pwx->d[i], float64_sqrt(pws->d[i],
+                    &env->active_tc.msa_fp_status), 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_frcp_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t ws)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_RECIPROCAL(pwx->w[i], pws->w[i], 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_RECIPROCAL(pwx->d[i], pws->d[i], 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_frint_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                         uint32_t ws)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_UNOP(pwx->w[i], round_to_int, pws->w[i], 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_UNOP(pwx->d[i], round_to_int, pws->d[i], 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+#define MSA_FLOAT_LOGB(DEST, ARG, BITS)                                     \
+    do {                                                                    \
+        int c;                                                              \
+                                                                            \
+        set_float_exception_flags(0, &env->active_tc.msa_fp_status);        \
+        set_float_rounding_mode(float_round_down,                           \
+                                &env->active_tc.msa_fp_status);             \
+        DEST = float ## BITS ## _ ## log2(ARG,                              \
+                                          &env->active_tc.msa_fp_status);   \
+        DEST = float ## BITS ## _ ## round_to_int(DEST,                     \
+                                          &env->active_tc.msa_fp_status);   \
+        set_float_rounding_mode(ieee_rm[(env->active_tc.msacsr &            \
+                                         MSACSR_RM_MASK) >> MSACSR_RM],     \
+                                &env->active_tc.msa_fp_status);             \
+                                                                            \
+        set_float_exception_flags(                                          \
+            get_float_exception_flags(&env->active_tc.msa_fp_status)        \
+                                                & (~float_flag_inexact),    \
+            &env->active_tc.msa_fp_status);                                 \
+                                                                            \
+        c = update_msacsr(env, 0, IS_DENORMAL(DEST, BITS));                 \
+                                                                            \
+        if (get_enabled_exceptions(env, c)) {                               \
+            DEST = ((FLOAT_SNAN ## BITS >> 6) << 6) | c;                    \
+        }                                                                   \
+    } while (0)
+
+void helper_msa_flog2_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                         uint32_t ws)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_LOGB(pwx->w[i], pws->w[i], 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_LOGB(pwx->d[i], pws->d[i], 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_fexupl_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                          uint32_t ws)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    uint32_t i;
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            /* Half precision floats come in two formats: standard
+               IEEE and "ARM" format.  The latter gains extra exponent
+               range by omitting the NaN/Inf encodings.  */
+            flag ieee = 1;
+
+            MSA_FLOAT_BINOP(pwx->w[i], from_float16, Lh(pws, i), ieee, 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_UNOP(pwx->d[i], from_float32, Lw(pws, i), 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_fexupr_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                          uint32_t ws)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    uint32_t i;
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            /* Half precision floats come in two formats: standard
+               IEEE and "ARM" format.  The latter gains extra exponent
+               range by omitting the NaN/Inf encodings.  */
+            flag ieee = 1;
+
+            MSA_FLOAT_BINOP(pwx->w[i], from_float16, Rh(pws, i), ieee, 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_UNOP(pwx->d[i], from_float32, Rw(pws, i), 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_ffql_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t ws)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    uint32_t i;
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_UNOP(pwx->w[i], from_q16, Lh(pws, i), 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_UNOP(pwx->d[i], from_q32, Lw(pws, i), 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_ffqr_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                        uint32_t ws)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    uint32_t i;
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_UNOP(pwx->w[i], from_q16, Rh(pws, i), 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_UNOP(pwx->d[i], from_q32, Rw(pws, i), 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_ftint_s_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                           uint32_t ws)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_UNOP0(pwx->w[i], to_int32, pws->w[i], 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_UNOP0(pwx->d[i], to_int64, pws->d[i], 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_ftint_u_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                           uint32_t ws)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_UNOP0(pwx->w[i], to_uint32, pws->w[i], 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_UNOP0(pwx->d[i], to_uint64, pws->d[i], 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+#define float32_from_int32 int32_to_float32
+#define float32_from_uint32 uint32_to_float32
+
+#define float64_from_int64 int64_to_float64
+#define float64_from_uint64 uint64_to_float64
+
+void helper_msa_ffint_s_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                           uint32_t ws)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_UNOP(pwx->w[i], from_int32, pws->w[i], 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_UNOP(pwx->d[i], from_int64, pws->d[i], 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
+
+void helper_msa_ffint_u_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
+                           uint32_t ws)
+{
+    wr_t wx, *pwx = &wx;
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    wr_t *pws = &(env->active_fpu.fpr[ws].wr);
+    uint32_t i;
+
+    clear_msacsr_cause(env);
+
+    switch (df) {
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            MSA_FLOAT_UNOP(pwx->w[i], from_uint32, pws->w[i], 32);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            MSA_FLOAT_UNOP(pwx->d[i], from_uint64, pws->d[i], 64);
+        }
+        break;
+    default:
+        assert(0);
+    }
+
+    check_msacsr_cause(env);
+
+    msa_move_v(pwd, pwx);
+}
diff --git a/target-mips/op_helper.c b/target-mips/op_helper.c
index 27651a4a0..638c9f9df 100644
--- a/target-mips/op_helper.c
+++ b/target-mips/op_helper.c
@@ -21,6 +21,7 @@
 #include "qemu/host-utils.h"
 #include "exec/helper-proto.h"
 #include "exec/cpu_ldst.h"
+#include "sysemu/kvm.h"
 
 #ifndef CONFIG_USER_ONLY
 static inline void cpu_mips_tlb_flush (CPUMIPSState *env, int flush_global);
@@ -90,10 +91,9 @@ static inline type do_##name(CPUMIPSState *env, target_ulong addr,      \
 }
 #endif
 HELPER_LD(lbu, ldub, uint8_t)
+HELPER_LD(lhu, lduw, uint16_t)
 HELPER_LD(lw, ldl, int32_t)
-#ifdef TARGET_MIPS64
 HELPER_LD(ld, ldq, int64_t)
-#endif
 #undef HELPER_LD
 
 #if defined(CONFIG_USER_ONLY)
@@ -118,10 +118,9 @@ static inline void do_##name(CPUMIPSState *env, target_ulong addr,      \
 }
 #endif
 HELPER_ST(sb, stb, uint8_t)
+HELPER_ST(sh, stw, uint16_t)
 HELPER_ST(sw, stl, uint32_t)
-#ifdef TARGET_MIPS64
 HELPER_ST(sd, stq, uint64_t)
-#endif
 #undef HELPER_ST
 
 target_ulong helper_clo (target_ulong arg1)
@@ -265,6 +264,29 @@ target_ulong helper_mulshiu(CPUMIPSState *env, target_ulong arg1,
                        (uint64_t)(uint32_t)arg2);
 }
 
+static inline target_ulong bitswap(target_ulong v)
+{
+    v = ((v >> 1) & (target_ulong)0x5555555555555555ULL) |
+              ((v & (target_ulong)0x5555555555555555ULL) << 1);
+    v = ((v >> 2) & (target_ulong)0x3333333333333333ULL) |
+              ((v & (target_ulong)0x3333333333333333ULL) << 2);
+    v = ((v >> 4) & (target_ulong)0x0F0F0F0F0F0F0F0FULL) |
+              ((v & (target_ulong)0x0F0F0F0F0F0F0F0FULL) << 4);
+    return v;
+}
+
+#ifdef TARGET_MIPS64
+target_ulong helper_dbitswap(target_ulong rt)
+{
+    return bitswap(rt);
+}
+#endif
+
+target_ulong helper_bitswap(target_ulong rt)
+{
+    return (int32_t)bitswap(rt);
+}
+
 #ifndef CONFIG_USER_ONLY
 
 static inline hwaddr do_translate_address(CPUMIPSState *env,
@@ -936,14 +958,14 @@ target_ulong helper_dmfc0_watchlo(CPUMIPSState *env, uint32_t sel)
 
 void helper_mtc0_index(CPUMIPSState *env, target_ulong arg1)
 {
-    int num = 1;
-    unsigned int tmp = env->tlb->nb_tlb;
-
-    do {
-        tmp >>= 1;
-        num <<= 1;
-    } while (tmp);
-    env->CP0_Index = (env->CP0_Index & 0x80000000) | (arg1 & (num - 1));
+    uint32_t index_p = env->CP0_Index & 0x80000000;
+    uint32_t tlb_index = arg1 & 0x7fffffff;
+    if (tlb_index < env->tlb->nb_tlb) {
+        if (env->insn_flags & ISA_MIPS32R6) {
+            index_p |= arg1 & 0x80000000;
+        }
+        env->CP0_Index = index_p | tlb_index;
+    }
 }
 
 void helper_mtc0_mvpcontrol(CPUMIPSState *env, target_ulong arg1)
@@ -1076,8 +1098,17 @@ void helper_mtc0_entrylo0(CPUMIPSState *env, target_ulong arg1)
 {
     /* Large physaddr (PABITS) not implemented */
     /* 1k pages not implemented */
-    env->CP0_EntryLo0 = arg1 & 0x3FFFFFFF;
+    target_ulong rxi = arg1 & (env->CP0_PageGrain & (3u << CP0PG_XIE));
+    env->CP0_EntryLo0 = (arg1 & 0x3FFFFFFF) | (rxi << (CP0EnLo_XI - 30));
+}
+
+#if defined(TARGET_MIPS64)
+void helper_dmtc0_entrylo0(CPUMIPSState *env, uint64_t arg1)
+{
+    uint64_t rxi = arg1 & ((env->CP0_PageGrain & (3ull << CP0PG_XIE)) << 32);
+    env->CP0_EntryLo0 = (arg1 & 0x3FFFFFFF) | rxi;
 }
+#endif
 
 void helper_mtc0_tcstatus(CPUMIPSState *env, target_ulong arg1)
 {
@@ -1243,9 +1274,18 @@ void helper_mtc0_entrylo1(CPUMIPSState *env, target_ulong arg1)
 {
     /* Large physaddr (PABITS) not implemented */
     /* 1k pages not implemented */
-    env->CP0_EntryLo1 = arg1 & 0x3FFFFFFF;
+    target_ulong rxi = arg1 & (env->CP0_PageGrain & (3u << CP0PG_XIE));
+    env->CP0_EntryLo1 = (arg1 & 0x3FFFFFFF) | (rxi << (CP0EnLo_XI - 30));
 }
 
+#if defined(TARGET_MIPS64)
+void helper_dmtc0_entrylo1(CPUMIPSState *env, uint64_t arg1)
+{
+    uint64_t rxi = arg1 & ((env->CP0_PageGrain & (3ull << CP0PG_XIE)) << 32);
+    env->CP0_EntryLo1 = (arg1 & 0x3FFFFFFF) | rxi;
+}
+#endif
+
 void helper_mtc0_context(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_Context = (env->CP0_Context & 0x007FFFFF) | (arg1 & ~0x007FFFFF);
@@ -1253,8 +1293,13 @@ void helper_mtc0_context(CPUMIPSState *env, target_ulong arg1)
 
 void helper_mtc0_pagemask(CPUMIPSState *env, target_ulong arg1)
 {
-    /* 1k pages not implemented */
-    env->CP0_PageMask = arg1 & (0x1FFFFFFF & (TARGET_PAGE_MASK << 1));
+    uint64_t mask = arg1 >> (TARGET_PAGE_BITS + 1);
+    if (!(env->insn_flags & ISA_MIPS32R6) || (arg1 == ~0) ||
+        (mask == 0x0000 || mask == 0x0003 || mask == 0x000F ||
+         mask == 0x003F || mask == 0x00FF || mask == 0x03FF ||
+         mask == 0x0FFF || mask == 0x3FFF || mask == 0xFFFF)) {
+        env->CP0_PageMask = arg1 & (0x1FFFFFFF & (TARGET_PAGE_MASK << 1));
+    }
 }
 
 void helper_mtc0_pagegrain(CPUMIPSState *env, target_ulong arg1)
@@ -1262,12 +1307,19 @@ void helper_mtc0_pagegrain(CPUMIPSState *env, target_ulong arg1)
     /* SmartMIPS not implemented */
     /* Large physaddr (PABITS) not implemented */
     /* 1k pages not implemented */
-    env->CP0_PageGrain = 0;
+    env->CP0_PageGrain = (arg1 & env->CP0_PageGrain_rw_bitmask) |
+                         (env->CP0_PageGrain & ~env->CP0_PageGrain_rw_bitmask);
 }
 
 void helper_mtc0_wired(CPUMIPSState *env, target_ulong arg1)
 {
-    env->CP0_Wired = arg1 % env->tlb->nb_tlb;
+    if (env->insn_flags & ISA_MIPS32R6) {
+        if (arg1 < env->tlb->nb_tlb) {
+            env->CP0_Wired = arg1;
+        }
+    } else {
+        env->CP0_Wired = arg1 % env->tlb->nb_tlb;
+    }
 }
 
 void helper_mtc0_srsconf0(CPUMIPSState *env, target_ulong arg1)
@@ -1319,14 +1371,28 @@ void helper_mtc0_count(CPUMIPSState *env, target_ulong arg1)
 
 void helper_mtc0_entryhi(CPUMIPSState *env, target_ulong arg1)
 {
-    target_ulong old, val;
+    target_ulong old, val, mask;
+    mask = (TARGET_PAGE_MASK << 1) | 0xFF;
+    if (((env->CP0_Config4 >> CP0C4_IE) & 0x3) >= 2) {
+        mask |= 1 << CP0EnHi_EHINV;
+    }
 
     /* 1k pages not implemented */
-    val = arg1 & ((TARGET_PAGE_MASK << 1) | 0xFF);
 #if defined(TARGET_MIPS64)
-    val &= env->SEGMask;
+    if (env->insn_flags & ISA_MIPS32R6) {
+        int entryhi_r = extract64(arg1, 62, 2);
+        int config0_at = extract32(env->CP0_Config0, 13, 2);
+        bool no_supervisor = (env->CP0_Status_rw_bitmask & 0x8) == 0;
+        if ((entryhi_r == 2) ||
+            (entryhi_r == 1 && (no_supervisor || config0_at == 1))) {
+            /* skip EntryHi.R field if new value is reserved */
+            mask &= ~(0x3ull << 62);
+        }
+    }
+    mask &= env->SEGMask;
 #endif
     old = env->CP0_EntryHi;
+    val = (arg1 & mask) | (old & ~mask);
     env->CP0_EntryHi = val;
     if (env->CP0_Config3 & (1 << CP0C3_MT)) {
         sync_c0_entryhi(env, env->current_tc);
@@ -1356,6 +1422,13 @@ void helper_mtc0_status(CPUMIPSState *env, target_ulong arg1)
     uint32_t val, old;
     uint32_t mask = env->CP0_Status_rw_bitmask;
 
+    if (env->insn_flags & ISA_MIPS32R6) {
+        if (extract32(env->CP0_Status, CP0St_KSU, 2) == 0x3) {
+            mask &= ~(3 << CP0St_KSU);
+        }
+        mask &= ~(0x00180000 & arg1);
+    }
+
     val = arg1 & mask;
     old = env->CP0_Status;
     env->CP0_Status = (env->CP0_Status & ~mask) | val;
@@ -1411,6 +1484,9 @@ static void mtc0_cause(CPUMIPSState *cpu, target_ulong arg1)
     if (cpu->insn_flags & ISA_MIPS32R2) {
         mask |= 1 << CP0Ca_DC;
     }
+    if (cpu->insn_flags & ISA_MIPS32R6) {
+        mask &= ~((1 << CP0Ca_WP) & arg1);
+    }
 
     cpu->CP0_Cause = (cpu->CP0_Cause & ~mask) | (arg1 & mask);
 
@@ -1512,6 +1588,7 @@ void helper_mtc0_config5(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_Config5 = (env->CP0_Config5 & (~env->CP0_Config5_rw_bitmask)) |
                        (arg1 & env->CP0_Config5_rw_bitmask);
+    compute_hflags(env);
 }
 
 void helper_mtc0_lladdr(CPUMIPSState *env, target_ulong arg1)
@@ -1816,6 +1893,11 @@ static void r4k_fill_tlb(CPUMIPSState *env, int idx)
 
     /* XXX: detect conflicting TLBs and raise a MCHECK exception when needed */
     tlb = &env->tlb->mmu.r4k.tlb[idx];
+    if (env->CP0_EntryHi & (1 << CP0EnHi_EHINV)) {
+        tlb->EHINV = 1;
+        return;
+    }
+    tlb->EHINV = 0;
     tlb->VPN = env->CP0_EntryHi & (TARGET_PAGE_MASK << 1);
 #if defined(TARGET_MIPS64)
     tlb->VPN &= env->SEGMask;
@@ -1826,13 +1908,42 @@ static void r4k_fill_tlb(CPUMIPSState *env, int idx)
     tlb->V0 = (env->CP0_EntryLo0 & 2) != 0;
     tlb->D0 = (env->CP0_EntryLo0 & 4) != 0;
     tlb->C0 = (env->CP0_EntryLo0 >> 3) & 0x7;
+    tlb->XI0 = (env->CP0_EntryLo0 >> CP0EnLo_XI) & 1;
+    tlb->RI0 = (env->CP0_EntryLo0 >> CP0EnLo_RI) & 1;
     tlb->PFN[0] = (env->CP0_EntryLo0 >> 6) << 12;
     tlb->V1 = (env->CP0_EntryLo1 & 2) != 0;
     tlb->D1 = (env->CP0_EntryLo1 & 4) != 0;
     tlb->C1 = (env->CP0_EntryLo1 >> 3) & 0x7;
+    tlb->XI1 = (env->CP0_EntryLo1 >> CP0EnLo_XI) & 1;
+    tlb->RI1 = (env->CP0_EntryLo1 >> CP0EnLo_RI) & 1;
     tlb->PFN[1] = (env->CP0_EntryLo1 >> 6) << 12;
 }
 
+void r4k_helper_tlbinv(CPUMIPSState *env)
+{
+    int idx;
+    r4k_tlb_t *tlb;
+    uint8_t ASID = env->CP0_EntryHi & 0xFF;
+
+    for (idx = 0; idx < env->tlb->nb_tlb; idx++) {
+        tlb = &env->tlb->mmu.r4k.tlb[idx];
+        if (!tlb->G && tlb->ASID == ASID) {
+            tlb->EHINV = 1;
+        }
+    }
+    cpu_mips_tlb_flush(env, 1);
+}
+
+void r4k_helper_tlbinvf(CPUMIPSState *env)
+{
+    int idx;
+
+    for (idx = 0; idx < env->tlb->nb_tlb; idx++) {
+        env->tlb->mmu.r4k.tlb[idx].EHINV = 1;
+    }
+    cpu_mips_tlb_flush(env, 1);
+}
+
 void r4k_helper_tlbwi(CPUMIPSState *env)
 {
     r4k_tlb_t *tlb;
@@ -1894,7 +2005,7 @@ void r4k_helper_tlbp(CPUMIPSState *env)
         tag &= env->SEGMask;
 #endif
         /* Check ASID, virtual page number & size */
-        if ((tlb->G == 1 || tlb->ASID == ASID) && VPN == tag) {
+        if ((tlb->G == 1 || tlb->ASID == ASID) && VPN == tag && !tlb->EHINV) {
             /* TLB match */
             env->CP0_Index = i;
             break;
@@ -1938,12 +2049,23 @@ void r4k_helper_tlbr(CPUMIPSState *env)
 
     r4k_mips_tlb_flush_extra(env, env->tlb->nb_tlb);
 
-    env->CP0_EntryHi = tlb->VPN | tlb->ASID;
-    env->CP0_PageMask = tlb->PageMask;
-    env->CP0_EntryLo0 = tlb->G | (tlb->V0 << 1) | (tlb->D0 << 2) |
+    if (tlb->EHINV) {
+        env->CP0_EntryHi = 1 << CP0EnHi_EHINV;
+        env->CP0_PageMask = 0;
+        env->CP0_EntryLo0 = 0;
+        env->CP0_EntryLo1 = 0;
+    } else {
+        env->CP0_EntryHi = tlb->VPN | tlb->ASID;
+        env->CP0_PageMask = tlb->PageMask;
+        env->CP0_EntryLo0 = tlb->G | (tlb->V0 << 1) | (tlb->D0 << 2) |
+                        ((target_ulong)tlb->RI0 << CP0EnLo_RI) |
+                        ((target_ulong)tlb->XI0 << CP0EnLo_XI) |
                         (tlb->C0 << 3) | (tlb->PFN[0] >> 6);
-    env->CP0_EntryLo1 = tlb->G | (tlb->V1 << 1) | (tlb->D1 << 2) |
+        env->CP0_EntryLo1 = tlb->G | (tlb->V1 << 1) | (tlb->D1 << 2) |
+                        ((target_ulong)tlb->RI1 << CP0EnLo_RI) |
+                        ((target_ulong)tlb->XI1 << CP0EnLo_XI) |
                         (tlb->C1 << 3) | (tlb->PFN[1] >> 6);
+    }
 }
 
 void helper_tlbwi(CPUMIPSState *env)
@@ -1966,6 +2088,16 @@ void helper_tlbr(CPUMIPSState *env)
     env->tlb->helper_tlbr(env);
 }
 
+void helper_tlbinv(CPUMIPSState *env)
+{
+    env->tlb->helper_tlbinv(env);
+}
+
+void helper_tlbinvf(CPUMIPSState *env)
+{
+    env->tlb->helper_tlbinvf(env);
+}
+
 /* Specials */
 target_ulong helper_di(CPUMIPSState *env)
 {
@@ -2137,13 +2269,26 @@ void helper_wait(CPUMIPSState *env)
 #if !defined(CONFIG_USER_ONLY)
 
 void mips_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
-                                  int is_write, int is_user, uintptr_t retaddr)
+                                  int access_type, int is_user,
+                                  uintptr_t retaddr)
 {
     MIPSCPU *cpu = MIPS_CPU(cs);
     CPUMIPSState *env = &cpu->env;
+    int error_code = 0;
+    int excp;
 
     env->CP0_BadVAddr = addr;
-    do_raise_exception(env, (is_write == 1) ? EXCP_AdES : EXCP_AdEL, retaddr);
+
+    if (access_type == MMU_DATA_STORE) {
+        excp = EXCP_AdES;
+    } else {
+        excp = EXCP_AdEL;
+        if (access_type == MMU_INST_FETCH) {
+            error_code |= EXCP_INST_NOTAVAIL;
+        }
+    }
+
+    do_raise_exception_err(env, excp, error_code, retaddr);
 }
 
 void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
@@ -2168,6 +2313,16 @@ void mips_cpu_unassigned_access(CPUState *cs, hwaddr addr,
     MIPSCPU *cpu = MIPS_CPU(cs);
     CPUMIPSState *env = &cpu->env;
 
+    /*
+     * Raising an exception with KVM enabled will crash because it won't be from
+     * the main execution loop so the longjmp won't have a matching setjmp.
+     * Until we can trigger a bus error exception through KVM lets just ignore
+     * the access.
+     */
+    if (kvm_enabled()) {
+        return;
+    }
+
     if (is_exec) {
         helper_raise_exception(env, EXCP_IBE);
     } else {
@@ -2184,7 +2339,7 @@ void mips_cpu_unassigned_access(CPUState *cs, hwaddr addr,
 #define FP_TO_INT64_OVERFLOW 0x7fffffffffffffffULL
 
 /* convert MIPS rounding mode in FCR31 to IEEE library */
-static unsigned int ieee_rm[] = {
+unsigned int ieee_rm[] = {
     float_round_nearest_even,
     float_round_to_zero,
     float_round_up,
@@ -2267,8 +2422,9 @@ void helper_ctc1(CPUMIPSState *env, target_ulong arg1, uint32_t fs, uint32_t rt)
         }
         break;
     case 25:
-        if (arg1 & 0xffffff00)
+        if ((env->insn_flags & ISA_MIPS32R6) || (arg1 & 0xffffff00)) {
             return;
+        }
         env->active_fpu.fcr31 = (env->active_fpu.fcr31 & 0x017fffff) | ((arg1 & 0xfe) << 24) |
                      ((arg1 & 0x1) << 23);
         break;
@@ -2284,9 +2440,13 @@ void helper_ctc1(CPUMIPSState *env, target_ulong arg1, uint32_t fs, uint32_t rt)
                      ((arg1 & 0x4) << 22);
         break;
     case 31:
-        if (arg1 & 0x007c0000)
-            return;
-        env->active_fpu.fcr31 = arg1;
+        if (env->insn_flags & ISA_MIPS32R6) {
+            uint32_t mask = 0xfefc0000;
+            env->active_fpu.fcr31 = (arg1 & ~mask) |
+                (env->active_fpu.fcr31 & mask);
+        } else if (!(arg1 & 0x007c0000)) {
+            env->active_fpu.fcr31 = arg1;
+        }
         break;
     default:
         return;
@@ -2300,7 +2460,7 @@ void helper_ctc1(CPUMIPSState *env, target_ulong arg1, uint32_t fs, uint32_t rt)
         do_raise_exception(env, EXCP_FPE, GETPC());
 }
 
-static inline int ieee_ex_to_mips(int xcpt)
+int ieee_ex_to_mips(int xcpt)
 {
     int ret = 0;
     if (xcpt) {
@@ -2775,6 +2935,110 @@ FLOAT_UNOP(abs)
 FLOAT_UNOP(chs)
 #undef FLOAT_UNOP
 
+#define FLOAT_FMADDSUB(name, bits, muladd_arg)                          \
+uint ## bits ## _t helper_float_ ## name (CPUMIPSState *env,            \
+                                          uint ## bits ## _t fs,        \
+                                          uint ## bits ## _t ft,        \
+                                          uint ## bits ## _t fd)        \
+{                                                                       \
+    uint ## bits ## _t fdret;                                           \
+                                                                        \
+    fdret = float ## bits ## _muladd(fs, ft, fd, muladd_arg,            \
+                                     &env->active_fpu.fp_status);       \
+    update_fcr31(env, GETPC());                                         \
+    return fdret;                                                       \
+}
+
+FLOAT_FMADDSUB(maddf_s, 32, 0)
+FLOAT_FMADDSUB(maddf_d, 64, 0)
+FLOAT_FMADDSUB(msubf_s, 32, float_muladd_negate_product)
+FLOAT_FMADDSUB(msubf_d, 64, float_muladd_negate_product)
+#undef FLOAT_FMADDSUB
+
+#define FLOAT_MINMAX(name, bits, minmaxfunc)                            \
+uint ## bits ## _t helper_float_ ## name (CPUMIPSState *env,            \
+                                          uint ## bits ## _t fs,        \
+                                          uint ## bits ## _t ft)        \
+{                                                                       \
+    uint ## bits ## _t fdret;                                           \
+                                                                        \
+    fdret = float ## bits ## _ ## minmaxfunc(fs, ft,                    \
+                                           &env->active_fpu.fp_status); \
+    update_fcr31(env, GETPC());                                         \
+    return fdret;                                                       \
+}
+
+FLOAT_MINMAX(max_s, 32, maxnum)
+FLOAT_MINMAX(max_d, 64, maxnum)
+FLOAT_MINMAX(maxa_s, 32, maxnummag)
+FLOAT_MINMAX(maxa_d, 64, maxnummag)
+
+FLOAT_MINMAX(min_s, 32, minnum)
+FLOAT_MINMAX(min_d, 64, minnum)
+FLOAT_MINMAX(mina_s, 32, minnummag)
+FLOAT_MINMAX(mina_d, 64, minnummag)
+#undef FLOAT_MINMAX
+
+#define FLOAT_RINT(name, bits)                                              \
+uint ## bits ## _t helper_float_ ## name (CPUMIPSState *env,                \
+                                          uint ## bits ## _t fs)            \
+{                                                                           \
+    uint ## bits ## _t fdret;                                               \
+                                                                            \
+    fdret = float ## bits ## _round_to_int(fs, &env->active_fpu.fp_status); \
+    update_fcr31(env, GETPC());                                             \
+    return fdret;                                                           \
+}
+
+FLOAT_RINT(rint_s, 32)
+FLOAT_RINT(rint_d, 64)
+#undef FLOAT_RINT
+
+#define FLOAT_CLASS_SIGNALING_NAN      0x001
+#define FLOAT_CLASS_QUIET_NAN          0x002
+#define FLOAT_CLASS_NEGATIVE_INFINITY  0x004
+#define FLOAT_CLASS_NEGATIVE_NORMAL    0x008
+#define FLOAT_CLASS_NEGATIVE_SUBNORMAL 0x010
+#define FLOAT_CLASS_NEGATIVE_ZERO      0x020
+#define FLOAT_CLASS_POSITIVE_INFINITY  0x040
+#define FLOAT_CLASS_POSITIVE_NORMAL    0x080
+#define FLOAT_CLASS_POSITIVE_SUBNORMAL 0x100
+#define FLOAT_CLASS_POSITIVE_ZERO      0x200
+
+#define FLOAT_CLASS(name, bits)                                      \
+uint ## bits ## _t helper_float_ ## name (uint ## bits ## _t arg)    \
+{                                                                    \
+    if (float ## bits ## _is_signaling_nan(arg)) {                   \
+        return FLOAT_CLASS_SIGNALING_NAN;                            \
+    } else if (float ## bits ## _is_quiet_nan(arg)) {                \
+        return FLOAT_CLASS_QUIET_NAN;                                \
+    } else if (float ## bits ## _is_neg(arg)) {                      \
+        if (float ## bits ## _is_infinity(arg)) {                    \
+            return FLOAT_CLASS_NEGATIVE_INFINITY;                    \
+        } else if (float ## bits ## _is_zero(arg)) {                 \
+            return FLOAT_CLASS_NEGATIVE_ZERO;                        \
+        } else if (float ## bits ## _is_zero_or_denormal(arg)) {     \
+            return FLOAT_CLASS_NEGATIVE_SUBNORMAL;                   \
+        } else {                                                     \
+            return FLOAT_CLASS_NEGATIVE_NORMAL;                      \
+        }                                                            \
+    } else {                                                         \
+        if (float ## bits ## _is_infinity(arg)) {                    \
+            return FLOAT_CLASS_POSITIVE_INFINITY;                    \
+        } else if (float ## bits ## _is_zero(arg)) {                 \
+            return FLOAT_CLASS_POSITIVE_ZERO;                        \
+        } else if (float ## bits ## _is_zero_or_denormal(arg)) {     \
+            return FLOAT_CLASS_POSITIVE_SUBNORMAL;                   \
+        } else {                                                     \
+            return FLOAT_CLASS_POSITIVE_NORMAL;                      \
+        }                                                            \
+    }                                                                \
+}
+
+FLOAT_CLASS(class_s, 32)
+FLOAT_CLASS(class_d, 64)
+#undef FLOAT_CLASS
+
 /* MIPS specific unary operations */
 uint64_t helper_float_recip_d(CPUMIPSState *env, uint64_t fdt0)
 {
@@ -3250,3 +3514,191 @@ FOP_COND_PS(le,  float32_le(fst0, fst1, &env->active_fpu.fp_status),
                  float32_le(fsth0, fsth1, &env->active_fpu.fp_status))
 FOP_COND_PS(ngt, float32_unordered(fst1, fst0, &env->active_fpu.fp_status)    || float32_le(fst0, fst1, &env->active_fpu.fp_status),
                  float32_unordered(fsth1, fsth0, &env->active_fpu.fp_status)  || float32_le(fsth0, fsth1, &env->active_fpu.fp_status))
+
+/* R6 compare operations */
+#define FOP_CONDN_D(op, cond)                                       \
+uint64_t helper_r6_cmp_d_ ## op(CPUMIPSState * env, uint64_t fdt0,  \
+                         uint64_t fdt1)                             \
+{                                                                   \
+    uint64_t c;                                                     \
+    c = cond;                                                       \
+    update_fcr31(env, GETPC());                                     \
+    if (c) {                                                        \
+        return -1;                                                  \
+    } else {                                                        \
+        return 0;                                                   \
+    }                                                               \
+}
+
+/* NOTE: the comma operator will make "cond" to eval to false,
+ * but float64_unordered_quiet() is still called. */
+FOP_CONDN_D(af,  (float64_unordered_quiet(fdt1, fdt0, &env->active_fpu.fp_status), 0))
+FOP_CONDN_D(un,  (float64_unordered_quiet(fdt1, fdt0, &env->active_fpu.fp_status)))
+FOP_CONDN_D(eq,  (float64_eq_quiet(fdt0, fdt1, &env->active_fpu.fp_status)))
+FOP_CONDN_D(ueq, (float64_unordered_quiet(fdt1, fdt0, &env->active_fpu.fp_status)
+                  || float64_eq_quiet(fdt0, fdt1, &env->active_fpu.fp_status)))
+FOP_CONDN_D(lt,  (float64_lt_quiet(fdt0, fdt1, &env->active_fpu.fp_status)))
+FOP_CONDN_D(ult, (float64_unordered_quiet(fdt1, fdt0, &env->active_fpu.fp_status)
+                  || float64_lt_quiet(fdt0, fdt1, &env->active_fpu.fp_status)))
+FOP_CONDN_D(le,  (float64_le_quiet(fdt0, fdt1, &env->active_fpu.fp_status)))
+FOP_CONDN_D(ule, (float64_unordered_quiet(fdt1, fdt0, &env->active_fpu.fp_status)
+                  || float64_le_quiet(fdt0, fdt1, &env->active_fpu.fp_status)))
+/* NOTE: the comma operator will make "cond" to eval to false,
+ * but float64_unordered() is still called. */
+FOP_CONDN_D(saf,  (float64_unordered(fdt1, fdt0, &env->active_fpu.fp_status), 0))
+FOP_CONDN_D(sun,  (float64_unordered(fdt1, fdt0, &env->active_fpu.fp_status)))
+FOP_CONDN_D(seq,  (float64_eq(fdt0, fdt1, &env->active_fpu.fp_status)))
+FOP_CONDN_D(sueq, (float64_unordered(fdt1, fdt0, &env->active_fpu.fp_status)
+                   || float64_eq(fdt0, fdt1, &env->active_fpu.fp_status)))
+FOP_CONDN_D(slt,  (float64_lt(fdt0, fdt1, &env->active_fpu.fp_status)))
+FOP_CONDN_D(sult, (float64_unordered(fdt1, fdt0, &env->active_fpu.fp_status)
+                   || float64_lt(fdt0, fdt1, &env->active_fpu.fp_status)))
+FOP_CONDN_D(sle,  (float64_le(fdt0, fdt1, &env->active_fpu.fp_status)))
+FOP_CONDN_D(sule, (float64_unordered(fdt1, fdt0, &env->active_fpu.fp_status)
+                   || float64_le(fdt0, fdt1, &env->active_fpu.fp_status)))
+FOP_CONDN_D(or,   (float64_le_quiet(fdt1, fdt0, &env->active_fpu.fp_status)
+                   || float64_le_quiet(fdt0, fdt1, &env->active_fpu.fp_status)))
+FOP_CONDN_D(une,  (float64_unordered_quiet(fdt1, fdt0, &env->active_fpu.fp_status)
+                   || float64_lt_quiet(fdt1, fdt0, &env->active_fpu.fp_status)
+                   || float64_lt_quiet(fdt0, fdt1, &env->active_fpu.fp_status)))
+FOP_CONDN_D(ne,   (float64_lt_quiet(fdt1, fdt0, &env->active_fpu.fp_status)
+                   || float64_lt_quiet(fdt0, fdt1, &env->active_fpu.fp_status)))
+FOP_CONDN_D(sor,  (float64_le(fdt1, fdt0, &env->active_fpu.fp_status)
+                   || float64_le(fdt0, fdt1, &env->active_fpu.fp_status)))
+FOP_CONDN_D(sune, (float64_unordered(fdt1, fdt0, &env->active_fpu.fp_status)
+                   || float64_lt(fdt1, fdt0, &env->active_fpu.fp_status)
+                   || float64_lt(fdt0, fdt1, &env->active_fpu.fp_status)))
+FOP_CONDN_D(sne,  (float64_lt(fdt1, fdt0, &env->active_fpu.fp_status)
+                   || float64_lt(fdt0, fdt1, &env->active_fpu.fp_status)))
+
+#define FOP_CONDN_S(op, cond)                                       \
+uint32_t helper_r6_cmp_s_ ## op(CPUMIPSState * env, uint32_t fst0,  \
+                         uint32_t fst1)                             \
+{                                                                   \
+    uint64_t c;                                                     \
+    c = cond;                                                       \
+    update_fcr31(env, GETPC());                                     \
+    if (c) {                                                        \
+        return -1;                                                  \
+    } else {                                                        \
+        return 0;                                                   \
+    }                                                               \
+}
+
+/* NOTE: the comma operator will make "cond" to eval to false,
+ * but float32_unordered_quiet() is still called. */
+FOP_CONDN_S(af,   (float32_unordered_quiet(fst1, fst0, &env->active_fpu.fp_status), 0))
+FOP_CONDN_S(un,   (float32_unordered_quiet(fst1, fst0, &env->active_fpu.fp_status)))
+FOP_CONDN_S(eq,   (float32_eq_quiet(fst0, fst1, &env->active_fpu.fp_status)))
+FOP_CONDN_S(ueq,  (float32_unordered_quiet(fst1, fst0, &env->active_fpu.fp_status)
+                   || float32_eq_quiet(fst0, fst1, &env->active_fpu.fp_status)))
+FOP_CONDN_S(lt,   (float32_lt_quiet(fst0, fst1, &env->active_fpu.fp_status)))
+FOP_CONDN_S(ult,  (float32_unordered_quiet(fst1, fst0, &env->active_fpu.fp_status)
+                   || float32_lt_quiet(fst0, fst1, &env->active_fpu.fp_status)))
+FOP_CONDN_S(le,   (float32_le_quiet(fst0, fst1, &env->active_fpu.fp_status)))
+FOP_CONDN_S(ule,  (float32_unordered_quiet(fst1, fst0, &env->active_fpu.fp_status)
+                   || float32_le_quiet(fst0, fst1, &env->active_fpu.fp_status)))
+/* NOTE: the comma operator will make "cond" to eval to false,
+ * but float32_unordered() is still called. */
+FOP_CONDN_S(saf,  (float32_unordered(fst1, fst0, &env->active_fpu.fp_status), 0))
+FOP_CONDN_S(sun,  (float32_unordered(fst1, fst0, &env->active_fpu.fp_status)))
+FOP_CONDN_S(seq,  (float32_eq(fst0, fst1, &env->active_fpu.fp_status)))
+FOP_CONDN_S(sueq, (float32_unordered(fst1, fst0, &env->active_fpu.fp_status)
+                   || float32_eq(fst0, fst1, &env->active_fpu.fp_status)))
+FOP_CONDN_S(slt,  (float32_lt(fst0, fst1, &env->active_fpu.fp_status)))
+FOP_CONDN_S(sult, (float32_unordered(fst1, fst0, &env->active_fpu.fp_status)
+                   || float32_lt(fst0, fst1, &env->active_fpu.fp_status)))
+FOP_CONDN_S(sle,  (float32_le(fst0, fst1, &env->active_fpu.fp_status)))
+FOP_CONDN_S(sule, (float32_unordered(fst1, fst0, &env->active_fpu.fp_status)
+                   || float32_le(fst0, fst1, &env->active_fpu.fp_status)))
+FOP_CONDN_S(or,   (float32_le_quiet(fst1, fst0, &env->active_fpu.fp_status)
+                   || float32_le_quiet(fst0, fst1, &env->active_fpu.fp_status)))
+FOP_CONDN_S(une,  (float32_unordered_quiet(fst1, fst0, &env->active_fpu.fp_status)
+                   || float32_lt_quiet(fst1, fst0, &env->active_fpu.fp_status)
+                   || float32_lt_quiet(fst0, fst1, &env->active_fpu.fp_status)))
+FOP_CONDN_S(ne,   (float32_lt_quiet(fst1, fst0, &env->active_fpu.fp_status)
+                   || float32_lt_quiet(fst0, fst1, &env->active_fpu.fp_status)))
+FOP_CONDN_S(sor,  (float32_le(fst1, fst0, &env->active_fpu.fp_status)
+                   || float32_le(fst0, fst1, &env->active_fpu.fp_status)))
+FOP_CONDN_S(sune, (float32_unordered(fst1, fst0, &env->active_fpu.fp_status)
+                   || float32_lt(fst1, fst0, &env->active_fpu.fp_status)
+                   || float32_lt(fst0, fst1, &env->active_fpu.fp_status)))
+FOP_CONDN_S(sne,  (float32_lt(fst1, fst0, &env->active_fpu.fp_status)
+                   || float32_lt(fst0, fst1, &env->active_fpu.fp_status)))
+
+/* MSA */
+/* Data format min and max values */
+#define DF_BITS(df) (1 << ((df) + 3))
+
+/* Element-by-element access macros */
+#define DF_ELEMENTS(df) (MSA_WRLEN / DF_BITS(df))
+
+void helper_msa_ld_df(CPUMIPSState *env, uint32_t df, uint32_t wd, uint32_t rs,
+                     int32_t s10)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    target_ulong addr = env->active_tc.gpr[rs] + (s10 << df);
+    int i;
+
+    switch (df) {
+    case DF_BYTE:
+        for (i = 0; i < DF_ELEMENTS(DF_BYTE); i++) {
+            pwd->b[i] = do_lbu(env, addr + (i << DF_BYTE),
+                                env->hflags & MIPS_HFLAG_KSU);
+        }
+        break;
+    case DF_HALF:
+        for (i = 0; i < DF_ELEMENTS(DF_HALF); i++) {
+            pwd->h[i] = do_lhu(env, addr + (i << DF_HALF),
+                                env->hflags & MIPS_HFLAG_KSU);
+        }
+        break;
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            pwd->w[i] = do_lw(env, addr + (i << DF_WORD),
+                                env->hflags & MIPS_HFLAG_KSU);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            pwd->d[i] = do_ld(env, addr + (i << DF_DOUBLE),
+                                env->hflags & MIPS_HFLAG_KSU);
+        }
+        break;
+    }
+}
+
+void helper_msa_st_df(CPUMIPSState *env, uint32_t df, uint32_t wd, uint32_t rs,
+                     int32_t s10)
+{
+    wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
+    target_ulong addr = env->active_tc.gpr[rs] + (s10 << df);
+    int i;
+
+    switch (df) {
+    case DF_BYTE:
+        for (i = 0; i < DF_ELEMENTS(DF_BYTE); i++) {
+            do_sb(env, addr + (i << DF_BYTE), pwd->b[i],
+                    env->hflags & MIPS_HFLAG_KSU);
+        }
+        break;
+    case DF_HALF:
+        for (i = 0; i < DF_ELEMENTS(DF_HALF); i++) {
+            do_sh(env, addr + (i << DF_HALF), pwd->h[i],
+                    env->hflags & MIPS_HFLAG_KSU);
+        }
+        break;
+    case DF_WORD:
+        for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
+            do_sw(env, addr + (i << DF_WORD), pwd->w[i],
+                    env->hflags & MIPS_HFLAG_KSU);
+        }
+        break;
+    case DF_DOUBLE:
+        for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
+            do_sd(env, addr + (i << DF_DOUBLE), pwd->d[i],
+                    env->hflags & MIPS_HFLAG_KSU);
+        }
+        break;
+    }
+}
diff --git a/target-mips/translate.c b/target-mips/translate.c
index c38136650..f0b8e6ffe 100644
--- a/target-mips/translate.c
+++ b/target-mips/translate.c
@@ -30,6 +30,9 @@
 #include "exec/helper-gen.h"
 #include "sysemu/kvm.h"
 
+#include "trace-tcg.h"
+
+
 #define MIPS_DEBUG_DISAS 0
 //#define MIPS_DEBUG_SIGN_EXTENSIONS
 
@@ -62,7 +65,6 @@ enum {
     /* Jump and branches */
     OPC_J        = (0x02 << 26),
     OPC_JAL      = (0x03 << 26),
-    OPC_JALS     = OPC_JAL | 0x5,
     OPC_BEQ      = (0x04 << 26),  /* Unconditional if rs = rt = 0 (B) */
     OPC_BEQL     = (0x14 << 26),
     OPC_BNE      = (0x05 << 26),
@@ -71,8 +73,8 @@ enum {
     OPC_BLEZL    = (0x16 << 26),
     OPC_BGTZ     = (0x07 << 26),
     OPC_BGTZL    = (0x17 << 26),
-    OPC_JALX     = (0x1D << 26),  /* MIPS 16 only */
-    OPC_JALXS    = OPC_JALX | 0x5,
+    OPC_JALX     = (0x1D << 26),
+    OPC_DAUI     = (0x1D << 26),
     /* Load and stores */
     OPC_LDL      = (0x1A << 26),
     OPC_LDR      = (0x1B << 26),
@@ -108,13 +110,57 @@ enum {
     OPC_SWC2     = (0x3A << 26),
     OPC_SDC1     = (0x3D << 26),
     OPC_SDC2     = (0x3E << 26),
+    /* Compact Branches */
+    OPC_BLEZALC  = (0x06 << 26),
+    OPC_BGEZALC  = (0x06 << 26),
+    OPC_BGEUC    = (0x06 << 26),
+    OPC_BGTZALC  = (0x07 << 26),
+    OPC_BLTZALC  = (0x07 << 26),
+    OPC_BLTUC    = (0x07 << 26),
+    OPC_BOVC     = (0x08 << 26),
+    OPC_BEQZALC  = (0x08 << 26),
+    OPC_BEQC     = (0x08 << 26),
+    OPC_BLEZC    = (0x16 << 26),
+    OPC_BGEZC    = (0x16 << 26),
+    OPC_BGEC     = (0x16 << 26),
+    OPC_BGTZC    = (0x17 << 26),
+    OPC_BLTZC    = (0x17 << 26),
+    OPC_BLTC     = (0x17 << 26),
+    OPC_BNVC     = (0x18 << 26),
+    OPC_BNEZALC  = (0x18 << 26),
+    OPC_BNEC     = (0x18 << 26),
+    OPC_BC       = (0x32 << 26),
+    OPC_BEQZC    = (0x36 << 26),
+    OPC_JIC      = (0x36 << 26),
+    OPC_BALC     = (0x3A << 26),
+    OPC_BNEZC    = (0x3E << 26),
+    OPC_JIALC    = (0x3E << 26),
     /* MDMX ASE specific */
     OPC_MDMX     = (0x1E << 26),
+    /* MSA ASE, same as MDMX */
+    OPC_MSA      = OPC_MDMX,
     /* Cache and prefetch */
     OPC_CACHE    = (0x2F << 26),
     OPC_PREF     = (0x33 << 26),
-    /* Reserved major opcode */
-    OPC_MAJOR3B_RESERVED = (0x3B << 26),
+    /* PC-relative address computation / loads */
+    OPC_PCREL    = (0x3B << 26),
+};
+
+/* PC-relative address computation / loads  */
+#define MASK_OPC_PCREL_TOP2BITS(op)  (MASK_OP_MAJOR(op) | (op & (3 << 19)))
+#define MASK_OPC_PCREL_TOP5BITS(op)  (MASK_OP_MAJOR(op) | (op & (0x1f << 16)))
+enum {
+    /* Instructions determined by bits 19 and 20 */
+    OPC_ADDIUPC = OPC_PCREL | (0 << 19),
+    R6_OPC_LWPC = OPC_PCREL | (1 << 19),
+    OPC_LWUPC   = OPC_PCREL | (2 << 19),
+
+    /* Instructions determined by bits 16 ... 20 */
+    OPC_AUIPC   = OPC_PCREL | (0x1e << 16),
+    OPC_ALUIPC  = OPC_PCREL | (0x1f << 16),
+
+    /* Other */
+    R6_OPC_LDPC = OPC_PCREL | (6 << 18),
 };
 
 /* MIPS special opcodes */
@@ -154,6 +200,7 @@ enum {
     OPC_DMULTU   = 0x1D | OPC_SPECIAL,
     OPC_DDIV     = 0x1E | OPC_SPECIAL,
     OPC_DDIVU    = 0x1F | OPC_SPECIAL,
+
     /* 2 registers arithmetic / logic */
     OPC_ADD      = 0x20 | OPC_SPECIAL,
     OPC_ADDU     = 0x21 | OPC_SPECIAL,
@@ -172,8 +219,6 @@ enum {
     /* Jumps */
     OPC_JR       = 0x08 | OPC_SPECIAL, /* Also JR.HB */
     OPC_JALR     = 0x09 | OPC_SPECIAL, /* Also JALR.HB */
-    OPC_JALRC    = OPC_JALR | (0x5 << 6),
-    OPC_JALRS    = 0x10 | OPC_SPECIAL | (0x5 << 6),
     /* Traps */
     OPC_TGE      = 0x30 | OPC_SPECIAL,
     OPC_TGEU     = 0x31 | OPC_SPECIAL,
@@ -190,6 +235,9 @@ enum {
     OPC_MOVZ     = 0x0A | OPC_SPECIAL,
     OPC_MOVN     = 0x0B | OPC_SPECIAL,
 
+    OPC_SELEQZ   = 0x35 | OPC_SPECIAL,
+    OPC_SELNEZ   = 0x37 | OPC_SPECIAL,
+
     OPC_MOVCI    = 0x01 | OPC_SPECIAL,
 
     /* Special */
@@ -199,15 +247,45 @@ enum {
     OPC_SPIM     = 0x0E | OPC_SPECIAL, /* unofficial */
     OPC_SYNC     = 0x0F | OPC_SPECIAL,
 
-    OPC_SPECIAL15_RESERVED = 0x15 | OPC_SPECIAL,
     OPC_SPECIAL28_RESERVED = 0x28 | OPC_SPECIAL,
     OPC_SPECIAL29_RESERVED = 0x29 | OPC_SPECIAL,
-    OPC_SPECIAL35_RESERVED = 0x35 | OPC_SPECIAL,
-    OPC_SPECIAL37_RESERVED = 0x37 | OPC_SPECIAL,
     OPC_SPECIAL39_RESERVED = 0x39 | OPC_SPECIAL,
     OPC_SPECIAL3D_RESERVED = 0x3D | OPC_SPECIAL,
 };
 
+/* R6 Multiply and Divide instructions have the same Opcode
+   and function field as legacy OPC_MULT[U]/OPC_DIV[U] */
+#define MASK_R6_MULDIV(op)   (MASK_SPECIAL(op) | (op & (0x7ff)))
+
+enum {
+    R6_OPC_MUL   = OPC_MULT  | (2 << 6),
+    R6_OPC_MUH   = OPC_MULT  | (3 << 6),
+    R6_OPC_MULU  = OPC_MULTU | (2 << 6),
+    R6_OPC_MUHU  = OPC_MULTU | (3 << 6),
+    R6_OPC_DIV   = OPC_DIV   | (2 << 6),
+    R6_OPC_MOD   = OPC_DIV   | (3 << 6),
+    R6_OPC_DIVU  = OPC_DIVU  | (2 << 6),
+    R6_OPC_MODU  = OPC_DIVU  | (3 << 6),
+
+    R6_OPC_DMUL   = OPC_DMULT  | (2 << 6),
+    R6_OPC_DMUH   = OPC_DMULT  | (3 << 6),
+    R6_OPC_DMULU  = OPC_DMULTU | (2 << 6),
+    R6_OPC_DMUHU  = OPC_DMULTU | (3 << 6),
+    R6_OPC_DDIV   = OPC_DDIV   | (2 << 6),
+    R6_OPC_DMOD   = OPC_DDIV   | (3 << 6),
+    R6_OPC_DDIVU  = OPC_DDIVU  | (2 << 6),
+    R6_OPC_DMODU  = OPC_DDIVU  | (3 << 6),
+
+    R6_OPC_CLZ      = 0x10 | OPC_SPECIAL,
+    R6_OPC_CLO      = 0x11 | OPC_SPECIAL,
+    R6_OPC_DCLZ     = 0x12 | OPC_SPECIAL,
+    R6_OPC_DCLO     = 0x13 | OPC_SPECIAL,
+    R6_OPC_SDBBP    = 0x0e | OPC_SPECIAL,
+
+    OPC_LSA  = 0x05 | OPC_SPECIAL,
+    OPC_DLSA = 0x15 | OPC_SPECIAL,
+};
+
 /* Multiplication variants of the vr54xx. */
 #define MASK_MUL_VR54XX(op)   MASK_SPECIAL(op) | (op & (0x1F << 6))
 
@@ -237,10 +315,8 @@ enum {
     OPC_BGEZ     = (0x01 << 16) | OPC_REGIMM,
     OPC_BGEZL    = (0x03 << 16) | OPC_REGIMM,
     OPC_BLTZAL   = (0x10 << 16) | OPC_REGIMM,
-    OPC_BLTZALS  = OPC_BLTZAL | 0x5, /* microMIPS */
     OPC_BLTZALL  = (0x12 << 16) | OPC_REGIMM,
     OPC_BGEZAL   = (0x11 << 16) | OPC_REGIMM,
-    OPC_BGEZALS  = OPC_BGEZAL | 0x5, /* microMIPS */
     OPC_BGEZALL  = (0x13 << 16) | OPC_REGIMM,
     OPC_TGEI     = (0x08 << 16) | OPC_REGIMM,
     OPC_TGEIU    = (0x09 << 16) | OPC_REGIMM,
@@ -249,6 +325,9 @@ enum {
     OPC_TEQI     = (0x0C << 16) | OPC_REGIMM,
     OPC_TNEI     = (0x0E << 16) | OPC_REGIMM,
     OPC_SYNCI    = (0x1F << 16) | OPC_REGIMM,
+
+    OPC_DAHI     = (0x06 << 16) | OPC_REGIMM,
+    OPC_DATI     = (0x1e << 16) | OPC_REGIMM,
 };
 
 /* Special2 opcodes */
@@ -343,23 +422,37 @@ enum {
     /* MIPS DSP Accumulator and DSPControl Access Sub-class */
     OPC_EXTR_W_DSP     = 0x38 | OPC_SPECIAL3,
     OPC_DEXTR_W_DSP    = 0x3C | OPC_SPECIAL3,
+
+    /* R6 */
+    R6_OPC_PREF        = 0x35 | OPC_SPECIAL3,
+    R6_OPC_CACHE       = 0x25 | OPC_SPECIAL3,
+    R6_OPC_LL          = 0x36 | OPC_SPECIAL3,
+    R6_OPC_SC          = 0x26 | OPC_SPECIAL3,
+    R6_OPC_LLD         = 0x37 | OPC_SPECIAL3,
+    R6_OPC_SCD         = 0x27 | OPC_SPECIAL3,
 };
 
 /* BSHFL opcodes */
 #define MASK_BSHFL(op)     MASK_SPECIAL3(op) | (op & (0x1F << 6))
 
 enum {
-    OPC_WSBH     = (0x02 << 6) | OPC_BSHFL,
-    OPC_SEB      = (0x10 << 6) | OPC_BSHFL,
-    OPC_SEH      = (0x18 << 6) | OPC_BSHFL,
+    OPC_WSBH      = (0x02 << 6) | OPC_BSHFL,
+    OPC_SEB       = (0x10 << 6) | OPC_BSHFL,
+    OPC_SEH       = (0x18 << 6) | OPC_BSHFL,
+    OPC_ALIGN     = (0x08 << 6) | OPC_BSHFL, /* 010.bp */
+    OPC_ALIGN_END = (0x0B << 6) | OPC_BSHFL, /* 010.00 to 010.11 */
+    OPC_BITSWAP   = (0x00 << 6) | OPC_BSHFL  /* 00000 */
 };
 
 /* DBSHFL opcodes */
 #define MASK_DBSHFL(op)    MASK_SPECIAL3(op) | (op & (0x1F << 6))
 
 enum {
-    OPC_DSBH     = (0x02 << 6) | OPC_DBSHFL,
-    OPC_DSHD     = (0x05 << 6) | OPC_DBSHFL,
+    OPC_DSBH       = (0x02 << 6) | OPC_DBSHFL,
+    OPC_DSHD       = (0x05 << 6) | OPC_DBSHFL,
+    OPC_DALIGN     = (0x08 << 6) | OPC_DBSHFL, /* 01.bp */
+    OPC_DALIGN_END = (0x0F << 6) | OPC_DBSHFL, /* 01.000 to 01.111 */
+    OPC_DBITSWAP   = (0x00 << 6) | OPC_DBSHFL, /* 00000 */
 };
 
 /* MIPS DSP REGIMM opcodes */
@@ -805,6 +898,8 @@ enum {
 enum {
     OPC_TLBR     = 0x01 | OPC_C0,
     OPC_TLBWI    = 0x02 | OPC_C0,
+    OPC_TLBINV   = 0x03 | OPC_C0,
+    OPC_TLBINVF  = 0x04 | OPC_C0,
     OPC_TLBWR    = 0x06 | OPC_C0,
     OPC_TLBP     = 0x08 | OPC_C0,
     OPC_RFE      = 0x10 | OPC_C0,
@@ -841,6 +936,8 @@ enum {
     OPC_BC1      = (0x08 << 21) | OPC_CP1, /* bc */
     OPC_BC1ANY2  = (0x09 << 21) | OPC_CP1,
     OPC_BC1ANY4  = (0x0A << 21) | OPC_CP1,
+    OPC_BZ_V     = (0x0B << 21) | OPC_CP1,
+    OPC_BNZ_V    = (0x0F << 21) | OPC_CP1,
     OPC_S_FMT    = (FMT_S << 21) | OPC_CP1,
     OPC_D_FMT    = (FMT_D << 21) | OPC_CP1,
     OPC_E_FMT    = (FMT_E << 21) | OPC_CP1,
@@ -848,6 +945,16 @@ enum {
     OPC_W_FMT    = (FMT_W << 21) | OPC_CP1,
     OPC_L_FMT    = (FMT_L << 21) | OPC_CP1,
     OPC_PS_FMT   = (FMT_PS << 21) | OPC_CP1,
+    OPC_BC1EQZ   = (0x09 << 21) | OPC_CP1,
+    OPC_BC1NEZ   = (0x0D << 21) | OPC_CP1,
+    OPC_BZ_B     = (0x18 << 21) | OPC_CP1,
+    OPC_BZ_H     = (0x19 << 21) | OPC_CP1,
+    OPC_BZ_W     = (0x1A << 21) | OPC_CP1,
+    OPC_BZ_D     = (0x1B << 21) | OPC_CP1,
+    OPC_BNZ_B    = (0x1C << 21) | OPC_CP1,
+    OPC_BNZ_H    = (0x1D << 21) | OPC_CP1,
+    OPC_BNZ_W    = (0x1E << 21) | OPC_CP1,
+    OPC_BNZ_D    = (0x1F << 21) | OPC_CP1,
 };
 
 #define MASK_CP1_FUNC(op)       MASK_CP1(op) | (op & 0x3F)
@@ -882,6 +989,8 @@ enum {
     OPC_CTC2    = (0x06 << 21) | OPC_CP2,
     OPC_MTHC2   = (0x07 << 21) | OPC_CP2,
     OPC_BC2     = (0x08 << 21) | OPC_CP2,
+    OPC_BC2EQZ  = (0x09 << 21) | OPC_CP2,
+    OPC_BC2NEZ  = (0x0D << 21) | OPC_CP2,
 };
 
 #define MASK_LMI(op)  (MASK_OP_MAJOR(op) | (op & (0x1F << 21)) | (op & 0x1F))
@@ -1006,14 +1115,248 @@ enum {
     OPC_NMSUB_PS= 0x3E | OPC_CP3,
 };
 
+/* MSA Opcodes */
+#define MASK_MSA_MINOR(op)    (MASK_OP_MAJOR(op) | (op & 0x3F))
+enum {
+    OPC_MSA_I8_00   = 0x00 | OPC_MSA,
+    OPC_MSA_I8_01   = 0x01 | OPC_MSA,
+    OPC_MSA_I8_02   = 0x02 | OPC_MSA,
+    OPC_MSA_I5_06   = 0x06 | OPC_MSA,
+    OPC_MSA_I5_07   = 0x07 | OPC_MSA,
+    OPC_MSA_BIT_09  = 0x09 | OPC_MSA,
+    OPC_MSA_BIT_0A  = 0x0A | OPC_MSA,
+    OPC_MSA_3R_0D   = 0x0D | OPC_MSA,
+    OPC_MSA_3R_0E   = 0x0E | OPC_MSA,
+    OPC_MSA_3R_0F   = 0x0F | OPC_MSA,
+    OPC_MSA_3R_10   = 0x10 | OPC_MSA,
+    OPC_MSA_3R_11   = 0x11 | OPC_MSA,
+    OPC_MSA_3R_12   = 0x12 | OPC_MSA,
+    OPC_MSA_3R_13   = 0x13 | OPC_MSA,
+    OPC_MSA_3R_14   = 0x14 | OPC_MSA,
+    OPC_MSA_3R_15   = 0x15 | OPC_MSA,
+    OPC_MSA_ELM     = 0x19 | OPC_MSA,
+    OPC_MSA_3RF_1A  = 0x1A | OPC_MSA,
+    OPC_MSA_3RF_1B  = 0x1B | OPC_MSA,
+    OPC_MSA_3RF_1C  = 0x1C | OPC_MSA,
+    OPC_MSA_VEC     = 0x1E | OPC_MSA,
+
+    /* MI10 instruction */
+    OPC_LD_B    = (0x20) | OPC_MSA,
+    OPC_LD_H    = (0x21) | OPC_MSA,
+    OPC_LD_W    = (0x22) | OPC_MSA,
+    OPC_LD_D    = (0x23) | OPC_MSA,
+    OPC_ST_B    = (0x24) | OPC_MSA,
+    OPC_ST_H    = (0x25) | OPC_MSA,
+    OPC_ST_W    = (0x26) | OPC_MSA,
+    OPC_ST_D    = (0x27) | OPC_MSA,
+};
+
+enum {
+    /* I5 instruction df(bits 22..21) = _b, _h, _w, _d */
+    OPC_ADDVI_df    = (0x0 << 23) | OPC_MSA_I5_06,
+    OPC_CEQI_df     = (0x0 << 23) | OPC_MSA_I5_07,
+    OPC_SUBVI_df    = (0x1 << 23) | OPC_MSA_I5_06,
+    OPC_MAXI_S_df   = (0x2 << 23) | OPC_MSA_I5_06,
+    OPC_CLTI_S_df   = (0x2 << 23) | OPC_MSA_I5_07,
+    OPC_MAXI_U_df   = (0x3 << 23) | OPC_MSA_I5_06,
+    OPC_CLTI_U_df   = (0x3 << 23) | OPC_MSA_I5_07,
+    OPC_MINI_S_df   = (0x4 << 23) | OPC_MSA_I5_06,
+    OPC_CLEI_S_df   = (0x4 << 23) | OPC_MSA_I5_07,
+    OPC_MINI_U_df   = (0x5 << 23) | OPC_MSA_I5_06,
+    OPC_CLEI_U_df   = (0x5 << 23) | OPC_MSA_I5_07,
+    OPC_LDI_df      = (0x6 << 23) | OPC_MSA_I5_07,
+
+    /* I8 instruction */
+    OPC_ANDI_B  = (0x0 << 24) | OPC_MSA_I8_00,
+    OPC_BMNZI_B = (0x0 << 24) | OPC_MSA_I8_01,
+    OPC_SHF_B   = (0x0 << 24) | OPC_MSA_I8_02,
+    OPC_ORI_B   = (0x1 << 24) | OPC_MSA_I8_00,
+    OPC_BMZI_B  = (0x1 << 24) | OPC_MSA_I8_01,
+    OPC_SHF_H   = (0x1 << 24) | OPC_MSA_I8_02,
+    OPC_NORI_B  = (0x2 << 24) | OPC_MSA_I8_00,
+    OPC_BSELI_B = (0x2 << 24) | OPC_MSA_I8_01,
+    OPC_SHF_W   = (0x2 << 24) | OPC_MSA_I8_02,
+    OPC_XORI_B  = (0x3 << 24) | OPC_MSA_I8_00,
+
+    /* VEC/2R/2RF instruction */
+    OPC_AND_V   = (0x00 << 21) | OPC_MSA_VEC,
+    OPC_OR_V    = (0x01 << 21) | OPC_MSA_VEC,
+    OPC_NOR_V   = (0x02 << 21) | OPC_MSA_VEC,
+    OPC_XOR_V   = (0x03 << 21) | OPC_MSA_VEC,
+    OPC_BMNZ_V  = (0x04 << 21) | OPC_MSA_VEC,
+    OPC_BMZ_V   = (0x05 << 21) | OPC_MSA_VEC,
+    OPC_BSEL_V  = (0x06 << 21) | OPC_MSA_VEC,
+
+    OPC_MSA_2R      = (0x18 << 21) | OPC_MSA_VEC,
+    OPC_MSA_2RF     = (0x19 << 21) | OPC_MSA_VEC,
+
+    /* 2R instruction df(bits 17..16) = _b, _h, _w, _d */
+    OPC_FILL_df = (0x00 << 18) | OPC_MSA_2R,
+    OPC_PCNT_df = (0x01 << 18) | OPC_MSA_2R,
+    OPC_NLOC_df = (0x02 << 18) | OPC_MSA_2R,
+    OPC_NLZC_df = (0x03 << 18) | OPC_MSA_2R,
+
+    /* 2RF instruction df(bit 16) = _w, _d */
+    OPC_FCLASS_df   = (0x00 << 17) | OPC_MSA_2RF,
+    OPC_FTRUNC_S_df = (0x01 << 17) | OPC_MSA_2RF,
+    OPC_FTRUNC_U_df = (0x02 << 17) | OPC_MSA_2RF,
+    OPC_FSQRT_df    = (0x03 << 17) | OPC_MSA_2RF,
+    OPC_FRSQRT_df   = (0x04 << 17) | OPC_MSA_2RF,
+    OPC_FRCP_df     = (0x05 << 17) | OPC_MSA_2RF,
+    OPC_FRINT_df    = (0x06 << 17) | OPC_MSA_2RF,
+    OPC_FLOG2_df    = (0x07 << 17) | OPC_MSA_2RF,
+    OPC_FEXUPL_df   = (0x08 << 17) | OPC_MSA_2RF,
+    OPC_FEXUPR_df   = (0x09 << 17) | OPC_MSA_2RF,
+    OPC_FFQL_df     = (0x0A << 17) | OPC_MSA_2RF,
+    OPC_FFQR_df     = (0x0B << 17) | OPC_MSA_2RF,
+    OPC_FTINT_S_df  = (0x0C << 17) | OPC_MSA_2RF,
+    OPC_FTINT_U_df  = (0x0D << 17) | OPC_MSA_2RF,
+    OPC_FFINT_S_df  = (0x0E << 17) | OPC_MSA_2RF,
+    OPC_FFINT_U_df  = (0x0F << 17) | OPC_MSA_2RF,
+
+    /* 3R instruction df(bits 22..21) = _b, _h, _w, d */
+    OPC_SLL_df      = (0x0 << 23) | OPC_MSA_3R_0D,
+    OPC_ADDV_df     = (0x0 << 23) | OPC_MSA_3R_0E,
+    OPC_CEQ_df      = (0x0 << 23) | OPC_MSA_3R_0F,
+    OPC_ADD_A_df    = (0x0 << 23) | OPC_MSA_3R_10,
+    OPC_SUBS_S_df   = (0x0 << 23) | OPC_MSA_3R_11,
+    OPC_MULV_df     = (0x0 << 23) | OPC_MSA_3R_12,
+    OPC_DOTP_S_df   = (0x0 << 23) | OPC_MSA_3R_13,
+    OPC_SLD_df      = (0x0 << 23) | OPC_MSA_3R_14,
+    OPC_VSHF_df     = (0x0 << 23) | OPC_MSA_3R_15,
+    OPC_SRA_df      = (0x1 << 23) | OPC_MSA_3R_0D,
+    OPC_SUBV_df     = (0x1 << 23) | OPC_MSA_3R_0E,
+    OPC_ADDS_A_df   = (0x1 << 23) | OPC_MSA_3R_10,
+    OPC_SUBS_U_df   = (0x1 << 23) | OPC_MSA_3R_11,
+    OPC_MADDV_df    = (0x1 << 23) | OPC_MSA_3R_12,
+    OPC_DOTP_U_df   = (0x1 << 23) | OPC_MSA_3R_13,
+    OPC_SPLAT_df    = (0x1 << 23) | OPC_MSA_3R_14,
+    OPC_SRAR_df     = (0x1 << 23) | OPC_MSA_3R_15,
+    OPC_SRL_df      = (0x2 << 23) | OPC_MSA_3R_0D,
+    OPC_MAX_S_df    = (0x2 << 23) | OPC_MSA_3R_0E,
+    OPC_CLT_S_df    = (0x2 << 23) | OPC_MSA_3R_0F,
+    OPC_ADDS_S_df   = (0x2 << 23) | OPC_MSA_3R_10,
+    OPC_SUBSUS_U_df = (0x2 << 23) | OPC_MSA_3R_11,
+    OPC_MSUBV_df    = (0x2 << 23) | OPC_MSA_3R_12,
+    OPC_DPADD_S_df  = (0x2 << 23) | OPC_MSA_3R_13,
+    OPC_PCKEV_df    = (0x2 << 23) | OPC_MSA_3R_14,
+    OPC_SRLR_df     = (0x2 << 23) | OPC_MSA_3R_15,
+    OPC_BCLR_df     = (0x3 << 23) | OPC_MSA_3R_0D,
+    OPC_MAX_U_df    = (0x3 << 23) | OPC_MSA_3R_0E,
+    OPC_CLT_U_df    = (0x3 << 23) | OPC_MSA_3R_0F,
+    OPC_ADDS_U_df   = (0x3 << 23) | OPC_MSA_3R_10,
+    OPC_SUBSUU_S_df = (0x3 << 23) | OPC_MSA_3R_11,
+    OPC_DPADD_U_df  = (0x3 << 23) | OPC_MSA_3R_13,
+    OPC_PCKOD_df    = (0x3 << 23) | OPC_MSA_3R_14,
+    OPC_BSET_df     = (0x4 << 23) | OPC_MSA_3R_0D,
+    OPC_MIN_S_df    = (0x4 << 23) | OPC_MSA_3R_0E,
+    OPC_CLE_S_df    = (0x4 << 23) | OPC_MSA_3R_0F,
+    OPC_AVE_S_df    = (0x4 << 23) | OPC_MSA_3R_10,
+    OPC_ASUB_S_df   = (0x4 << 23) | OPC_MSA_3R_11,
+    OPC_DIV_S_df    = (0x4 << 23) | OPC_MSA_3R_12,
+    OPC_DPSUB_S_df  = (0x4 << 23) | OPC_MSA_3R_13,
+    OPC_ILVL_df     = (0x4 << 23) | OPC_MSA_3R_14,
+    OPC_HADD_S_df   = (0x4 << 23) | OPC_MSA_3R_15,
+    OPC_BNEG_df     = (0x5 << 23) | OPC_MSA_3R_0D,
+    OPC_MIN_U_df    = (0x5 << 23) | OPC_MSA_3R_0E,
+    OPC_CLE_U_df    = (0x5 << 23) | OPC_MSA_3R_0F,
+    OPC_AVE_U_df    = (0x5 << 23) | OPC_MSA_3R_10,
+    OPC_ASUB_U_df   = (0x5 << 23) | OPC_MSA_3R_11,
+    OPC_DIV_U_df    = (0x5 << 23) | OPC_MSA_3R_12,
+    OPC_DPSUB_U_df  = (0x5 << 23) | OPC_MSA_3R_13,
+    OPC_ILVR_df     = (0x5 << 23) | OPC_MSA_3R_14,
+    OPC_HADD_U_df   = (0x5 << 23) | OPC_MSA_3R_15,
+    OPC_BINSL_df    = (0x6 << 23) | OPC_MSA_3R_0D,
+    OPC_MAX_A_df    = (0x6 << 23) | OPC_MSA_3R_0E,
+    OPC_AVER_S_df   = (0x6 << 23) | OPC_MSA_3R_10,
+    OPC_MOD_S_df    = (0x6 << 23) | OPC_MSA_3R_12,
+    OPC_ILVEV_df    = (0x6 << 23) | OPC_MSA_3R_14,
+    OPC_HSUB_S_df   = (0x6 << 23) | OPC_MSA_3R_15,
+    OPC_BINSR_df    = (0x7 << 23) | OPC_MSA_3R_0D,
+    OPC_MIN_A_df    = (0x7 << 23) | OPC_MSA_3R_0E,
+    OPC_AVER_U_df   = (0x7 << 23) | OPC_MSA_3R_10,
+    OPC_MOD_U_df    = (0x7 << 23) | OPC_MSA_3R_12,
+    OPC_ILVOD_df    = (0x7 << 23) | OPC_MSA_3R_14,
+    OPC_HSUB_U_df   = (0x7 << 23) | OPC_MSA_3R_15,
+
+    /* ELM instructions df(bits 21..16) = _b, _h, _w, _d */
+    OPC_SLDI_df     = (0x0 << 22) | (0x00 << 16) | OPC_MSA_ELM,
+    OPC_CTCMSA      = (0x0 << 22) | (0x3E << 16) | OPC_MSA_ELM,
+    OPC_SPLATI_df   = (0x1 << 22) | (0x00 << 16) | OPC_MSA_ELM,
+    OPC_CFCMSA      = (0x1 << 22) | (0x3E << 16) | OPC_MSA_ELM,
+    OPC_COPY_S_df   = (0x2 << 22) | (0x00 << 16) | OPC_MSA_ELM,
+    OPC_MOVE_V      = (0x2 << 22) | (0x3E << 16) | OPC_MSA_ELM,
+    OPC_COPY_U_df   = (0x3 << 22) | (0x00 << 16) | OPC_MSA_ELM,
+    OPC_INSERT_df   = (0x4 << 22) | (0x00 << 16) | OPC_MSA_ELM,
+    OPC_INSVE_df    = (0x5 << 22) | (0x00 << 16) | OPC_MSA_ELM,
+
+    /* 3RF instruction _df(bit 21) = _w, _d */
+    OPC_FCAF_df     = (0x0 << 22) | OPC_MSA_3RF_1A,
+    OPC_FADD_df     = (0x0 << 22) | OPC_MSA_3RF_1B,
+    OPC_FCUN_df     = (0x1 << 22) | OPC_MSA_3RF_1A,
+    OPC_FSUB_df     = (0x1 << 22) | OPC_MSA_3RF_1B,
+    OPC_FCOR_df     = (0x1 << 22) | OPC_MSA_3RF_1C,
+    OPC_FCEQ_df     = (0x2 << 22) | OPC_MSA_3RF_1A,
+    OPC_FMUL_df     = (0x2 << 22) | OPC_MSA_3RF_1B,
+    OPC_FCUNE_df    = (0x2 << 22) | OPC_MSA_3RF_1C,
+    OPC_FCUEQ_df    = (0x3 << 22) | OPC_MSA_3RF_1A,
+    OPC_FDIV_df     = (0x3 << 22) | OPC_MSA_3RF_1B,
+    OPC_FCNE_df     = (0x3 << 22) | OPC_MSA_3RF_1C,
+    OPC_FCLT_df     = (0x4 << 22) | OPC_MSA_3RF_1A,
+    OPC_FMADD_df    = (0x4 << 22) | OPC_MSA_3RF_1B,
+    OPC_MUL_Q_df    = (0x4 << 22) | OPC_MSA_3RF_1C,
+    OPC_FCULT_df    = (0x5 << 22) | OPC_MSA_3RF_1A,
+    OPC_FMSUB_df    = (0x5 << 22) | OPC_MSA_3RF_1B,
+    OPC_MADD_Q_df   = (0x5 << 22) | OPC_MSA_3RF_1C,
+    OPC_FCLE_df     = (0x6 << 22) | OPC_MSA_3RF_1A,
+    OPC_MSUB_Q_df   = (0x6 << 22) | OPC_MSA_3RF_1C,
+    OPC_FCULE_df    = (0x7 << 22) | OPC_MSA_3RF_1A,
+    OPC_FEXP2_df    = (0x7 << 22) | OPC_MSA_3RF_1B,
+    OPC_FSAF_df     = (0x8 << 22) | OPC_MSA_3RF_1A,
+    OPC_FEXDO_df    = (0x8 << 22) | OPC_MSA_3RF_1B,
+    OPC_FSUN_df     = (0x9 << 22) | OPC_MSA_3RF_1A,
+    OPC_FSOR_df     = (0x9 << 22) | OPC_MSA_3RF_1C,
+    OPC_FSEQ_df     = (0xA << 22) | OPC_MSA_3RF_1A,
+    OPC_FTQ_df      = (0xA << 22) | OPC_MSA_3RF_1B,
+    OPC_FSUNE_df    = (0xA << 22) | OPC_MSA_3RF_1C,
+    OPC_FSUEQ_df    = (0xB << 22) | OPC_MSA_3RF_1A,
+    OPC_FSNE_df     = (0xB << 22) | OPC_MSA_3RF_1C,
+    OPC_FSLT_df     = (0xC << 22) | OPC_MSA_3RF_1A,
+    OPC_FMIN_df     = (0xC << 22) | OPC_MSA_3RF_1B,
+    OPC_MULR_Q_df   = (0xC << 22) | OPC_MSA_3RF_1C,
+    OPC_FSULT_df    = (0xD << 22) | OPC_MSA_3RF_1A,
+    OPC_FMIN_A_df   = (0xD << 22) | OPC_MSA_3RF_1B,
+    OPC_MADDR_Q_df  = (0xD << 22) | OPC_MSA_3RF_1C,
+    OPC_FSLE_df     = (0xE << 22) | OPC_MSA_3RF_1A,
+    OPC_FMAX_df     = (0xE << 22) | OPC_MSA_3RF_1B,
+    OPC_MSUBR_Q_df  = (0xE << 22) | OPC_MSA_3RF_1C,
+    OPC_FSULE_df    = (0xF << 22) | OPC_MSA_3RF_1A,
+    OPC_FMAX_A_df   = (0xF << 22) | OPC_MSA_3RF_1B,
+
+    /* BIT instruction df(bits 22..16) = _B _H _W _D */
+    OPC_SLLI_df     = (0x0 << 23) | OPC_MSA_BIT_09,
+    OPC_SAT_S_df    = (0x0 << 23) | OPC_MSA_BIT_0A,
+    OPC_SRAI_df     = (0x1 << 23) | OPC_MSA_BIT_09,
+    OPC_SAT_U_df    = (0x1 << 23) | OPC_MSA_BIT_0A,
+    OPC_SRLI_df     = (0x2 << 23) | OPC_MSA_BIT_09,
+    OPC_SRARI_df    = (0x2 << 23) | OPC_MSA_BIT_0A,
+    OPC_BCLRI_df    = (0x3 << 23) | OPC_MSA_BIT_09,
+    OPC_SRLRI_df    = (0x3 << 23) | OPC_MSA_BIT_0A,
+    OPC_BSETI_df    = (0x4 << 23) | OPC_MSA_BIT_09,
+    OPC_BNEGI_df    = (0x5 << 23) | OPC_MSA_BIT_09,
+    OPC_BINSLI_df   = (0x6 << 23) | OPC_MSA_BIT_09,
+    OPC_BINSRI_df   = (0x7 << 23) | OPC_MSA_BIT_09,
+};
+
 /* global register indices */
 static TCGv_ptr cpu_env;
 static TCGv cpu_gpr[32], cpu_PC;
-static TCGv cpu_HI[MIPS_DSP_ACC], cpu_LO[MIPS_DSP_ACC], cpu_ACX[MIPS_DSP_ACC];
+static TCGv cpu_HI[MIPS_DSP_ACC], cpu_LO[MIPS_DSP_ACC];
 static TCGv cpu_dspctrl, btarget, bcond;
 static TCGv_i32 hflags;
 static TCGv_i32 fpu_fcr0, fpu_fcr31;
 static TCGv_i64 fpu_f64[32];
+static TCGv_i64 msa_wr_d[64];
 
 static uint32_t gen_opc_hflags[OPC_BUF_SIZE];
 static target_ulong gen_opc_btarget[OPC_BUF_SIZE];
@@ -1075,6 +1418,11 @@ typedef struct DisasContext {
     int bstate;
     target_ulong btarget;
     bool ulri;
+    int kscrexist;
+    bool rxi;
+    int ie;
+    bool bi;
+    bool bp;
 } DisasContext;
 
 enum {
@@ -1100,10 +1448,6 @@ static const char * const regnames_LO[] = {
     "LO0", "LO1", "LO2", "LO3",
 };
 
-static const char * const regnames_ACX[] = {
-    "ACX0", "ACX1", "ACX2", "ACX3",
-};
-
 static const char * const fregnames[] = {
     "f0",  "f1",  "f2",  "f3",  "f4",  "f5",  "f6",  "f7",
     "f8",  "f9",  "f10", "f11", "f12", "f13", "f14", "f15",
@@ -1111,6 +1455,25 @@ static const char * const fregnames[] = {
     "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31",
 };
 
+static const char * const msaregnames[] = {
+    "w0.d0",  "w0.d1",  "w1.d0",  "w1.d1",
+    "w2.d0",  "w2.d1",  "w3.d0",  "w3.d1",
+    "w4.d0",  "w4.d1",  "w5.d0",  "w5.d1",
+    "w6.d0",  "w6.d1",  "w7.d0",  "w7.d1",
+    "w8.d0",  "w8.d1",  "w9.d0",  "w9.d1",
+    "w10.d0", "w10.d1", "w11.d0", "w11.d1",
+    "w12.d0", "w12.d1", "w13.d0", "w13.d1",
+    "w14.d0", "w14.d1", "w15.d0", "w15.d1",
+    "w16.d0", "w16.d1", "w17.d0", "w17.d1",
+    "w18.d0", "w18.d1", "w19.d0", "w19.d1",
+    "w20.d0", "w20.d1", "w21.d0", "w21.d1",
+    "w22.d0", "w22.d1", "w23.d0", "w23.d1",
+    "w24.d0", "w24.d1", "w25.d0", "w25.d1",
+    "w26.d0", "w26.d1", "w27.d0", "w27.d1",
+    "w28.d0", "w28.d1", "w29.d0", "w29.d1",
+    "w30.d0", "w30.d1", "w31.d0", "w31.d1",
+};
+
 #define MIPS_DEBUG(fmt, ...)                                                  \
     do {                                                                      \
         if (MIPS_DEBUG_DISAS) {                                               \
@@ -1146,17 +1509,6 @@ static inline void gen_store_gpr (TCGv t, int reg)
         tcg_gen_mov_tl(cpu_gpr[reg], t);
 }
 
-/* Moves to/from ACX register.  */
-static inline void gen_load_ACX (TCGv t, int reg)
-{
-    tcg_gen_mov_tl(t, cpu_ACX[reg]);
-}
-
-static inline void gen_store_ACX (TCGv t, int reg)
-{
-    tcg_gen_mov_tl(cpu_ACX[reg], t);
-}
-
 /* Moves to/from shadow registers. */
 static inline void gen_load_srsgpr (int from, int to)
 {
@@ -1340,16 +1692,26 @@ static inline void gen_op_addr_add (DisasContext *ctx, TCGv ret, TCGv arg0, TCGv
     tcg_gen_add_tl(ret, arg0, arg1);
 
 #if defined(TARGET_MIPS64)
-    /* For compatibility with 32-bit code, data reference in user mode
-       with Status_UX = 0 should be casted to 32-bit and sign extended.
-       See the MIPS64 PRA manual, section 4.10. */
-    if (((ctx->hflags & MIPS_HFLAG_KSU) == MIPS_HFLAG_UM) &&
-        !(ctx->hflags & MIPS_HFLAG_UX)) {
+    if (ctx->hflags & MIPS_HFLAG_AWRAP) {
         tcg_gen_ext32s_i64(ret, ret);
     }
 #endif
 }
 
+/* Addresses computation (translation time) */
+static target_long addr_add(DisasContext *ctx, target_long base,
+                            target_long offset)
+{
+    target_long sum = base + offset;
+
+#if defined(TARGET_MIPS64)
+    if (ctx->hflags & MIPS_HFLAG_AWRAP) {
+        sum = (int32_t)sum;
+    }
+#endif
+    return sum;
+}
+
 static inline void check_cp0_enabled(DisasContext *ctx)
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_CP0)))
@@ -1433,6 +1795,17 @@ static inline void check_insn(DisasContext *ctx, int flags)
     }
 }
 
+/* This code generates a "reserved instruction" exception if the
+   CPU has corresponding flag set which indicates that the instruction
+   has been removed. */
+static inline void check_insn_opc_removed(DisasContext *ctx, int flags)
+{
+    if (unlikely(ctx->insn_flags & flags)) {
+        generate_exception(ctx, EXCP_RI);
+    }
+}
+
+#ifdef TARGET_MIPS64
 /* This code generates a "reserved instruction" exception if 64-bit
    instructions are not enabled. */
 static inline void check_mips_64(DisasContext *ctx)
@@ -1440,6 +1813,7 @@ static inline void check_mips_64(DisasContext *ctx)
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_64)))
         generate_exception(ctx, EXCP_RI);
 }
+#endif
 
 /* Define small wrappers for gen_load_fpr* so that we have a uniform
    calling interface for 32 and 64-bit FPRs.  No sense in changing
@@ -1501,6 +1875,98 @@ FOP_CONDS(abs, 1, s, FMT_S, 32)
 FOP_CONDS(, 0, ps, FMT_PS, 64)
 FOP_CONDS(abs, 1, ps, FMT_PS, 64)
 #undef FOP_CONDS
+
+#define FOP_CONDNS(fmt, ifmt, bits, STORE)                              \
+static inline void gen_r6_cmp_ ## fmt(DisasContext * ctx, int n,        \
+                                      int ft, int fs, int fd)           \
+{                                                                       \
+    TCGv_i ## bits fp0 = tcg_temp_new_i ## bits();                      \
+    TCGv_i ## bits fp1 = tcg_temp_new_i ## bits();                      \
+    switch (ifmt) {                                                     \
+    case FMT_D:                                                         \
+        check_cp1_registers(ctx, fs | ft | fd);                         \
+        break;                                                          \
+    }                                                                   \
+    gen_ldcmp_fpr ## bits(ctx, fp0, fs);                                \
+    gen_ldcmp_fpr ## bits(ctx, fp1, ft);                                \
+    switch (n) {                                                        \
+    case  0:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _af(fp0, cpu_env, fp0, fp1);       \
+        break;                                                          \
+    case  1:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _un(fp0, cpu_env, fp0, fp1);       \
+        break;                                                          \
+    case  2:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _eq(fp0, cpu_env, fp0, fp1);       \
+        break;                                                          \
+    case  3:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _ueq(fp0, cpu_env, fp0, fp1);      \
+        break;                                                          \
+    case  4:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _lt(fp0, cpu_env, fp0, fp1);       \
+        break;                                                          \
+    case  5:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _ult(fp0, cpu_env, fp0, fp1);      \
+        break;                                                          \
+    case  6:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _le(fp0, cpu_env, fp0, fp1);       \
+        break;                                                          \
+    case  7:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _ule(fp0, cpu_env, fp0, fp1);      \
+        break;                                                          \
+    case  8:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _saf(fp0, cpu_env, fp0, fp1);      \
+        break;                                                          \
+    case  9:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _sun(fp0, cpu_env, fp0, fp1);      \
+        break;                                                          \
+    case 10:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _seq(fp0, cpu_env, fp0, fp1);      \
+        break;                                                          \
+    case 11:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _sueq(fp0, cpu_env, fp0, fp1);     \
+        break;                                                          \
+    case 12:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _slt(fp0, cpu_env, fp0, fp1);      \
+        break;                                                          \
+    case 13:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _sult(fp0, cpu_env, fp0, fp1);     \
+        break;                                                          \
+    case 14:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _sle(fp0, cpu_env, fp0, fp1);      \
+        break;                                                          \
+    case 15:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _sule(fp0, cpu_env, fp0, fp1);     \
+        break;                                                          \
+    case 17:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _or(fp0, cpu_env, fp0, fp1);       \
+        break;                                                          \
+    case 18:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _une(fp0, cpu_env, fp0, fp1);      \
+        break;                                                          \
+    case 19:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _ne(fp0, cpu_env, fp0, fp1);       \
+        break;                                                          \
+    case 25:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _sor(fp0, cpu_env, fp0, fp1);      \
+        break;                                                          \
+    case 26:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _sune(fp0, cpu_env, fp0, fp1);     \
+        break;                                                          \
+    case 27:                                                            \
+        gen_helper_r6_cmp_ ## fmt ## _sne(fp0, cpu_env, fp0, fp1);      \
+        break;                                                          \
+    default:                                                            \
+        abort();                                                        \
+    }                                                                   \
+    STORE;                                                              \
+    tcg_temp_free_i ## bits (fp0);                                      \
+    tcg_temp_free_i ## bits (fp1);                                      \
+}
+
+FOP_CONDNS(d, FMT_D, 64, gen_store_fpr64(ctx, fp0, fd))
+FOP_CONDNS(s, FMT_S, 32, gen_store_fpr32(fp0, fd))
+#undef FOP_CONDNS
 #undef gen_ldcmp_fpr32
 #undef gen_ldcmp_fpr64
 
@@ -1627,6 +2093,7 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
         opn = "ld";
         break;
     case OPC_LLD:
+    case R6_OPC_LLD:
         save_cpu_state(ctx, 1);
         op_ld_lld(t0, t0, ctx);
         gen_store_gpr(t0, rt);
@@ -1761,6 +2228,7 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
         opn = "lwr";
         break;
     case OPC_LL:
+    case R6_OPC_LL:
         save_cpu_state(ctx, 1);
         op_ld_ll(t0, t0, ctx);
         gen_store_gpr(t0, rt);
@@ -1848,12 +2316,14 @@ static void gen_st_cond (DisasContext *ctx, uint32_t opc, int rt,
     switch (opc) {
 #if defined(TARGET_MIPS64)
     case OPC_SCD:
+    case R6_OPC_SCD:
         save_cpu_state(ctx, 1);
         op_st_scd(t1, t0, rt, ctx);
         opn = "scd";
         break;
 #endif
     case OPC_SC:
+    case R6_OPC_SC:
         save_cpu_state(ctx, 1);
         op_st_sc(t1, t0, rt, ctx);
         opn = "sc";
@@ -2060,8 +2530,15 @@ static void gen_logic_imm(DisasContext *ctx, uint32_t opc,
                    regnames[rs], uimm);
         break;
     case OPC_LUI:
-        tcg_gen_movi_tl(cpu_gpr[rt], imm << 16);
-        MIPS_DEBUG("lui %s, " TARGET_FMT_lx, regnames[rt], uimm);
+        if (rs != 0 && (ctx->insn_flags & ISA_MIPS32R6)) {
+            /* OPC_AUI */
+            tcg_gen_addi_tl(cpu_gpr[rt], cpu_gpr[rs], imm << 16);
+            tcg_gen_ext32s_tl(cpu_gpr[rt], cpu_gpr[rt]);
+            MIPS_DEBUG("aui %s, %s, %04x", regnames[rt], regnames[rs], imm);
+        } else {
+            tcg_gen_movi_tl(cpu_gpr[rt], imm << 16);
+            MIPS_DEBUG("lui %s, " TARGET_FMT_lx, regnames[rt], uimm);
+        }
         break;
 
     default:
@@ -2399,6 +2876,14 @@ static void gen_cond_move(DisasContext *ctx, uint32_t opc,
         tcg_gen_movcond_tl(TCG_COND_EQ, cpu_gpr[rd], t0, t1, t2, cpu_gpr[rd]);
         opn = "movz";
         break;
+    case OPC_SELNEZ:
+        tcg_gen_movcond_tl(TCG_COND_NE, cpu_gpr[rd], t0, t1, t2, t1);
+        opn = "selnez";
+        break;
+    case OPC_SELEQZ:
+        tcg_gen_movcond_tl(TCG_COND_EQ, cpu_gpr[rd], t0, t1, t2, t1);
+        opn = "seleqz";
+        break;
     }
     tcg_temp_free(t2);
     tcg_temp_free(t1);
@@ -2657,6 +3142,309 @@ static void gen_HILO(DisasContext *ctx, uint32_t opc, int acc, int reg)
     MIPS_DEBUG("%s %s", opn, regnames[reg]);
 }
 
+static inline void gen_r6_ld(target_long addr, int reg, int memidx,
+                             TCGMemOp memop)
+{
+    TCGv t0 = tcg_const_tl(addr);
+    tcg_gen_qemu_ld_tl(t0, t0, memidx, memop);
+    gen_store_gpr(t0, reg);
+    tcg_temp_free(t0);
+}
+
+static inline void gen_pcrel(DisasContext *ctx, int rs, int16_t imm)
+{
+    target_long offset;
+    target_long addr;
+
+    switch (MASK_OPC_PCREL_TOP2BITS(ctx->opcode)) {
+    case OPC_ADDIUPC:
+        if (rs != 0) {
+            offset = sextract32(ctx->opcode << 2, 0, 21);
+            addr = addr_add(ctx, ctx->pc, offset);
+            tcg_gen_movi_tl(cpu_gpr[rs], addr);
+        }
+        break;
+    case R6_OPC_LWPC:
+        offset = sextract32(ctx->opcode << 2, 0, 21);
+        addr = addr_add(ctx, ctx->pc, offset);
+        gen_r6_ld(addr, rs, ctx->mem_idx, MO_TESL);
+        break;
+#if defined(TARGET_MIPS64)
+    case OPC_LWUPC:
+        check_mips_64(ctx);
+        offset = sextract32(ctx->opcode << 2, 0, 21);
+        addr = addr_add(ctx, ctx->pc, offset);
+        gen_r6_ld(addr, rs, ctx->mem_idx, MO_TEUL);
+        break;
+#endif
+    default:
+        switch (MASK_OPC_PCREL_TOP5BITS(ctx->opcode)) {
+        case OPC_AUIPC:
+            if (rs != 0) {
+                offset = imm << 16;
+                addr = addr_add(ctx, ctx->pc, offset);
+                tcg_gen_movi_tl(cpu_gpr[rs], addr);
+            }
+            break;
+        case OPC_ALUIPC:
+            if (rs != 0) {
+                offset = imm << 16;
+                addr = ~0xFFFF & addr_add(ctx, ctx->pc, offset);
+                tcg_gen_movi_tl(cpu_gpr[rs], addr);
+            }
+            break;
+#if defined(TARGET_MIPS64)
+        case R6_OPC_LDPC: /* bits 16 and 17 are part of immediate */
+        case R6_OPC_LDPC + (1 << 16):
+        case R6_OPC_LDPC + (2 << 16):
+        case R6_OPC_LDPC + (3 << 16):
+            check_mips_64(ctx);
+            offset = sextract32(ctx->opcode << 3, 0, 21);
+            addr = addr_add(ctx, (ctx->pc & ~0x7), offset);
+            gen_r6_ld(addr, rs, ctx->mem_idx, MO_TEQ);
+            break;
+#endif
+        default:
+            MIPS_INVAL("OPC_PCREL");
+            generate_exception(ctx, EXCP_RI);
+            break;
+        }
+        break;
+    }
+}
+
+static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
+{
+    const char *opn = "r6 mul/div";
+    TCGv t0, t1;
+
+    if (rd == 0) {
+        /* Treat as NOP. */
+        MIPS_DEBUG("NOP");
+        return;
+    }
+
+    t0 = tcg_temp_new();
+    t1 = tcg_temp_new();
+
+    gen_load_gpr(t0, rs);
+    gen_load_gpr(t1, rt);
+
+    switch (opc) {
+    case R6_OPC_DIV:
+        {
+            TCGv t2 = tcg_temp_new();
+            TCGv t3 = tcg_temp_new();
+            tcg_gen_ext32s_tl(t0, t0);
+            tcg_gen_ext32s_tl(t1, t1);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t2, t0, INT_MIN);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t3, t1, -1);
+            tcg_gen_and_tl(t2, t2, t3);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t3, t1, 0);
+            tcg_gen_or_tl(t2, t2, t3);
+            tcg_gen_movi_tl(t3, 0);
+            tcg_gen_movcond_tl(TCG_COND_NE, t1, t2, t3, t2, t1);
+            tcg_gen_div_tl(cpu_gpr[rd], t0, t1);
+            tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
+            tcg_temp_free(t3);
+            tcg_temp_free(t2);
+        }
+        opn = "div";
+        break;
+    case R6_OPC_MOD:
+        {
+            TCGv t2 = tcg_temp_new();
+            TCGv t3 = tcg_temp_new();
+            tcg_gen_ext32s_tl(t0, t0);
+            tcg_gen_ext32s_tl(t1, t1);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t2, t0, INT_MIN);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t3, t1, -1);
+            tcg_gen_and_tl(t2, t2, t3);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t3, t1, 0);
+            tcg_gen_or_tl(t2, t2, t3);
+            tcg_gen_movi_tl(t3, 0);
+            tcg_gen_movcond_tl(TCG_COND_NE, t1, t2, t3, t2, t1);
+            tcg_gen_rem_tl(cpu_gpr[rd], t0, t1);
+            tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
+            tcg_temp_free(t3);
+            tcg_temp_free(t2);
+        }
+        opn = "mod";
+        break;
+    case R6_OPC_DIVU:
+        {
+            TCGv t2 = tcg_const_tl(0);
+            TCGv t3 = tcg_const_tl(1);
+            tcg_gen_ext32u_tl(t0, t0);
+            tcg_gen_ext32u_tl(t1, t1);
+            tcg_gen_movcond_tl(TCG_COND_EQ, t1, t1, t2, t3, t1);
+            tcg_gen_divu_tl(cpu_gpr[rd], t0, t1);
+            tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
+            tcg_temp_free(t3);
+            tcg_temp_free(t2);
+        }
+        opn = "divu";
+        break;
+    case R6_OPC_MODU:
+        {
+            TCGv t2 = tcg_const_tl(0);
+            TCGv t3 = tcg_const_tl(1);
+            tcg_gen_ext32u_tl(t0, t0);
+            tcg_gen_ext32u_tl(t1, t1);
+            tcg_gen_movcond_tl(TCG_COND_EQ, t1, t1, t2, t3, t1);
+            tcg_gen_remu_tl(cpu_gpr[rd], t0, t1);
+            tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
+            tcg_temp_free(t3);
+            tcg_temp_free(t2);
+        }
+        opn = "modu";
+        break;
+    case R6_OPC_MUL:
+        {
+            TCGv_i32 t2 = tcg_temp_new_i32();
+            TCGv_i32 t3 = tcg_temp_new_i32();
+            tcg_gen_trunc_tl_i32(t2, t0);
+            tcg_gen_trunc_tl_i32(t3, t1);
+            tcg_gen_mul_i32(t2, t2, t3);
+            tcg_gen_ext_i32_tl(cpu_gpr[rd], t2);
+            tcg_temp_free_i32(t2);
+            tcg_temp_free_i32(t3);
+        }
+        opn = "mul";
+        break;
+    case R6_OPC_MUH:
+        {
+            TCGv_i32 t2 = tcg_temp_new_i32();
+            TCGv_i32 t3 = tcg_temp_new_i32();
+            tcg_gen_trunc_tl_i32(t2, t0);
+            tcg_gen_trunc_tl_i32(t3, t1);
+            tcg_gen_muls2_i32(t2, t3, t2, t3);
+            tcg_gen_ext_i32_tl(cpu_gpr[rd], t3);
+            tcg_temp_free_i32(t2);
+            tcg_temp_free_i32(t3);
+        }
+        opn = "muh";
+        break;
+    case R6_OPC_MULU:
+        {
+            TCGv_i32 t2 = tcg_temp_new_i32();
+            TCGv_i32 t3 = tcg_temp_new_i32();
+            tcg_gen_trunc_tl_i32(t2, t0);
+            tcg_gen_trunc_tl_i32(t3, t1);
+            tcg_gen_mul_i32(t2, t2, t3);
+            tcg_gen_ext_i32_tl(cpu_gpr[rd], t2);
+            tcg_temp_free_i32(t2);
+            tcg_temp_free_i32(t3);
+        }
+        opn = "mulu";
+        break;
+    case R6_OPC_MUHU:
+        {
+            TCGv_i32 t2 = tcg_temp_new_i32();
+            TCGv_i32 t3 = tcg_temp_new_i32();
+            tcg_gen_trunc_tl_i32(t2, t0);
+            tcg_gen_trunc_tl_i32(t3, t1);
+            tcg_gen_mulu2_i32(t2, t3, t2, t3);
+            tcg_gen_ext_i32_tl(cpu_gpr[rd], t3);
+            tcg_temp_free_i32(t2);
+            tcg_temp_free_i32(t3);
+        }
+        opn = "muhu";
+        break;
+#if defined(TARGET_MIPS64)
+    case R6_OPC_DDIV:
+        {
+            TCGv t2 = tcg_temp_new();
+            TCGv t3 = tcg_temp_new();
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t2, t0, -1LL << 63);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t3, t1, -1LL);
+            tcg_gen_and_tl(t2, t2, t3);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t3, t1, 0);
+            tcg_gen_or_tl(t2, t2, t3);
+            tcg_gen_movi_tl(t3, 0);
+            tcg_gen_movcond_tl(TCG_COND_NE, t1, t2, t3, t2, t1);
+            tcg_gen_div_tl(cpu_gpr[rd], t0, t1);
+            tcg_temp_free(t3);
+            tcg_temp_free(t2);
+        }
+        opn = "ddiv";
+        break;
+    case R6_OPC_DMOD:
+        {
+            TCGv t2 = tcg_temp_new();
+            TCGv t3 = tcg_temp_new();
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t2, t0, -1LL << 63);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t3, t1, -1LL);
+            tcg_gen_and_tl(t2, t2, t3);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t3, t1, 0);
+            tcg_gen_or_tl(t2, t2, t3);
+            tcg_gen_movi_tl(t3, 0);
+            tcg_gen_movcond_tl(TCG_COND_NE, t1, t2, t3, t2, t1);
+            tcg_gen_rem_tl(cpu_gpr[rd], t0, t1);
+            tcg_temp_free(t3);
+            tcg_temp_free(t2);
+        }
+        opn = "dmod";
+        break;
+    case R6_OPC_DDIVU:
+        {
+            TCGv t2 = tcg_const_tl(0);
+            TCGv t3 = tcg_const_tl(1);
+            tcg_gen_movcond_tl(TCG_COND_EQ, t1, t1, t2, t3, t1);
+            tcg_gen_divu_i64(cpu_gpr[rd], t0, t1);
+            tcg_temp_free(t3);
+            tcg_temp_free(t2);
+        }
+        opn = "ddivu";
+        break;
+    case R6_OPC_DMODU:
+        {
+            TCGv t2 = tcg_const_tl(0);
+            TCGv t3 = tcg_const_tl(1);
+            tcg_gen_movcond_tl(TCG_COND_EQ, t1, t1, t2, t3, t1);
+            tcg_gen_remu_i64(cpu_gpr[rd], t0, t1);
+            tcg_temp_free(t3);
+            tcg_temp_free(t2);
+        }
+        opn = "dmodu";
+        break;
+    case R6_OPC_DMUL:
+        tcg_gen_mul_i64(cpu_gpr[rd], t0, t1);
+        opn = "dmul";
+        break;
+    case R6_OPC_DMUH:
+        {
+            TCGv t2 = tcg_temp_new();
+            tcg_gen_muls2_i64(t2, cpu_gpr[rd], t0, t1);
+            tcg_temp_free(t2);
+        }
+        opn = "dmuh";
+        break;
+    case R6_OPC_DMULU:
+        tcg_gen_mul_i64(cpu_gpr[rd], t0, t1);
+        opn = "dmulu";
+        break;
+    case R6_OPC_DMUHU:
+        {
+            TCGv t2 = tcg_temp_new();
+            tcg_gen_mulu2_i64(t2, cpu_gpr[rd], t0, t1);
+            tcg_temp_free(t2);
+        }
+        opn = "dmuhu";
+        break;
+#endif
+    default:
+        MIPS_INVAL(opn);
+        generate_exception(ctx, EXCP_RI);
+        goto out;
+    }
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s %s %s", opn, regnames[rs], regnames[rt]);
+ out:
+    tcg_temp_free(t0);
+    tcg_temp_free(t1);
+}
+
 static void gen_muldiv(DisasContext *ctx, uint32_t opc,
                        int acc, int rs, int rt)
 {
@@ -2972,19 +3760,23 @@ static void gen_cl (DisasContext *ctx, uint32_t opc,
     gen_load_gpr(t0, rs);
     switch (opc) {
     case OPC_CLO:
+    case R6_OPC_CLO:
         gen_helper_clo(cpu_gpr[rd], t0);
         opn = "clo";
         break;
     case OPC_CLZ:
+    case R6_OPC_CLZ:
         gen_helper_clz(cpu_gpr[rd], t0);
         opn = "clz";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DCLO:
+    case R6_OPC_DCLO:
         gen_helper_dclo(cpu_gpr[rd], t0);
         opn = "dclo";
         break;
     case OPC_DCLZ:
+    case R6_OPC_DCLZ:
         gen_helper_dclz(cpu_gpr[rd], t0);
         opn = "dclz";
         break;
@@ -3598,7 +4390,8 @@ static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest)
 /* Branches (before delay slot) */
 static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
                                 int insn_bytes,
-                                int rs, int rt, int32_t offset)
+                                int rs, int rt, int32_t offset,
+                                int delayslot_size)
 {
     target_ulong btgt = -1;
     int blink = 0;
@@ -3608,7 +4401,8 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
 
     if (ctx->hflags & MIPS_HFLAG_BMASK) {
 #ifdef MIPS_DEBUG_DISAS
-        LOG_DISAS("Branch in delay slot at PC 0x" TARGET_FMT_lx "\n", ctx->pc);
+        LOG_DISAS("Branch in delay / forbidden slot at PC 0x"
+                  TARGET_FMT_lx "\n", ctx->pc);
 #endif
         generate_exception(ctx, EXCP_RI);
         goto out;
@@ -3630,7 +4424,6 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
         break;
     case OPC_BGEZ:
     case OPC_BGEZAL:
-    case OPC_BGEZALS:
     case OPC_BGEZALL:
     case OPC_BGEZL:
     case OPC_BGTZ:
@@ -3639,7 +4432,6 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
     case OPC_BLEZL:
     case OPC_BLTZ:
     case OPC_BLTZAL:
-    case OPC_BLTZALS:
     case OPC_BLTZALL:
     case OPC_BLTZL:
         /* Compare to zero */
@@ -3662,15 +4454,11 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
     case OPC_J:
     case OPC_JAL:
     case OPC_JALX:
-    case OPC_JALS:
-    case OPC_JALXS:
         /* Jump to immediate */
         btgt = ((ctx->pc + insn_bytes) & (int32_t)0xF0000000) | (uint32_t)offset;
         break;
     case OPC_JR:
     case OPC_JALR:
-    case OPC_JALRC:
-    case OPC_JALRS:
         /* Jump to register */
         if (offset != 0 && offset != 16) {
             /* Hint = 0 is JR/JALR, hint 16 is JR.HB/JALR.HB, the
@@ -3699,12 +4487,8 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
             ctx->hflags |= MIPS_HFLAG_B;
             MIPS_DEBUG("balways");
             break;
-        case OPC_BGEZALS:
         case OPC_BGEZAL:  /* 0 >= 0          */
         case OPC_BGEZALL: /* 0 >= 0 likely   */
-            ctx->hflags |= (opc == OPC_BGEZALS
-                            ? MIPS_HFLAG_BDS16
-                            : MIPS_HFLAG_BDS32);
             /* Always take and link */
             blink = 31;
             ctx->hflags |= MIPS_HFLAG_B;
@@ -3716,15 +4500,11 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
             /* Treat as NOP. */
             MIPS_DEBUG("bnever (NOP)");
             goto out;
-        case OPC_BLTZALS:
         case OPC_BLTZAL:  /* 0 < 0           */
-            ctx->hflags |= (opc == OPC_BLTZALS
-                            ? MIPS_HFLAG_BDS16
-                            : MIPS_HFLAG_BDS32);
             /* Handle as an unconditional branch to get correct delay
                slot checking.  */
             blink = 31;
-            btgt = ctx->pc + (opc == OPC_BLTZALS ? 6 : 8);
+            btgt = ctx->pc + insn_bytes + delayslot_size;
             ctx->hflags |= MIPS_HFLAG_B;
             MIPS_DEBUG("bnever and link");
             break;
@@ -3745,33 +4525,21 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
             ctx->hflags |= MIPS_HFLAG_B;
             MIPS_DEBUG("j " TARGET_FMT_lx, btgt);
             break;
-        case OPC_JALXS:
         case OPC_JALX:
             ctx->hflags |= MIPS_HFLAG_BX;
             /* Fallthrough */
-        case OPC_JALS:
         case OPC_JAL:
             blink = 31;
             ctx->hflags |= MIPS_HFLAG_B;
-            ctx->hflags |= ((opc == OPC_JALS || opc == OPC_JALXS)
-                            ? MIPS_HFLAG_BDS16
-                            : MIPS_HFLAG_BDS32);
             MIPS_DEBUG("jal " TARGET_FMT_lx, btgt);
             break;
         case OPC_JR:
             ctx->hflags |= MIPS_HFLAG_BR;
-            if (insn_bytes == 4)
-                ctx->hflags |= MIPS_HFLAG_BDS32;
             MIPS_DEBUG("jr %s", regnames[rs]);
             break;
-        case OPC_JALRS:
         case OPC_JALR:
-        case OPC_JALRC:
             blink = rt;
             ctx->hflags |= MIPS_HFLAG_BR;
-            ctx->hflags |= (opc == OPC_JALRS
-                            ? MIPS_HFLAG_BDS16
-                            : MIPS_HFLAG_BDS32);
             MIPS_DEBUG("jalr %s, %s", regnames[rt], regnames[rs]);
             break;
         default:
@@ -3809,11 +4577,7 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 0);
             MIPS_DEBUG("bgezl %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto likely;
-        case OPC_BGEZALS:
         case OPC_BGEZAL:
-            ctx->hflags |= (opc == OPC_BGEZALS
-                            ? MIPS_HFLAG_BDS16
-                            : MIPS_HFLAG_BDS32);
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 0);
             MIPS_DEBUG("bgezal %s, " TARGET_FMT_lx, regnames[rs], btgt);
             blink = 31;
@@ -3857,11 +4621,7 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
             MIPS_DEBUG("bposge64 " TARGET_FMT_lx, btgt);
             goto not_likely;
 #endif
-        case OPC_BLTZALS:
         case OPC_BLTZAL:
-            ctx->hflags |= (opc == OPC_BLTZALS
-                            ? MIPS_HFLAG_BDS16
-                            : MIPS_HFLAG_BDS32);
             tcg_gen_setcondi_tl(TCG_COND_LT, bcond, t0, 0);
             blink = 31;
             MIPS_DEBUG("bltzal %s, " TARGET_FMT_lx, regnames[rs], btgt);
@@ -3885,13 +4645,20 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
                blink, ctx->hflags, btgt);
 
     ctx->btarget = btgt;
+
+    switch (delayslot_size) {
+    case 2:
+        ctx->hflags |= MIPS_HFLAG_BDS16;
+        break;
+    case 4:
+        ctx->hflags |= MIPS_HFLAG_BDS32;
+        break;
+    }
+
     if (blink > 0) {
-        int post_delay = insn_bytes;
+        int post_delay = insn_bytes + delayslot_size;
         int lowbit = !!(ctx->hflags & MIPS_HFLAG_M16);
 
-        if (opc != OPC_JALRC)
-            post_delay += ((ctx->hflags & MIPS_HFLAG_BDS16) ? 2 : 4);
-
         tcg_gen_movi_tl(cpu_gpr[blink], ctx->pc + post_delay + lowbit);
     }
 
@@ -4073,6 +4840,22 @@ static inline void gen_mtc0_store64 (TCGv arg, target_ulong off)
     tcg_gen_st_tl(arg, cpu_env, off);
 }
 
+static inline void gen_mfc0_unimplemented(DisasContext *ctx, TCGv arg)
+{
+    if (ctx->insn_flags & ISA_MIPS32R6) {
+        tcg_gen_movi_tl(arg, 0);
+    } else {
+        tcg_gen_movi_tl(arg, ~0);
+    }
+}
+
+#define CP0_CHECK(c)                            \
+    do {                                        \
+        if (!(c)) {                             \
+            goto cp0_unimplemented;             \
+        }                                       \
+    } while (0)
+
 static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
 {
     const char *rn = "invalid";
@@ -4088,124 +4871,143 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "Index";
             break;
         case 1:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mfc0_mvpcontrol(arg, cpu_env);
             rn = "MVPControl";
             break;
         case 2:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mfc0_mvpconf0(arg, cpu_env);
             rn = "MVPConf0";
             break;
         case 3:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mfc0_mvpconf1(arg, cpu_env);
             rn = "MVPConf1";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 1:
         switch (sel) {
         case 0:
+            CP0_CHECK(!(ctx->insn_flags & ISA_MIPS32R6));
             gen_helper_mfc0_random(arg, cpu_env);
             rn = "Random";
             break;
         case 1:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPEControl));
             rn = "VPEControl";
             break;
         case 2:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPEConf0));
             rn = "VPEConf0";
             break;
         case 3:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPEConf1));
             rn = "VPEConf1";
             break;
         case 4:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_mfc0_load64(arg, offsetof(CPUMIPSState, CP0_YQMask));
             rn = "YQMask";
             break;
         case 5:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_mfc0_load64(arg, offsetof(CPUMIPSState, CP0_VPESchedule));
             rn = "VPESchedule";
             break;
         case 6:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_mfc0_load64(arg, offsetof(CPUMIPSState, CP0_VPEScheFBack));
             rn = "VPEScheFBack";
             break;
         case 7:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPEOpt));
             rn = "VPEOpt";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 2:
         switch (sel) {
         case 0:
             tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_EntryLo0));
+#if defined(TARGET_MIPS64)
+            if (ctx->rxi) {
+                TCGv tmp = tcg_temp_new();
+                tcg_gen_andi_tl(tmp, arg, (3ull << 62));
+                tcg_gen_shri_tl(tmp, tmp, 32);
+                tcg_gen_or_tl(arg, arg, tmp);
+                tcg_temp_free(tmp);
+            }
+#endif
             tcg_gen_ext32s_tl(arg, arg);
             rn = "EntryLo0";
             break;
         case 1:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mfc0_tcstatus(arg, cpu_env);
             rn = "TCStatus";
             break;
         case 2:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mfc0_tcbind(arg, cpu_env);
             rn = "TCBind";
             break;
         case 3:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mfc0_tcrestart(arg, cpu_env);
             rn = "TCRestart";
             break;
         case 4:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mfc0_tchalt(arg, cpu_env);
             rn = "TCHalt";
             break;
         case 5:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mfc0_tccontext(arg, cpu_env);
             rn = "TCContext";
             break;
         case 6:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mfc0_tcschedule(arg, cpu_env);
             rn = "TCSchedule";
             break;
         case 7:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mfc0_tcschefback(arg, cpu_env);
             rn = "TCScheFBack";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 3:
         switch (sel) {
         case 0:
             tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_EntryLo1));
+#if defined(TARGET_MIPS64)
+            if (ctx->rxi) {
+                TCGv tmp = tcg_temp_new();
+                tcg_gen_andi_tl(tmp, arg, (3ull << 62));
+                tcg_gen_shri_tl(tmp, tmp, 32);
+                tcg_gen_or_tl(arg, arg, tmp);
+                tcg_temp_free(tmp);
+            }
+#endif
             tcg_gen_ext32s_tl(arg, arg);
             rn = "EntryLo1";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 4:
@@ -4218,20 +5020,16 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
         case 1:
 //            gen_helper_mfc0_contextconfig(arg); /* SmartMIPS ASE */
             rn = "ContextConfig";
-            goto die;
+            goto cp0_unimplemented;
 //            break;
         case 2:
-            if (ctx->ulri) {
-                tcg_gen_ld32s_tl(arg, cpu_env,
-                                 offsetof(CPUMIPSState,
-                                          active_tc.CP0_UserLocal));
-                rn = "UserLocal";
-            } else {
-                tcg_gen_movi_tl(arg, 0);
-            }
+            CP0_CHECK(ctx->ulri);
+            tcg_gen_ld32s_tl(arg, cpu_env,
+                             offsetof(CPUMIPSState, active_tc.CP0_UserLocal));
+            rn = "UserLocal";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 5:
@@ -4246,7 +5044,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "PageGrain";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 6:
@@ -4281,7 +5079,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "SRSConf4";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 7:
@@ -4292,7 +5090,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "HWREna";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 8:
@@ -4302,9 +5100,19 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             tcg_gen_ext32s_tl(arg, arg);
             rn = "BadVAddr";
             break;
+        case 1:
+            CP0_CHECK(ctx->bi);
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_BadInstr));
+            rn = "BadInstr";
+            break;
+        case 2:
+            CP0_CHECK(ctx->bp);
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_BadInstrP));
+            rn = "BadInstrP";
+            break;
         default:
-            goto die;
-       }
+            goto cp0_unimplemented;
+        }
         break;
     case 9:
         switch (sel) {
@@ -4323,7 +5131,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             break;
         /* 6,7 are implementation dependent */
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 10:
@@ -4334,7 +5142,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EntryHi";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 11:
@@ -4345,7 +5153,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             break;
         /* 6,7 are implementation dependent */
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 12:
@@ -4370,7 +5178,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "SRSMap";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
        }
         break;
     case 13:
@@ -4380,7 +5188,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "Cause";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
        }
         break;
     case 14:
@@ -4391,7 +5199,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EPC";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 15:
@@ -4406,7 +5214,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EBase";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
        }
         break;
     case 16:
@@ -4445,7 +5253,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "Config7";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 17:
@@ -4455,7 +5263,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "LLAddr";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 18:
@@ -4465,7 +5273,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "WatchLo";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 19:
@@ -4475,7 +5283,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "WatchHi";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 20:
@@ -4489,18 +5297,19 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             break;
 #endif
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 21:
        /* Officially reserved, but sel 0 is used for R1x000 framemask */
+        CP0_CHECK(!(ctx->insn_flags & ISA_MIPS32R6));
         switch (sel) {
         case 0:
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_Framemask));
             rn = "Framemask";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 22:
@@ -4530,7 +5339,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "TraceBPC";
 //            break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 24:
@@ -4542,7 +5351,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "DEPC";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 25:
@@ -4580,7 +5389,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "Performance7";
 //            break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 26:
@@ -4594,7 +5403,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "CacheErr";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 28:
@@ -4614,7 +5423,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "DataLo";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 29:
@@ -4634,7 +5443,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "DataHi";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 30:
@@ -4645,7 +5454,7 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "ErrorEPC";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 31:
@@ -4655,20 +5464,27 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_DESAVE));
             rn = "DESAVE";
             break;
+        case 2 ... 7:
+            CP0_CHECK(ctx->kscrexist & (1 << sel));
+            tcg_gen_ld_tl(arg, cpu_env,
+                          offsetof(CPUMIPSState, CP0_KScratch[sel-2]));
+            tcg_gen_ext32s_tl(arg, arg);
+            rn = "KScratch";
+            break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     default:
-       goto die;
+       goto cp0_unimplemented;
     }
     (void)rn; /* avoid a compiler warning */
     LOG_DISAS("mfc0 %s (reg %d sel %d)\n", rn, reg, sel);
     return;
 
-die:
+cp0_unimplemented:
     LOG_DISAS("mfc0 %s (reg %d sel %d)\n", rn, reg, sel);
-    generate_exception(ctx, EXCP_RI);
+    gen_mfc0_unimplemented(ctx, arg);
 }
 
 static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
@@ -4689,22 +5505,22 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "Index";
             break;
         case 1:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_mvpcontrol(cpu_env, arg);
             rn = "MVPControl";
             break;
         case 2:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             /* ignored */
             rn = "MVPConf0";
             break;
         case 3:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             /* ignored */
             rn = "MVPConf1";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 1:
@@ -4714,42 +5530,42 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "Random";
             break;
         case 1:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_vpecontrol(cpu_env, arg);
             rn = "VPEControl";
             break;
         case 2:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_vpeconf0(cpu_env, arg);
             rn = "VPEConf0";
             break;
         case 3:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_vpeconf1(cpu_env, arg);
             rn = "VPEConf1";
             break;
         case 4:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_yqmask(cpu_env, arg);
             rn = "YQMask";
             break;
         case 5:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_VPESchedule));
             rn = "VPESchedule";
             break;
         case 6:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_VPEScheFBack));
             rn = "VPEScheFBack";
             break;
         case 7:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_vpeopt(cpu_env, arg);
             rn = "VPEOpt";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 2:
@@ -4759,42 +5575,42 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EntryLo0";
             break;
         case 1:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_tcstatus(cpu_env, arg);
             rn = "TCStatus";
             break;
         case 2:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_tcbind(cpu_env, arg);
             rn = "TCBind";
             break;
         case 3:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_tcrestart(cpu_env, arg);
             rn = "TCRestart";
             break;
         case 4:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_tchalt(cpu_env, arg);
             rn = "TCHalt";
             break;
         case 5:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_tccontext(cpu_env, arg);
             rn = "TCContext";
             break;
         case 6:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_tcschedule(cpu_env, arg);
             rn = "TCSchedule";
             break;
         case 7:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_tcschefback(cpu_env, arg);
             rn = "TCScheFBack";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 3:
@@ -4804,7 +5620,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EntryLo1";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 4:
@@ -4816,17 +5632,16 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
         case 1:
 //            gen_helper_mtc0_contextconfig(cpu_env, arg); /* SmartMIPS ASE */
             rn = "ContextConfig";
-            goto die;
+            goto cp0_unimplemented;
 //            break;
         case 2:
-            if (ctx->ulri) {
-                tcg_gen_st_tl(arg, cpu_env,
-                              offsetof(CPUMIPSState, active_tc.CP0_UserLocal));
-                rn = "UserLocal";
-            }
+            CP0_CHECK(ctx->ulri);
+            tcg_gen_st_tl(arg, cpu_env,
+                          offsetof(CPUMIPSState, active_tc.CP0_UserLocal));
+            rn = "UserLocal";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 5:
@@ -4841,7 +5656,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "PageGrain";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 6:
@@ -4876,7 +5691,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "SRSConf4";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 7:
@@ -4888,12 +5703,26 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "HWREna";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 8:
-        /* ignored */
-        rn = "BadVAddr";
+        switch (sel) {
+        case 0:
+            /* ignored */
+            rn = "BadVAddr";
+            break;
+        case 1:
+            /* ignored */
+            rn = "BadInstr";
+            break;
+        case 2:
+            /* ignored */
+            rn = "BadInstrP";
+            break;
+        default:
+            goto cp0_unimplemented;
+        }
         break;
     case 9:
         switch (sel) {
@@ -4903,7 +5732,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             break;
         /* 6,7 are implementation dependent */
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 10:
@@ -4913,7 +5742,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EntryHi";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 11:
@@ -4924,7 +5753,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             break;
         /* 6,7 are implementation dependent */
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 12:
@@ -4959,7 +5788,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "SRSMap";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 13:
@@ -4970,7 +5799,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "Cause";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 14:
@@ -4980,7 +5809,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EPC";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 15:
@@ -4995,7 +5824,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EBase";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 16:
@@ -5042,7 +5871,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             break;
         default:
             rn = "Invalid config selector";
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 17:
@@ -5052,7 +5881,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "LLAddr";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 18:
@@ -5062,7 +5891,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "WatchLo";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 19:
@@ -5072,7 +5901,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "WatchHi";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 20:
@@ -5085,18 +5914,19 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             break;
 #endif
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 21:
        /* Officially reserved, but sel 0 is used for R1x000 framemask */
+        CP0_CHECK(!(ctx->insn_flags & ISA_MIPS32R6));
         switch (sel) {
         case 0:
             gen_helper_mtc0_framemask(cpu_env, arg);
             rn = "Framemask";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 22:
@@ -5139,7 +5969,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "TraceBPC";
 //            break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 24:
@@ -5150,7 +5980,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "DEPC";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 25:
@@ -5188,7 +6018,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "Performance7";
 //            break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
        break;
     case 26:
@@ -5202,7 +6032,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "CacheErr";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
        break;
     case 28:
@@ -5222,7 +6052,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "DataLo";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 29:
@@ -5243,7 +6073,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             break;
         default:
             rn = "invalid sel";
-            goto die;
+            goto cp0_unimplemented;
         }
        break;
     case 30:
@@ -5253,7 +6083,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "ErrorEPC";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 31:
@@ -5263,14 +6093,20 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             gen_mtc0_store32(arg, offsetof(CPUMIPSState, CP0_DESAVE));
             rn = "DESAVE";
             break;
+        case 2 ... 7:
+            CP0_CHECK(ctx->kscrexist & (1 << sel));
+            tcg_gen_st_tl(arg, cpu_env,
+                          offsetof(CPUMIPSState, CP0_KScratch[sel-2]));
+            rn = "KScratch";
+            break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         /* Stop translation as we may have switched the execution mode */
         ctx->bstate = BS_STOP;
         break;
     default:
-       goto die;
+       goto cp0_unimplemented;
     }
     (void)rn; /* avoid a compiler warning */
     LOG_DISAS("mtc0 %s (reg %d sel %d)\n", rn, reg, sel);
@@ -5281,9 +6117,8 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
     }
     return;
 
-die:
+cp0_unimplemented:
     LOG_DISAS("mtc0 %s (reg %d sel %d)\n", rn, reg, sel);
-    generate_exception(ctx, EXCP_RI);
 }
 
 #if defined(TARGET_MIPS64)
@@ -5302,67 +6137,68 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "Index";
             break;
         case 1:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mfc0_mvpcontrol(arg, cpu_env);
             rn = "MVPControl";
             break;
         case 2:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mfc0_mvpconf0(arg, cpu_env);
             rn = "MVPConf0";
             break;
         case 3:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mfc0_mvpconf1(arg, cpu_env);
             rn = "MVPConf1";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 1:
         switch (sel) {
         case 0:
+            CP0_CHECK(!(ctx->insn_flags & ISA_MIPS32R6));
             gen_helper_mfc0_random(arg, cpu_env);
             rn = "Random";
             break;
         case 1:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPEControl));
             rn = "VPEControl";
             break;
         case 2:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPEConf0));
             rn = "VPEConf0";
             break;
         case 3:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPEConf1));
             rn = "VPEConf1";
             break;
         case 4:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_YQMask));
             rn = "YQMask";
             break;
         case 5:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_VPESchedule));
             rn = "VPESchedule";
             break;
         case 6:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_VPEScheFBack));
             rn = "VPEScheFBack";
             break;
         case 7:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPEOpt));
             rn = "VPEOpt";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 2:
@@ -5372,42 +6208,42 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EntryLo0";
             break;
         case 1:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mfc0_tcstatus(arg, cpu_env);
             rn = "TCStatus";
             break;
         case 2:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mfc0_tcbind(arg, cpu_env);
             rn = "TCBind";
             break;
         case 3:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_dmfc0_tcrestart(arg, cpu_env);
             rn = "TCRestart";
             break;
         case 4:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_dmfc0_tchalt(arg, cpu_env);
             rn = "TCHalt";
             break;
         case 5:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_dmfc0_tccontext(arg, cpu_env);
             rn = "TCContext";
             break;
         case 6:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_dmfc0_tcschedule(arg, cpu_env);
             rn = "TCSchedule";
             break;
         case 7:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_dmfc0_tcschefback(arg, cpu_env);
             rn = "TCScheFBack";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 3:
@@ -5417,7 +6253,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EntryLo1";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 4:
@@ -5429,19 +6265,16 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
         case 1:
 //            gen_helper_dmfc0_contextconfig(arg); /* SmartMIPS ASE */
             rn = "ContextConfig";
-            goto die;
+            goto cp0_unimplemented;
 //            break;
         case 2:
-            if (ctx->ulri) {
-                tcg_gen_ld_tl(arg, cpu_env,
-                              offsetof(CPUMIPSState, active_tc.CP0_UserLocal));
-                rn = "UserLocal";
-            } else {
-                tcg_gen_movi_tl(arg, 0);
-            }
+            CP0_CHECK(ctx->ulri);
+            tcg_gen_ld_tl(arg, cpu_env,
+                          offsetof(CPUMIPSState, active_tc.CP0_UserLocal));
+            rn = "UserLocal";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 5:
@@ -5456,7 +6289,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "PageGrain";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 6:
@@ -5491,7 +6324,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "SRSConf4";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 7:
@@ -5502,7 +6335,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "HWREna";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 8:
@@ -5511,8 +6344,18 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_BadVAddr));
             rn = "BadVAddr";
             break;
+        case 1:
+            CP0_CHECK(ctx->bi);
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_BadInstr));
+            rn = "BadInstr";
+            break;
+        case 2:
+            CP0_CHECK(ctx->bp);
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_BadInstrP));
+            rn = "BadInstrP";
+            break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 9:
@@ -5532,7 +6375,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             break;
         /* 6,7 are implementation dependent */
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 10:
@@ -5542,7 +6385,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EntryHi";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 11:
@@ -5553,7 +6396,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             break;
         /* 6,7 are implementation dependent */
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 12:
@@ -5578,7 +6421,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "SRSMap";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 13:
@@ -5588,7 +6431,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "Cause";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 14:
@@ -5598,7 +6441,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EPC";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 15:
@@ -5613,7 +6456,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EBase";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 16:
@@ -5634,6 +6477,14 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_Config3));
             rn = "Config3";
             break;
+        case 4:
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_Config4));
+            rn = "Config4";
+            break;
+        case 5:
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_Config5));
+            rn = "Config5";
+            break;
        /* 6,7 are implementation dependent */
         case 6:
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_Config6));
@@ -5644,7 +6495,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "Config7";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 17:
@@ -5654,7 +6505,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "LLAddr";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 18:
@@ -5664,7 +6515,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "WatchLo";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 19:
@@ -5674,7 +6525,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "WatchHi";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 20:
@@ -5685,18 +6536,19 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "XContext";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 21:
        /* Officially reserved, but sel 0 is used for R1x000 framemask */
+        CP0_CHECK(!(ctx->insn_flags & ISA_MIPS32R6));
         switch (sel) {
         case 0:
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_Framemask));
             rn = "Framemask";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 22:
@@ -5726,7 +6578,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "TraceBPC";
 //            break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 24:
@@ -5737,7 +6589,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "DEPC";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 25:
@@ -5775,7 +6627,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "Performance7";
 //            break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 26:
@@ -5790,7 +6642,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "CacheErr";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 28:
@@ -5810,7 +6662,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "DataLo";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 29:
@@ -5830,7 +6682,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "DataHi";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 30:
@@ -5840,7 +6692,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "ErrorEPC";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 31:
@@ -5850,20 +6702,26 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_DESAVE));
             rn = "DESAVE";
             break;
+        case 2 ... 7:
+            CP0_CHECK(ctx->kscrexist & (1 << sel));
+            tcg_gen_ld_tl(arg, cpu_env,
+                          offsetof(CPUMIPSState, CP0_KScratch[sel-2]));
+            rn = "KScratch";
+            break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     default:
-        goto die;
+        goto cp0_unimplemented;
     }
     (void)rn; /* avoid a compiler warning */
     LOG_DISAS("dmfc0 %s (reg %d sel %d)\n", rn, reg, sel);
     return;
 
-die:
+cp0_unimplemented:
     LOG_DISAS("dmfc0 %s (reg %d sel %d)\n", rn, reg, sel);
-    generate_exception(ctx, EXCP_RI);
+    gen_mfc0_unimplemented(ctx, arg);
 }
 
 static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
@@ -5884,22 +6742,22 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "Index";
             break;
         case 1:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_mvpcontrol(cpu_env, arg);
             rn = "MVPControl";
             break;
         case 2:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             /* ignored */
             rn = "MVPConf0";
             break;
         case 3:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             /* ignored */
             rn = "MVPConf1";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 1:
@@ -5909,97 +6767,97 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "Random";
             break;
         case 1:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_vpecontrol(cpu_env, arg);
             rn = "VPEControl";
             break;
         case 2:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_vpeconf0(cpu_env, arg);
             rn = "VPEConf0";
             break;
         case 3:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_vpeconf1(cpu_env, arg);
             rn = "VPEConf1";
             break;
         case 4:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_yqmask(cpu_env, arg);
             rn = "YQMask";
             break;
         case 5:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             tcg_gen_st_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_VPESchedule));
             rn = "VPESchedule";
             break;
         case 6:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             tcg_gen_st_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_VPEScheFBack));
             rn = "VPEScheFBack";
             break;
         case 7:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_vpeopt(cpu_env, arg);
             rn = "VPEOpt";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 2:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_entrylo0(cpu_env, arg);
+            gen_helper_dmtc0_entrylo0(cpu_env, arg);
             rn = "EntryLo0";
             break;
         case 1:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_tcstatus(cpu_env, arg);
             rn = "TCStatus";
             break;
         case 2:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_tcbind(cpu_env, arg);
             rn = "TCBind";
             break;
         case 3:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_tcrestart(cpu_env, arg);
             rn = "TCRestart";
             break;
         case 4:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_tchalt(cpu_env, arg);
             rn = "TCHalt";
             break;
         case 5:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_tccontext(cpu_env, arg);
             rn = "TCContext";
             break;
         case 6:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_tcschedule(cpu_env, arg);
             rn = "TCSchedule";
             break;
         case 7:
-            check_insn(ctx, ASE_MT);
+            CP0_CHECK(ctx->insn_flags & ASE_MT);
             gen_helper_mtc0_tcschefback(cpu_env, arg);
             rn = "TCScheFBack";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 3:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_entrylo1(cpu_env, arg);
+            gen_helper_dmtc0_entrylo1(cpu_env, arg);
             rn = "EntryLo1";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 4:
@@ -6011,17 +6869,16 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
         case 1:
 //           gen_helper_mtc0_contextconfig(cpu_env, arg); /* SmartMIPS ASE */
             rn = "ContextConfig";
-            goto die;
+            goto cp0_unimplemented;
 //           break;
         case 2:
-            if (ctx->ulri) {
-                tcg_gen_st_tl(arg, cpu_env,
-                              offsetof(CPUMIPSState, active_tc.CP0_UserLocal));
-                rn = "UserLocal";
-            }
+            CP0_CHECK(ctx->ulri);
+            tcg_gen_st_tl(arg, cpu_env,
+                          offsetof(CPUMIPSState, active_tc.CP0_UserLocal));
+            rn = "UserLocal";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 5:
@@ -6036,7 +6893,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "PageGrain";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 6:
@@ -6071,7 +6928,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "SRSConf4";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 7:
@@ -6083,12 +6940,26 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "HWREna";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 8:
-        /* ignored */
-        rn = "BadVAddr";
+        switch (sel) {
+        case 0:
+            /* ignored */
+            rn = "BadVAddr";
+            break;
+        case 1:
+            /* ignored */
+            rn = "BadInstr";
+            break;
+        case 2:
+            /* ignored */
+            rn = "BadInstrP";
+            break;
+        default:
+            goto cp0_unimplemented;
+        }
         break;
     case 9:
         switch (sel) {
@@ -6098,7 +6969,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             break;
         /* 6,7 are implementation dependent */
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         /* Stop translation as we may have switched the execution mode */
         ctx->bstate = BS_STOP;
@@ -6110,7 +6981,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EntryHi";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 11:
@@ -6121,7 +6992,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             break;
         /* 6,7 are implementation dependent */
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         /* Stop translation as we may have switched the execution mode */
         ctx->bstate = BS_STOP;
@@ -6158,7 +7029,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "SRSMap";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 13:
@@ -6179,7 +7050,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "Cause";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 14:
@@ -6189,7 +7060,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EPC";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 15:
@@ -6204,7 +7075,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EBase";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 16:
@@ -6229,10 +7100,20 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             /* ignored */
             rn = "Config3";
             break;
+        case 4:
+            /* currently ignored */
+            rn = "Config4";
+            break;
+        case 5:
+            gen_helper_mtc0_config5(cpu_env, arg);
+            rn = "Config5";
+            /* Stop translation as we may have switched the execution mode */
+            ctx->bstate = BS_STOP;
+            break;
         /* 6,7 are implementation dependent */
         default:
             rn = "Invalid config selector";
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 17:
@@ -6242,7 +7123,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "LLAddr";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 18:
@@ -6252,7 +7133,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "WatchLo";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 19:
@@ -6262,7 +7143,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "WatchHi";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 20:
@@ -6273,18 +7154,19 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "XContext";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 21:
        /* Officially reserved, but sel 0 is used for R1x000 framemask */
+        CP0_CHECK(!(ctx->insn_flags & ISA_MIPS32R6));
         switch (sel) {
         case 0:
             gen_helper_mtc0_framemask(cpu_env, arg);
             rn = "Framemask";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 22:
@@ -6325,7 +7207,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "TraceBPC";
 //            break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 24:
@@ -6336,7 +7218,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "DEPC";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 25:
@@ -6374,7 +7256,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "Performance7";
 //            break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 26:
@@ -6388,7 +7270,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "CacheErr";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 28:
@@ -6408,7 +7290,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "DataLo";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 29:
@@ -6429,7 +7311,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             break;
         default:
             rn = "invalid sel";
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 30:
@@ -6439,7 +7321,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "ErrorEPC";
             break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         break;
     case 31:
@@ -6449,14 +7331,20 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             gen_mtc0_store32(arg, offsetof(CPUMIPSState, CP0_DESAVE));
             rn = "DESAVE";
             break;
+        case 2 ... 7:
+            CP0_CHECK(ctx->kscrexist & (1 << sel));
+            tcg_gen_st_tl(arg, cpu_env,
+                          offsetof(CPUMIPSState, CP0_KScratch[sel-2]));
+            rn = "KScratch";
+            break;
         default:
-            goto die;
+            goto cp0_unimplemented;
         }
         /* Stop translation as we may have switched the execution mode */
         ctx->bstate = BS_STOP;
         break;
     default:
-        goto die;
+        goto cp0_unimplemented;
     }
     (void)rn; /* avoid a compiler warning */
     LOG_DISAS("dmtc0 %s (reg %d sel %d)\n", rn, reg, sel);
@@ -6467,9 +7355,8 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
     }
     return;
 
-die:
+cp0_unimplemented:
     LOG_DISAS("dmtc0 %s (reg %d sel %d)\n", rn, reg, sel);
-    generate_exception(ctx, EXCP_RI);
 }
 #endif /* TARGET_MIPS64 */
 
@@ -6868,12 +7755,15 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         break;
     case 3:
         /* XXX: For now we support only a single FPU context. */
+        save_cpu_state(ctx, 1);
         {
             TCGv_i32 fs_tmp = tcg_const_i32(rd);
 
             gen_helper_0e2i(ctc1, t0, fs_tmp, rt);
             tcg_temp_free_i32(fs_tmp);
         }
+        /* Stop translation as we may have changed hflags */
+        ctx->bstate = BS_STOP;
         break;
     /* COP2: Not implemented. */
     case 4:
@@ -6960,6 +7850,24 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
             goto die;
         gen_helper_tlbwi(cpu_env);
         break;
+    case OPC_TLBINV:
+        opn = "tlbinv";
+        if (ctx->ie >= 2) {
+            if (!env->tlb->helper_tlbinv) {
+                goto die;
+            }
+            gen_helper_tlbinv(cpu_env);
+        } /* treat as nop if TLBINV not supported */
+        break;
+    case OPC_TLBINVF:
+        opn = "tlbinvf";
+        if (ctx->ie >= 2) {
+            if (!env->tlb->helper_tlbinvf) {
+                goto die;
+            }
+            gen_helper_tlbinvf(cpu_env);
+        } /* treat as nop if TLBINV not supported */
+        break;
     case OPC_TLBWR:
         opn = "tlbwr";
         if (!env->tlb->helper_tlbwr)
@@ -6981,12 +7889,22 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
     case OPC_ERET:
         opn = "eret";
         check_insn(ctx, ISA_MIPS2);
+        if ((ctx->insn_flags & ISA_MIPS32R6) &&
+            (ctx->hflags & MIPS_HFLAG_BMASK)) {
+            MIPS_DEBUG("CTI in delay / forbidden slot");
+            goto die;
+        }
         gen_helper_eret(cpu_env);
         ctx->bstate = BS_EXCP;
         break;
     case OPC_DERET:
         opn = "deret";
         check_insn(ctx, ISA_MIPS32);
+        if ((ctx->insn_flags & ISA_MIPS32R6) &&
+            (ctx->hflags & MIPS_HFLAG_BMASK)) {
+            MIPS_DEBUG("CTI in delay / forbidden slot");
+            goto die;
+        }
         if (!(ctx->hflags & MIPS_HFLAG_DM)) {
             MIPS_INVAL(opn);
             generate_exception(ctx, EXCP_RI);
@@ -6998,6 +7916,11 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
     case OPC_WAIT:
         opn = "wait";
         check_insn(ctx, ISA_MIPS3 | ISA_MIPS32);
+        if ((ctx->insn_flags & ISA_MIPS32R6) &&
+            (ctx->hflags & MIPS_HFLAG_BMASK)) {
+            MIPS_DEBUG("CTI in delay / forbidden slot");
+            goto die;
+        }
         /* If we get an exception, we want to restart at next instruction */
         ctx->pc += 4;
         save_cpu_state(ctx, 1);
@@ -7024,6 +7947,12 @@ static void gen_compute_branch1(DisasContext *ctx, uint32_t op,
     const char *opn = "cp1 cond branch";
     TCGv_i32 t0 = tcg_temp_new_i32();
 
+    if ((ctx->insn_flags & ISA_MIPS32R6) && (ctx->hflags & MIPS_HFLAG_BMASK)) {
+        MIPS_DEBUG("CTI in delay / forbidden slot");
+        generate_exception(ctx, EXCP_RI);
+        goto out;
+    }
+
     if (cc != 0)
         check_insn(ctx, ISA_MIPS4 | ISA_MIPS32);
 
@@ -7125,11 +8054,62 @@ static void gen_compute_branch1(DisasContext *ctx, uint32_t op,
     MIPS_DEBUG("%s: cond %02x target " TARGET_FMT_lx, opn,
                ctx->hflags, btarget);
     ctx->btarget = btarget;
-
+    ctx->hflags |= MIPS_HFLAG_BDS32;
  out:
     tcg_temp_free_i32(t0);
 }
 
+/* R6 CP1 Branches */
+static void gen_compute_branch1_r6(DisasContext *ctx, uint32_t op,
+                                   int32_t ft, int32_t offset)
+{
+    target_ulong btarget;
+    const char *opn = "cp1 cond branch";
+    TCGv_i64 t0 = tcg_temp_new_i64();
+
+    if (ctx->hflags & MIPS_HFLAG_BMASK) {
+#ifdef MIPS_DEBUG_DISAS
+        LOG_DISAS("Branch in delay / forbidden slot at PC 0x" TARGET_FMT_lx
+                  "\n", ctx->pc);
+#endif
+        generate_exception(ctx, EXCP_RI);
+        goto out;
+    }
+
+    gen_load_fpr64(ctx, t0, ft);
+    tcg_gen_andi_i64(t0, t0, 1);
+
+    btarget = addr_add(ctx, ctx->pc + 4, offset);
+
+    switch (op) {
+    case OPC_BC1EQZ:
+        tcg_gen_xori_i64(t0, t0, 1);
+        opn = "bc1eqz";
+        ctx->hflags |= MIPS_HFLAG_BC;
+        break;
+    case OPC_BC1NEZ:
+        /* t0 already set */
+        opn = "bc1nez";
+        ctx->hflags |= MIPS_HFLAG_BC;
+        break;
+    default:
+        MIPS_INVAL(opn);
+        generate_exception(ctx, EXCP_RI);
+        goto out;
+    }
+
+    tcg_gen_trunc_i64_tl(bcond, t0);
+
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s: cond %02x target " TARGET_FMT_lx, opn,
+               ctx->hflags, btarget);
+    ctx->btarget = btarget;
+    ctx->hflags |= MIPS_HFLAG_BDS32;
+
+out:
+    tcg_temp_free_i64(t0);
+}
+
 /* Coprocessor 1 (FPU) */
 
 #define FOP(func, fmt) (((fmt) << 21) | (func))
@@ -7151,14 +8131,25 @@ enum fopcode {
     OPC_TRUNC_W_S = FOP(13, FMT_S),
     OPC_CEIL_W_S = FOP(14, FMT_S),
     OPC_FLOOR_W_S = FOP(15, FMT_S),
+    OPC_SEL_S = FOP(16, FMT_S),
     OPC_MOVCF_S = FOP(17, FMT_S),
     OPC_MOVZ_S = FOP(18, FMT_S),
     OPC_MOVN_S = FOP(19, FMT_S),
+    OPC_SELEQZ_S = FOP(20, FMT_S),
     OPC_RECIP_S = FOP(21, FMT_S),
     OPC_RSQRT_S = FOP(22, FMT_S),
+    OPC_SELNEZ_S = FOP(23, FMT_S),
+    OPC_MADDF_S = FOP(24, FMT_S),
+    OPC_MSUBF_S = FOP(25, FMT_S),
+    OPC_RINT_S = FOP(26, FMT_S),
+    OPC_CLASS_S = FOP(27, FMT_S),
+    OPC_MIN_S = FOP(28, FMT_S),
     OPC_RECIP2_S = FOP(28, FMT_S),
+    OPC_MINA_S = FOP(29, FMT_S),
     OPC_RECIP1_S = FOP(29, FMT_S),
+    OPC_MAX_S = FOP(30, FMT_S),
     OPC_RSQRT1_S = FOP(30, FMT_S),
+    OPC_MAXA_S = FOP(31, FMT_S),
     OPC_RSQRT2_S = FOP(31, FMT_S),
     OPC_CVT_D_S = FOP(33, FMT_S),
     OPC_CVT_W_S = FOP(36, FMT_S),
@@ -7197,14 +8188,25 @@ enum fopcode {
     OPC_TRUNC_W_D = FOP(13, FMT_D),
     OPC_CEIL_W_D = FOP(14, FMT_D),
     OPC_FLOOR_W_D = FOP(15, FMT_D),
+    OPC_SEL_D = FOP(16, FMT_D),
     OPC_MOVCF_D = FOP(17, FMT_D),
     OPC_MOVZ_D = FOP(18, FMT_D),
     OPC_MOVN_D = FOP(19, FMT_D),
+    OPC_SELEQZ_D = FOP(20, FMT_D),
     OPC_RECIP_D = FOP(21, FMT_D),
     OPC_RSQRT_D = FOP(22, FMT_D),
+    OPC_SELNEZ_D = FOP(23, FMT_D),
+    OPC_MADDF_D = FOP(24, FMT_D),
+    OPC_MSUBF_D = FOP(25, FMT_D),
+    OPC_RINT_D = FOP(26, FMT_D),
+    OPC_CLASS_D = FOP(27, FMT_D),
+    OPC_MIN_D = FOP(28, FMT_D),
     OPC_RECIP2_D = FOP(28, FMT_D),
+    OPC_MINA_D = FOP(29, FMT_D),
     OPC_RECIP1_D = FOP(29, FMT_D),
+    OPC_MAX_D = FOP(30, FMT_D),
     OPC_RSQRT1_D = FOP(30, FMT_D),
+    OPC_MAXA_D = FOP(31, FMT_D),
     OPC_RSQRT2_D = FOP(31, FMT_D),
     OPC_CVT_S_D = FOP(32, FMT_D),
     OPC_CVT_W_D = FOP(36, FMT_D),
@@ -7274,6 +8276,53 @@ enum fopcode {
     OPC_CMP_NGT_PS = FOP (63, FMT_PS),
 };
 
+enum r6_f_cmp_op {
+    R6_OPC_CMP_AF_S   = FOP(0, FMT_W),
+    R6_OPC_CMP_UN_S   = FOP(1, FMT_W),
+    R6_OPC_CMP_EQ_S   = FOP(2, FMT_W),
+    R6_OPC_CMP_UEQ_S  = FOP(3, FMT_W),
+    R6_OPC_CMP_LT_S   = FOP(4, FMT_W),
+    R6_OPC_CMP_ULT_S  = FOP(5, FMT_W),
+    R6_OPC_CMP_LE_S   = FOP(6, FMT_W),
+    R6_OPC_CMP_ULE_S  = FOP(7, FMT_W),
+    R6_OPC_CMP_SAF_S  = FOP(8, FMT_W),
+    R6_OPC_CMP_SUN_S  = FOP(9, FMT_W),
+    R6_OPC_CMP_SEQ_S  = FOP(10, FMT_W),
+    R6_OPC_CMP_SEUQ_S = FOP(11, FMT_W),
+    R6_OPC_CMP_SLT_S  = FOP(12, FMT_W),
+    R6_OPC_CMP_SULT_S = FOP(13, FMT_W),
+    R6_OPC_CMP_SLE_S  = FOP(14, FMT_W),
+    R6_OPC_CMP_SULE_S = FOP(15, FMT_W),
+    R6_OPC_CMP_OR_S   = FOP(17, FMT_W),
+    R6_OPC_CMP_UNE_S  = FOP(18, FMT_W),
+    R6_OPC_CMP_NE_S   = FOP(19, FMT_W),
+    R6_OPC_CMP_SOR_S  = FOP(25, FMT_W),
+    R6_OPC_CMP_SUNE_S = FOP(26, FMT_W),
+    R6_OPC_CMP_SNE_S  = FOP(27, FMT_W),
+
+    R6_OPC_CMP_AF_D   = FOP(0, FMT_L),
+    R6_OPC_CMP_UN_D   = FOP(1, FMT_L),
+    R6_OPC_CMP_EQ_D   = FOP(2, FMT_L),
+    R6_OPC_CMP_UEQ_D  = FOP(3, FMT_L),
+    R6_OPC_CMP_LT_D   = FOP(4, FMT_L),
+    R6_OPC_CMP_ULT_D  = FOP(5, FMT_L),
+    R6_OPC_CMP_LE_D   = FOP(6, FMT_L),
+    R6_OPC_CMP_ULE_D  = FOP(7, FMT_L),
+    R6_OPC_CMP_SAF_D  = FOP(8, FMT_L),
+    R6_OPC_CMP_SUN_D  = FOP(9, FMT_L),
+    R6_OPC_CMP_SEQ_D  = FOP(10, FMT_L),
+    R6_OPC_CMP_SEUQ_D = FOP(11, FMT_L),
+    R6_OPC_CMP_SLT_D  = FOP(12, FMT_L),
+    R6_OPC_CMP_SULT_D = FOP(13, FMT_L),
+    R6_OPC_CMP_SLE_D  = FOP(14, FMT_L),
+    R6_OPC_CMP_SULE_D = FOP(15, FMT_L),
+    R6_OPC_CMP_OR_D   = FOP(17, FMT_L),
+    R6_OPC_CMP_UNE_D  = FOP(18, FMT_L),
+    R6_OPC_CMP_NE_D   = FOP(19, FMT_L),
+    R6_OPC_CMP_SOR_D  = FOP(25, FMT_L),
+    R6_OPC_CMP_SUNE_D = FOP(26, FMT_L),
+    R6_OPC_CMP_SNE_D  = FOP(27, FMT_L),
+};
 static void gen_cp1 (DisasContext *ctx, uint32_t opc, int rt, int fs)
 {
     const char *opn = "cp1 move";
@@ -7309,12 +8358,15 @@ static void gen_cp1 (DisasContext *ctx, uint32_t opc, int rt, int fs)
         break;
     case OPC_CTC1:
         gen_load_gpr(t0, rt);
+        save_cpu_state(ctx, 1);
         {
             TCGv_i32 fs_tmp = tcg_const_i32(fs);
 
             gen_helper_0e2i(ctc1, t0, fs_tmp, rt);
             tcg_temp_free_i32(fs_tmp);
         }
+        /* Stop translation as we may have changed hflags */
+        ctx->bstate = BS_STOP;
         opn = "ctc1";
         break;
 #if defined(TARGET_MIPS64)
@@ -7460,6 +8512,79 @@ static inline void gen_movcf_ps(DisasContext *ctx, int fs, int fd,
     gen_set_label(l2);
 }
 
+static void gen_sel_s(DisasContext *ctx, enum fopcode op1, int fd, int ft,
+                      int fs)
+{
+    TCGv_i32 t1 = tcg_const_i32(0);
+    TCGv_i32 fp0 = tcg_temp_new_i32();
+    TCGv_i32 fp1 = tcg_temp_new_i32();
+    TCGv_i32 fp2 = tcg_temp_new_i32();
+    gen_load_fpr32(fp0, fd);
+    gen_load_fpr32(fp1, ft);
+    gen_load_fpr32(fp2, fs);
+
+    switch (op1) {
+    case OPC_SEL_S:
+        tcg_gen_andi_i32(fp0, fp0, 1);
+        tcg_gen_movcond_i32(TCG_COND_NE, fp0, fp0, t1, fp1, fp2);
+        break;
+    case OPC_SELEQZ_S:
+        tcg_gen_andi_i32(fp1, fp1, 1);
+        tcg_gen_movcond_i32(TCG_COND_EQ, fp0, fp1, t1, fp2, t1);
+        break;
+    case OPC_SELNEZ_S:
+        tcg_gen_andi_i32(fp1, fp1, 1);
+        tcg_gen_movcond_i32(TCG_COND_NE, fp0, fp1, t1, fp2, t1);
+        break;
+    default:
+        MIPS_INVAL("gen_sel_s");
+        generate_exception (ctx, EXCP_RI);
+        break;
+    }
+
+    gen_store_fpr32(fp0, fd);
+    tcg_temp_free_i32(fp2);
+    tcg_temp_free_i32(fp1);
+    tcg_temp_free_i32(fp0);
+    tcg_temp_free_i32(t1);
+}
+
+static void gen_sel_d(DisasContext *ctx, enum fopcode op1, int fd, int ft,
+                      int fs)
+{
+    TCGv_i64 t1 = tcg_const_i64(0);
+    TCGv_i64 fp0 = tcg_temp_new_i64();
+    TCGv_i64 fp1 = tcg_temp_new_i64();
+    TCGv_i64 fp2 = tcg_temp_new_i64();
+    gen_load_fpr64(ctx, fp0, fd);
+    gen_load_fpr64(ctx, fp1, ft);
+    gen_load_fpr64(ctx, fp2, fs);
+
+    switch (op1) {
+    case OPC_SEL_D:
+        tcg_gen_andi_i64(fp0, fp0, 1);
+        tcg_gen_movcond_i64(TCG_COND_NE, fp0, fp0, t1, fp1, fp2);
+        break;
+    case OPC_SELEQZ_D:
+        tcg_gen_andi_i64(fp1, fp1, 1);
+        tcg_gen_movcond_i64(TCG_COND_EQ, fp0, fp1, t1, fp2, t1);
+        break;
+    case OPC_SELNEZ_D:
+        tcg_gen_andi_i64(fp1, fp1, 1);
+        tcg_gen_movcond_i64(TCG_COND_NE, fp0, fp1, t1, fp2, t1);
+        break;
+    default:
+        MIPS_INVAL("gen_sel_d");
+        generate_exception (ctx, EXCP_RI);
+        break;
+    }
+
+    gen_store_fpr64(ctx, fp0, fd);
+    tcg_temp_free_i64(fp2);
+    tcg_temp_free_i64(fp1);
+    tcg_temp_free_i64(fp0);
+    tcg_temp_free_i64(t1);
+}
 
 static void gen_farith (DisasContext *ctx, enum fopcode op1,
                         int ft, int fs, int fd, int cc)
@@ -7708,11 +8833,28 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
         }
         opn = "floor.w.s";
         break;
+    case OPC_SEL_S:
+        check_insn(ctx, ISA_MIPS32R6);
+        gen_sel_s(ctx, op1, fd, ft, fs);
+        opn = "sel.s";
+        break;
+    case OPC_SELEQZ_S:
+        check_insn(ctx, ISA_MIPS32R6);
+        gen_sel_s(ctx, op1, fd, ft, fs);
+        opn = "seleqz.s";
+        break;
+    case OPC_SELNEZ_S:
+        check_insn(ctx, ISA_MIPS32R6);
+        gen_sel_s(ctx, op1, fd, ft, fs);
+        opn = "selnez.s";
+        break;
     case OPC_MOVCF_S:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         gen_movcf_s(fs, fd, (ft >> 2) & 0x7, ft & 0x1);
         opn = "movcf.s";
         break;
     case OPC_MOVZ_S:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         {
             int l1 = gen_new_label();
             TCGv_i32 fp0;
@@ -7729,6 +8871,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
         opn = "movz.s";
         break;
     case OPC_MOVN_S:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         {
             int l1 = gen_new_label();
             TCGv_i32 fp0;
@@ -7768,59 +8911,175 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
         }
         opn = "rsqrt.s";
         break;
-    case OPC_RECIP2_S:
-        check_cp1_64bitmode(ctx);
+    case OPC_MADDF_S:
+        check_insn(ctx, ISA_MIPS32R6);
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
             TCGv_i32 fp1 = tcg_temp_new_i32();
-
+            TCGv_i32 fp2 = tcg_temp_new_i32();
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
-            gen_helper_float_recip2_s(fp0, cpu_env, fp0, fp1);
+            gen_load_fpr32(fp2, fd);
+            gen_helper_float_maddf_s(fp2, cpu_env, fp0, fp1, fp2);
+            gen_store_fpr32(fp2, fd);
+            tcg_temp_free_i32(fp2);
             tcg_temp_free_i32(fp1);
-            gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
+            opn = "maddf.s";
         }
-        opn = "recip2.s";
         break;
-    case OPC_RECIP1_S:
-        check_cp1_64bitmode(ctx);
+    case OPC_MSUBF_S:
+        check_insn(ctx, ISA_MIPS32R6);
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
-
+            TCGv_i32 fp1 = tcg_temp_new_i32();
+            TCGv_i32 fp2 = tcg_temp_new_i32();
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_recip1_s(fp0, cpu_env, fp0);
-            gen_store_fpr32(fp0, fd);
+            gen_load_fpr32(fp1, ft);
+            gen_load_fpr32(fp2, fd);
+            gen_helper_float_msubf_s(fp2, cpu_env, fp0, fp1, fp2);
+            gen_store_fpr32(fp2, fd);
+            tcg_temp_free_i32(fp2);
+            tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
+            opn = "msubf.s";
         }
-        opn = "recip1.s";
         break;
-    case OPC_RSQRT1_S:
-        check_cp1_64bitmode(ctx);
+    case OPC_RINT_S:
+        check_insn(ctx, ISA_MIPS32R6);
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
-
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_rsqrt1_s(fp0, cpu_env, fp0);
+            gen_helper_float_rint_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
+            opn = "rint.s";
         }
-        opn = "rsqrt1.s";
         break;
-    case OPC_RSQRT2_S:
-        check_cp1_64bitmode(ctx);
+    case OPC_CLASS_S:
+        check_insn(ctx, ISA_MIPS32R6);
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
+            gen_load_fpr32(fp0, fs);
+            gen_helper_float_class_s(fp0, fp0);
+            gen_store_fpr32(fp0, fd);
+            tcg_temp_free_i32(fp0);
+            opn = "class.s";
+        }
+        break;
+    case OPC_MIN_S: /* OPC_RECIP2_S */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            /* OPC_MIN_S */
+            TCGv_i32 fp0 = tcg_temp_new_i32();
             TCGv_i32 fp1 = tcg_temp_new_i32();
+            TCGv_i32 fp2 = tcg_temp_new_i32();
+            gen_load_fpr32(fp0, fs);
+            gen_load_fpr32(fp1, ft);
+            gen_helper_float_min_s(fp2, cpu_env, fp0, fp1);
+            gen_store_fpr32(fp2, fd);
+            tcg_temp_free_i32(fp2);
+            tcg_temp_free_i32(fp1);
+            tcg_temp_free_i32(fp0);
+            opn = "min.s";
+        } else {
+            /* OPC_RECIP2_S */
+            check_cp1_64bitmode(ctx);
+            {
+                TCGv_i32 fp0 = tcg_temp_new_i32();
+                TCGv_i32 fp1 = tcg_temp_new_i32();
 
+                gen_load_fpr32(fp0, fs);
+                gen_load_fpr32(fp1, ft);
+                gen_helper_float_recip2_s(fp0, cpu_env, fp0, fp1);
+                tcg_temp_free_i32(fp1);
+                gen_store_fpr32(fp0, fd);
+                tcg_temp_free_i32(fp0);
+            }
+            opn = "recip2.s";
+        }
+        break;
+    case OPC_MINA_S: /* OPC_RECIP1_S */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            /* OPC_MINA_S */
+            TCGv_i32 fp0 = tcg_temp_new_i32();
+            TCGv_i32 fp1 = tcg_temp_new_i32();
+            TCGv_i32 fp2 = tcg_temp_new_i32();
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
-            gen_helper_float_rsqrt2_s(fp0, cpu_env, fp0, fp1);
+            gen_helper_float_mina_s(fp2, cpu_env, fp0, fp1);
+            gen_store_fpr32(fp2, fd);
+            tcg_temp_free_i32(fp2);
             tcg_temp_free_i32(fp1);
-            gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
+            opn = "mina.s";
+        } else {
+            /* OPC_RECIP1_S */
+            check_cp1_64bitmode(ctx);
+            {
+                TCGv_i32 fp0 = tcg_temp_new_i32();
+
+                gen_load_fpr32(fp0, fs);
+                gen_helper_float_recip1_s(fp0, cpu_env, fp0);
+                gen_store_fpr32(fp0, fd);
+                tcg_temp_free_i32(fp0);
+            }
+            opn = "recip1.s";
+        }
+        break;
+    case OPC_MAX_S: /* OPC_RSQRT1_S */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            /* OPC_MAX_S */
+            TCGv_i32 fp0 = tcg_temp_new_i32();
+            TCGv_i32 fp1 = tcg_temp_new_i32();
+            gen_load_fpr32(fp0, fs);
+            gen_load_fpr32(fp1, ft);
+            gen_helper_float_max_s(fp1, cpu_env, fp0, fp1);
+            gen_store_fpr32(fp1, fd);
+            tcg_temp_free_i32(fp1);
+            tcg_temp_free_i32(fp0);
+            opn = "max.s";
+        } else {
+            /* OPC_RSQRT1_S */
+            check_cp1_64bitmode(ctx);
+            {
+                TCGv_i32 fp0 = tcg_temp_new_i32();
+
+                gen_load_fpr32(fp0, fs);
+                gen_helper_float_rsqrt1_s(fp0, cpu_env, fp0);
+                gen_store_fpr32(fp0, fd);
+                tcg_temp_free_i32(fp0);
+            }
+            opn = "rsqrt1.s";
+        }
+        break;
+    case OPC_MAXA_S: /* OPC_RSQRT2_S */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            /* OPC_MAXA_S */
+            TCGv_i32 fp0 = tcg_temp_new_i32();
+            TCGv_i32 fp1 = tcg_temp_new_i32();
+            gen_load_fpr32(fp0, fs);
+            gen_load_fpr32(fp1, ft);
+            gen_helper_float_maxa_s(fp1, cpu_env, fp0, fp1);
+            gen_store_fpr32(fp1, fd);
+            tcg_temp_free_i32(fp1);
+            tcg_temp_free_i32(fp0);
+            opn = "maxa.s";
+        } else {
+            /* OPC_RSQRT2_S */
+            check_cp1_64bitmode(ctx);
+            {
+                TCGv_i32 fp0 = tcg_temp_new_i32();
+                TCGv_i32 fp1 = tcg_temp_new_i32();
+
+                gen_load_fpr32(fp0, fs);
+                gen_load_fpr32(fp1, ft);
+                gen_helper_float_rsqrt2_s(fp0, cpu_env, fp0, fp1);
+                tcg_temp_free_i32(fp1);
+                gen_store_fpr32(fp0, fd);
+                tcg_temp_free_i32(fp0);
+            }
+            opn = "rsqrt2.s";
         }
-        opn = "rsqrt2.s";
         break;
     case OPC_CVT_D_S:
         check_cp1_registers(ctx, fd);
@@ -7862,6 +9121,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
         opn = "cvt.l.s";
         break;
     case OPC_CVT_PS_S:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         check_cp1_64bitmode(ctx);
         {
             TCGv_i64 fp64 = tcg_temp_new_i64();
@@ -7894,6 +9154,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
     case OPC_CMP_NGE_S:
     case OPC_CMP_LE_S:
     case OPC_CMP_NGT_S:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         if (ctx->opcode & (1 << 6)) {
             gen_cmpabs_s(ctx, func-48, ft, fs, cc);
             opn = condnames_abs[func-48];
@@ -8117,11 +9378,28 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
         }
         opn = "floor.w.d";
         break;
+    case OPC_SEL_D:
+        check_insn(ctx, ISA_MIPS32R6);
+        gen_sel_d(ctx, op1, fd, ft, fs);
+        opn = "sel.d";
+        break;
+    case OPC_SELEQZ_D:
+        check_insn(ctx, ISA_MIPS32R6);
+        gen_sel_d(ctx, op1, fd, ft, fs);
+        opn = "seleqz.d";
+        break;
+    case OPC_SELNEZ_D:
+        check_insn(ctx, ISA_MIPS32R6);
+        gen_sel_d(ctx, op1, fd, ft, fs);
+        opn = "selnez.d";
+        break;
     case OPC_MOVCF_D:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         gen_movcf_d(ctx, fs, fd, (ft >> 2) & 0x7, ft & 0x1);
         opn = "movcf.d";
         break;
     case OPC_MOVZ_D:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         {
             int l1 = gen_new_label();
             TCGv_i64 fp0;
@@ -8138,6 +9416,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
         opn = "movz.d";
         break;
     case OPC_MOVN_D:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         {
             int l1 = gen_new_label();
             TCGv_i64 fp0;
@@ -8177,59 +9456,171 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
         }
         opn = "rsqrt.d";
         break;
-    case OPC_RECIP2_D:
-        check_cp1_64bitmode(ctx);
+    case OPC_MADDF_D:
+        check_insn(ctx, ISA_MIPS32R6);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
             TCGv_i64 fp1 = tcg_temp_new_i64();
-
+            TCGv_i64 fp2 = tcg_temp_new_i64();
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_recip2_d(fp0, cpu_env, fp0, fp1);
+            gen_load_fpr64(ctx, fp2, fd);
+            gen_helper_float_maddf_d(fp2, cpu_env, fp0, fp1, fp2);
+            gen_store_fpr64(ctx, fp2, fd);
+            tcg_temp_free_i64(fp2);
             tcg_temp_free_i64(fp1);
-            gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
+            opn = "maddf.d";
         }
-        opn = "recip2.d";
         break;
-    case OPC_RECIP1_D:
-        check_cp1_64bitmode(ctx);
+    case OPC_MSUBF_D:
+        check_insn(ctx, ISA_MIPS32R6);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
-
+            TCGv_i64 fp1 = tcg_temp_new_i64();
+            TCGv_i64 fp2 = tcg_temp_new_i64();
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_recip1_d(fp0, cpu_env, fp0);
-            gen_store_fpr64(ctx, fp0, fd);
+            gen_load_fpr64(ctx, fp1, ft);
+            gen_load_fpr64(ctx, fp2, fd);
+            gen_helper_float_msubf_d(fp2, cpu_env, fp0, fp1, fp2);
+            gen_store_fpr64(ctx, fp2, fd);
+            tcg_temp_free_i64(fp2);
+            tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
+            opn = "msubf.d";
         }
-        opn = "recip1.d";
         break;
-    case OPC_RSQRT1_D:
-        check_cp1_64bitmode(ctx);
+    case OPC_RINT_D:
+        check_insn(ctx, ISA_MIPS32R6);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
-
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_rsqrt1_d(fp0, cpu_env, fp0);
+            gen_helper_float_rint_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
+            opn = "rint.d";
         }
-        opn = "rsqrt1.d";
         break;
-    case OPC_RSQRT2_D:
-        check_cp1_64bitmode(ctx);
+    case OPC_CLASS_D:
+        check_insn(ctx, ISA_MIPS32R6);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
+            gen_load_fpr64(ctx, fp0, fs);
+            gen_helper_float_class_d(fp0, fp0);
+            gen_store_fpr64(ctx, fp0, fd);
+            tcg_temp_free_i64(fp0);
+            opn = "class.d";
+        }
+        break;
+    case OPC_MIN_D: /* OPC_RECIP2_D */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            /* OPC_MIN_D */
+            TCGv_i64 fp0 = tcg_temp_new_i64();
+            TCGv_i64 fp1 = tcg_temp_new_i64();
+            gen_load_fpr64(ctx, fp0, fs);
+            gen_load_fpr64(ctx, fp1, ft);
+            gen_helper_float_min_d(fp1, cpu_env, fp0, fp1);
+            gen_store_fpr64(ctx, fp1, fd);
+            tcg_temp_free_i64(fp1);
+            tcg_temp_free_i64(fp0);
+            opn = "min.d";
+        } else {
+            /* OPC_RECIP2_D */
+            check_cp1_64bitmode(ctx);
+            {
+                TCGv_i64 fp0 = tcg_temp_new_i64();
+                TCGv_i64 fp1 = tcg_temp_new_i64();
+
+                gen_load_fpr64(ctx, fp0, fs);
+                gen_load_fpr64(ctx, fp1, ft);
+                gen_helper_float_recip2_d(fp0, cpu_env, fp0, fp1);
+                tcg_temp_free_i64(fp1);
+                gen_store_fpr64(ctx, fp0, fd);
+                tcg_temp_free_i64(fp0);
+            }
+            opn = "recip2.d";
+        }
+        break;
+    case OPC_MINA_D: /* OPC_RECIP1_D */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            /* OPC_MINA_D */
+            TCGv_i64 fp0 = tcg_temp_new_i64();
             TCGv_i64 fp1 = tcg_temp_new_i64();
+            gen_load_fpr64(ctx, fp0, fs);
+            gen_load_fpr64(ctx, fp1, ft);
+            gen_helper_float_mina_d(fp1, cpu_env, fp0, fp1);
+            gen_store_fpr64(ctx, fp1, fd);
+            tcg_temp_free_i64(fp1);
+            tcg_temp_free_i64(fp0);
+            opn = "mina.d";
+        } else {
+            /* OPC_RECIP1_D */
+            check_cp1_64bitmode(ctx);
+            {
+                TCGv_i64 fp0 = tcg_temp_new_i64();
 
+                gen_load_fpr64(ctx, fp0, fs);
+                gen_helper_float_recip1_d(fp0, cpu_env, fp0);
+                gen_store_fpr64(ctx, fp0, fd);
+                tcg_temp_free_i64(fp0);
+            }
+            opn = "recip1.d";
+        }
+        break;
+    case OPC_MAX_D: /*  OPC_RSQRT1_D */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            /* OPC_MAX_D */
+            TCGv_i64 fp0 = tcg_temp_new_i64();
+            TCGv_i64 fp1 = tcg_temp_new_i64();
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_rsqrt2_d(fp0, cpu_env, fp0, fp1);
+            gen_helper_float_max_d(fp1, cpu_env, fp0, fp1);
+            gen_store_fpr64(ctx, fp1, fd);
+            tcg_temp_free_i64(fp1);
+            tcg_temp_free_i64(fp0);
+            opn = "max.d";
+        } else {
+            /* OPC_RSQRT1_D */
+            check_cp1_64bitmode(ctx);
+            {
+                TCGv_i64 fp0 = tcg_temp_new_i64();
+
+                gen_load_fpr64(ctx, fp0, fs);
+                gen_helper_float_rsqrt1_d(fp0, cpu_env, fp0);
+                gen_store_fpr64(ctx, fp0, fd);
+                tcg_temp_free_i64(fp0);
+            }
+            opn = "rsqrt1.d";
+        }
+        break;
+    case OPC_MAXA_D: /* OPC_RSQRT2_D */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            /* OPC_MAXA_D */
+            TCGv_i64 fp0 = tcg_temp_new_i64();
+            TCGv_i64 fp1 = tcg_temp_new_i64();
+            gen_load_fpr64(ctx, fp0, fs);
+            gen_load_fpr64(ctx, fp1, ft);
+            gen_helper_float_maxa_d(fp1, cpu_env, fp0, fp1);
+            gen_store_fpr64(ctx, fp1, fd);
             tcg_temp_free_i64(fp1);
-            gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
+            opn = "maxa.d";
+        } else {
+            /* OPC_RSQRT2_D */
+            check_cp1_64bitmode(ctx);
+            {
+                TCGv_i64 fp0 = tcg_temp_new_i64();
+                TCGv_i64 fp1 = tcg_temp_new_i64();
+
+                gen_load_fpr64(ctx, fp0, fs);
+                gen_load_fpr64(ctx, fp1, ft);
+                gen_helper_float_rsqrt2_d(fp0, cpu_env, fp0, fp1);
+                tcg_temp_free_i64(fp1);
+                gen_store_fpr64(ctx, fp0, fd);
+                tcg_temp_free_i64(fp0);
+            }
+            opn = "rsqrt2.d";
         }
-        opn = "rsqrt2.d";
         break;
     case OPC_CMP_F_D:
     case OPC_CMP_UN_D:
@@ -8247,6 +9638,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
     case OPC_CMP_NGE_D:
     case OPC_CMP_LE_D:
     case OPC_CMP_NGT_D:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         if (ctx->opcode & (1 << 6)) {
             gen_cmpabs_d(ctx, func-48, ft, fs, cc);
             opn = condnames_abs[func-48];
@@ -8347,6 +9739,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
         opn = "cvt.d.l";
         break;
     case OPC_CVT_PS_PW:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         check_cp1_64bitmode(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
@@ -9125,7 +10518,7 @@ static void gen_rdhwr(DisasContext *ctx, int rt, int rd)
     tcg_temp_free(t0);
 }
 
-static void handle_delay_slot(DisasContext *ctx, int insn_bytes)
+static void gen_branch(DisasContext *ctx, int insn_bytes)
 {
     if (ctx->hflags & MIPS_HFLAG_BMASK) {
         int proc_hflags = ctx->hflags & MIPS_HFLAG_BMASK;
@@ -9135,6 +10528,10 @@ static void handle_delay_slot(DisasContext *ctx, int insn_bytes)
         save_cpu_state(ctx, 0);
         /* FIXME: Need to clear can_do_io.  */
         switch (proc_hflags & MIPS_HFLAG_BMASK_BASE) {
+        case MIPS_HFLAG_FBNSLOT:
+            MIPS_DEBUG("forbidden slot");
+            gen_goto_tb(ctx, 0, ctx->pc + insn_bytes);
+            break;
         case MIPS_HFLAG_B:
             /* unconditional branch */
             MIPS_DEBUG("unconditional branch");
@@ -9668,15 +11065,15 @@ static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
         gen_addiupc(ctx, rx, imm, 0, 1);
         break;
     case M16_OPC_B:
-        gen_compute_branch(ctx, OPC_BEQ, 4, 0, 0, offset << 1);
+        gen_compute_branch(ctx, OPC_BEQ, 4, 0, 0, offset << 1, 0);
         /* No delay slot, so just process as a normal instruction */
         break;
     case M16_OPC_BEQZ:
-        gen_compute_branch(ctx, OPC_BEQ, 4, rx, 0, offset << 1);
+        gen_compute_branch(ctx, OPC_BEQ, 4, rx, 0, offset << 1, 0);
         /* No delay slot, so just process as a normal instruction */
         break;
     case M16_OPC_BNEQZ:
-        gen_compute_branch(ctx, OPC_BNE, 4, rx, 0, offset << 1);
+        gen_compute_branch(ctx, OPC_BNE, 4, rx, 0, offset << 1, 0);
         /* No delay slot, so just process as a normal instruction */
         break;
     case M16_OPC_SHIFT:
@@ -9734,10 +11131,10 @@ static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
     case M16_OPC_I8:
         switch (funct) {
         case I8_BTEQZ:
-            gen_compute_branch(ctx, OPC_BEQ, 4, 24, 0, offset << 1);
+            gen_compute_branch(ctx, OPC_BEQ, 4, 24, 0, offset << 1, 0);
             break;
         case I8_BTNEZ:
-            gen_compute_branch(ctx, OPC_BNE, 4, 24, 0, offset << 1);
+            gen_compute_branch(ctx, OPC_BNE, 4, 24, 0, offset << 1, 0);
             break;
         case I8_SWRASP:
             gen_st(ctx, OPC_SW, 31, 29, imm);
@@ -9865,7 +11262,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
     case M16_OPC_B:
         offset = (ctx->opcode & 0x7ff) << 1;
         offset = (int16_t)(offset << 4) >> 4;
-        gen_compute_branch(ctx, OPC_BEQ, 2, 0, 0, offset);
+        gen_compute_branch(ctx, OPC_BEQ, 2, 0, 0, offset, 0);
         /* No delay slot, so just process as a normal instruction */
         break;
     case M16_OPC_JAL:
@@ -9873,16 +11270,18 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
         offset = (((ctx->opcode & 0x1f) << 21)
                   | ((ctx->opcode >> 5) & 0x1f) << 16
                   | offset) << 2;
-        op = ((ctx->opcode >> 10) & 0x1) ? OPC_JALXS : OPC_JALS;
-        gen_compute_branch(ctx, op, 4, rx, ry, offset);
+        op = ((ctx->opcode >> 10) & 0x1) ? OPC_JALX : OPC_JAL;
+        gen_compute_branch(ctx, op, 4, rx, ry, offset, 2);
         n_bytes = 4;
         break;
     case M16_OPC_BEQZ:
-        gen_compute_branch(ctx, OPC_BEQ, 2, rx, 0, ((int8_t)ctx->opcode) << 1);
+        gen_compute_branch(ctx, OPC_BEQ, 2, rx, 0,
+                           ((int8_t)ctx->opcode) << 1, 0);
         /* No delay slot, so just process as a normal instruction */
         break;
     case M16_OPC_BNEQZ:
-        gen_compute_branch(ctx, OPC_BNE, 2, rx, 0, ((int8_t)ctx->opcode) << 1);
+        gen_compute_branch(ctx, OPC_BNE, 2, rx, 0,
+                           ((int8_t)ctx->opcode) << 1, 0);
         /* No delay slot, so just process as a normal instruction */
         break;
     case M16_OPC_SHIFT:
@@ -9955,11 +11354,11 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
             switch (funct) {
             case I8_BTEQZ:
                 gen_compute_branch(ctx, OPC_BEQ, 2, 24, 0,
-                                   ((int8_t)ctx->opcode) << 1);
+                                   ((int8_t)ctx->opcode) << 1, 0);
                 break;
             case I8_BTNEZ:
                 gen_compute_branch(ctx, OPC_BNE, 2, 24, 0,
-                                   ((int8_t)ctx->opcode) << 1);
+                                   ((int8_t)ctx->opcode) << 1, 0);
                 break;
             case I8_SWRASP:
                 gen_st(ctx, OPC_SW, 31, 29, (ctx->opcode & 0xff) << 2);
@@ -10108,12 +11507,13 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
                 int ra = (ctx->opcode >> 5) & 0x1;
 
                 if (link) {
-                    op = nd ? OPC_JALRC : OPC_JALRS;
+                    op = OPC_JALR;
                 } else {
                     op = OPC_JR;
                 }
 
-                gen_compute_branch(ctx, op, 2, ra ? 31 : rx, 31, 0);
+                gen_compute_branch(ctx, op, 2, ra ? 31 : rx, 31, 0,
+                                   (nd ? 0 : 2));
             }
             break;
         case RR_SDBBP:
@@ -10871,7 +12271,6 @@ static void gen_pool16c_insn(DisasContext *ctx)
 {
     int rd = mmreg((ctx->opcode >> 3) & 0x7);
     int rs = mmreg(ctx->opcode & 0x7);
-    int opc;
 
     switch (((ctx->opcode) >> 4) & 0x3f) {
     case NOT16 + 0:
@@ -10927,32 +12326,27 @@ static void gen_pool16c_insn(DisasContext *ctx)
         {
             int reg = ctx->opcode & 0x1f;
 
-            gen_compute_branch(ctx, OPC_JR, 2, reg, 0, 0);
+            gen_compute_branch(ctx, OPC_JR, 2, reg, 0, 0, 4);
         }
         break;
     case JRC16 + 0:
     case JRC16 + 1:
         {
             int reg = ctx->opcode & 0x1f;
-
-            gen_compute_branch(ctx, OPC_JR, 2, reg, 0, 0);
+            gen_compute_branch(ctx, OPC_JR, 2, reg, 0, 0, 0);
             /* Let normal delay slot handling in our caller take us
                to the branch target.  */
         }
         break;
     case JALR16 + 0:
     case JALR16 + 1:
-        opc = OPC_JALR;
-        goto do_jalr;
+        gen_compute_branch(ctx, OPC_JALR, 2, ctx->opcode & 0x1f, 31, 0, 4);
+        ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
+        break;
     case JALR16S + 0:
     case JALR16S + 1:
-        opc = OPC_JALRS;
-    do_jalr:
-        {
-            int reg = ctx->opcode & 0x1f;
-
-            gen_compute_branch(ctx, opc, 2, reg, 31, 0);
-        }
+        gen_compute_branch(ctx, OPC_JALR, 2, ctx->opcode & 0x1f, 31, 0, 2);
+        ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
         break;
     case MFHI16 + 0:
     case MFHI16 + 1:
@@ -10980,8 +12374,7 @@ static void gen_pool16c_insn(DisasContext *ctx)
     case JRADDIUSP + 1:
         {
             int imm = ZIMM(ctx->opcode, 0, 5);
-
-            gen_compute_branch(ctx, OPC_JR, 2, 31, 0, 0);
+            gen_compute_branch(ctx, OPC_JR, 2, 31, 0, 0, 0);
             gen_arith_imm(ctx, OPC_ADDIU, 29, 29, imm << 2);
             /* Let normal delay slot handling in our caller take us
                to the branch target.  */
@@ -11238,11 +12631,13 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs)
         switch (minor) {
         case JALR:
         case JALR_HB:
-            gen_compute_branch (ctx, OPC_JALR, 4, rs, rt, 0);
+            gen_compute_branch(ctx, OPC_JALR, 4, rs, rt, 0, 4);
+            ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
             break;
         case JALRS:
         case JALRS_HB:
-            gen_compute_branch (ctx, OPC_JALRS, 4, rs, rt, 0);
+            gen_compute_branch(ctx, OPC_JALR, 4, rs, rt, 0, 2);
+            ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
             break;
         default:
             goto pool32axf_invalid;
@@ -12132,30 +13527,32 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
         minor = (ctx->opcode >> 21) & 0x1f;
         switch (minor) {
         case BLTZ:
-            mips32_op = OPC_BLTZ;
-            goto do_branch;
+            gen_compute_branch(ctx, OPC_BLTZ, 4, rs, -1, imm << 1, 4);
+            break;
         case BLTZAL:
-            mips32_op = OPC_BLTZAL;
-            goto do_branch;
+            gen_compute_branch(ctx, OPC_BLTZAL, 4, rs, -1, imm << 1, 4);
+            ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
+            break;
         case BLTZALS:
-            mips32_op = OPC_BLTZALS;
-            goto do_branch;
+            gen_compute_branch(ctx, OPC_BLTZAL, 4, rs, -1, imm << 1, 2);
+            ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
+            break;
         case BGEZ:
-            mips32_op = OPC_BGEZ;
-            goto do_branch;
+            gen_compute_branch(ctx, OPC_BGEZ, 4, rs, -1, imm << 1, 4);
+            break;
         case BGEZAL:
-            mips32_op = OPC_BGEZAL;
-            goto do_branch;
+            gen_compute_branch(ctx, OPC_BGEZAL, 4, rs, -1, imm << 1, 4);
+            ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
+            break;
         case BGEZALS:
-            mips32_op = OPC_BGEZALS;
-            goto do_branch;
+            gen_compute_branch(ctx, OPC_BGEZAL, 4, rs, -1, imm << 1, 2);
+            ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
+            break;
         case BLEZ:
-            mips32_op = OPC_BLEZ;
-            goto do_branch;
+            gen_compute_branch(ctx, OPC_BLEZ, 4, rs, -1, imm << 1, 4);
+            break;
         case BGTZ:
-            mips32_op = OPC_BGTZ;
-        do_branch:
-            gen_compute_branch(ctx, mips32_op, 4, rs, -1, imm << 1);
+            gen_compute_branch(ctx, OPC_BGTZ, 4, rs, -1, imm << 1, 4);
             break;
 
             /* Traps */
@@ -12183,7 +13580,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
         case BNEZC:
         case BEQZC:
             gen_compute_branch(ctx, minor == BNEZC ? OPC_BNE : OPC_BEQ,
-                               4, rs, 0, imm << 1);
+                               4, rs, 0, imm << 1, 0);
             /* Compact branches don't have a delay slot, so just let
                the normal delay slot handling take us to the branch
                target. */
@@ -12192,6 +13589,9 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
             gen_logic_imm(ctx, OPC_LUI, rs, -1, imm);
             break;
         case SYNCI:
+            /* Break the TB to be able to sync copied instructions
+               immediately */
+            ctx->bstate = BS_STOP;
             break;
         case BC2F:
         case BC2T:
@@ -12214,8 +13614,13 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
             check_insn(ctx, ASE_MIPS3D);
             /* Fall through */
         do_cp1branch:
-            gen_compute_branch1(ctx, mips32_op,
-                                (ctx->opcode >> 18) & 0x7, imm << 1);
+            if (env->CP0_Config1 & (1 << CP0C1_FP)) {
+                check_cp1_enabled(ctx);
+                gen_compute_branch1(ctx, mips32_op,
+                                    (ctx->opcode >> 18) & 0x7, imm << 1);
+            } else {
+                generate_exception_err(ctx, EXCP_CpU, 1);
+            }
             break;
         case BPOSGE64:
         case BPOSGE32:
@@ -12321,25 +13726,28 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
         break;
     case JALX32:
         offset = (int32_t)(ctx->opcode & 0x3FFFFFF) << 2;
-        gen_compute_branch(ctx, OPC_JALX, 4, rt, rs, offset);
+        gen_compute_branch(ctx, OPC_JALX, 4, rt, rs, offset, 4);
+        ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
         break;
     case JALS32:
         offset = (int32_t)(ctx->opcode & 0x3FFFFFF) << 1;
-        gen_compute_branch(ctx, OPC_JALS, 4, rt, rs, offset);
+        gen_compute_branch(ctx, OPC_JAL, 4, rt, rs, offset, 2);
+        ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
         break;
     case BEQ32:
-        gen_compute_branch(ctx, OPC_BEQ, 4, rt, rs, imm << 1);
+        gen_compute_branch(ctx, OPC_BEQ, 4, rt, rs, imm << 1, 4);
         break;
     case BNE32:
-        gen_compute_branch(ctx, OPC_BNE, 4, rt, rs, imm << 1);
+        gen_compute_branch(ctx, OPC_BNE, 4, rt, rs, imm << 1, 4);
         break;
     case J32:
         gen_compute_branch(ctx, OPC_J, 4, rt, rs,
-                           (int32_t)(ctx->opcode & 0x3FFFFFF) << 1);
+                           (int32_t)(ctx->opcode & 0x3FFFFFF) << 1, 4);
         break;
     case JAL32:
         gen_compute_branch(ctx, OPC_JAL, 4, rt, rs,
-                           (int32_t)(ctx->opcode & 0x3FFFFFF) << 1);
+                           (int32_t)(ctx->opcode & 0x3FFFFFF) << 1, 4);
+        ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
         break;
         /* Floating point (COP1) */
     case LWC132:
@@ -12423,84 +13831,41 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx)
 
     op = (ctx->opcode >> 10) & 0x3f;
     /* Enforce properly-sized instructions in a delay slot */
-    if (ctx->hflags & MIPS_HFLAG_BMASK) {
-        int bits = ctx->hflags & MIPS_HFLAG_BMASK_EXT;
-
-        switch (op) {
-        case POOL32A:
-        case POOL32B:
-        case POOL32I:
-        case POOL32C:
-        case ADDI32:
-        case ADDIU32:
-        case ORI32:
-        case XORI32:
-        case SLTI32:
-        case SLTIU32:
-        case ANDI32:
-        case JALX32:
-        case LBU32:
-        case LHU32:
-        case POOL32F:
-        case JALS32:
-        case BEQ32:
-        case BNE32:
-        case J32:
-        case JAL32:
-        case SB32:
-        case SH32:
-        case POOL32S:
-        case ADDIUPC:
-        case SWC132:
-        case SDC132:
-        case SD32:
-        case SW32:
-        case LB32:
-        case LH32:
-        case DADDIU32:
-        case LWC132:
-        case LDC132:
-        case LD32:
-        case LW32:
-            if (bits & MIPS_HFLAG_BDS16) {
+    if (ctx->hflags & MIPS_HFLAG_BDS_STRICT) {
+        switch (op & 0x7) { /* MSB-3..MSB-5 */
+        case 0:
+        /* POOL32A, POOL32B, POOL32I, POOL32C */
+        case 4:
+        /* ADDI32, ADDIU32, ORI32, XORI32, SLTI32, SLTIU32, ANDI32, JALX32 */
+        case 5:
+        /* LBU32, LHU32, POOL32F, JALS32, BEQ32, BNE32, J32, JAL32 */
+        case 6:
+        /* SB32, SH32, ADDIUPC, SWC132, SDC132, SW32 */
+        case 7:
+        /* LB32, LH32, LWC132, LDC132, LW32 */
+            if (ctx->hflags & MIPS_HFLAG_BDS16) {
                 generate_exception(ctx, EXCP_RI);
                 /* Just stop translation; the user is confused.  */
                 ctx->bstate = BS_STOP;
                 return 2;
             }
             break;
-        case POOL16A:
-        case POOL16B:
-        case POOL16C:
-        case LWGP16:
-        case POOL16F:
-        case LBU16:
-        case LHU16:
-        case LWSP16:
-        case LW16:
-        case SB16:
-        case SH16:
-        case SWSP16:
-        case SW16:
-        case MOVE16:
-        case ANDI16:
-        case POOL16D:
-        case POOL16E:
-        case BEQZ16:
-        case BNEZ16:
-        case B16:
-        case LI16:
-            if (bits & MIPS_HFLAG_BDS32) {
+        case 1:
+        /* POOL16A, POOL16B, POOL16C, LWGP16, POOL16F */
+        case 2:
+        /* LBU16, LHU16, LWSP16, LW16, SB16, SH16, SWSP16, SW16 */
+        case 3:
+        /* MOVE16, ANDI16, POOL16D, POOL16E, BEQZ16, BNEZ16, B16, LI16 */
+            if (ctx->hflags & MIPS_HFLAG_BDS32) {
                 generate_exception(ctx, EXCP_RI);
                 /* Just stop translation; the user is confused.  */
                 ctx->bstate = BS_STOP;
                 return 2;
             }
             break;
-        default:
-            break;
         }
     }
+
     switch (op) {
     case POOL16A:
         {
@@ -12681,13 +14046,13 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx)
         break;
     case B16:
         gen_compute_branch(ctx, OPC_BEQ, 2, 0, 0,
-                           SIMM(ctx->opcode, 0, 10) << 1);
+                           SIMM(ctx->opcode, 0, 10) << 1, 4);
         break;
     case BNEZ16:
     case BEQZ16:
         gen_compute_branch(ctx, op == BNEZ16 ? OPC_BNE : OPC_BEQ, 2,
                            mmreg(uMIPS_RD(ctx->opcode)),
-                           0, SIMM(ctx->opcode, 0, 7) << 1);
+                           0, SIMM(ctx->opcode, 0, 7) << 1, 4);
         break;
     case LI16:
         {
@@ -14444,905 +15809,2643 @@ static void gen_mipsdsp_accinsn(DisasContext *ctx, uint32_t op1, uint32_t op2,
 
 /* End MIPSDSP functions. */
 
-static void decode_opc (CPUMIPSState *env, DisasContext *ctx)
+/* Compact Branches */
+static void gen_compute_compact_branch(DisasContext *ctx, uint32_t opc,
+                                       int rs, int rt, int32_t offset)
 {
-    int32_t offset;
-    int rs, rt, rd, sa;
-    uint32_t op, op1, op2;
-    int16_t imm;
+    int bcond_compute = 0;
+    TCGv t0 = tcg_temp_new();
+    TCGv t1 = tcg_temp_new();
 
-    /* make sure instructions are on a word boundary */
-    if (ctx->pc & 0x3) {
-        env->CP0_BadVAddr = ctx->pc;
-        generate_exception(ctx, EXCP_AdEL);
-        return;
+    if (ctx->hflags & MIPS_HFLAG_BMASK) {
+#ifdef MIPS_DEBUG_DISAS
+        LOG_DISAS("Branch in delay / forbidden slot at PC 0x" TARGET_FMT_lx
+                  "\n", ctx->pc);
+#endif
+        generate_exception(ctx, EXCP_RI);
+        goto out;
     }
 
-    /* Handle blikely not taken case */
-    if ((ctx->hflags & MIPS_HFLAG_BMASK_BASE) == MIPS_HFLAG_BL) {
-        int l1 = gen_new_label();
+    /* Load needed operands and calculate btarget */
+    switch (opc) {
+    /* compact branch */
+    case OPC_BOVC: /* OPC_BEQZALC, OPC_BEQC */
+    case OPC_BNVC: /* OPC_BNEZALC, OPC_BNEC */
+        gen_load_gpr(t0, rs);
+        gen_load_gpr(t1, rt);
+        bcond_compute = 1;
+        ctx->btarget = addr_add(ctx, ctx->pc + 4, offset);
+        if (rs <= rt && rs == 0) {
+            /* OPC_BEQZALC, OPC_BNEZALC */
+            tcg_gen_movi_tl(cpu_gpr[31], ctx->pc + 4);
+        }
+        break;
+    case OPC_BLEZC: /* OPC_BGEZC, OPC_BGEC */
+    case OPC_BGTZC: /* OPC_BLTZC, OPC_BLTC */
+        gen_load_gpr(t0, rs);
+        gen_load_gpr(t1, rt);
+        bcond_compute = 1;
+        ctx->btarget = addr_add(ctx, ctx->pc + 4, offset);
+        break;
+    case OPC_BLEZALC: /* OPC_BGEZALC, OPC_BGEUC */
+    case OPC_BGTZALC: /* OPC_BLTZALC, OPC_BLTUC */
+        if (rs == 0 || rs == rt) {
+            /* OPC_BLEZALC, OPC_BGEZALC */
+            /* OPC_BGTZALC, OPC_BLTZALC */
+            tcg_gen_movi_tl(cpu_gpr[31], ctx->pc + 4);
+        }
+        gen_load_gpr(t0, rs);
+        gen_load_gpr(t1, rt);
+        bcond_compute = 1;
+        ctx->btarget = addr_add(ctx, ctx->pc + 4, offset);
+        break;
+    case OPC_BC:
+    case OPC_BALC:
+        ctx->btarget = addr_add(ctx, ctx->pc + 4, offset);
+        break;
+    case OPC_BEQZC:
+    case OPC_BNEZC:
+        if (rs != 0) {
+            /* OPC_BEQZC, OPC_BNEZC */
+            gen_load_gpr(t0, rs);
+            bcond_compute = 1;
+            ctx->btarget = addr_add(ctx, ctx->pc + 4, offset);
+        } else {
+            /* OPC_JIC, OPC_JIALC */
+            TCGv tbase = tcg_temp_new();
+            TCGv toffset = tcg_temp_new();
 
-        MIPS_DEBUG("blikely condition (" TARGET_FMT_lx ")", ctx->pc + 4);
-        tcg_gen_brcondi_tl(TCG_COND_NE, bcond, 0, l1);
-        tcg_gen_movi_i32(hflags, ctx->hflags & ~MIPS_HFLAG_BMASK);
-        gen_goto_tb(ctx, 1, ctx->pc + 4);
-        gen_set_label(l1);
+            gen_load_gpr(tbase, rt);
+            tcg_gen_movi_tl(toffset, offset);
+            gen_op_addr_add(ctx, btarget, tbase, toffset);
+            tcg_temp_free(tbase);
+            tcg_temp_free(toffset);
+        }
+        break;
+    default:
+        MIPS_INVAL("Compact branch/jump");
+        generate_exception(ctx, EXCP_RI);
+        goto out;
     }
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(ctx->pc);
+    if (bcond_compute == 0) {
+        /* Uncoditional compact branch */
+        switch (opc) {
+        case OPC_JIALC:
+            tcg_gen_movi_tl(cpu_gpr[31], ctx->pc + 4);
+            /* Fallthrough */
+        case OPC_JIC:
+            ctx->hflags |= MIPS_HFLAG_BR;
+            break;
+        case OPC_BALC:
+            tcg_gen_movi_tl(cpu_gpr[31], ctx->pc + 4);
+            /* Fallthrough */
+        case OPC_BC:
+            ctx->hflags |= MIPS_HFLAG_B;
+            break;
+        default:
+            MIPS_INVAL("Compact branch/jump");
+            generate_exception(ctx, EXCP_RI);
+            goto out;
+        }
+
+        /* Generating branch here as compact branches don't have delay slot */
+        gen_branch(ctx, 4);
+    } else {
+        /* Conditional compact branch */
+        int fs = gen_new_label();
+        save_cpu_state(ctx, 0);
+
+        switch (opc) {
+        case OPC_BLEZALC: /* OPC_BGEZALC, OPC_BGEUC */
+            if (rs == 0 && rt != 0) {
+                /* OPC_BLEZALC */
+                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_LE), t1, 0, fs);
+            } else if (rs != 0 && rt != 0 && rs == rt) {
+                /* OPC_BGEZALC */
+                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_GE), t1, 0, fs);
+            } else {
+                /* OPC_BGEUC */
+                tcg_gen_brcond_tl(tcg_invert_cond(TCG_COND_GEU), t0, t1, fs);
+            }
+            break;
+        case OPC_BGTZALC: /* OPC_BLTZALC, OPC_BLTUC */
+            if (rs == 0 && rt != 0) {
+                /* OPC_BGTZALC */
+                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_GT), t1, 0, fs);
+            } else if (rs != 0 && rt != 0 && rs == rt) {
+                /* OPC_BLTZALC */
+                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_LT), t1, 0, fs);
+            } else {
+                /* OPC_BLTUC */
+                tcg_gen_brcond_tl(tcg_invert_cond(TCG_COND_LTU), t0, t1, fs);
+            }
+            break;
+        case OPC_BLEZC: /* OPC_BGEZC, OPC_BGEC */
+            if (rs == 0 && rt != 0) {
+                /* OPC_BLEZC */
+                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_LE), t1, 0, fs);
+            } else if (rs != 0 && rt != 0 && rs == rt) {
+                /* OPC_BGEZC */
+                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_GE), t1, 0, fs);
+            } else {
+                /* OPC_BGEC */
+                tcg_gen_brcond_tl(tcg_invert_cond(TCG_COND_GE), t0, t1, fs);
+            }
+            break;
+        case OPC_BGTZC: /* OPC_BLTZC, OPC_BLTC */
+            if (rs == 0 && rt != 0) {
+                /* OPC_BGTZC */
+                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_GT), t1, 0, fs);
+            } else if (rs != 0 && rt != 0 && rs == rt) {
+                /* OPC_BLTZC */
+                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_LT), t1, 0, fs);
+            } else {
+                /* OPC_BLTC */
+                tcg_gen_brcond_tl(tcg_invert_cond(TCG_COND_LT), t0, t1, fs);
+            }
+            break;
+        case OPC_BOVC: /* OPC_BEQZALC, OPC_BEQC */
+        case OPC_BNVC: /* OPC_BNEZALC, OPC_BNEC */
+            if (rs >= rt) {
+                /* OPC_BOVC, OPC_BNVC */
+                TCGv t2 = tcg_temp_new();
+                TCGv t3 = tcg_temp_new();
+                TCGv t4 = tcg_temp_new();
+                TCGv input_overflow = tcg_temp_new();
+
+                gen_load_gpr(t0, rs);
+                gen_load_gpr(t1, rt);
+                tcg_gen_ext32s_tl(t2, t0);
+                tcg_gen_setcond_tl(TCG_COND_NE, input_overflow, t2, t0);
+                tcg_gen_ext32s_tl(t3, t1);
+                tcg_gen_setcond_tl(TCG_COND_NE, t4, t3, t1);
+                tcg_gen_or_tl(input_overflow, input_overflow, t4);
+
+                tcg_gen_add_tl(t4, t2, t3);
+                tcg_gen_ext32s_tl(t4, t4);
+                tcg_gen_xor_tl(t2, t2, t3);
+                tcg_gen_xor_tl(t3, t4, t3);
+                tcg_gen_andc_tl(t2, t3, t2);
+                tcg_gen_setcondi_tl(TCG_COND_LT, t4, t2, 0);
+                tcg_gen_or_tl(t4, t4, input_overflow);
+                if (opc == OPC_BOVC) {
+                    /* OPC_BOVC */
+                    tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_NE), t4, 0, fs);
+                } else {
+                    /* OPC_BNVC */
+                    tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_EQ), t4, 0, fs);
+                }
+                tcg_temp_free(input_overflow);
+                tcg_temp_free(t4);
+                tcg_temp_free(t3);
+                tcg_temp_free(t2);
+            } else if (rs < rt && rs == 0) {
+                /* OPC_BEQZALC, OPC_BNEZALC */
+                if (opc == OPC_BEQZALC) {
+                    /* OPC_BEQZALC */
+                    tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_EQ), t1, 0, fs);
+                } else {
+                    /* OPC_BNEZALC */
+                    tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_NE), t1, 0, fs);
+                }
+            } else {
+                /* OPC_BEQC, OPC_BNEC */
+                if (opc == OPC_BEQC) {
+                    /* OPC_BEQC */
+                    tcg_gen_brcond_tl(tcg_invert_cond(TCG_COND_EQ), t0, t1, fs);
+                } else {
+                    /* OPC_BNEC */
+                    tcg_gen_brcond_tl(tcg_invert_cond(TCG_COND_NE), t0, t1, fs);
+                }
+            }
+            break;
+        case OPC_BEQZC:
+            tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_EQ), t0, 0, fs);
+            break;
+        case OPC_BNEZC:
+            tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_NE), t0, 0, fs);
+            break;
+        default:
+            MIPS_INVAL("Compact conditional branch/jump");
+            generate_exception(ctx, EXCP_RI);
+            goto out;
+        }
+
+        /* Generating branch here as compact branches don't have delay slot */
+        gen_goto_tb(ctx, 1, ctx->btarget);
+        gen_set_label(fs);
+
+        ctx->hflags |= MIPS_HFLAG_FBNSLOT;
+        MIPS_DEBUG("Compact conditional branch");
     }
 
-    op = MASK_OP_MAJOR(ctx->opcode);
+out:
+    tcg_temp_free(t0);
+    tcg_temp_free(t1);
+}
+
+static void decode_opc_special_r6(CPUMIPSState *env, DisasContext *ctx)
+{
+    int rs, rt, rd, sa;
+    uint32_t op1, op2;
+
     rs = (ctx->opcode >> 21) & 0x1f;
     rt = (ctx->opcode >> 16) & 0x1f;
     rd = (ctx->opcode >> 11) & 0x1f;
     sa = (ctx->opcode >> 6) & 0x1f;
-    imm = (int16_t)ctx->opcode;
-    switch (op) {
-    case OPC_SPECIAL:
-        op1 = MASK_SPECIAL(ctx->opcode);
-        switch (op1) {
-        case OPC_SLL:          /* Shift with immediate */
-        case OPC_SRA:
-            gen_shift_imm(ctx, op1, rd, rt, sa);
-            break;
-        case OPC_SRL:
-            switch ((ctx->opcode >> 21) & 0x1f) {
-            case 1:
-                /* rotr is decoded as srl on non-R2 CPUs */
-                if (ctx->insn_flags & ISA_MIPS32R2) {
-                    op1 = OPC_ROTR;
-                }
-                /* Fallthrough */
-            case 0:
-                gen_shift_imm(ctx, op1, rd, rt, sa);
-                break;
-            default:
-                generate_exception(ctx, EXCP_RI);
-                break;
-            }
+
+    op1 = MASK_SPECIAL(ctx->opcode);
+    switch (op1) {
+    case OPC_LSA:
+        if (rd != 0) {
+            int imm2 = extract32(ctx->opcode, 6, 3);
+            TCGv t0 = tcg_temp_new();
+            TCGv t1 = tcg_temp_new();
+            gen_load_gpr(t0, rs);
+            gen_load_gpr(t1, rt);
+            tcg_gen_shli_tl(t0, t0, imm2 + 1);
+            tcg_gen_add_tl(t0, t0, t1);
+            tcg_gen_ext32s_tl(cpu_gpr[rd], t0);
+            tcg_temp_free(t1);
+            tcg_temp_free(t0);
+        }
+        break;
+    case OPC_MULT ... OPC_DIVU:
+        op2 = MASK_R6_MULDIV(ctx->opcode);
+        switch (op2) {
+        case R6_OPC_MUL:
+        case R6_OPC_MUH:
+        case R6_OPC_MULU:
+        case R6_OPC_MUHU:
+        case R6_OPC_DIV:
+        case R6_OPC_MOD:
+        case R6_OPC_DIVU:
+        case R6_OPC_MODU:
+            gen_r6_muldiv(ctx, op2, rd, rs, rt);
             break;
-        case OPC_MOVN:         /* Conditional move */
-        case OPC_MOVZ:
-            check_insn(ctx, ISA_MIPS4 | ISA_MIPS32 |
-                                 INSN_LOONGSON2E | INSN_LOONGSON2F);
-            gen_cond_move(ctx, op1, rd, rs, rt);
+        default:
+            MIPS_INVAL("special_r6 muldiv");
+            generate_exception(ctx, EXCP_RI);
             break;
-        case OPC_ADD ... OPC_SUBU:
-            gen_arith(ctx, op1, rd, rs, rt);
+        }
+        break;
+    case OPC_SELEQZ:
+    case OPC_SELNEZ:
+        gen_cond_move(ctx, op1, rd, rs, rt);
+        break;
+    case R6_OPC_CLO:
+    case R6_OPC_CLZ:
+        if (rt == 0 && sa == 1) {
+            /* Major opcode and function field is shared with preR6 MFHI/MTHI.
+               We need additionally to check other fields */
+            gen_cl(ctx, op1, rd, rs);
+        } else {
+            generate_exception(ctx, EXCP_RI);
+        }
+        break;
+    case R6_OPC_SDBBP:
+        if (ctx->hflags & MIPS_HFLAG_SBRI) {
+            generate_exception(ctx, EXCP_RI);
+        } else {
+            generate_exception(ctx, EXCP_DBp);
+        }
+        break;
+#if defined(TARGET_MIPS64)
+    case OPC_DLSA:
+        check_mips_64(ctx);
+        if (rd != 0) {
+            int imm2 = extract32(ctx->opcode, 6, 3);
+            TCGv t0 = tcg_temp_new();
+            TCGv t1 = tcg_temp_new();
+            gen_load_gpr(t0, rs);
+            gen_load_gpr(t1, rt);
+            tcg_gen_shli_tl(t0, t0, imm2 + 1);
+            tcg_gen_add_tl(cpu_gpr[rd], t0, t1);
+            tcg_temp_free(t1);
+            tcg_temp_free(t0);
+        }
+        break;
+    case R6_OPC_DCLO:
+    case R6_OPC_DCLZ:
+        if (rt == 0 && sa == 1) {
+            /* Major opcode and function field is shared with preR6 MFHI/MTHI.
+               We need additionally to check other fields */
+            check_mips_64(ctx);
+            gen_cl(ctx, op1, rd, rs);
+        } else {
+            generate_exception(ctx, EXCP_RI);
+        }
+        break;
+    case OPC_DMULT ... OPC_DDIVU:
+        op2 = MASK_R6_MULDIV(ctx->opcode);
+        switch (op2) {
+        case R6_OPC_DMUL:
+        case R6_OPC_DMUH:
+        case R6_OPC_DMULU:
+        case R6_OPC_DMUHU:
+        case R6_OPC_DDIV:
+        case R6_OPC_DMOD:
+        case R6_OPC_DDIVU:
+        case R6_OPC_DMODU:
+            check_mips_64(ctx);
+            gen_r6_muldiv(ctx, op2, rd, rs, rt);
             break;
-        case OPC_SLLV:         /* Shifts */
-        case OPC_SRAV:
-            gen_shift(ctx, op1, rd, rs, rt);
+        default:
+            MIPS_INVAL("special_r6 muldiv");
+            generate_exception(ctx, EXCP_RI);
             break;
-        case OPC_SRLV:
-            switch ((ctx->opcode >> 6) & 0x1f) {
-            case 1:
-                /* rotrv is decoded as srlv on non-R2 CPUs */
-                if (ctx->insn_flags & ISA_MIPS32R2) {
-                    op1 = OPC_ROTRV;
-                }
-                /* Fallthrough */
-            case 0:
-                gen_shift(ctx, op1, rd, rs, rt);
-                break;
-            default:
+        }
+        break;
+#endif
+    default:            /* Invalid */
+        MIPS_INVAL("special_r6");
+        generate_exception(ctx, EXCP_RI);
+        break;
+    }
+}
+
+static void decode_opc_special_legacy(CPUMIPSState *env, DisasContext *ctx)
+{
+    int rs, rt, rd, sa;
+    uint32_t op1;
+
+    rs = (ctx->opcode >> 21) & 0x1f;
+    rt = (ctx->opcode >> 16) & 0x1f;
+    rd = (ctx->opcode >> 11) & 0x1f;
+    sa = (ctx->opcode >> 6) & 0x1f;
+
+    op1 = MASK_SPECIAL(ctx->opcode);
+    switch (op1) {
+    case OPC_MOVN:         /* Conditional move */
+    case OPC_MOVZ:
+        check_insn(ctx, ISA_MIPS4 | ISA_MIPS32 |
+                   INSN_LOONGSON2E | INSN_LOONGSON2F);
+        gen_cond_move(ctx, op1, rd, rs, rt);
+        break;
+    case OPC_MFHI:          /* Move from HI/LO */
+    case OPC_MFLO:
+        gen_HILO(ctx, op1, rs & 3, rd);
+        break;
+    case OPC_MTHI:
+    case OPC_MTLO:          /* Move to HI/LO */
+        gen_HILO(ctx, op1, rd & 3, rs);
+        break;
+    case OPC_MOVCI:
+        check_insn(ctx, ISA_MIPS4 | ISA_MIPS32);
+        if (env->CP0_Config1 & (1 << CP0C1_FP)) {
+            check_cp1_enabled(ctx);
+            gen_movci(ctx, rd, rs, (ctx->opcode >> 18) & 0x7,
+                      (ctx->opcode >> 16) & 1);
+        } else {
+            generate_exception_err(ctx, EXCP_CpU, 1);
+        }
+        break;
+    case OPC_MULT:
+    case OPC_MULTU:
+        if (sa) {
+            check_insn(ctx, INSN_VR54XX);
+            op1 = MASK_MUL_VR54XX(ctx->opcode);
+            gen_mul_vr54xx(ctx, op1, rd, rs, rt);
+        } else {
+            gen_muldiv(ctx, op1, rd & 3, rs, rt);
+        }
+        break;
+    case OPC_DIV:
+    case OPC_DIVU:
+        gen_muldiv(ctx, op1, 0, rs, rt);
+        break;
+#if defined(TARGET_MIPS64)
+    case OPC_DMULT ... OPC_DDIVU:
+        check_insn(ctx, ISA_MIPS3);
+        check_mips_64(ctx);
+        gen_muldiv(ctx, op1, 0, rs, rt);
+        break;
+#endif
+    case OPC_JR:
+        gen_compute_branch(ctx, op1, 4, rs, rd, sa, 4);
+        break;
+    case OPC_SPIM:
+#ifdef MIPS_STRICT_STANDARD
+        MIPS_INVAL("SPIM");
+        generate_exception(ctx, EXCP_RI);
+#else
+        /* Implemented as RI exception for now. */
+        MIPS_INVAL("spim (unofficial)");
+        generate_exception(ctx, EXCP_RI);
+#endif
+        break;
+    default:            /* Invalid */
+        MIPS_INVAL("special_legacy");
+        generate_exception(ctx, EXCP_RI);
+        break;
+    }
+}
+
+static void decode_opc_special(CPUMIPSState *env, DisasContext *ctx)
+{
+    int rs, rt, rd, sa;
+    uint32_t op1;
+
+    rs = (ctx->opcode >> 21) & 0x1f;
+    rt = (ctx->opcode >> 16) & 0x1f;
+    rd = (ctx->opcode >> 11) & 0x1f;
+    sa = (ctx->opcode >> 6) & 0x1f;
+
+    op1 = MASK_SPECIAL(ctx->opcode);
+    switch (op1) {
+    case OPC_SLL:          /* Shift with immediate */
+        if (sa == 5 && rd == 0 &&
+            rs == 0 && rt == 0) { /* PAUSE */
+            if ((ctx->insn_flags & ISA_MIPS32R6) &&
+                (ctx->hflags & MIPS_HFLAG_BMASK)) {
+                MIPS_DEBUG("CTI in delay / forbidden slot");
                 generate_exception(ctx, EXCP_RI);
                 break;
             }
-            break;
-        case OPC_SLT:          /* Set on less than */
-        case OPC_SLTU:
-            gen_slt(ctx, op1, rd, rs, rt);
-            break;
-        case OPC_AND:          /* Logic*/
-        case OPC_OR:
-        case OPC_NOR:
-        case OPC_XOR:
-            gen_logic(ctx, op1, rd, rs, rt);
-            break;
-        case OPC_MULT:
-        case OPC_MULTU:
-            if (sa) {
-                check_insn(ctx, INSN_VR54XX);
-                op1 = MASK_MUL_VR54XX(ctx->opcode);
-                gen_mul_vr54xx(ctx, op1, rd, rs, rt);
-            } else {
-                gen_muldiv(ctx, op1, rd & 3, rs, rt);
+        }
+        /* Fallthrough */
+    case OPC_SRA:
+        gen_shift_imm(ctx, op1, rd, rt, sa);
+        break;
+    case OPC_SRL:
+        switch ((ctx->opcode >> 21) & 0x1f) {
+        case 1:
+            /* rotr is decoded as srl on non-R2 CPUs */
+            if (ctx->insn_flags & ISA_MIPS32R2) {
+                op1 = OPC_ROTR;
             }
+            /* Fallthrough */
+        case 0:
+            gen_shift_imm(ctx, op1, rd, rt, sa);
             break;
-        case OPC_DIV:
-        case OPC_DIVU:
-            gen_muldiv(ctx, op1, 0, rs, rt);
-            break;
-        case OPC_JR ... OPC_JALR:
-            gen_compute_branch(ctx, op1, 4, rs, rd, sa);
-            break;
-        case OPC_TGE ... OPC_TEQ: /* Traps */
-        case OPC_TNE:
-            gen_trap(ctx, op1, rs, rt, -1);
+        default:
+            generate_exception(ctx, EXCP_RI);
             break;
-        case OPC_MFHI:          /* Move from HI/LO */
-        case OPC_MFLO:
-            gen_HILO(ctx, op1, rs & 3, rd);
+        }
+        break;
+    case OPC_ADD ... OPC_SUBU:
+        gen_arith(ctx, op1, rd, rs, rt);
+        break;
+    case OPC_SLLV:         /* Shifts */
+    case OPC_SRAV:
+        gen_shift(ctx, op1, rd, rs, rt);
+        break;
+    case OPC_SRLV:
+        switch ((ctx->opcode >> 6) & 0x1f) {
+        case 1:
+            /* rotrv is decoded as srlv on non-R2 CPUs */
+            if (ctx->insn_flags & ISA_MIPS32R2) {
+                op1 = OPC_ROTRV;
+            }
+            /* Fallthrough */
+        case 0:
+            gen_shift(ctx, op1, rd, rs, rt);
             break;
-        case OPC_MTHI:
-        case OPC_MTLO:          /* Move to HI/LO */
-            gen_HILO(ctx, op1, rd & 3, rs);
+        default:
+            generate_exception(ctx, EXCP_RI);
             break;
-        case OPC_PMON:          /* Pmon entry point, also R4010 selsl */
+        }
+        break;
+    case OPC_SLT:          /* Set on less than */
+    case OPC_SLTU:
+        gen_slt(ctx, op1, rd, rs, rt);
+        break;
+    case OPC_AND:          /* Logic*/
+    case OPC_OR:
+    case OPC_NOR:
+    case OPC_XOR:
+        gen_logic(ctx, op1, rd, rs, rt);
+        break;
+    case OPC_JALR:
+        gen_compute_branch(ctx, op1, 4, rs, rd, sa, 4);
+        break;
+    case OPC_TGE ... OPC_TEQ: /* Traps */
+    case OPC_TNE:
+        gen_trap(ctx, op1, rs, rt, -1);
+        break;
+    case OPC_LSA: /* OPC_PMON */
+        if ((ctx->insn_flags & ISA_MIPS32R6) ||
+            (env->CP0_Config3 & (1 << CP0C3_MSAP))) {
+            decode_opc_special_r6(env, ctx);
+        } else {
+            /* Pmon entry point, also R4010 selsl */
 #ifdef MIPS_STRICT_STANDARD
             MIPS_INVAL("PMON / selsl");
             generate_exception(ctx, EXCP_RI);
 #else
             gen_helper_0e0i(pmon, sa);
 #endif
+        }
+        break;
+    case OPC_SYSCALL:
+        generate_exception(ctx, EXCP_SYSCALL);
+        ctx->bstate = BS_STOP;
+        break;
+    case OPC_BREAK:
+        generate_exception(ctx, EXCP_BREAK);
+        break;
+    case OPC_SYNC:
+        /* Treat as NOP. */
+        break;
+
+#if defined(TARGET_MIPS64)
+        /* MIPS64 specific opcodes */
+    case OPC_DSLL:
+    case OPC_DSRA:
+    case OPC_DSLL32:
+    case OPC_DSRA32:
+        check_insn(ctx, ISA_MIPS3);
+        check_mips_64(ctx);
+        gen_shift_imm(ctx, op1, rd, rt, sa);
+        break;
+    case OPC_DSRL:
+        switch ((ctx->opcode >> 21) & 0x1f) {
+        case 1:
+            /* drotr is decoded as dsrl on non-R2 CPUs */
+            if (ctx->insn_flags & ISA_MIPS32R2) {
+                op1 = OPC_DROTR;
+            }
+            /* Fallthrough */
+        case 0:
+            check_insn(ctx, ISA_MIPS3);
+            check_mips_64(ctx);
+            gen_shift_imm(ctx, op1, rd, rt, sa);
             break;
-        case OPC_SYSCALL:
-            generate_exception(ctx, EXCP_SYSCALL);
-            ctx->bstate = BS_STOP;
+        default:
+            generate_exception(ctx, EXCP_RI);
             break;
-        case OPC_BREAK:
-            generate_exception(ctx, EXCP_BREAK);
+        }
+        break;
+    case OPC_DSRL32:
+        switch ((ctx->opcode >> 21) & 0x1f) {
+        case 1:
+            /* drotr32 is decoded as dsrl32 on non-R2 CPUs */
+            if (ctx->insn_flags & ISA_MIPS32R2) {
+                op1 = OPC_DROTR32;
+            }
+            /* Fallthrough */
+        case 0:
+            check_insn(ctx, ISA_MIPS3);
+            check_mips_64(ctx);
+            gen_shift_imm(ctx, op1, rd, rt, sa);
             break;
-        case OPC_SPIM:
-#ifdef MIPS_STRICT_STANDARD
-            MIPS_INVAL("SPIM");
-            generate_exception(ctx, EXCP_RI);
-#else
-           /* Implemented as RI exception for now. */
-            MIPS_INVAL("spim (unofficial)");
+        default:
             generate_exception(ctx, EXCP_RI);
-#endif
             break;
-        case OPC_SYNC:
-            /* Treat as NOP. */
+        }
+        break;
+    case OPC_DADD ... OPC_DSUBU:
+        check_insn(ctx, ISA_MIPS3);
+        check_mips_64(ctx);
+        gen_arith(ctx, op1, rd, rs, rt);
+        break;
+    case OPC_DSLLV:
+    case OPC_DSRAV:
+        check_insn(ctx, ISA_MIPS3);
+        check_mips_64(ctx);
+        gen_shift(ctx, op1, rd, rs, rt);
+        break;
+    case OPC_DSRLV:
+        switch ((ctx->opcode >> 6) & 0x1f) {
+        case 1:
+            /* drotrv is decoded as dsrlv on non-R2 CPUs */
+            if (ctx->insn_flags & ISA_MIPS32R2) {
+                op1 = OPC_DROTRV;
+            }
+            /* Fallthrough */
+        case 0:
+            check_insn(ctx, ISA_MIPS3);
+            check_mips_64(ctx);
+            gen_shift(ctx, op1, rd, rs, rt);
             break;
+        default:
+            generate_exception(ctx, EXCP_RI);
+            break;
+        }
+        break;
+    case OPC_DLSA:
+        if ((ctx->insn_flags & ISA_MIPS32R6) ||
+            (env->CP0_Config3 & (1 << CP0C3_MSAP))) {
+            decode_opc_special_r6(env, ctx);
+        }
+        break;
+#endif
+    default:
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            decode_opc_special_r6(env, ctx);
+        } else {
+            decode_opc_special_legacy(env, ctx);
+        }
+    }
+}
 
-        case OPC_MOVCI:
-            check_insn(ctx, ISA_MIPS4 | ISA_MIPS32);
-            if (ctx->CP0_Config1 & (1 << CP0C1_FP)) {
-                check_cp1_enabled(ctx);
-                gen_movci(ctx, rd, rs, (ctx->opcode >> 18) & 0x7,
-                          (ctx->opcode >> 16) & 1);
-            } else {
-                generate_exception_err(ctx, EXCP_CpU, 1);
+static void decode_opc_special2_legacy(CPUMIPSState *env, DisasContext *ctx)
+{
+    int rs, rt, rd;
+    uint32_t op1;
+
+    check_insn_opc_removed(ctx, ISA_MIPS32R6);
+
+    rs = (ctx->opcode >> 21) & 0x1f;
+    rt = (ctx->opcode >> 16) & 0x1f;
+    rd = (ctx->opcode >> 11) & 0x1f;
+
+    op1 = MASK_SPECIAL2(ctx->opcode);
+    switch (op1) {
+    case OPC_MADD ... OPC_MADDU: /* Multiply and add/sub */
+    case OPC_MSUB ... OPC_MSUBU:
+        check_insn(ctx, ISA_MIPS32);
+        gen_muldiv(ctx, op1, rd & 3, rs, rt);
+        break;
+    case OPC_MUL:
+        gen_arith(ctx, op1, rd, rs, rt);
+        break;
+    case OPC_DIV_G_2F:
+    case OPC_DIVU_G_2F:
+    case OPC_MULT_G_2F:
+    case OPC_MULTU_G_2F:
+    case OPC_MOD_G_2F:
+    case OPC_MODU_G_2F:
+        check_insn(ctx, INSN_LOONGSON2F);
+        gen_loongson_integer(ctx, op1, rd, rs, rt);
+        break;
+    case OPC_CLO:
+    case OPC_CLZ:
+        check_insn(ctx, ISA_MIPS32);
+        gen_cl(ctx, op1, rd, rs);
+        break;
+    case OPC_SDBBP:
+        /* XXX: not clear which exception should be raised
+         *      when in debug mode...
+         */
+        check_insn(ctx, ISA_MIPS32);
+        if (!(ctx->hflags & MIPS_HFLAG_DM)) {
+            generate_exception(ctx, EXCP_DBp);
+        } else {
+            generate_exception(ctx, EXCP_DBp);
+        }
+        /* Treat as NOP. */
+        break;
+#if defined(TARGET_MIPS64)
+    case OPC_DCLO:
+    case OPC_DCLZ:
+        check_insn(ctx, ISA_MIPS64);
+        check_mips_64(ctx);
+        gen_cl(ctx, op1, rd, rs);
+        break;
+    case OPC_DMULT_G_2F:
+    case OPC_DMULTU_G_2F:
+    case OPC_DDIV_G_2F:
+    case OPC_DDIVU_G_2F:
+    case OPC_DMOD_G_2F:
+    case OPC_DMODU_G_2F:
+        check_insn(ctx, INSN_LOONGSON2F);
+        gen_loongson_integer(ctx, op1, rd, rs, rt);
+        break;
+#endif
+    default:            /* Invalid */
+        MIPS_INVAL("special2_legacy");
+        generate_exception(ctx, EXCP_RI);
+        break;
+    }
+}
+
+static void decode_opc_special3_r6(CPUMIPSState *env, DisasContext *ctx)
+{
+    int rs, rt, rd, sa;
+    uint32_t op1, op2;
+    int16_t imm;
+
+    rs = (ctx->opcode >> 21) & 0x1f;
+    rt = (ctx->opcode >> 16) & 0x1f;
+    rd = (ctx->opcode >> 11) & 0x1f;
+    sa = (ctx->opcode >> 6) & 0x1f;
+    imm = (int16_t)ctx->opcode >> 7;
+
+    op1 = MASK_SPECIAL3(ctx->opcode);
+    switch (op1) {
+    case R6_OPC_PREF:
+        if (rt >= 24) {
+            /* hint codes 24-31 are reserved and signal RI */
+            generate_exception(ctx, EXCP_RI);
+        }
+        /* Treat as NOP. */
+        break;
+    case R6_OPC_CACHE:
+        /* Treat as NOP. */
+        break;
+    case R6_OPC_SC:
+        gen_st_cond(ctx, op1, rt, rs, imm);
+        break;
+    case R6_OPC_LL:
+        gen_ld(ctx, op1, rt, rs, imm);
+        break;
+    case OPC_BSHFL:
+        {
+            if (rd == 0) {
+                /* Treat as NOP. */
+                break;
             }
-            break;
+            TCGv t0 = tcg_temp_new();
+            gen_load_gpr(t0, rt);
 
+            op2 = MASK_BSHFL(ctx->opcode);
+            switch (op2) {
+            case OPC_ALIGN ... OPC_ALIGN_END:
+                sa &= 3;
+                if (sa == 0) {
+                    tcg_gen_mov_tl(cpu_gpr[rd], t0);
+                } else {
+                    TCGv t1 = tcg_temp_new();
+                    TCGv_i64 t2 = tcg_temp_new_i64();
+                    gen_load_gpr(t1, rs);
+                    tcg_gen_concat_tl_i64(t2, t1, t0);
+                    tcg_gen_shri_i64(t2, t2, 8 * (4 - sa));
 #if defined(TARGET_MIPS64)
-       /* MIPS64 specific opcodes */
-        case OPC_DSLL:
-        case OPC_DSRA:
-        case OPC_DSLL32:
-        case OPC_DSRA32:
-            check_insn(ctx, ISA_MIPS3);
-            check_mips_64(ctx);
-            gen_shift_imm(ctx, op1, rd, rt, sa);
-            break;
-        case OPC_DSRL:
-            switch ((ctx->opcode >> 21) & 0x1f) {
-            case 1:
-                /* drotr is decoded as dsrl on non-R2 CPUs */
-                if (ctx->insn_flags & ISA_MIPS32R2) {
-                    op1 = OPC_DROTR;
+                    tcg_gen_ext32s_i64(cpu_gpr[rd], t2);
+#else
+                    tcg_gen_trunc_i64_i32(cpu_gpr[rd], t2);
+#endif
+                    tcg_temp_free_i64(t2);
+                    tcg_temp_free(t1);
                 }
-                /* Fallthrough */
-            case 0:
-                check_insn(ctx, ISA_MIPS3);
-                check_mips_64(ctx);
-                gen_shift_imm(ctx, op1, rd, rt, sa);
                 break;
-            default:
-                generate_exception(ctx, EXCP_RI);
+            case OPC_BITSWAP:
+                gen_helper_bitswap(cpu_gpr[rd], t0);
                 break;
             }
-            break;
-        case OPC_DSRL32:
-            switch ((ctx->opcode >> 21) & 0x1f) {
-            case 1:
-                /* drotr32 is decoded as dsrl32 on non-R2 CPUs */
-                if (ctx->insn_flags & ISA_MIPS32R2) {
-                    op1 = OPC_DROTR32;
+            tcg_temp_free(t0);
+        }
+        break;
+#if defined(TARGET_MIPS64)
+    case R6_OPC_SCD:
+        gen_st_cond(ctx, op1, rt, rs, imm);
+        break;
+    case R6_OPC_LLD:
+        gen_ld(ctx, op1, rt, rs, imm);
+        break;
+    case OPC_DBSHFL:
+        check_mips_64(ctx);
+        {
+            if (rd == 0) {
+                /* Treat as NOP. */
+                break;
+            }
+            TCGv t0 = tcg_temp_new();
+            gen_load_gpr(t0, rt);
+
+            op2 = MASK_DBSHFL(ctx->opcode);
+            switch (op2) {
+            case OPC_DALIGN ... OPC_DALIGN_END:
+                sa &= 7;
+                if (sa == 0) {
+                    tcg_gen_mov_tl(cpu_gpr[rd], t0);
+                } else {
+                    TCGv t1 = tcg_temp_new();
+                    gen_load_gpr(t1, rs);
+                    tcg_gen_shli_tl(t0, t0, 8 * sa);
+                    tcg_gen_shri_tl(t1, t1, 8 * (8 - sa));
+                    tcg_gen_or_tl(cpu_gpr[rd], t1, t0);
+                    tcg_temp_free(t1);
                 }
-                /* Fallthrough */
-            case 0:
-                check_insn(ctx, ISA_MIPS3);
-                check_mips_64(ctx);
-                gen_shift_imm(ctx, op1, rd, rt, sa);
                 break;
-            default:
-                generate_exception(ctx, EXCP_RI);
+            case OPC_DBITSWAP:
+                gen_helper_dbitswap(cpu_gpr[rd], t0);
                 break;
             }
-            break;
-        case OPC_DADD ... OPC_DSUBU:
-            check_insn(ctx, ISA_MIPS3);
-            check_mips_64(ctx);
-            gen_arith(ctx, op1, rd, rs, rt);
-            break;
-        case OPC_DSLLV:
-        case OPC_DSRAV:
-            check_insn(ctx, ISA_MIPS3);
-            check_mips_64(ctx);
-            gen_shift(ctx, op1, rd, rs, rt);
-            break;
-        case OPC_DSRLV:
-            switch ((ctx->opcode >> 6) & 0x1f) {
-            case 1:
-                /* drotrv is decoded as dsrlv on non-R2 CPUs */
-                if (ctx->insn_flags & ISA_MIPS32R2) {
-                    op1 = OPC_DROTRV;
-                }
-                /* Fallthrough */
-            case 0:
-                check_insn(ctx, ISA_MIPS3);
-                check_mips_64(ctx);
-                gen_shift(ctx, op1, rd, rs, rt);
+            tcg_temp_free(t0);
+        }
+        break;
+#endif
+    default:            /* Invalid */
+        MIPS_INVAL("special3_r6");
+        generate_exception(ctx, EXCP_RI);
+        break;
+    }
+}
+
+static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
+{
+    int rs, rt, rd;
+    uint32_t op1, op2;
+
+    rs = (ctx->opcode >> 21) & 0x1f;
+    rt = (ctx->opcode >> 16) & 0x1f;
+    rd = (ctx->opcode >> 11) & 0x1f;
+
+    op1 = MASK_SPECIAL3(ctx->opcode);
+    switch (op1) {
+    case OPC_DIV_G_2E ... OPC_DIVU_G_2E:
+    case OPC_MOD_G_2E ... OPC_MODU_G_2E:
+    case OPC_MULT_G_2E ... OPC_MULTU_G_2E:
+        /* OPC_MULT_G_2E, OPC_ADDUH_QB_DSP, OPC_MUL_PH_DSP have
+         * the same mask and op1. */
+        if ((ctx->insn_flags & ASE_DSPR2) && (op1 == OPC_MULT_G_2E)) {
+            op2 = MASK_ADDUH_QB(ctx->opcode);
+            switch (op2) {
+            case OPC_ADDUH_QB:
+            case OPC_ADDUH_R_QB:
+            case OPC_ADDQH_PH:
+            case OPC_ADDQH_R_PH:
+            case OPC_ADDQH_W:
+            case OPC_ADDQH_R_W:
+            case OPC_SUBUH_QB:
+            case OPC_SUBUH_R_QB:
+            case OPC_SUBQH_PH:
+            case OPC_SUBQH_R_PH:
+            case OPC_SUBQH_W:
+            case OPC_SUBQH_R_W:
+                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
+                break;
+            case OPC_MUL_PH:
+            case OPC_MUL_S_PH:
+            case OPC_MULQ_S_W:
+            case OPC_MULQ_RS_W:
+                gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 1);
                 break;
             default:
+                MIPS_INVAL("MASK ADDUH.QB");
                 generate_exception(ctx, EXCP_RI);
                 break;
             }
-            break;
-        case OPC_DMULT ... OPC_DDIVU:
-            check_insn(ctx, ISA_MIPS3);
-            check_mips_64(ctx);
-            gen_muldiv(ctx, op1, 0, rs, rt);
-            break;
+        } else if (ctx->insn_flags & INSN_LOONGSON2E) {
+            gen_loongson_integer(ctx, op1, rd, rs, rt);
+        } else {
+            generate_exception(ctx, EXCP_RI);
+        }
+        break;
+    case OPC_LX_DSP:
+        op2 = MASK_LX(ctx->opcode);
+        switch (op2) {
+#if defined(TARGET_MIPS64)
+        case OPC_LDX:
 #endif
+        case OPC_LBUX:
+        case OPC_LHX:
+        case OPC_LWX:
+            gen_mipsdsp_ld(ctx, op2, rd, rs, rt);
+            break;
         default:            /* Invalid */
-            MIPS_INVAL("special");
+            MIPS_INVAL("MASK LX");
             generate_exception(ctx, EXCP_RI);
             break;
         }
         break;
-    case OPC_SPECIAL2:
-        op1 = MASK_SPECIAL2(ctx->opcode);
-        switch (op1) {
-        case OPC_MADD ... OPC_MADDU: /* Multiply and add/sub */
-        case OPC_MSUB ... OPC_MSUBU:
-            check_insn(ctx, ISA_MIPS32);
-            gen_muldiv(ctx, op1, rd & 3, rs, rt);
+    case OPC_ABSQ_S_PH_DSP:
+        op2 = MASK_ABSQ_S_PH(ctx->opcode);
+        switch (op2) {
+        case OPC_ABSQ_S_QB:
+        case OPC_ABSQ_S_PH:
+        case OPC_ABSQ_S_W:
+        case OPC_PRECEQ_W_PHL:
+        case OPC_PRECEQ_W_PHR:
+        case OPC_PRECEQU_PH_QBL:
+        case OPC_PRECEQU_PH_QBR:
+        case OPC_PRECEQU_PH_QBLA:
+        case OPC_PRECEQU_PH_QBRA:
+        case OPC_PRECEU_PH_QBL:
+        case OPC_PRECEU_PH_QBR:
+        case OPC_PRECEU_PH_QBLA:
+        case OPC_PRECEU_PH_QBRA:
+            gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
             break;
-        case OPC_MUL:
-            gen_arith(ctx, op1, rd, rs, rt);
+        case OPC_BITREV:
+        case OPC_REPL_QB:
+        case OPC_REPLV_QB:
+        case OPC_REPL_PH:
+        case OPC_REPLV_PH:
+            gen_mipsdsp_bitinsn(ctx, op1, op2, rd, rt);
             break;
-        case OPC_CLO:
-        case OPC_CLZ:
-            check_insn(ctx, ISA_MIPS32);
-            gen_cl(ctx, op1, rd, rs);
+        default:
+            MIPS_INVAL("MASK ABSQ_S.PH");
+            generate_exception(ctx, EXCP_RI);
             break;
-        case OPC_SDBBP:
-            /* XXX: not clear which exception should be raised
-             *      when in debug mode...
-             */
-            check_insn(ctx, ISA_MIPS32);
-            if (!(ctx->hflags & MIPS_HFLAG_DM)) {
-                generate_exception(ctx, EXCP_DBp);
-            } else {
-                generate_exception(ctx, EXCP_DBp);
-            }
-            /* Treat as NOP. */
+        }
+        break;
+    case OPC_ADDU_QB_DSP:
+        op2 = MASK_ADDU_QB(ctx->opcode);
+        switch (op2) {
+        case OPC_ADDQ_PH:
+        case OPC_ADDQ_S_PH:
+        case OPC_ADDQ_S_W:
+        case OPC_ADDU_QB:
+        case OPC_ADDU_S_QB:
+        case OPC_ADDU_PH:
+        case OPC_ADDU_S_PH:
+        case OPC_SUBQ_PH:
+        case OPC_SUBQ_S_PH:
+        case OPC_SUBQ_S_W:
+        case OPC_SUBU_QB:
+        case OPC_SUBU_S_QB:
+        case OPC_SUBU_PH:
+        case OPC_SUBU_S_PH:
+        case OPC_ADDSC:
+        case OPC_ADDWC:
+        case OPC_MODSUB:
+        case OPC_RADDU_W_QB:
+            gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
             break;
-        case OPC_DIV_G_2F:
-        case OPC_DIVU_G_2F:
-        case OPC_MULT_G_2F:
-        case OPC_MULTU_G_2F:
-        case OPC_MOD_G_2F:
-        case OPC_MODU_G_2F:
-            check_insn(ctx, INSN_LOONGSON2F);
-            gen_loongson_integer(ctx, op1, rd, rs, rt);
+        case OPC_MULEU_S_PH_QBL:
+        case OPC_MULEU_S_PH_QBR:
+        case OPC_MULQ_RS_PH:
+        case OPC_MULEQ_S_W_PHL:
+        case OPC_MULEQ_S_W_PHR:
+        case OPC_MULQ_S_PH:
+            gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 1);
             break;
-#if defined(TARGET_MIPS64)
-        case OPC_DCLO:
-        case OPC_DCLZ:
-            check_insn(ctx, ISA_MIPS64);
-            check_mips_64(ctx);
-            gen_cl(ctx, op1, rd, rs);
+        default:            /* Invalid */
+            MIPS_INVAL("MASK ADDU.QB");
+            generate_exception(ctx, EXCP_RI);
             break;
-        case OPC_DMULT_G_2F:
-        case OPC_DMULTU_G_2F:
-        case OPC_DDIV_G_2F:
-        case OPC_DDIVU_G_2F:
-        case OPC_DMOD_G_2F:
-        case OPC_DMODU_G_2F:
-            check_insn(ctx, INSN_LOONGSON2F);
-            gen_loongson_integer(ctx, op1, rd, rs, rt);
+
+        }
+        break;
+    case OPC_CMPU_EQ_QB_DSP:
+        op2 = MASK_CMPU_EQ_QB(ctx->opcode);
+        switch (op2) {
+        case OPC_PRECR_SRA_PH_W:
+        case OPC_PRECR_SRA_R_PH_W:
+            gen_mipsdsp_arith(ctx, op1, op2, rt, rs, rd);
+            break;
+        case OPC_PRECR_QB_PH:
+        case OPC_PRECRQ_QB_PH:
+        case OPC_PRECRQ_PH_W:
+        case OPC_PRECRQ_RS_PH_W:
+        case OPC_PRECRQU_S_QB_PH:
+            gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
+            break;
+        case OPC_CMPU_EQ_QB:
+        case OPC_CMPU_LT_QB:
+        case OPC_CMPU_LE_QB:
+        case OPC_CMP_EQ_PH:
+        case OPC_CMP_LT_PH:
+        case OPC_CMP_LE_PH:
+            gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rd, rs, rt, 0);
+            break;
+        case OPC_CMPGU_EQ_QB:
+        case OPC_CMPGU_LT_QB:
+        case OPC_CMPGU_LE_QB:
+        case OPC_CMPGDU_EQ_QB:
+        case OPC_CMPGDU_LT_QB:
+        case OPC_CMPGDU_LE_QB:
+        case OPC_PICK_QB:
+        case OPC_PICK_PH:
+        case OPC_PACKRL_PH:
+            gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rd, rs, rt, 1);
             break;
-#endif
         default:            /* Invalid */
-            MIPS_INVAL("special2");
+            MIPS_INVAL("MASK CMPU.EQ.QB");
             generate_exception(ctx, EXCP_RI);
             break;
         }
         break;
-    case OPC_SPECIAL3:
-        op1 = MASK_SPECIAL3(ctx->opcode);
-        switch (op1) {
-        case OPC_EXT:
-        case OPC_INS:
-            check_insn(ctx, ISA_MIPS32R2);
-            gen_bitops(ctx, op1, rt, rs, sa, rd);
-            break;
-        case OPC_BSHFL:
-            check_insn(ctx, ISA_MIPS32R2);
-            op2 = MASK_BSHFL(ctx->opcode);
-            gen_bshfl(ctx, op2, rt, rd);
+    case OPC_SHLL_QB_DSP:
+        gen_mipsdsp_shift(ctx, op1, rd, rs, rt);
+        break;
+    case OPC_DPA_W_PH_DSP:
+        op2 = MASK_DPA_W_PH(ctx->opcode);
+        switch (op2) {
+        case OPC_DPAU_H_QBL:
+        case OPC_DPAU_H_QBR:
+        case OPC_DPSU_H_QBL:
+        case OPC_DPSU_H_QBR:
+        case OPC_DPA_W_PH:
+        case OPC_DPAX_W_PH:
+        case OPC_DPAQ_S_W_PH:
+        case OPC_DPAQX_S_W_PH:
+        case OPC_DPAQX_SA_W_PH:
+        case OPC_DPS_W_PH:
+        case OPC_DPSX_W_PH:
+        case OPC_DPSQ_S_W_PH:
+        case OPC_DPSQX_S_W_PH:
+        case OPC_DPSQX_SA_W_PH:
+        case OPC_MULSAQ_S_W_PH:
+        case OPC_DPAQ_SA_L_W:
+        case OPC_DPSQ_SA_L_W:
+        case OPC_MAQ_S_W_PHL:
+        case OPC_MAQ_S_W_PHR:
+        case OPC_MAQ_SA_W_PHL:
+        case OPC_MAQ_SA_W_PHR:
+        case OPC_MULSA_W_PH:
+            gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 0);
             break;
-        case OPC_RDHWR:
-            gen_rdhwr(ctx, rt, rd);
+        default:            /* Invalid */
+            MIPS_INVAL("MASK DPAW.PH");
+            generate_exception(ctx, EXCP_RI);
             break;
-        case OPC_FORK:
-            check_insn(ctx, ASE_MT);
+        }
+        break;
+    case OPC_INSV_DSP:
+        op2 = MASK_INSV(ctx->opcode);
+        switch (op2) {
+        case OPC_INSV:
+            check_dsp(ctx);
             {
-                TCGv t0 = tcg_temp_new();
-                TCGv t1 = tcg_temp_new();
+                TCGv t0, t1;
+
+                if (rt == 0) {
+                    MIPS_DEBUG("NOP");
+                    break;
+                }
+
+                t0 = tcg_temp_new();
+                t1 = tcg_temp_new();
 
                 gen_load_gpr(t0, rt);
                 gen_load_gpr(t1, rs);
-                gen_helper_fork(t0, t1);
+
+                gen_helper_insv(cpu_gpr[rt], cpu_env, t1, t0);
+
                 tcg_temp_free(t0);
                 tcg_temp_free(t1);
+                break;
             }
+        default:            /* Invalid */
+            MIPS_INVAL("MASK INSV");
+            generate_exception(ctx, EXCP_RI);
             break;
-        case OPC_YIELD:
-            check_insn(ctx, ASE_MT);
-            {
-                TCGv t0 = tcg_temp_new();
-
-                save_cpu_state(ctx, 1);
-                gen_load_gpr(t0, rs);
-                gen_helper_yield(t0, cpu_env, t0);
-                gen_store_gpr(t0, rd);
-                tcg_temp_free(t0);
-            }
+        }
+        break;
+    case OPC_APPEND_DSP:
+        gen_mipsdsp_append(env, ctx, op1, rt, rs, rd);
+        break;
+    case OPC_EXTR_W_DSP:
+        op2 = MASK_EXTR_W(ctx->opcode);
+        switch (op2) {
+        case OPC_EXTR_W:
+        case OPC_EXTR_R_W:
+        case OPC_EXTR_RS_W:
+        case OPC_EXTR_S_H:
+        case OPC_EXTRV_S_H:
+        case OPC_EXTRV_W:
+        case OPC_EXTRV_R_W:
+        case OPC_EXTRV_RS_W:
+        case OPC_EXTP:
+        case OPC_EXTPV:
+        case OPC_EXTPDP:
+        case OPC_EXTPDPV:
+            gen_mipsdsp_accinsn(ctx, op1, op2, rt, rs, rd, 1);
             break;
-        case OPC_DIV_G_2E ... OPC_DIVU_G_2E:
-        case OPC_MOD_G_2E ... OPC_MODU_G_2E:
-        case OPC_MULT_G_2E ... OPC_MULTU_G_2E:
-        /* OPC_MULT_G_2E, OPC_ADDUH_QB_DSP, OPC_MUL_PH_DSP have
-         * the same mask and op1. */
-            if ((ctx->insn_flags & ASE_DSPR2) && (op1 == OPC_MULT_G_2E)) {
-                op2 = MASK_ADDUH_QB(ctx->opcode);
-                switch (op2) {
-                case OPC_ADDUH_QB:
-                case OPC_ADDUH_R_QB:
-                case OPC_ADDQH_PH:
-                case OPC_ADDQH_R_PH:
-                case OPC_ADDQH_W:
-                case OPC_ADDQH_R_W:
-                case OPC_SUBUH_QB:
-                case OPC_SUBUH_R_QB:
-                case OPC_SUBQH_PH:
-                case OPC_SUBQH_R_PH:
-                case OPC_SUBQH_W:
-                case OPC_SUBQH_R_W:
-                    gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
-                    break;
-                case OPC_MUL_PH:
-                case OPC_MUL_S_PH:
-                case OPC_MULQ_S_W:
-                case OPC_MULQ_RS_W:
-                    gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 1);
-                    break;
-                default:
-                    MIPS_INVAL("MASK ADDUH.QB");
-                    generate_exception(ctx, EXCP_RI);
-                    break;
-                }
-            } else if (ctx->insn_flags & INSN_LOONGSON2E) {
-                gen_loongson_integer(ctx, op1, rd, rs, rt);
-            } else {
-                generate_exception(ctx, EXCP_RI);
-            }
+        case OPC_RDDSP:
+            gen_mipsdsp_accinsn(ctx, op1, op2, rd, rs, rt, 1);
             break;
-        case OPC_LX_DSP:
-            op2 = MASK_LX(ctx->opcode);
-            switch (op2) {
+        case OPC_SHILO:
+        case OPC_SHILOV:
+        case OPC_MTHLIP:
+        case OPC_WRDSP:
+            gen_mipsdsp_accinsn(ctx, op1, op2, rd, rs, rt, 0);
+            break;
+        default:            /* Invalid */
+            MIPS_INVAL("MASK EXTR.W");
+            generate_exception(ctx, EXCP_RI);
+            break;
+        }
+        break;
 #if defined(TARGET_MIPS64)
-            case OPC_LDX:
-#endif
-            case OPC_LBUX:
-            case OPC_LHX:
-            case OPC_LWX:
-                gen_mipsdsp_ld(ctx, op2, rd, rs, rt);
-                break;
-            default:            /* Invalid */
-                MIPS_INVAL("MASK LX");
-                generate_exception(ctx, EXCP_RI);
-                break;
-            }
+    case OPC_DDIV_G_2E ... OPC_DDIVU_G_2E:
+    case OPC_DMULT_G_2E ... OPC_DMULTU_G_2E:
+    case OPC_DMOD_G_2E ... OPC_DMODU_G_2E:
+        check_insn(ctx, INSN_LOONGSON2E);
+        gen_loongson_integer(ctx, op1, rd, rs, rt);
+        break;
+    case OPC_ABSQ_S_QH_DSP:
+        op2 = MASK_ABSQ_S_QH(ctx->opcode);
+        switch (op2) {
+        case OPC_PRECEQ_L_PWL:
+        case OPC_PRECEQ_L_PWR:
+        case OPC_PRECEQ_PW_QHL:
+        case OPC_PRECEQ_PW_QHR:
+        case OPC_PRECEQ_PW_QHLA:
+        case OPC_PRECEQ_PW_QHRA:
+        case OPC_PRECEQU_QH_OBL:
+        case OPC_PRECEQU_QH_OBR:
+        case OPC_PRECEQU_QH_OBLA:
+        case OPC_PRECEQU_QH_OBRA:
+        case OPC_PRECEU_QH_OBL:
+        case OPC_PRECEU_QH_OBR:
+        case OPC_PRECEU_QH_OBLA:
+        case OPC_PRECEU_QH_OBRA:
+        case OPC_ABSQ_S_OB:
+        case OPC_ABSQ_S_PW:
+        case OPC_ABSQ_S_QH:
+            gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
             break;
-        case OPC_ABSQ_S_PH_DSP:
-            op2 = MASK_ABSQ_S_PH(ctx->opcode);
-            switch (op2) {
-            case OPC_ABSQ_S_QB:
-            case OPC_ABSQ_S_PH:
-            case OPC_ABSQ_S_W:
-            case OPC_PRECEQ_W_PHL:
-            case OPC_PRECEQ_W_PHR:
-            case OPC_PRECEQU_PH_QBL:
-            case OPC_PRECEQU_PH_QBR:
-            case OPC_PRECEQU_PH_QBLA:
-            case OPC_PRECEQU_PH_QBRA:
-            case OPC_PRECEU_PH_QBL:
-            case OPC_PRECEU_PH_QBR:
-            case OPC_PRECEU_PH_QBLA:
-            case OPC_PRECEU_PH_QBRA:
-                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
-                break;
-            case OPC_BITREV:
-            case OPC_REPL_QB:
-            case OPC_REPLV_QB:
-            case OPC_REPL_PH:
-            case OPC_REPLV_PH:
-                gen_mipsdsp_bitinsn(ctx, op1, op2, rd, rt);
-                break;
-            default:
-                MIPS_INVAL("MASK ABSQ_S.PH");
-                generate_exception(ctx, EXCP_RI);
-                break;
-            }
+        case OPC_REPL_OB:
+        case OPC_REPL_PW:
+        case OPC_REPL_QH:
+        case OPC_REPLV_OB:
+        case OPC_REPLV_PW:
+        case OPC_REPLV_QH:
+            gen_mipsdsp_bitinsn(ctx, op1, op2, rd, rt);
             break;
-        case OPC_ADDU_QB_DSP:
-            op2 = MASK_ADDU_QB(ctx->opcode);
-            switch (op2) {
-            case OPC_ADDQ_PH:
-            case OPC_ADDQ_S_PH:
-            case OPC_ADDQ_S_W:
-            case OPC_ADDU_QB:
-            case OPC_ADDU_S_QB:
-            case OPC_ADDU_PH:
-            case OPC_ADDU_S_PH:
-            case OPC_SUBQ_PH:
-            case OPC_SUBQ_S_PH:
-            case OPC_SUBQ_S_W:
-            case OPC_SUBU_QB:
-            case OPC_SUBU_S_QB:
-            case OPC_SUBU_PH:
-            case OPC_SUBU_S_PH:
-            case OPC_ADDSC:
-            case OPC_ADDWC:
-            case OPC_MODSUB:
-            case OPC_RADDU_W_QB:
-                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
-                break;
-            case OPC_MULEU_S_PH_QBL:
-            case OPC_MULEU_S_PH_QBR:
-            case OPC_MULQ_RS_PH:
-            case OPC_MULEQ_S_W_PHL:
-            case OPC_MULEQ_S_W_PHR:
-            case OPC_MULQ_S_PH:
-                gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 1);
-                break;
-            default:            /* Invalid */
-                MIPS_INVAL("MASK ADDU.QB");
-                generate_exception(ctx, EXCP_RI);
-                break;
-
-            }
+        default:            /* Invalid */
+            MIPS_INVAL("MASK ABSQ_S.QH");
+            generate_exception(ctx, EXCP_RI);
             break;
-        case OPC_CMPU_EQ_QB_DSP:
-            op2 = MASK_CMPU_EQ_QB(ctx->opcode);
-            switch (op2) {
-            case OPC_PRECR_SRA_PH_W:
-            case OPC_PRECR_SRA_R_PH_W:
-                gen_mipsdsp_arith(ctx, op1, op2, rt, rs, rd);
-                break;
-            case OPC_PRECR_QB_PH:
-            case OPC_PRECRQ_QB_PH:
-            case OPC_PRECRQ_PH_W:
-            case OPC_PRECRQ_RS_PH_W:
-            case OPC_PRECRQU_S_QB_PH:
-                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
-                break;
-            case OPC_CMPU_EQ_QB:
-            case OPC_CMPU_LT_QB:
-            case OPC_CMPU_LE_QB:
-            case OPC_CMP_EQ_PH:
-            case OPC_CMP_LT_PH:
-            case OPC_CMP_LE_PH:
-                gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rd, rs, rt, 0);
-                break;
-            case OPC_CMPGU_EQ_QB:
-            case OPC_CMPGU_LT_QB:
-            case OPC_CMPGU_LE_QB:
-            case OPC_CMPGDU_EQ_QB:
-            case OPC_CMPGDU_LT_QB:
-            case OPC_CMPGDU_LE_QB:
-            case OPC_PICK_QB:
-            case OPC_PICK_PH:
-            case OPC_PACKRL_PH:
-                gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rd, rs, rt, 1);
-                break;
-            default:            /* Invalid */
-                MIPS_INVAL("MASK CMPU.EQ.QB");
-                generate_exception(ctx, EXCP_RI);
-                break;
-            }
+        }
+        break;
+    case OPC_ADDU_OB_DSP:
+        op2 = MASK_ADDU_OB(ctx->opcode);
+        switch (op2) {
+        case OPC_RADDU_L_OB:
+        case OPC_SUBQ_PW:
+        case OPC_SUBQ_S_PW:
+        case OPC_SUBQ_QH:
+        case OPC_SUBQ_S_QH:
+        case OPC_SUBU_OB:
+        case OPC_SUBU_S_OB:
+        case OPC_SUBU_QH:
+        case OPC_SUBU_S_QH:
+        case OPC_SUBUH_OB:
+        case OPC_SUBUH_R_OB:
+        case OPC_ADDQ_PW:
+        case OPC_ADDQ_S_PW:
+        case OPC_ADDQ_QH:
+        case OPC_ADDQ_S_QH:
+        case OPC_ADDU_OB:
+        case OPC_ADDU_S_OB:
+        case OPC_ADDU_QH:
+        case OPC_ADDU_S_QH:
+        case OPC_ADDUH_OB:
+        case OPC_ADDUH_R_OB:
+            gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
+            break;
+        case OPC_MULEQ_S_PW_QHL:
+        case OPC_MULEQ_S_PW_QHR:
+        case OPC_MULEU_S_QH_OBL:
+        case OPC_MULEU_S_QH_OBR:
+        case OPC_MULQ_RS_QH:
+            gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 1);
             break;
-        case OPC_SHLL_QB_DSP:
-            gen_mipsdsp_shift(ctx, op1, rd, rs, rt);
+        default:            /* Invalid */
+            MIPS_INVAL("MASK ADDU.OB");
+            generate_exception(ctx, EXCP_RI);
             break;
-        case OPC_DPA_W_PH_DSP:
-            op2 = MASK_DPA_W_PH(ctx->opcode);
-            switch (op2) {
-            case OPC_DPAU_H_QBL:
-            case OPC_DPAU_H_QBR:
-            case OPC_DPSU_H_QBL:
-            case OPC_DPSU_H_QBR:
-            case OPC_DPA_W_PH:
-            case OPC_DPAX_W_PH:
-            case OPC_DPAQ_S_W_PH:
-            case OPC_DPAQX_S_W_PH:
-            case OPC_DPAQX_SA_W_PH:
-            case OPC_DPS_W_PH:
-            case OPC_DPSX_W_PH:
-            case OPC_DPSQ_S_W_PH:
-            case OPC_DPSQX_S_W_PH:
-            case OPC_DPSQX_SA_W_PH:
-            case OPC_MULSAQ_S_W_PH:
-            case OPC_DPAQ_SA_L_W:
-            case OPC_DPSQ_SA_L_W:
-            case OPC_MAQ_S_W_PHL:
-            case OPC_MAQ_S_W_PHR:
-            case OPC_MAQ_SA_W_PHL:
-            case OPC_MAQ_SA_W_PHR:
-            case OPC_MULSA_W_PH:
-                gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 0);
-                break;
-            default:            /* Invalid */
-                MIPS_INVAL("MASK DPAW.PH");
-                generate_exception(ctx, EXCP_RI);
-                break;
-            }
+        }
+        break;
+    case OPC_CMPU_EQ_OB_DSP:
+        op2 = MASK_CMPU_EQ_OB(ctx->opcode);
+        switch (op2) {
+        case OPC_PRECR_SRA_QH_PW:
+        case OPC_PRECR_SRA_R_QH_PW:
+            /* Return value is rt. */
+            gen_mipsdsp_arith(ctx, op1, op2, rt, rs, rd);
             break;
-        case OPC_INSV_DSP:
-            op2 = MASK_INSV(ctx->opcode);
-            switch (op2) {
-            case OPC_INSV:
-                check_dsp(ctx);
-                {
-                    TCGv t0, t1;
+        case OPC_PRECR_OB_QH:
+        case OPC_PRECRQ_OB_QH:
+        case OPC_PRECRQ_PW_L:
+        case OPC_PRECRQ_QH_PW:
+        case OPC_PRECRQ_RS_QH_PW:
+        case OPC_PRECRQU_S_OB_QH:
+            gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
+            break;
+        case OPC_CMPU_EQ_OB:
+        case OPC_CMPU_LT_OB:
+        case OPC_CMPU_LE_OB:
+        case OPC_CMP_EQ_QH:
+        case OPC_CMP_LT_QH:
+        case OPC_CMP_LE_QH:
+        case OPC_CMP_EQ_PW:
+        case OPC_CMP_LT_PW:
+        case OPC_CMP_LE_PW:
+            gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rd, rs, rt, 0);
+            break;
+        case OPC_CMPGDU_EQ_OB:
+        case OPC_CMPGDU_LT_OB:
+        case OPC_CMPGDU_LE_OB:
+        case OPC_CMPGU_EQ_OB:
+        case OPC_CMPGU_LT_OB:
+        case OPC_CMPGU_LE_OB:
+        case OPC_PACKRL_PW:
+        case OPC_PICK_OB:
+        case OPC_PICK_PW:
+        case OPC_PICK_QH:
+            gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rd, rs, rt, 1);
+            break;
+        default:            /* Invalid */
+            MIPS_INVAL("MASK CMPU_EQ.OB");
+            generate_exception(ctx, EXCP_RI);
+            break;
+        }
+        break;
+    case OPC_DAPPEND_DSP:
+        gen_mipsdsp_append(env, ctx, op1, rt, rs, rd);
+        break;
+    case OPC_DEXTR_W_DSP:
+        op2 = MASK_DEXTR_W(ctx->opcode);
+        switch (op2) {
+        case OPC_DEXTP:
+        case OPC_DEXTPDP:
+        case OPC_DEXTPDPV:
+        case OPC_DEXTPV:
+        case OPC_DEXTR_L:
+        case OPC_DEXTR_R_L:
+        case OPC_DEXTR_RS_L:
+        case OPC_DEXTR_W:
+        case OPC_DEXTR_R_W:
+        case OPC_DEXTR_RS_W:
+        case OPC_DEXTR_S_H:
+        case OPC_DEXTRV_L:
+        case OPC_DEXTRV_R_L:
+        case OPC_DEXTRV_RS_L:
+        case OPC_DEXTRV_S_H:
+        case OPC_DEXTRV_W:
+        case OPC_DEXTRV_R_W:
+        case OPC_DEXTRV_RS_W:
+            gen_mipsdsp_accinsn(ctx, op1, op2, rt, rs, rd, 1);
+            break;
+        case OPC_DMTHLIP:
+        case OPC_DSHILO:
+        case OPC_DSHILOV:
+            gen_mipsdsp_accinsn(ctx, op1, op2, rd, rs, rt, 0);
+            break;
+        default:            /* Invalid */
+            MIPS_INVAL("MASK EXTR.W");
+            generate_exception(ctx, EXCP_RI);
+            break;
+        }
+        break;
+    case OPC_DPAQ_W_QH_DSP:
+        op2 = MASK_DPAQ_W_QH(ctx->opcode);
+        switch (op2) {
+        case OPC_DPAU_H_OBL:
+        case OPC_DPAU_H_OBR:
+        case OPC_DPSU_H_OBL:
+        case OPC_DPSU_H_OBR:
+        case OPC_DPA_W_QH:
+        case OPC_DPAQ_S_W_QH:
+        case OPC_DPS_W_QH:
+        case OPC_DPSQ_S_W_QH:
+        case OPC_MULSAQ_S_W_QH:
+        case OPC_DPAQ_SA_L_PW:
+        case OPC_DPSQ_SA_L_PW:
+        case OPC_MULSAQ_S_L_PW:
+            gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 0);
+            break;
+        case OPC_MAQ_S_W_QHLL:
+        case OPC_MAQ_S_W_QHLR:
+        case OPC_MAQ_S_W_QHRL:
+        case OPC_MAQ_S_W_QHRR:
+        case OPC_MAQ_SA_W_QHLL:
+        case OPC_MAQ_SA_W_QHLR:
+        case OPC_MAQ_SA_W_QHRL:
+        case OPC_MAQ_SA_W_QHRR:
+        case OPC_MAQ_S_L_PWL:
+        case OPC_MAQ_S_L_PWR:
+        case OPC_DMADD:
+        case OPC_DMADDU:
+        case OPC_DMSUB:
+        case OPC_DMSUBU:
+            gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 0);
+            break;
+        default:            /* Invalid */
+            MIPS_INVAL("MASK DPAQ.W.QH");
+            generate_exception(ctx, EXCP_RI);
+            break;
+        }
+        break;
+    case OPC_DINSV_DSP:
+        op2 = MASK_INSV(ctx->opcode);
+        switch (op2) {
+        case OPC_DINSV:
+        {
+            TCGv t0, t1;
 
-                    if (rt == 0) {
-                        MIPS_DEBUG("NOP");
-                        break;
-                    }
+            if (rt == 0) {
+                MIPS_DEBUG("NOP");
+                break;
+            }
+            check_dsp(ctx);
 
-                    t0 = tcg_temp_new();
-                    t1 = tcg_temp_new();
+            t0 = tcg_temp_new();
+            t1 = tcg_temp_new();
 
-                    gen_load_gpr(t0, rt);
-                    gen_load_gpr(t1, rs);
+            gen_load_gpr(t0, rt);
+            gen_load_gpr(t1, rs);
 
-                    gen_helper_insv(cpu_gpr[rt], cpu_env, t1, t0);
+            gen_helper_dinsv(cpu_gpr[rt], cpu_env, t1, t0);
 
-                    tcg_temp_free(t0);
-                    tcg_temp_free(t1);
-                    break;
-                }
-            default:            /* Invalid */
-                MIPS_INVAL("MASK INSV");
-                generate_exception(ctx, EXCP_RI);
-                break;
-            }
+            tcg_temp_free(t0);
+            tcg_temp_free(t1);
             break;
-        case OPC_APPEND_DSP:
-            gen_mipsdsp_append(env, ctx, op1, rt, rs, rd);
+        }
+        default:            /* Invalid */
+            MIPS_INVAL("MASK DINSV");
+            generate_exception(ctx, EXCP_RI);
             break;
-        case OPC_EXTR_W_DSP:
-            op2 = MASK_EXTR_W(ctx->opcode);
-            switch (op2) {
-            case OPC_EXTR_W:
-            case OPC_EXTR_R_W:
-            case OPC_EXTR_RS_W:
-            case OPC_EXTR_S_H:
-            case OPC_EXTRV_S_H:
-            case OPC_EXTRV_W:
-            case OPC_EXTRV_R_W:
-            case OPC_EXTRV_RS_W:
-            case OPC_EXTP:
-            case OPC_EXTPV:
-            case OPC_EXTPDP:
-            case OPC_EXTPDPV:
-                gen_mipsdsp_accinsn(ctx, op1, op2, rt, rs, rd, 1);
-                break;
-            case OPC_RDDSP:
-                gen_mipsdsp_accinsn(ctx, op1, op2, rd, rs, rt, 1);
-                break;
-            case OPC_SHILO:
-            case OPC_SHILOV:
-            case OPC_MTHLIP:
-            case OPC_WRDSP:
-                gen_mipsdsp_accinsn(ctx, op1, op2, rd, rs, rt, 0);
-                break;
-            default:            /* Invalid */
-                MIPS_INVAL("MASK EXTR.W");
-                generate_exception(ctx, EXCP_RI);
-                break;
-            }
+        }
+        break;
+    case OPC_SHLL_OB_DSP:
+        gen_mipsdsp_shift(ctx, op1, rd, rs, rt);
+        break;
+#endif
+    default:            /* Invalid */
+        MIPS_INVAL("special3_legacy");
+        generate_exception(ctx, EXCP_RI);
+        break;
+    }
+}
+
+static void decode_opc_special3(CPUMIPSState *env, DisasContext *ctx)
+{
+    int rs, rt, rd, sa;
+    uint32_t op1, op2;
+
+    rs = (ctx->opcode >> 21) & 0x1f;
+    rt = (ctx->opcode >> 16) & 0x1f;
+    rd = (ctx->opcode >> 11) & 0x1f;
+    sa = (ctx->opcode >> 6) & 0x1f;
+
+    op1 = MASK_SPECIAL3(ctx->opcode);
+    switch (op1) {
+    case OPC_EXT:
+    case OPC_INS:
+        check_insn(ctx, ISA_MIPS32R2);
+        gen_bitops(ctx, op1, rt, rs, sa, rd);
+        break;
+    case OPC_BSHFL:
+        op2 = MASK_BSHFL(ctx->opcode);
+        switch (op2) {
+        case OPC_ALIGN ... OPC_ALIGN_END:
+        case OPC_BITSWAP:
+            check_insn(ctx, ISA_MIPS32R6);
+            decode_opc_special3_r6(env, ctx);
             break;
+        default:
+            check_insn(ctx, ISA_MIPS32R2);
+            gen_bshfl(ctx, op2, rt, rd);
+            break;
+        }
+        break;
 #if defined(TARGET_MIPS64)
-        case OPC_DEXTM ... OPC_DEXT:
-        case OPC_DINSM ... OPC_DINS:
-            check_insn(ctx, ISA_MIPS64R2);
-            check_mips_64(ctx);
-            gen_bitops(ctx, op1, rt, rs, sa, rd);
+    case OPC_DEXTM ... OPC_DEXT:
+    case OPC_DINSM ... OPC_DINS:
+        check_insn(ctx, ISA_MIPS64R2);
+        check_mips_64(ctx);
+        gen_bitops(ctx, op1, rt, rs, sa, rd);
+        break;
+    case OPC_DBSHFL:
+        op2 = MASK_DBSHFL(ctx->opcode);
+        switch (op2) {
+        case OPC_DALIGN ... OPC_DALIGN_END:
+        case OPC_DBITSWAP:
+            check_insn(ctx, ISA_MIPS32R6);
+            decode_opc_special3_r6(env, ctx);
             break;
-        case OPC_DBSHFL:
+        default:
             check_insn(ctx, ISA_MIPS64R2);
             check_mips_64(ctx);
             op2 = MASK_DBSHFL(ctx->opcode);
             gen_bshfl(ctx, op2, rt, rd);
             break;
-        case OPC_DDIV_G_2E ... OPC_DDIVU_G_2E:
-        case OPC_DMULT_G_2E ... OPC_DMULTU_G_2E:
-        case OPC_DMOD_G_2E ... OPC_DMODU_G_2E:
-            check_insn(ctx, INSN_LOONGSON2E);
-            gen_loongson_integer(ctx, op1, rd, rs, rt);
-            break;
-        case OPC_ABSQ_S_QH_DSP:
-            op2 = MASK_ABSQ_S_QH(ctx->opcode);
-            switch (op2) {
-            case OPC_PRECEQ_L_PWL:
-            case OPC_PRECEQ_L_PWR:
-            case OPC_PRECEQ_PW_QHL:
-            case OPC_PRECEQ_PW_QHR:
-            case OPC_PRECEQ_PW_QHLA:
-            case OPC_PRECEQ_PW_QHRA:
-            case OPC_PRECEQU_QH_OBL:
-            case OPC_PRECEQU_QH_OBR:
-            case OPC_PRECEQU_QH_OBLA:
-            case OPC_PRECEQU_QH_OBRA:
-            case OPC_PRECEU_QH_OBL:
-            case OPC_PRECEU_QH_OBR:
-            case OPC_PRECEU_QH_OBLA:
-            case OPC_PRECEU_QH_OBRA:
-            case OPC_ABSQ_S_OB:
-            case OPC_ABSQ_S_PW:
-            case OPC_ABSQ_S_QH:
-                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
-                break;
-            case OPC_REPL_OB:
-            case OPC_REPL_PW:
-            case OPC_REPL_QH:
-            case OPC_REPLV_OB:
-            case OPC_REPLV_PW:
-            case OPC_REPLV_QH:
-                gen_mipsdsp_bitinsn(ctx, op1, op2, rd, rt);
-                break;
-            default:            /* Invalid */
-                MIPS_INVAL("MASK ABSQ_S.QH");
+        }
+        break;
+#endif
+    case OPC_RDHWR:
+        gen_rdhwr(ctx, rt, rd);
+        break;
+    case OPC_FORK:
+        check_insn(ctx, ASE_MT);
+        {
+            TCGv t0 = tcg_temp_new();
+            TCGv t1 = tcg_temp_new();
+
+            gen_load_gpr(t0, rt);
+            gen_load_gpr(t1, rs);
+            gen_helper_fork(t0, t1);
+            tcg_temp_free(t0);
+            tcg_temp_free(t1);
+        }
+        break;
+    case OPC_YIELD:
+        check_insn(ctx, ASE_MT);
+        {
+            TCGv t0 = tcg_temp_new();
+
+            save_cpu_state(ctx, 1);
+            gen_load_gpr(t0, rs);
+            gen_helper_yield(t0, cpu_env, t0);
+            gen_store_gpr(t0, rd);
+            tcg_temp_free(t0);
+        }
+        break;
+    default:
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            decode_opc_special3_r6(env, ctx);
+        } else {
+            decode_opc_special3_legacy(env, ctx);
+        }
+    }
+}
+
+/* MIPS SIMD Architecture (MSA)  */
+static inline int check_msa_access(DisasContext *ctx)
+{
+    if (unlikely((ctx->hflags & MIPS_HFLAG_FPU) &&
+                 !(ctx->hflags & MIPS_HFLAG_F64))) {
+        generate_exception(ctx, EXCP_RI);
+        return 0;
+    }
+
+    if (unlikely(!(ctx->hflags & MIPS_HFLAG_MSA))) {
+        if (ctx->insn_flags & ASE_MSA) {
+            generate_exception(ctx, EXCP_MSADIS);
+            return 0;
+        } else {
+            generate_exception(ctx, EXCP_RI);
+            return 0;
+        }
+    }
+    return 1;
+}
+
+static void gen_check_zero_element(TCGv tresult, uint8_t df, uint8_t wt)
+{
+    /* generates tcg ops to check if any element is 0 */
+    /* Note this function only works with MSA_WRLEN = 128 */
+    uint64_t eval_zero_or_big = 0;
+    uint64_t eval_big = 0;
+    TCGv_i64 t0 = tcg_temp_new_i64();
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    switch (df) {
+    case DF_BYTE:
+        eval_zero_or_big = 0x0101010101010101ULL;
+        eval_big = 0x8080808080808080ULL;
+        break;
+    case DF_HALF:
+        eval_zero_or_big = 0x0001000100010001ULL;
+        eval_big = 0x8000800080008000ULL;
+        break;
+    case DF_WORD:
+        eval_zero_or_big = 0x0000000100000001ULL;
+        eval_big = 0x8000000080000000ULL;
+        break;
+    case DF_DOUBLE:
+        eval_zero_or_big = 0x0000000000000001ULL;
+        eval_big = 0x8000000000000000ULL;
+        break;
+    }
+    tcg_gen_subi_i64(t0, msa_wr_d[wt<<1], eval_zero_or_big);
+    tcg_gen_andc_i64(t0, t0, msa_wr_d[wt<<1]);
+    tcg_gen_andi_i64(t0, t0, eval_big);
+    tcg_gen_subi_i64(t1, msa_wr_d[(wt<<1)+1], eval_zero_or_big);
+    tcg_gen_andc_i64(t1, t1, msa_wr_d[(wt<<1)+1]);
+    tcg_gen_andi_i64(t1, t1, eval_big);
+    tcg_gen_or_i64(t0, t0, t1);
+    /* if all bits are zero then all elements are not zero */
+    /* if some bit is non-zero then some element is zero */
+    tcg_gen_setcondi_i64(TCG_COND_NE, t0, t0, 0);
+    tcg_gen_trunc_i64_tl(tresult, t0);
+    tcg_temp_free_i64(t0);
+    tcg_temp_free_i64(t1);
+}
+
+static void gen_msa_branch(CPUMIPSState *env, DisasContext *ctx, uint32_t op1)
+{
+    uint8_t df = (ctx->opcode >> 21) & 0x3;
+    uint8_t wt = (ctx->opcode >> 16) & 0x1f;
+    int64_t s16 = (int16_t)ctx->opcode;
+
+    check_msa_access(ctx);
+
+    if (ctx->insn_flags & ISA_MIPS32R6 && ctx->hflags & MIPS_HFLAG_BMASK) {
+        MIPS_DEBUG("CTI in delay / forbidden slot");
+        generate_exception(ctx, EXCP_RI);
+        return;
+    }
+    switch (op1) {
+    case OPC_BZ_V:
+    case OPC_BNZ_V:
+        {
+            TCGv_i64 t0 = tcg_temp_new_i64();
+            tcg_gen_or_i64(t0, msa_wr_d[wt<<1], msa_wr_d[(wt<<1)+1]);
+            tcg_gen_setcondi_i64((op1 == OPC_BZ_V) ?
+                    TCG_COND_EQ : TCG_COND_NE, t0, t0, 0);
+            tcg_gen_trunc_i64_tl(bcond, t0);
+            tcg_temp_free_i64(t0);
+        }
+        break;
+    case OPC_BZ_B:
+    case OPC_BZ_H:
+    case OPC_BZ_W:
+    case OPC_BZ_D:
+        gen_check_zero_element(bcond, df, wt);
+        break;
+    case OPC_BNZ_B:
+    case OPC_BNZ_H:
+    case OPC_BNZ_W:
+    case OPC_BNZ_D:
+        gen_check_zero_element(bcond, df, wt);
+        tcg_gen_setcondi_tl(TCG_COND_EQ, bcond, bcond, 0);
+        break;
+    }
+
+    ctx->btarget = ctx->pc + (s16 << 2) + 4;
+
+    ctx->hflags |= MIPS_HFLAG_BC;
+    ctx->hflags |= MIPS_HFLAG_BDS32;
+}
+
+static void gen_msa_i8(CPUMIPSState *env, DisasContext *ctx)
+{
+#define MASK_MSA_I8(op)    (MASK_MSA_MINOR(op) | (op & (0x03 << 24)))
+    uint8_t i8 = (ctx->opcode >> 16) & 0xff;
+    uint8_t ws = (ctx->opcode >> 11) & 0x1f;
+    uint8_t wd = (ctx->opcode >> 6) & 0x1f;
+
+    TCGv_i32 twd = tcg_const_i32(wd);
+    TCGv_i32 tws = tcg_const_i32(ws);
+    TCGv_i32 ti8 = tcg_const_i32(i8);
+
+    switch (MASK_MSA_I8(ctx->opcode)) {
+    case OPC_ANDI_B:
+        gen_helper_msa_andi_b(cpu_env, twd, tws, ti8);
+        break;
+    case OPC_ORI_B:
+        gen_helper_msa_ori_b(cpu_env, twd, tws, ti8);
+        break;
+    case OPC_NORI_B:
+        gen_helper_msa_nori_b(cpu_env, twd, tws, ti8);
+        break;
+    case OPC_XORI_B:
+        gen_helper_msa_xori_b(cpu_env, twd, tws, ti8);
+        break;
+    case OPC_BMNZI_B:
+        gen_helper_msa_bmnzi_b(cpu_env, twd, tws, ti8);
+        break;
+    case OPC_BMZI_B:
+        gen_helper_msa_bmzi_b(cpu_env, twd, tws, ti8);
+        break;
+    case OPC_BSELI_B:
+        gen_helper_msa_bseli_b(cpu_env, twd, tws, ti8);
+        break;
+    case OPC_SHF_B:
+    case OPC_SHF_H:
+    case OPC_SHF_W:
+        {
+            uint8_t df = (ctx->opcode >> 24) & 0x3;
+            if (df == DF_DOUBLE) {
                 generate_exception(ctx, EXCP_RI);
-                break;
+            } else {
+                TCGv_i32 tdf = tcg_const_i32(df);
+                gen_helper_msa_shf_df(cpu_env, tdf, twd, tws, ti8);
+                tcg_temp_free_i32(tdf);
             }
+        }
+        break;
+    default:
+        MIPS_INVAL("MSA instruction");
+        generate_exception(ctx, EXCP_RI);
+        break;
+    }
+
+    tcg_temp_free_i32(twd);
+    tcg_temp_free_i32(tws);
+    tcg_temp_free_i32(ti8);
+}
+
+static void gen_msa_i5(CPUMIPSState *env, DisasContext *ctx)
+{
+#define MASK_MSA_I5(op)    (MASK_MSA_MINOR(op) | (op & (0x7 << 23)))
+    uint8_t df = (ctx->opcode >> 21) & 0x3;
+    int8_t s5 = (int8_t) sextract32(ctx->opcode, 16, 5);
+    uint8_t u5 = (ctx->opcode >> 16) & 0x1f;
+    uint8_t ws = (ctx->opcode >> 11) & 0x1f;
+    uint8_t wd = (ctx->opcode >> 6) & 0x1f;
+
+    TCGv_i32 tdf = tcg_const_i32(df);
+    TCGv_i32 twd = tcg_const_i32(wd);
+    TCGv_i32 tws = tcg_const_i32(ws);
+    TCGv_i32 timm = tcg_temp_new_i32();
+    tcg_gen_movi_i32(timm, u5);
+
+    switch (MASK_MSA_I5(ctx->opcode)) {
+    case OPC_ADDVI_df:
+        gen_helper_msa_addvi_df(cpu_env, tdf, twd, tws, timm);
+        break;
+    case OPC_SUBVI_df:
+        gen_helper_msa_subvi_df(cpu_env, tdf, twd, tws, timm);
+        break;
+    case OPC_MAXI_S_df:
+        tcg_gen_movi_i32(timm, s5);
+        gen_helper_msa_maxi_s_df(cpu_env, tdf, twd, tws, timm);
+        break;
+    case OPC_MAXI_U_df:
+        gen_helper_msa_maxi_u_df(cpu_env, tdf, twd, tws, timm);
+        break;
+    case OPC_MINI_S_df:
+        tcg_gen_movi_i32(timm, s5);
+        gen_helper_msa_mini_s_df(cpu_env, tdf, twd, tws, timm);
+        break;
+    case OPC_MINI_U_df:
+        gen_helper_msa_mini_u_df(cpu_env, tdf, twd, tws, timm);
+        break;
+    case OPC_CEQI_df:
+        tcg_gen_movi_i32(timm, s5);
+        gen_helper_msa_ceqi_df(cpu_env, tdf, twd, tws, timm);
+        break;
+    case OPC_CLTI_S_df:
+        tcg_gen_movi_i32(timm, s5);
+        gen_helper_msa_clti_s_df(cpu_env, tdf, twd, tws, timm);
+        break;
+    case OPC_CLTI_U_df:
+        gen_helper_msa_clti_u_df(cpu_env, tdf, twd, tws, timm);
+        break;
+    case OPC_CLEI_S_df:
+        tcg_gen_movi_i32(timm, s5);
+        gen_helper_msa_clei_s_df(cpu_env, tdf, twd, tws, timm);
+        break;
+    case OPC_CLEI_U_df:
+        gen_helper_msa_clei_u_df(cpu_env, tdf, twd, tws, timm);
+        break;
+    case OPC_LDI_df:
+        {
+            int32_t s10 = sextract32(ctx->opcode, 11, 10);
+            tcg_gen_movi_i32(timm, s10);
+            gen_helper_msa_ldi_df(cpu_env, tdf, twd, timm);
+        }
+        break;
+    default:
+        MIPS_INVAL("MSA instruction");
+        generate_exception(ctx, EXCP_RI);
+        break;
+    }
+
+    tcg_temp_free_i32(tdf);
+    tcg_temp_free_i32(twd);
+    tcg_temp_free_i32(tws);
+    tcg_temp_free_i32(timm);
+}
+
+static void gen_msa_bit(CPUMIPSState *env, DisasContext *ctx)
+{
+#define MASK_MSA_BIT(op)    (MASK_MSA_MINOR(op) | (op & (0x7 << 23)))
+    uint8_t dfm = (ctx->opcode >> 16) & 0x7f;
+    uint32_t df = 0, m = 0;
+    uint8_t ws = (ctx->opcode >> 11) & 0x1f;
+    uint8_t wd = (ctx->opcode >> 6) & 0x1f;
+
+    TCGv_i32 tdf;
+    TCGv_i32 tm;
+    TCGv_i32 twd;
+    TCGv_i32 tws;
+
+    if ((dfm & 0x40) == 0x00) {
+        m = dfm & 0x3f;
+        df = DF_DOUBLE;
+    } else if ((dfm & 0x60) == 0x40) {
+        m = dfm & 0x1f;
+        df = DF_WORD;
+    } else if ((dfm & 0x70) == 0x60) {
+        m = dfm & 0x0f;
+        df = DF_HALF;
+    } else if ((dfm & 0x78) == 0x70) {
+        m = dfm & 0x7;
+        df = DF_BYTE;
+    } else {
+        generate_exception(ctx, EXCP_RI);
+        return;
+    }
+
+    tdf = tcg_const_i32(df);
+    tm  = tcg_const_i32(m);
+    twd = tcg_const_i32(wd);
+    tws = tcg_const_i32(ws);
+
+    switch (MASK_MSA_BIT(ctx->opcode)) {
+    case OPC_SLLI_df:
+        gen_helper_msa_slli_df(cpu_env, tdf, twd, tws, tm);
+        break;
+    case OPC_SRAI_df:
+        gen_helper_msa_srai_df(cpu_env, tdf, twd, tws, tm);
+        break;
+    case OPC_SRLI_df:
+        gen_helper_msa_srli_df(cpu_env, tdf, twd, tws, tm);
+        break;
+    case OPC_BCLRI_df:
+        gen_helper_msa_bclri_df(cpu_env, tdf, twd, tws, tm);
+        break;
+    case OPC_BSETI_df:
+        gen_helper_msa_bseti_df(cpu_env, tdf, twd, tws, tm);
+        break;
+    case OPC_BNEGI_df:
+        gen_helper_msa_bnegi_df(cpu_env, tdf, twd, tws, tm);
+        break;
+    case OPC_BINSLI_df:
+        gen_helper_msa_binsli_df(cpu_env, tdf, twd, tws, tm);
+        break;
+    case OPC_BINSRI_df:
+        gen_helper_msa_binsri_df(cpu_env, tdf, twd, tws, tm);
+        break;
+    case OPC_SAT_S_df:
+        gen_helper_msa_sat_s_df(cpu_env, tdf, twd, tws, tm);
+        break;
+    case OPC_SAT_U_df:
+        gen_helper_msa_sat_u_df(cpu_env, tdf, twd, tws, tm);
+        break;
+    case OPC_SRARI_df:
+        gen_helper_msa_srari_df(cpu_env, tdf, twd, tws, tm);
+        break;
+    case OPC_SRLRI_df:
+        gen_helper_msa_srlri_df(cpu_env, tdf, twd, tws, tm);
+        break;
+    default:
+        MIPS_INVAL("MSA instruction");
+        generate_exception(ctx, EXCP_RI);
+        break;
+    }
+
+    tcg_temp_free_i32(tdf);
+    tcg_temp_free_i32(tm);
+    tcg_temp_free_i32(twd);
+    tcg_temp_free_i32(tws);
+}
+
+static void gen_msa_3r(CPUMIPSState *env, DisasContext *ctx)
+{
+#define MASK_MSA_3R(op)    (MASK_MSA_MINOR(op) | (op & (0x7 << 23)))
+    uint8_t df = (ctx->opcode >> 21) & 0x3;
+    uint8_t wt = (ctx->opcode >> 16) & 0x1f;
+    uint8_t ws = (ctx->opcode >> 11) & 0x1f;
+    uint8_t wd = (ctx->opcode >> 6) & 0x1f;
+
+    TCGv_i32 tdf = tcg_const_i32(df);
+    TCGv_i32 twd = tcg_const_i32(wd);
+    TCGv_i32 tws = tcg_const_i32(ws);
+    TCGv_i32 twt = tcg_const_i32(wt);
+
+    switch (MASK_MSA_3R(ctx->opcode)) {
+    case OPC_SLL_df:
+        gen_helper_msa_sll_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_ADDV_df:
+        gen_helper_msa_addv_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_CEQ_df:
+        gen_helper_msa_ceq_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_ADD_A_df:
+        gen_helper_msa_add_a_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_SUBS_S_df:
+        gen_helper_msa_subs_s_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_MULV_df:
+        gen_helper_msa_mulv_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_SLD_df:
+        gen_helper_msa_sld_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_VSHF_df:
+        gen_helper_msa_vshf_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_SRA_df:
+        gen_helper_msa_sra_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_SUBV_df:
+        gen_helper_msa_subv_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_ADDS_A_df:
+        gen_helper_msa_adds_a_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_SUBS_U_df:
+        gen_helper_msa_subs_u_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_MADDV_df:
+        gen_helper_msa_maddv_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_SPLAT_df:
+        gen_helper_msa_splat_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_SRAR_df:
+        gen_helper_msa_srar_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_SRL_df:
+        gen_helper_msa_srl_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_MAX_S_df:
+        gen_helper_msa_max_s_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_CLT_S_df:
+        gen_helper_msa_clt_s_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_ADDS_S_df:
+        gen_helper_msa_adds_s_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_SUBSUS_U_df:
+        gen_helper_msa_subsus_u_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_MSUBV_df:
+        gen_helper_msa_msubv_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_PCKEV_df:
+        gen_helper_msa_pckev_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_SRLR_df:
+        gen_helper_msa_srlr_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_BCLR_df:
+        gen_helper_msa_bclr_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_MAX_U_df:
+        gen_helper_msa_max_u_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_CLT_U_df:
+        gen_helper_msa_clt_u_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_ADDS_U_df:
+        gen_helper_msa_adds_u_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_SUBSUU_S_df:
+        gen_helper_msa_subsuu_s_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_PCKOD_df:
+        gen_helper_msa_pckod_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_BSET_df:
+        gen_helper_msa_bset_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_MIN_S_df:
+        gen_helper_msa_min_s_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_CLE_S_df:
+        gen_helper_msa_cle_s_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_AVE_S_df:
+        gen_helper_msa_ave_s_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_ASUB_S_df:
+        gen_helper_msa_asub_s_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_DIV_S_df:
+        gen_helper_msa_div_s_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_ILVL_df:
+        gen_helper_msa_ilvl_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_BNEG_df:
+        gen_helper_msa_bneg_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_MIN_U_df:
+        gen_helper_msa_min_u_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_CLE_U_df:
+        gen_helper_msa_cle_u_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_AVE_U_df:
+        gen_helper_msa_ave_u_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_ASUB_U_df:
+        gen_helper_msa_asub_u_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_DIV_U_df:
+        gen_helper_msa_div_u_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_ILVR_df:
+        gen_helper_msa_ilvr_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_BINSL_df:
+        gen_helper_msa_binsl_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_MAX_A_df:
+        gen_helper_msa_max_a_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_AVER_S_df:
+        gen_helper_msa_aver_s_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_MOD_S_df:
+        gen_helper_msa_mod_s_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_ILVEV_df:
+        gen_helper_msa_ilvev_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_BINSR_df:
+        gen_helper_msa_binsr_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_MIN_A_df:
+        gen_helper_msa_min_a_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_AVER_U_df:
+        gen_helper_msa_aver_u_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_MOD_U_df:
+        gen_helper_msa_mod_u_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_ILVOD_df:
+        gen_helper_msa_ilvod_df(cpu_env, tdf, twd, tws, twt);
+        break;
+
+    case OPC_DOTP_S_df:
+    case OPC_DOTP_U_df:
+    case OPC_DPADD_S_df:
+    case OPC_DPADD_U_df:
+    case OPC_DPSUB_S_df:
+    case OPC_HADD_S_df:
+    case OPC_DPSUB_U_df:
+    case OPC_HADD_U_df:
+    case OPC_HSUB_S_df:
+    case OPC_HSUB_U_df:
+        if (df == DF_BYTE) {
+            generate_exception(ctx, EXCP_RI);
+        }
+        switch (MASK_MSA_3R(ctx->opcode)) {
+        case OPC_DOTP_S_df:
+            gen_helper_msa_dotp_s_df(cpu_env, tdf, twd, tws, twt);
             break;
-        case OPC_ADDU_OB_DSP:
-            op2 = MASK_ADDU_OB(ctx->opcode);
-            switch (op2) {
-            case OPC_RADDU_L_OB:
-            case OPC_SUBQ_PW:
-            case OPC_SUBQ_S_PW:
-            case OPC_SUBQ_QH:
-            case OPC_SUBQ_S_QH:
-            case OPC_SUBU_OB:
-            case OPC_SUBU_S_OB:
-            case OPC_SUBU_QH:
-            case OPC_SUBU_S_QH:
-            case OPC_SUBUH_OB:
-            case OPC_SUBUH_R_OB:
-            case OPC_ADDQ_PW:
-            case OPC_ADDQ_S_PW:
-            case OPC_ADDQ_QH:
-            case OPC_ADDQ_S_QH:
-            case OPC_ADDU_OB:
-            case OPC_ADDU_S_OB:
-            case OPC_ADDU_QH:
-            case OPC_ADDU_S_QH:
-            case OPC_ADDUH_OB:
-            case OPC_ADDUH_R_OB:
-                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
-                break;
-            case OPC_MULEQ_S_PW_QHL:
-            case OPC_MULEQ_S_PW_QHR:
-            case OPC_MULEU_S_QH_OBL:
-            case OPC_MULEU_S_QH_OBR:
-            case OPC_MULQ_RS_QH:
-                gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 1);
-                break;
-            default:            /* Invalid */
-                MIPS_INVAL("MASK ADDU.OB");
-                generate_exception(ctx, EXCP_RI);
-                break;
-            }
+        case OPC_DOTP_U_df:
+            gen_helper_msa_dotp_u_df(cpu_env, tdf, twd, tws, twt);
             break;
-        case OPC_CMPU_EQ_OB_DSP:
-            op2 = MASK_CMPU_EQ_OB(ctx->opcode);
-            switch (op2) {
-            case OPC_PRECR_SRA_QH_PW:
-            case OPC_PRECR_SRA_R_QH_PW:
-                /* Return value is rt. */
-                gen_mipsdsp_arith(ctx, op1, op2, rt, rs, rd);
-                break;
-            case OPC_PRECR_OB_QH:
-            case OPC_PRECRQ_OB_QH:
-            case OPC_PRECRQ_PW_L:
-            case OPC_PRECRQ_QH_PW:
-            case OPC_PRECRQ_RS_QH_PW:
-            case OPC_PRECRQU_S_OB_QH:
-                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
-                break;
-            case OPC_CMPU_EQ_OB:
-            case OPC_CMPU_LT_OB:
-            case OPC_CMPU_LE_OB:
-            case OPC_CMP_EQ_QH:
-            case OPC_CMP_LT_QH:
-            case OPC_CMP_LE_QH:
-            case OPC_CMP_EQ_PW:
-            case OPC_CMP_LT_PW:
-            case OPC_CMP_LE_PW:
-                gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rd, rs, rt, 0);
-                break;
-            case OPC_CMPGDU_EQ_OB:
-            case OPC_CMPGDU_LT_OB:
-            case OPC_CMPGDU_LE_OB:
-            case OPC_CMPGU_EQ_OB:
-            case OPC_CMPGU_LT_OB:
-            case OPC_CMPGU_LE_OB:
-            case OPC_PACKRL_PW:
-            case OPC_PICK_OB:
-            case OPC_PICK_PW:
-            case OPC_PICK_QH:
-                gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rd, rs, rt, 1);
-                break;
-            default:            /* Invalid */
-                MIPS_INVAL("MASK CMPU_EQ.OB");
-                generate_exception(ctx, EXCP_RI);
-                break;
-            }
+        case OPC_DPADD_S_df:
+            gen_helper_msa_dpadd_s_df(cpu_env, tdf, twd, tws, twt);
             break;
-        case OPC_DAPPEND_DSP:
-            gen_mipsdsp_append(env, ctx, op1, rt, rs, rd);
+        case OPC_DPADD_U_df:
+            gen_helper_msa_dpadd_u_df(cpu_env, tdf, twd, tws, twt);
             break;
-        case OPC_DEXTR_W_DSP:
-            op2 = MASK_DEXTR_W(ctx->opcode);
-            switch (op2) {
-            case OPC_DEXTP:
-            case OPC_DEXTPDP:
-            case OPC_DEXTPDPV:
-            case OPC_DEXTPV:
-            case OPC_DEXTR_L:
-            case OPC_DEXTR_R_L:
-            case OPC_DEXTR_RS_L:
-            case OPC_DEXTR_W:
-            case OPC_DEXTR_R_W:
-            case OPC_DEXTR_RS_W:
-            case OPC_DEXTR_S_H:
-            case OPC_DEXTRV_L:
-            case OPC_DEXTRV_R_L:
-            case OPC_DEXTRV_RS_L:
-            case OPC_DEXTRV_S_H:
-            case OPC_DEXTRV_W:
-            case OPC_DEXTRV_R_W:
-            case OPC_DEXTRV_RS_W:
-                gen_mipsdsp_accinsn(ctx, op1, op2, rt, rs, rd, 1);
-                break;
-            case OPC_DMTHLIP:
-            case OPC_DSHILO:
-            case OPC_DSHILOV:
-                gen_mipsdsp_accinsn(ctx, op1, op2, rd, rs, rt, 0);
-                break;
-            default:            /* Invalid */
-                MIPS_INVAL("MASK EXTR.W");
-                generate_exception(ctx, EXCP_RI);
-                break;
-            }
+        case OPC_DPSUB_S_df:
+            gen_helper_msa_dpsub_s_df(cpu_env, tdf, twd, tws, twt);
             break;
-        case OPC_DPAQ_W_QH_DSP:
-            op2 = MASK_DPAQ_W_QH(ctx->opcode);
-            switch (op2) {
-            case OPC_DPAU_H_OBL:
-            case OPC_DPAU_H_OBR:
-            case OPC_DPSU_H_OBL:
-            case OPC_DPSU_H_OBR:
-            case OPC_DPA_W_QH:
-            case OPC_DPAQ_S_W_QH:
-            case OPC_DPS_W_QH:
-            case OPC_DPSQ_S_W_QH:
-            case OPC_MULSAQ_S_W_QH:
-            case OPC_DPAQ_SA_L_PW:
-            case OPC_DPSQ_SA_L_PW:
-            case OPC_MULSAQ_S_L_PW:
-                gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 0);
-                break;
-            case OPC_MAQ_S_W_QHLL:
-            case OPC_MAQ_S_W_QHLR:
-            case OPC_MAQ_S_W_QHRL:
-            case OPC_MAQ_S_W_QHRR:
-            case OPC_MAQ_SA_W_QHLL:
-            case OPC_MAQ_SA_W_QHLR:
-            case OPC_MAQ_SA_W_QHRL:
-            case OPC_MAQ_SA_W_QHRR:
-            case OPC_MAQ_S_L_PWL:
-            case OPC_MAQ_S_L_PWR:
-            case OPC_DMADD:
-            case OPC_DMADDU:
-            case OPC_DMSUB:
-            case OPC_DMSUBU:
-                gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 0);
-                break;
-            default:            /* Invalid */
-                MIPS_INVAL("MASK DPAQ.W.QH");
-                generate_exception(ctx, EXCP_RI);
-                break;
-            }
+        case OPC_HADD_S_df:
+            gen_helper_msa_hadd_s_df(cpu_env, tdf, twd, tws, twt);
             break;
-        case OPC_DINSV_DSP:
-            op2 = MASK_INSV(ctx->opcode);
-            switch (op2) {
-            case OPC_DINSV:
-                {
-                    TCGv t0, t1;
-
-                    if (rt == 0) {
-                        MIPS_DEBUG("NOP");
-                        break;
-                    }
-                    check_dsp(ctx);
+        case OPC_DPSUB_U_df:
+            gen_helper_msa_dpsub_u_df(cpu_env, tdf, twd, tws, twt);
+            break;
+        case OPC_HADD_U_df:
+            gen_helper_msa_hadd_u_df(cpu_env, tdf, twd, tws, twt);
+            break;
+        case OPC_HSUB_S_df:
+            gen_helper_msa_hsub_s_df(cpu_env, tdf, twd, tws, twt);
+            break;
+        case OPC_HSUB_U_df:
+            gen_helper_msa_hsub_u_df(cpu_env, tdf, twd, tws, twt);
+            break;
+        }
+        break;
+    default:
+        MIPS_INVAL("MSA instruction");
+        generate_exception(ctx, EXCP_RI);
+        break;
+    }
+    tcg_temp_free_i32(twd);
+    tcg_temp_free_i32(tws);
+    tcg_temp_free_i32(twt);
+    tcg_temp_free_i32(tdf);
+}
 
-                    t0 = tcg_temp_new();
-                    t1 = tcg_temp_new();
+static void gen_msa_elm_3e(CPUMIPSState *env, DisasContext *ctx)
+{
+#define MASK_MSA_ELM_DF3E(op)   (MASK_MSA_MINOR(op) | (op & (0x3FF << 16)))
+    uint8_t source = (ctx->opcode >> 11) & 0x1f;
+    uint8_t dest = (ctx->opcode >> 6) & 0x1f;
+    TCGv telm = tcg_temp_new();
+    TCGv_i32 tsr = tcg_const_i32(source);
+    TCGv_i32 tdt = tcg_const_i32(dest);
 
-                    gen_load_gpr(t0, rt);
-                    gen_load_gpr(t1, rs);
+    switch (MASK_MSA_ELM_DF3E(ctx->opcode)) {
+    case OPC_CTCMSA:
+        gen_load_gpr(telm, source);
+        gen_helper_msa_ctcmsa(cpu_env, telm, tdt);
+        break;
+    case OPC_CFCMSA:
+        gen_helper_msa_cfcmsa(telm, cpu_env, tsr);
+        gen_store_gpr(telm, dest);
+        break;
+    case OPC_MOVE_V:
+        gen_helper_msa_move_v(cpu_env, tdt, tsr);
+        break;
+    default:
+        MIPS_INVAL("MSA instruction");
+        generate_exception(ctx, EXCP_RI);
+        break;
+    }
 
-                    gen_helper_dinsv(cpu_gpr[rt], cpu_env, t1, t0);
+    tcg_temp_free(telm);
+    tcg_temp_free_i32(tdt);
+    tcg_temp_free_i32(tsr);
+}
 
-                    tcg_temp_free(t0);
-                    tcg_temp_free(t1);
-                    break;
-                }
-            default:            /* Invalid */
-                MIPS_INVAL("MASK DINSV");
-                generate_exception(ctx, EXCP_RI);
-                break;
-            }
-            break;
-        case OPC_SHLL_OB_DSP:
-            gen_mipsdsp_shift(ctx, op1, rd, rs, rt);
+static void gen_msa_elm_df(CPUMIPSState *env, DisasContext *ctx, uint32_t df,
+        uint32_t n)
+{
+#define MASK_MSA_ELM(op)    (MASK_MSA_MINOR(op) | (op & (0xf << 22)))
+    uint8_t ws = (ctx->opcode >> 11) & 0x1f;
+    uint8_t wd = (ctx->opcode >> 6) & 0x1f;
+
+    TCGv_i32 tws = tcg_const_i32(ws);
+    TCGv_i32 twd = tcg_const_i32(wd);
+    TCGv_i32 tn  = tcg_const_i32(n);
+    TCGv_i32 tdf = tcg_const_i32(df);
+
+    switch (MASK_MSA_ELM(ctx->opcode)) {
+    case OPC_SLDI_df:
+        gen_helper_msa_sldi_df(cpu_env, tdf, twd, tws, tn);
+        break;
+    case OPC_SPLATI_df:
+        gen_helper_msa_splati_df(cpu_env, tdf, twd, tws, tn);
+        break;
+    case OPC_INSVE_df:
+        gen_helper_msa_insve_df(cpu_env, tdf, twd, tws, tn);
+        break;
+    case OPC_COPY_S_df:
+    case OPC_COPY_U_df:
+    case OPC_INSERT_df:
+#if !defined(TARGET_MIPS64)
+        /* Double format valid only for MIPS64 */
+        if (df == DF_DOUBLE) {
+            generate_exception(ctx, EXCP_RI);
             break;
+        }
 #endif
-        default:            /* Invalid */
-            MIPS_INVAL("special3");
+        switch (MASK_MSA_ELM(ctx->opcode)) {
+        case OPC_COPY_S_df:
+            gen_helper_msa_copy_s_df(cpu_env, tdf, twd, tws, tn);
+            break;
+        case OPC_COPY_U_df:
+            gen_helper_msa_copy_u_df(cpu_env, tdf, twd, tws, tn);
+            break;
+        case OPC_INSERT_df:
+            gen_helper_msa_insert_df(cpu_env, tdf, twd, tws, tn);
+            break;
+        }
+        break;
+    default:
+        MIPS_INVAL("MSA instruction");
+        generate_exception(ctx, EXCP_RI);
+    }
+    tcg_temp_free_i32(twd);
+    tcg_temp_free_i32(tws);
+    tcg_temp_free_i32(tn);
+    tcg_temp_free_i32(tdf);
+}
+
+static void gen_msa_elm(CPUMIPSState *env, DisasContext *ctx)
+{
+    uint8_t dfn = (ctx->opcode >> 16) & 0x3f;
+    uint32_t df = 0, n = 0;
+
+    if ((dfn & 0x30) == 0x00) {
+        n = dfn & 0x0f;
+        df = DF_BYTE;
+    } else if ((dfn & 0x38) == 0x20) {
+        n = dfn & 0x07;
+        df = DF_HALF;
+    } else if ((dfn & 0x3c) == 0x30) {
+        n = dfn & 0x03;
+        df = DF_WORD;
+    } else if ((dfn & 0x3e) == 0x38) {
+        n = dfn & 0x01;
+        df = DF_DOUBLE;
+    } else if (dfn == 0x3E) {
+        /* CTCMSA, CFCMSA, MOVE.V */
+        gen_msa_elm_3e(env, ctx);
+        return;
+    } else {
+        generate_exception(ctx, EXCP_RI);
+        return;
+    }
+
+    gen_msa_elm_df(env, ctx, df, n);
+}
+
+static void gen_msa_3rf(CPUMIPSState *env, DisasContext *ctx)
+{
+#define MASK_MSA_3RF(op)    (MASK_MSA_MINOR(op) | (op & (0xf << 22)))
+    uint8_t df = (ctx->opcode >> 21) & 0x1;
+    uint8_t wt = (ctx->opcode >> 16) & 0x1f;
+    uint8_t ws = (ctx->opcode >> 11) & 0x1f;
+    uint8_t wd = (ctx->opcode >> 6) & 0x1f;
+
+    TCGv_i32 twd = tcg_const_i32(wd);
+    TCGv_i32 tws = tcg_const_i32(ws);
+    TCGv_i32 twt = tcg_const_i32(wt);
+    TCGv_i32 tdf = tcg_temp_new_i32();
+
+    /* adjust df value for floating-point instruction */
+    tcg_gen_movi_i32(tdf, df + 2);
+
+    switch (MASK_MSA_3RF(ctx->opcode)) {
+    case OPC_FCAF_df:
+        gen_helper_msa_fcaf_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FADD_df:
+        gen_helper_msa_fadd_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FCUN_df:
+        gen_helper_msa_fcun_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FSUB_df:
+        gen_helper_msa_fsub_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FCOR_df:
+        gen_helper_msa_fcor_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FCEQ_df:
+        gen_helper_msa_fceq_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FMUL_df:
+        gen_helper_msa_fmul_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FCUNE_df:
+        gen_helper_msa_fcune_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FCUEQ_df:
+        gen_helper_msa_fcueq_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FDIV_df:
+        gen_helper_msa_fdiv_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FCNE_df:
+        gen_helper_msa_fcne_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FCLT_df:
+        gen_helper_msa_fclt_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FMADD_df:
+        gen_helper_msa_fmadd_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_MUL_Q_df:
+        tcg_gen_movi_i32(tdf, df + 1);
+        gen_helper_msa_mul_q_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FCULT_df:
+        gen_helper_msa_fcult_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FMSUB_df:
+        gen_helper_msa_fmsub_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_MADD_Q_df:
+        tcg_gen_movi_i32(tdf, df + 1);
+        gen_helper_msa_madd_q_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FCLE_df:
+        gen_helper_msa_fcle_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_MSUB_Q_df:
+        tcg_gen_movi_i32(tdf, df + 1);
+        gen_helper_msa_msub_q_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FCULE_df:
+        gen_helper_msa_fcule_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FEXP2_df:
+        gen_helper_msa_fexp2_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FSAF_df:
+        gen_helper_msa_fsaf_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FEXDO_df:
+        gen_helper_msa_fexdo_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FSUN_df:
+        gen_helper_msa_fsun_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FSOR_df:
+        gen_helper_msa_fsor_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FSEQ_df:
+        gen_helper_msa_fseq_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FTQ_df:
+        gen_helper_msa_ftq_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FSUNE_df:
+        gen_helper_msa_fsune_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FSUEQ_df:
+        gen_helper_msa_fsueq_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FSNE_df:
+        gen_helper_msa_fsne_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FSLT_df:
+        gen_helper_msa_fslt_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FMIN_df:
+        gen_helper_msa_fmin_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_MULR_Q_df:
+        tcg_gen_movi_i32(tdf, df + 1);
+        gen_helper_msa_mulr_q_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FSULT_df:
+        gen_helper_msa_fsult_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FMIN_A_df:
+        gen_helper_msa_fmin_a_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_MADDR_Q_df:
+        tcg_gen_movi_i32(tdf, df + 1);
+        gen_helper_msa_maddr_q_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FSLE_df:
+        gen_helper_msa_fsle_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FMAX_df:
+        gen_helper_msa_fmax_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_MSUBR_Q_df:
+        tcg_gen_movi_i32(tdf, df + 1);
+        gen_helper_msa_msubr_q_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FSULE_df:
+        gen_helper_msa_fsule_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    case OPC_FMAX_A_df:
+        gen_helper_msa_fmax_a_df(cpu_env, tdf, twd, tws, twt);
+        break;
+    default:
+        MIPS_INVAL("MSA instruction");
+        generate_exception(ctx, EXCP_RI);
+        break;
+    }
+
+    tcg_temp_free_i32(twd);
+    tcg_temp_free_i32(tws);
+    tcg_temp_free_i32(twt);
+    tcg_temp_free_i32(tdf);
+}
+
+static void gen_msa_2r(CPUMIPSState *env, DisasContext *ctx)
+{
+#define MASK_MSA_2R(op)     (MASK_MSA_MINOR(op) | (op & (0x1f << 21)) | \
+                            (op & (0x7 << 18)))
+    uint8_t wt = (ctx->opcode >> 16) & 0x1f;
+    uint8_t ws = (ctx->opcode >> 11) & 0x1f;
+    uint8_t wd = (ctx->opcode >> 6) & 0x1f;
+    uint8_t df = (ctx->opcode >> 16) & 0x3;
+    TCGv_i32 twd = tcg_const_i32(wd);
+    TCGv_i32 tws = tcg_const_i32(ws);
+    TCGv_i32 twt = tcg_const_i32(wt);
+    TCGv_i32 tdf = tcg_const_i32(df);
+
+    switch (MASK_MSA_2R(ctx->opcode)) {
+    case OPC_FILL_df:
+#if !defined(TARGET_MIPS64)
+        /* Double format valid only for MIPS64 */
+        if (df == DF_DOUBLE) {
             generate_exception(ctx, EXCP_RI);
             break;
         }
+#endif
+        gen_helper_msa_fill_df(cpu_env, tdf, twd, tws); /* trs */
+        break;
+    case OPC_PCNT_df:
+        gen_helper_msa_pcnt_df(cpu_env, tdf, twd, tws);
+        break;
+    case OPC_NLOC_df:
+        gen_helper_msa_nloc_df(cpu_env, tdf, twd, tws);
+        break;
+    case OPC_NLZC_df:
+        gen_helper_msa_nlzc_df(cpu_env, tdf, twd, tws);
+        break;
+    default:
+        MIPS_INVAL("MSA instruction");
+        generate_exception(ctx, EXCP_RI);
+        break;
+    }
+
+    tcg_temp_free_i32(twd);
+    tcg_temp_free_i32(tws);
+    tcg_temp_free_i32(twt);
+    tcg_temp_free_i32(tdf);
+}
+
+static void gen_msa_2rf(CPUMIPSState *env, DisasContext *ctx)
+{
+#define MASK_MSA_2RF(op)    (MASK_MSA_MINOR(op) | (op & (0x1f << 21)) | \
+                            (op & (0xf << 17)))
+    uint8_t wt = (ctx->opcode >> 16) & 0x1f;
+    uint8_t ws = (ctx->opcode >> 11) & 0x1f;
+    uint8_t wd = (ctx->opcode >> 6) & 0x1f;
+    uint8_t df = (ctx->opcode >> 16) & 0x1;
+    TCGv_i32 twd = tcg_const_i32(wd);
+    TCGv_i32 tws = tcg_const_i32(ws);
+    TCGv_i32 twt = tcg_const_i32(wt);
+    /* adjust df value for floating-point instruction */
+    TCGv_i32 tdf = tcg_const_i32(df + 2);
+
+    switch (MASK_MSA_2RF(ctx->opcode)) {
+    case OPC_FCLASS_df:
+        gen_helper_msa_fclass_df(cpu_env, tdf, twd, tws);
+        break;
+    case OPC_FTRUNC_S_df:
+        gen_helper_msa_ftrunc_s_df(cpu_env, tdf, twd, tws);
+        break;
+    case OPC_FTRUNC_U_df:
+        gen_helper_msa_ftrunc_u_df(cpu_env, tdf, twd, tws);
+        break;
+    case OPC_FSQRT_df:
+        gen_helper_msa_fsqrt_df(cpu_env, tdf, twd, tws);
+        break;
+    case OPC_FRSQRT_df:
+        gen_helper_msa_frsqrt_df(cpu_env, tdf, twd, tws);
+        break;
+    case OPC_FRCP_df:
+        gen_helper_msa_frcp_df(cpu_env, tdf, twd, tws);
+        break;
+    case OPC_FRINT_df:
+        gen_helper_msa_frint_df(cpu_env, tdf, twd, tws);
+        break;
+    case OPC_FLOG2_df:
+        gen_helper_msa_flog2_df(cpu_env, tdf, twd, tws);
+        break;
+    case OPC_FEXUPL_df:
+        gen_helper_msa_fexupl_df(cpu_env, tdf, twd, tws);
+        break;
+    case OPC_FEXUPR_df:
+        gen_helper_msa_fexupr_df(cpu_env, tdf, twd, tws);
+        break;
+    case OPC_FFQL_df:
+        gen_helper_msa_ffql_df(cpu_env, tdf, twd, tws);
+        break;
+    case OPC_FFQR_df:
+        gen_helper_msa_ffqr_df(cpu_env, tdf, twd, tws);
+        break;
+    case OPC_FTINT_S_df:
+        gen_helper_msa_ftint_s_df(cpu_env, tdf, twd, tws);
+        break;
+    case OPC_FTINT_U_df:
+        gen_helper_msa_ftint_u_df(cpu_env, tdf, twd, tws);
+        break;
+    case OPC_FFINT_S_df:
+        gen_helper_msa_ffint_s_df(cpu_env, tdf, twd, tws);
+        break;
+    case OPC_FFINT_U_df:
+        gen_helper_msa_ffint_u_df(cpu_env, tdf, twd, tws);
+        break;
+    }
+
+    tcg_temp_free_i32(twd);
+    tcg_temp_free_i32(tws);
+    tcg_temp_free_i32(twt);
+    tcg_temp_free_i32(tdf);
+}
+
+static void gen_msa_vec_v(CPUMIPSState *env, DisasContext *ctx)
+{
+#define MASK_MSA_VEC(op)    (MASK_MSA_MINOR(op) | (op & (0x1f << 21)))
+    uint8_t wt = (ctx->opcode >> 16) & 0x1f;
+    uint8_t ws = (ctx->opcode >> 11) & 0x1f;
+    uint8_t wd = (ctx->opcode >> 6) & 0x1f;
+    TCGv_i32 twd = tcg_const_i32(wd);
+    TCGv_i32 tws = tcg_const_i32(ws);
+    TCGv_i32 twt = tcg_const_i32(wt);
+
+    switch (MASK_MSA_VEC(ctx->opcode)) {
+    case OPC_AND_V:
+        gen_helper_msa_and_v(cpu_env, twd, tws, twt);
+        break;
+    case OPC_OR_V:
+        gen_helper_msa_or_v(cpu_env, twd, tws, twt);
+        break;
+    case OPC_NOR_V:
+        gen_helper_msa_nor_v(cpu_env, twd, tws, twt);
+        break;
+    case OPC_XOR_V:
+        gen_helper_msa_xor_v(cpu_env, twd, tws, twt);
+        break;
+    case OPC_BMNZ_V:
+        gen_helper_msa_bmnz_v(cpu_env, twd, tws, twt);
+        break;
+    case OPC_BMZ_V:
+        gen_helper_msa_bmz_v(cpu_env, twd, tws, twt);
+        break;
+    case OPC_BSEL_V:
+        gen_helper_msa_bsel_v(cpu_env, twd, tws, twt);
+        break;
+    default:
+        MIPS_INVAL("MSA instruction");
+        generate_exception(ctx, EXCP_RI);
+        break;
+    }
+
+    tcg_temp_free_i32(twd);
+    tcg_temp_free_i32(tws);
+    tcg_temp_free_i32(twt);
+}
+
+static void gen_msa_vec(CPUMIPSState *env, DisasContext *ctx)
+{
+    switch (MASK_MSA_VEC(ctx->opcode)) {
+    case OPC_AND_V:
+    case OPC_OR_V:
+    case OPC_NOR_V:
+    case OPC_XOR_V:
+    case OPC_BMNZ_V:
+    case OPC_BMZ_V:
+    case OPC_BSEL_V:
+        gen_msa_vec_v(env, ctx);
+        break;
+    case OPC_MSA_2R:
+        gen_msa_2r(env, ctx);
+        break;
+    case OPC_MSA_2RF:
+        gen_msa_2rf(env, ctx);
+        break;
+    default:
+        MIPS_INVAL("MSA instruction");
+        generate_exception(ctx, EXCP_RI);
+        break;
+    }
+}
+
+static void gen_msa(CPUMIPSState *env, DisasContext *ctx)
+{
+    uint32_t opcode = ctx->opcode;
+    check_insn(ctx, ASE_MSA);
+    check_msa_access(ctx);
+
+    switch (MASK_MSA_MINOR(opcode)) {
+    case OPC_MSA_I8_00:
+    case OPC_MSA_I8_01:
+    case OPC_MSA_I8_02:
+        gen_msa_i8(env, ctx);
+        break;
+    case OPC_MSA_I5_06:
+    case OPC_MSA_I5_07:
+        gen_msa_i5(env, ctx);
+        break;
+    case OPC_MSA_BIT_09:
+    case OPC_MSA_BIT_0A:
+        gen_msa_bit(env, ctx);
+        break;
+    case OPC_MSA_3R_0D:
+    case OPC_MSA_3R_0E:
+    case OPC_MSA_3R_0F:
+    case OPC_MSA_3R_10:
+    case OPC_MSA_3R_11:
+    case OPC_MSA_3R_12:
+    case OPC_MSA_3R_13:
+    case OPC_MSA_3R_14:
+    case OPC_MSA_3R_15:
+        gen_msa_3r(env, ctx);
+        break;
+    case OPC_MSA_ELM:
+        gen_msa_elm(env, ctx);
+        break;
+    case OPC_MSA_3RF_1A:
+    case OPC_MSA_3RF_1B:
+    case OPC_MSA_3RF_1C:
+        gen_msa_3rf(env, ctx);
+        break;
+    case OPC_MSA_VEC:
+        gen_msa_vec(env, ctx);
+        break;
+    case OPC_LD_B:
+    case OPC_LD_H:
+    case OPC_LD_W:
+    case OPC_LD_D:
+    case OPC_ST_B:
+    case OPC_ST_H:
+    case OPC_ST_W:
+    case OPC_ST_D:
+        {
+            int32_t s10 = sextract32(ctx->opcode, 16, 10);
+            uint8_t rs = (ctx->opcode >> 11) & 0x1f;
+            uint8_t wd = (ctx->opcode >> 6) & 0x1f;
+            uint8_t df = (ctx->opcode >> 0) & 0x3;
+
+            TCGv_i32 tdf = tcg_const_i32(df);
+            TCGv_i32 twd = tcg_const_i32(wd);
+            TCGv_i32 trs = tcg_const_i32(rs);
+            TCGv_i32 ts10 = tcg_const_i32(s10);
+
+            switch (MASK_MSA_MINOR(opcode)) {
+            case OPC_LD_B:
+            case OPC_LD_H:
+            case OPC_LD_W:
+            case OPC_LD_D:
+                gen_helper_msa_ld_df(cpu_env, tdf, twd, trs, ts10);
+                break;
+            case OPC_ST_B:
+            case OPC_ST_H:
+            case OPC_ST_W:
+            case OPC_ST_D:
+                gen_helper_msa_st_df(cpu_env, tdf, twd, trs, ts10);
+                break;
+            }
+
+            tcg_temp_free_i32(twd);
+            tcg_temp_free_i32(tdf);
+            tcg_temp_free_i32(trs);
+            tcg_temp_free_i32(ts10);
+        }
+        break;
+    default:
+        MIPS_INVAL("MSA instruction");
+        generate_exception(ctx, EXCP_RI);
+        break;
+    }
+
+}
+
+static void decode_opc (CPUMIPSState *env, DisasContext *ctx)
+{
+    int32_t offset;
+    int rs, rt, rd, sa;
+    uint32_t op, op1;
+    int16_t imm;
+
+    /* make sure instructions are on a word boundary */
+    if (ctx->pc & 0x3) {
+        env->CP0_BadVAddr = ctx->pc;
+        generate_exception_err(ctx, EXCP_AdEL, EXCP_INST_NOTAVAIL);
+        return;
+    }
+
+    /* Handle blikely not taken case */
+    if ((ctx->hflags & MIPS_HFLAG_BMASK_BASE) == MIPS_HFLAG_BL) {
+        int l1 = gen_new_label();
+
+        MIPS_DEBUG("blikely condition (" TARGET_FMT_lx ")", ctx->pc + 4);
+        tcg_gen_brcondi_tl(TCG_COND_NE, bcond, 0, l1);
+        tcg_gen_movi_i32(hflags, ctx->hflags & ~MIPS_HFLAG_BMASK);
+        gen_goto_tb(ctx, 1, ctx->pc + 4);
+        gen_set_label(l1);
+    }
+
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
+        tcg_gen_debug_insn_start(ctx->pc);
+    }
+
+    op = MASK_OP_MAJOR(ctx->opcode);
+    rs = (ctx->opcode >> 21) & 0x1f;
+    rt = (ctx->opcode >> 16) & 0x1f;
+    rd = (ctx->opcode >> 11) & 0x1f;
+    sa = (ctx->opcode >> 6) & 0x1f;
+    imm = (int16_t)ctx->opcode;
+    switch (op) {
+    case OPC_SPECIAL:
+        decode_opc_special(env, ctx);
+        break;
+    case OPC_SPECIAL2:
+        decode_opc_special2_legacy(env, ctx);
+        break;
+    case OPC_SPECIAL3:
+        decode_opc_special3(env, ctx);
         break;
     case OPC_REGIMM:
         op1 = MASK_REGIMM(ctx->opcode);
         switch (op1) {
-        case OPC_BLTZ ... OPC_BGEZL: /* REGIMM branches */
-        case OPC_BLTZAL ... OPC_BGEZALL:
-            gen_compute_branch(ctx, op1, 4, rs, -1, imm << 2);
+        case OPC_BLTZL: /* REGIMM branches */
+        case OPC_BGEZL:
+        case OPC_BLTZALL:
+        case OPC_BGEZALL:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
+        case OPC_BLTZ:
+        case OPC_BGEZ:
+            gen_compute_branch(ctx, op1, 4, rs, -1, imm << 2, 4);
+            break;
+        case OPC_BLTZAL:
+        case OPC_BGEZAL:
+            if (ctx->insn_flags & ISA_MIPS32R6) {
+                if (rs == 0) {
+                    /* OPC_NAL, OPC_BAL */
+                    gen_compute_branch(ctx, op1, 4, 0, -1, imm << 2, 4);
+                } else {
+                    generate_exception(ctx, EXCP_RI);
+                }
+            } else {
+                gen_compute_branch(ctx, op1, 4, rs, -1, imm << 2, 4);
+            }
             break;
         case OPC_TGEI ... OPC_TEQI: /* REGIMM traps */
         case OPC_TNEI:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             gen_trap(ctx, op1, rs, -1, imm);
             break;
         case OPC_SYNCI:
             check_insn(ctx, ISA_MIPS32R2);
-            /* Treat as NOP. */
+            /* Break the TB to be able to sync copied instructions
+               immediately */
+            ctx->bstate = BS_STOP;
             break;
         case OPC_BPOSGE32:    /* MIPS DSP branch */
 #if defined(TARGET_MIPS64)
         case OPC_BPOSGE64:
 #endif
             check_dsp(ctx);
-            gen_compute_branch(ctx, op1, 4, -1, -2, (int32_t)imm << 2);
+            gen_compute_branch(ctx, op1, 4, -1, -2, (int32_t)imm << 2, 4);
+            break;
+#if defined(TARGET_MIPS64)
+        case OPC_DAHI:
+            check_insn(ctx, ISA_MIPS32R6);
+            check_mips_64(ctx);
+            if (rs != 0) {
+                tcg_gen_addi_tl(cpu_gpr[rs], cpu_gpr[rs], (int64_t)imm << 32);
+            }
+            MIPS_DEBUG("dahi %s, %04x", regnames[rs], imm);
             break;
+        case OPC_DATI:
+            check_insn(ctx, ISA_MIPS32R6);
+            check_mips_64(ctx);
+            if (rs != 0) {
+                tcg_gen_addi_tl(cpu_gpr[rs], cpu_gpr[rs], (int64_t)imm << 48);
+            }
+            MIPS_DEBUG("dati %s, %04x", regnames[rs], imm);
+            break;
+#endif
         default:            /* Invalid */
             MIPS_INVAL("regimm");
             generate_exception(ctx, EXCP_RI);
@@ -15373,6 +18476,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx)
         case OPC_MFMC0:
 #ifndef CONFIG_USER_ONLY
             {
+                uint32_t op2;
                 TCGv t0 = tcg_temp_new();
 
                 op2 = MASK_MFMC0(ctx->opcode);
@@ -15436,7 +18540,16 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx)
             break;
         }
         break;
-    case OPC_ADDI: /* Arithmetic with immediate opcode */
+    case OPC_BOVC: /* OPC_BEQZALC, OPC_BEQC, OPC_ADDI */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            /* OPC_BOVC, OPC_BEQZALC, OPC_BEQC */
+            gen_compute_compact_branch(ctx, op, rs, rt, imm << 2);
+        } else {
+            /* OPC_ADDI */
+            /* Arithmetic with immediate opcode */
+            gen_arith_imm(ctx, op, rt, rs, imm);
+        }
+        break;
     case OPC_ADDIU:
          gen_arith_imm(ctx, op, rt, rs, imm);
          break;
@@ -15445,36 +18558,96 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx)
          gen_slt_imm(ctx, op, rt, rs, imm);
          break;
     case OPC_ANDI: /* Arithmetic with immediate opcode */
-    case OPC_LUI:
+    case OPC_LUI: /* OPC_AUI */
     case OPC_ORI:
     case OPC_XORI:
          gen_logic_imm(ctx, op, rt, rs, imm);
          break;
     case OPC_J ... OPC_JAL: /* Jump */
          offset = (int32_t)(ctx->opcode & 0x3FFFFFF) << 2;
-         gen_compute_branch(ctx, op, 4, rs, rt, offset);
+         gen_compute_branch(ctx, op, 4, rs, rt, offset, 4);
          break;
-    case OPC_BEQ ... OPC_BGTZ: /* Branch */
-    case OPC_BEQL ... OPC_BGTZL:
-         gen_compute_branch(ctx, op, 4, rs, rt, imm << 2);
+    /* Branch */
+    case OPC_BLEZC: /* OPC_BGEZC, OPC_BGEC, OPC_BLEZL */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            if (rt == 0) {
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            /* OPC_BLEZC, OPC_BGEZC, OPC_BGEC */
+            gen_compute_compact_branch(ctx, op, rs, rt, imm << 2);
+        } else {
+            /* OPC_BLEZL */
+            gen_compute_branch(ctx, op, 4, rs, rt, imm << 2, 4);
+        }
+        break;
+    case OPC_BGTZC: /* OPC_BLTZC, OPC_BLTC, OPC_BGTZL */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            if (rt == 0) {
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            /* OPC_BGTZC, OPC_BLTZC, OPC_BLTC */
+            gen_compute_compact_branch(ctx, op, rs, rt, imm << 2);
+        } else {
+            /* OPC_BGTZL */
+            gen_compute_branch(ctx, op, 4, rs, rt, imm << 2, 4);
+        }
+        break;
+    case OPC_BLEZALC: /* OPC_BGEZALC, OPC_BGEUC, OPC_BLEZ */
+        if (rt == 0) {
+            /* OPC_BLEZ */
+            gen_compute_branch(ctx, op, 4, rs, rt, imm << 2, 4);
+        } else {
+            check_insn(ctx, ISA_MIPS32R6);
+            /* OPC_BLEZALC, OPC_BGEZALC, OPC_BGEUC */
+            gen_compute_compact_branch(ctx, op, rs, rt, imm << 2);
+        }
+        break;
+    case OPC_BGTZALC: /* OPC_BLTZALC, OPC_BLTUC, OPC_BGTZ */
+        if (rt == 0) {
+            /* OPC_BGTZ */
+            gen_compute_branch(ctx, op, 4, rs, rt, imm << 2, 4);
+        } else {
+            check_insn(ctx, ISA_MIPS32R6);
+            /* OPC_BGTZALC, OPC_BLTZALC, OPC_BLTUC */
+            gen_compute_compact_branch(ctx, op, rs, rt, imm << 2);
+        }
+        break;
+    case OPC_BEQL:
+    case OPC_BNEL:
+         check_insn_opc_removed(ctx, ISA_MIPS32R6);
+    case OPC_BEQ:
+    case OPC_BNE:
+         gen_compute_branch(ctx, op, 4, rs, rt, imm << 2, 4);
          break;
-    case OPC_LB ... OPC_LWR: /* Load and stores */
+    case OPC_LWL: /* Load and stores */
+    case OPC_LWR:
     case OPC_LL:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
+    case OPC_LB ... OPC_LH:
+    case OPC_LW ... OPC_LHU:
          gen_ld(ctx, op, rt, rs, imm);
          break;
-    case OPC_SB ... OPC_SW:
+    case OPC_SWL:
     case OPC_SWR:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
+    case OPC_SB ... OPC_SH:
+    case OPC_SW:
          gen_st(ctx, op, rt, rs, imm);
          break;
     case OPC_SC:
+         check_insn_opc_removed(ctx, ISA_MIPS32R6);
          gen_st_cond(ctx, op, rt, rs, imm);
          break;
     case OPC_CACHE:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         check_cp0_enabled(ctx);
         check_insn(ctx, ISA_MIPS3 | ISA_MIPS32);
         /* Treat as NOP. */
         break;
     case OPC_PREF:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         check_insn(ctx, ISA_MIPS4 | ISA_MIPS32);
         /* Treat as NOP. */
         break;
@@ -15488,60 +18661,184 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx)
         break;
 
     case OPC_CP1:
-        if (ctx->CP0_Config1 & (1 << CP0C1_FP)) {
+        op1 = MASK_CP1(ctx->opcode);
+
+        switch (op1) {
+        case OPC_MFHC1:
+        case OPC_MTHC1:
             check_cp1_enabled(ctx);
-            op1 = MASK_CP1(ctx->opcode);
-            switch (op1) {
-            case OPC_MFHC1:
-            case OPC_MTHC1:
-                check_insn(ctx, ISA_MIPS32R2);
-            case OPC_MFC1:
-            case OPC_CFC1:
-            case OPC_MTC1:
-            case OPC_CTC1:
-                gen_cp1(ctx, op1, rt, rd);
-                break;
+            check_insn(ctx, ISA_MIPS32R2);
+        case OPC_MFC1:
+        case OPC_CFC1:
+        case OPC_MTC1:
+        case OPC_CTC1:
+            check_cp1_enabled(ctx);
+            gen_cp1(ctx, op1, rt, rd);
+            break;
 #if defined(TARGET_MIPS64)
-            case OPC_DMFC1:
-            case OPC_DMTC1:
-                check_insn(ctx, ISA_MIPS3);
-                gen_cp1(ctx, op1, rt, rd);
-                break;
+        case OPC_DMFC1:
+        case OPC_DMTC1:
+            check_cp1_enabled(ctx);
+            check_insn(ctx, ISA_MIPS3);
+            gen_cp1(ctx, op1, rt, rd);
+            break;
 #endif
-            case OPC_BC1ANY2:
-            case OPC_BC1ANY4:
+        case OPC_BC1EQZ: /* OPC_BC1ANY2 */
+            check_cp1_enabled(ctx);
+            if (ctx->insn_flags & ISA_MIPS32R6) {
+                /* OPC_BC1EQZ */
+                gen_compute_branch1_r6(ctx, MASK_CP1(ctx->opcode),
+                                rt, imm << 2);
+            } else {
+                /* OPC_BC1ANY2 */
                 check_cop1x(ctx);
                 check_insn(ctx, ASE_MIPS3D);
-                /* fall through */
-            case OPC_BC1:
                 gen_compute_branch1(ctx, MASK_BC1(ctx->opcode),
                                     (rt >> 2) & 0x7, imm << 2);
-                break;
-            case OPC_S_FMT:
-            case OPC_D_FMT:
-            case OPC_W_FMT:
-            case OPC_L_FMT:
-            case OPC_PS_FMT:
+            }
+            break;
+        case OPC_BC1NEZ:
+            check_cp1_enabled(ctx);
+            check_insn(ctx, ISA_MIPS32R6);
+            gen_compute_branch1_r6(ctx, MASK_CP1(ctx->opcode),
+                            rt, imm << 2);
+            break;
+        case OPC_BC1ANY4:
+            check_cp1_enabled(ctx);
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
+            check_cop1x(ctx);
+            check_insn(ctx, ASE_MIPS3D);
+            /* fall through */
+        case OPC_BC1:
+            check_cp1_enabled(ctx);
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
+            gen_compute_branch1(ctx, MASK_BC1(ctx->opcode),
+                                (rt >> 2) & 0x7, imm << 2);
+            break;
+        case OPC_PS_FMT:
+            check_cp1_enabled(ctx);
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
+        case OPC_S_FMT:
+        case OPC_D_FMT:
+            check_cp1_enabled(ctx);
+            gen_farith(ctx, ctx->opcode & FOP(0x3f, 0x1f), rt, rd, sa,
+                       (imm >> 8) & 0x7);
+            break;
+        case OPC_W_FMT:
+        case OPC_L_FMT:
+        {
+            int r6_op = ctx->opcode & FOP(0x3f, 0x1f);
+            check_cp1_enabled(ctx);
+            if (ctx->insn_flags & ISA_MIPS32R6) {
+                switch (r6_op) {
+                case R6_OPC_CMP_AF_S:
+                case R6_OPC_CMP_UN_S:
+                case R6_OPC_CMP_EQ_S:
+                case R6_OPC_CMP_UEQ_S:
+                case R6_OPC_CMP_LT_S:
+                case R6_OPC_CMP_ULT_S:
+                case R6_OPC_CMP_LE_S:
+                case R6_OPC_CMP_ULE_S:
+                case R6_OPC_CMP_SAF_S:
+                case R6_OPC_CMP_SUN_S:
+                case R6_OPC_CMP_SEQ_S:
+                case R6_OPC_CMP_SEUQ_S:
+                case R6_OPC_CMP_SLT_S:
+                case R6_OPC_CMP_SULT_S:
+                case R6_OPC_CMP_SLE_S:
+                case R6_OPC_CMP_SULE_S:
+                case R6_OPC_CMP_OR_S:
+                case R6_OPC_CMP_UNE_S:
+                case R6_OPC_CMP_NE_S:
+                case R6_OPC_CMP_SOR_S:
+                case R6_OPC_CMP_SUNE_S:
+                case R6_OPC_CMP_SNE_S:
+                    gen_r6_cmp_s(ctx, ctx->opcode & 0x1f, rt, rd, sa);
+                    break;
+                case R6_OPC_CMP_AF_D:
+                case R6_OPC_CMP_UN_D:
+                case R6_OPC_CMP_EQ_D:
+                case R6_OPC_CMP_UEQ_D:
+                case R6_OPC_CMP_LT_D:
+                case R6_OPC_CMP_ULT_D:
+                case R6_OPC_CMP_LE_D:
+                case R6_OPC_CMP_ULE_D:
+                case R6_OPC_CMP_SAF_D:
+                case R6_OPC_CMP_SUN_D:
+                case R6_OPC_CMP_SEQ_D:
+                case R6_OPC_CMP_SEUQ_D:
+                case R6_OPC_CMP_SLT_D:
+                case R6_OPC_CMP_SULT_D:
+                case R6_OPC_CMP_SLE_D:
+                case R6_OPC_CMP_SULE_D:
+                case R6_OPC_CMP_OR_D:
+                case R6_OPC_CMP_UNE_D:
+                case R6_OPC_CMP_NE_D:
+                case R6_OPC_CMP_SOR_D:
+                case R6_OPC_CMP_SUNE_D:
+                case R6_OPC_CMP_SNE_D:
+                    gen_r6_cmp_d(ctx, ctx->opcode & 0x1f, rt, rd, sa);
+                    break;
+                default:
+                    gen_farith(ctx, ctx->opcode & FOP(0x3f, 0x1f), rt, rd, sa,
+                               (imm >> 8) & 0x7);
+                    break;
+                }
+            } else {
                 gen_farith(ctx, ctx->opcode & FOP(0x3f, 0x1f), rt, rd, sa,
                            (imm >> 8) & 0x7);
-                break;
-            default:
-                MIPS_INVAL("cp1");
-                generate_exception (ctx, EXCP_RI);
-                break;
             }
-        } else {
-            generate_exception_err(ctx, EXCP_CpU, 1);
+            break;
+        }
+        case OPC_BZ_V:
+        case OPC_BNZ_V:
+        case OPC_BZ_B:
+        case OPC_BZ_H:
+        case OPC_BZ_W:
+        case OPC_BZ_D:
+        case OPC_BNZ_B:
+        case OPC_BNZ_H:
+        case OPC_BNZ_W:
+        case OPC_BNZ_D:
+            check_insn(ctx, ASE_MSA);
+            gen_msa_branch(env, ctx, op1);
+            break;
+        default:
+            MIPS_INVAL("cp1");
+            generate_exception(ctx, EXCP_RI);
+            break;
         }
         break;
 
-    /* COP2.  */
-    case OPC_LWC2:
-    case OPC_LDC2:
-    case OPC_SWC2:
-    case OPC_SDC2:
-        /* COP2: Not implemented. */
-        generate_exception_err(ctx, EXCP_CpU, 2);
+    /* Compact branches [R6] and COP2 [non-R6] */
+    case OPC_BC: /* OPC_LWC2 */
+    case OPC_BALC: /* OPC_SWC2 */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            /* OPC_BC, OPC_BALC */
+            gen_compute_compact_branch(ctx, op, 0, 0,
+                                       sextract32(ctx->opcode << 2, 0, 28));
+        } else {
+            /* OPC_LWC2, OPC_SWC2 */
+            /* COP2: Not implemented. */
+            generate_exception_err(ctx, EXCP_CpU, 2);
+        }
+        break;
+    case OPC_BEQZC: /* OPC_JIC, OPC_LDC2 */
+    case OPC_BNEZC: /* OPC_JIALC, OPC_SDC2 */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            if (rs != 0) {
+                /* OPC_BEQZC, OPC_BNEZC */
+                gen_compute_compact_branch(ctx, op, rs, 0,
+                                           sextract32(ctx->opcode << 2, 0, 23));
+            } else {
+                /* OPC_JIC, OPC_JIALC */
+                gen_compute_compact_branch(ctx, op, 0, rt, imm);
+            }
+        } else {
+            /* OPC_LWC2, OPC_SWC2 */
+            /* COP2: Not implemented. */
+            generate_exception_err(ctx, EXCP_CpU, 2);
+        }
         break;
     case OPC_CP2:
         check_insn(ctx, INSN_LOONGSON2F);
@@ -15550,6 +18847,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx)
         break;
 
     case OPC_CP3:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         if (ctx->CP0_Config1 & (1 << CP0C1_FP)) {
             check_cp1_enabled(ctx);
             op1 = MASK_CP3(ctx->opcode);
@@ -15592,40 +18890,85 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx)
 
 #if defined(TARGET_MIPS64)
     /* MIPS64 opcodes */
-    case OPC_LWU:
     case OPC_LDL ... OPC_LDR:
     case OPC_LLD:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
+    case OPC_LWU:
     case OPC_LD:
         check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
         gen_ld(ctx, op, rt, rs, imm);
         break;
     case OPC_SDL ... OPC_SDR:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
     case OPC_SD:
         check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
         gen_st(ctx, op, rt, rs, imm);
         break;
     case OPC_SCD:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
         gen_st_cond(ctx, op, rt, rs, imm);
         break;
-    case OPC_DADDI:
+    case OPC_BNVC: /* OPC_BNEZALC, OPC_BNEC, OPC_DADDI */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            /* OPC_BNVC, OPC_BNEZALC, OPC_BNEC */
+            gen_compute_compact_branch(ctx, op, rs, rt, imm << 2);
+        } else {
+            /* OPC_DADDI */
+            check_insn(ctx, ISA_MIPS3);
+            check_mips_64(ctx);
+            gen_arith_imm(ctx, op, rt, rs, imm);
+        }
+        break;
     case OPC_DADDIU:
         check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
         gen_arith_imm(ctx, op, rt, rs, imm);
         break;
+#else
+    case OPC_BNVC: /* OPC_BNEZALC, OPC_BNEC */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            gen_compute_compact_branch(ctx, op, rs, rt, imm << 2);
+        } else {
+            MIPS_INVAL("major opcode");
+            generate_exception(ctx, EXCP_RI);
+        }
+        break;
 #endif
-    case OPC_JALX:
-        check_insn(ctx, ASE_MIPS16 | ASE_MICROMIPS);
-        offset = (int32_t)(ctx->opcode & 0x3FFFFFF) << 2;
-        gen_compute_branch(ctx, op, 4, rs, rt, offset);
+    case OPC_DAUI: /* OPC_JALX */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+#if defined(TARGET_MIPS64)
+            /* OPC_DAUI */
+            check_mips_64(ctx);
+            if (rt != 0) {
+                TCGv t0 = tcg_temp_new();
+                gen_load_gpr(t0, rs);
+                tcg_gen_addi_tl(cpu_gpr[rt], t0, imm << 16);
+                tcg_temp_free(t0);
+            }
+            MIPS_DEBUG("daui %s, %s, %04x", regnames[rt], regnames[rs], imm);
+#else
+            generate_exception(ctx, EXCP_RI);
+            MIPS_INVAL("major opcode");
+#endif
+        } else {
+            /* OPC_JALX */
+            check_insn(ctx, ASE_MIPS16 | ASE_MICROMIPS);
+            offset = (int32_t)(ctx->opcode & 0x3FFFFFF) << 2;
+            gen_compute_branch(ctx, op, 4, rs, rt, offset, 4);
+        }
         break;
-    case OPC_MDMX:
-        check_insn(ctx, ASE_MDMX);
+    case OPC_MSA: /* OPC_MDMX */
         /* MDMX: Not implemented. */
+        gen_msa(env, ctx);
+        break;
+    case OPC_PCREL:
+        check_insn(ctx, ISA_MIPS32R6);
+        gen_pcrel(ctx, rs, imm);
+        break;
     default:            /* Invalid */
         MIPS_INVAL("major opcode");
         generate_exception(ctx, EXCP_RI);
@@ -15647,7 +18990,7 @@ gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
     int num_insns;
     int max_insns;
     int insn_bytes;
-    int is_delay;
+    int is_slot;
 
     if (search_pc)
         qemu_log("search pc %d\n", search_pc);
@@ -15661,6 +19004,11 @@ gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
     ctx.CP0_Config1 = env->CP0_Config1;
     ctx.tb = tb;
     ctx.bstate = BS_NONE;
+    ctx.kscrexist = (env->CP0_Config4 >> CP0C4_KScrExist) & 0xff;
+    ctx.rxi = (env->CP0_Config3 >> CP0C3_RXI) & 1;
+    ctx.ie = (env->CP0_Config4 >> CP0C4_IE) & 3;
+    ctx.bi = (env->CP0_Config3 >> CP0C3_BI) & 1;
+    ctx.bp = (env->CP0_Config3 >> CP0C3_BP) & 1;
     /* Restore delay slot state from the tb context.  */
     ctx.hflags = (uint32_t)tb->flags; /* FIXME: maybe use 64 bits here? */
     ctx.ulri = env->CP0_Config3 & (1 << CP0C3_ULRI);
@@ -15707,7 +19055,7 @@ gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
             gen_io_start();
 
-        is_delay = ctx.hflags & MIPS_HFLAG_BMASK;
+        is_slot = ctx.hflags & MIPS_HFLAG_BMASK;
         if (!(ctx.hflags & MIPS_HFLAG_M16)) {
             ctx.opcode = cpu_ldl_code(env, ctx.pc);
             insn_bytes = 4;
@@ -15723,8 +19071,17 @@ gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
             ctx.bstate = BS_STOP;
             break;
         }
-        if (is_delay) {
-            handle_delay_slot(&ctx, insn_bytes);
+
+        if (ctx.hflags & MIPS_HFLAG_BMASK) {
+            if (!(ctx.hflags & (MIPS_HFLAG_BDS16 | MIPS_HFLAG_BDS32 |
+                                MIPS_HFLAG_FBNSLOT))) {
+                /* force to generate branch as there is neither delay nor
+                   forbidden slot */
+                is_slot = 1;
+            }
+        }
+        if (is_slot) {
+            gen_branch(&ctx, insn_bytes);
         }
         ctx.pc += insn_bytes;
 
@@ -15755,7 +19112,7 @@ gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
         gen_io_end();
     }
     if (cs->singlestep_enabled && ctx.bstate != BS_BRANCH) {
-        save_cpu_state(&ctx, ctx.bstate == BS_NONE);
+        save_cpu_state(&ctx, ctx.bstate != BS_EXCP);
         gen_helper_0e0i(raise_exception, EXCP_DEBUG);
     } else {
         switch (ctx.bstate) {
@@ -15928,8 +19285,15 @@ void mips_tcg_init(void)
                                         regnames[i]);
 
     for (i = 0; i < 32; i++) {
-        int off = offsetof(CPUMIPSState, active_fpu.fpr[i]);
-        fpu_f64[i] = tcg_global_mem_new_i64(TCG_AREG0, off, fregnames[i]);
+        int off = offsetof(CPUMIPSState, active_fpu.fpr[i].wr.d[0]);
+        msa_wr_d[i * 2] =
+                tcg_global_mem_new_i64(TCG_AREG0, off, msaregnames[i * 2]);
+        /* The scalar floating-point unit (FPU) registers are mapped on
+         * the MSA vector registers. */
+        fpu_f64[i] = msa_wr_d[i * 2];
+        off = offsetof(CPUMIPSState, active_fpu.fpr[i].wr.d[1]);
+        msa_wr_d[i * 2 + 1] =
+                tcg_global_mem_new_i64(TCG_AREG0, off, msaregnames[i * 2 + 1]);
     }
 
     cpu_PC = tcg_global_mem_new(TCG_AREG0,
@@ -15941,9 +19305,6 @@ void mips_tcg_init(void)
         cpu_LO[i] = tcg_global_mem_new(TCG_AREG0,
                                        offsetof(CPUMIPSState, active_tc.LO[i]),
                                        regnames_LO[i]);
-        cpu_ACX[i] = tcg_global_mem_new(TCG_AREG0,
-                                        offsetof(CPUMIPSState, active_tc.ACX[i]),
-                                        regnames_ACX[i]);
     }
     cpu_dspctrl = tcg_global_mem_new(TCG_AREG0,
                                      offsetof(CPUMIPSState, active_tc.DSPControl),
@@ -16039,7 +19400,10 @@ void cpu_state_reset(CPUMIPSState *env)
     env->CP0_SRSConf3 = env->cpu_model->CP0_SRSConf3;
     env->CP0_SRSConf4_rw_bitmask = env->cpu_model->CP0_SRSConf4_rw_bitmask;
     env->CP0_SRSConf4 = env->cpu_model->CP0_SRSConf4;
+    env->CP0_PageGrain_rw_bitmask = env->cpu_model->CP0_PageGrain_rw_bitmask;
+    env->CP0_PageGrain = env->cpu_model->CP0_PageGrain;
     env->active_fpu.fcr0 = env->cpu_model->CP1_fcr0;
+    env->msair = env->cpu_model->MSAIR;
     env->insn_flags = env->cpu_model->insn_flags;
 
 #if defined(CONFIG_USER_ONLY)
@@ -16131,6 +19495,17 @@ void cpu_state_reset(CPUMIPSState *env)
         }
     }
 #endif
+    if ((env->insn_flags & ISA_MIPS32R6) &&
+        (env->active_fpu.fcr0 & (1 << FCR0_F64))) {
+        /* Status.FR = 0 mode in 64-bit FPU not allowed in R6 */
+        env->CP0_Status |= (1 << CP0St_FR);
+    }
+
+    /* MSA */
+    if (env->CP0_Config3 & (1 << CP0C3_MSAP)) {
+        msa_reset(env);
+    }
+
     compute_hflags(env);
     cs->exception_index = EXCP_NONE;
 }
diff --git a/target-mips/translate_init.c b/target-mips/translate_init.c
index 29dc2ef73..148b394cf 100644
--- a/target-mips/translate_init.c
+++ b/target-mips/translate_init.c
@@ -84,6 +84,7 @@ struct mips_def_t {
     int32_t CP0_TCStatus_rw_bitmask;
     int32_t CP0_SRSCtl;
     int32_t CP1_fcr0;
+    int32_t MSAIR;
     int32_t SEGBITS;
     int32_t PABITS;
     int32_t CP0_SRSConf0_rw_bitmask;
@@ -96,6 +97,8 @@ struct mips_def_t {
     int32_t CP0_SRSConf3;
     int32_t CP0_SRSConf4_rw_bitmask;
     int32_t CP0_SRSConf4;
+    int32_t CP0_PageGrain_rw_bitmask;
+    int32_t CP0_PageGrain;
     int insn_flags;
     enum mips_mmu_types mmu_type;
 };
@@ -330,7 +333,8 @@ static const mips_def_t mips_defs[] =
                        (0 << CP0C1_DS) | (3 << CP0C1_DL) | (1 << CP0C1_DA) |
                        (1 << CP0C1_CA),
         .CP0_Config2 = MIPS_CONFIG2,
-        .CP0_Config3 = MIPS_CONFIG3 | (0 << CP0C3_VInt) | (1 << CP0C3_DSPP),
+        .CP0_Config3 = MIPS_CONFIG3 | (1 << CP0C3_DSP2P) | (1 << CP0C3_DSPP) |
+                       (0 << CP0C3_VInt),
         .CP0_LLAddr_rw_bitmask = 0,
         .CP0_LLAddr_shift = 4,
         .SYNCI_Step = 32,
@@ -355,7 +359,7 @@ static const mips_def_t mips_defs[] =
                        (0 << CP0C1_DS) | (3 << CP0C1_DL) | (1 << CP0C1_DA) |
                        (1 << CP0C1_CA),
         .CP0_Config2 = MIPS_CONFIG2,
-        .CP0_Config3 = MIPS_CONFIG3 | (1U << CP0C3_M),
+        .CP0_Config3 = MIPS_CONFIG3 | (1U << CP0C3_M) | (1 << CP0C3_MSAP),
         .CP0_Config4 = MIPS_CONFIG4 | (1U << CP0C4_M),
         .CP0_Config4_rw_bitmask = 0,
         .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_UFR),
@@ -373,7 +377,7 @@ static const mips_def_t mips_defs[] =
                     (0x93 << FCR0_PRID),
         .SEGBITS = 32,
         .PABITS = 32,
-        .insn_flags = CPU_MIPS32R5 | ASE_MIPS16 | ASE_DSP | ASE_DSPR2,
+        .insn_flags = CPU_MIPS32R5 | ASE_MIPS16 | ASE_MSA,
         .mmu_type = MMU_TYPE_R4000,
     },
 #if defined(TARGET_MIPS64)
@@ -516,6 +520,43 @@ static const mips_def_t mips_defs[] =
         .mmu_type = MMU_TYPE_R4000,
     },
     {
+        /* A generic CPU supporting MIPS64 Release 6 ISA.
+           FIXME: Support IEEE 754-2008 FP and misaligned memory accesses.
+                  Eventually this should be replaced by a real CPU model. */
+        .name = "MIPS64R6-generic",
+        .CP0_PRid = 0x00010000,
+        .CP0_Config0 = MIPS_CONFIG0 | (0x2 << CP0C0_AR) | (0x2 << CP0C0_AT) |
+                       (MMU_TYPE_R4000 << CP0C0_MT),
+        .CP0_Config1 = MIPS_CONFIG1 | (1 << CP0C1_FP) | (63 << CP0C1_MMU) |
+                       (2 << CP0C1_IS) | (4 << CP0C1_IL) | (3 << CP0C1_IA) |
+                       (2 << CP0C1_DS) | (4 << CP0C1_DL) | (3 << CP0C1_DA) |
+                       (0 << CP0C1_PC) | (1 << CP0C1_WR) | (1 << CP0C1_EP),
+        .CP0_Config2 = MIPS_CONFIG2,
+        .CP0_Config3 = MIPS_CONFIG3 | (1 << CP0C3_RXI) | (1 << CP0C3_BP) |
+                       (1 << CP0C3_BI) | (1 << CP0C3_ULRI) | (1U << CP0C3_M),
+        .CP0_Config4 = MIPS_CONFIG4 | (0xfc << CP0C4_KScrExist) |
+                       (3 << CP0C4_IE) | (1 << CP0C4_M),
+        .CP0_Config5_rw_bitmask = (1 << CP0C5_SBRI),
+        .CP0_LLAddr_rw_bitmask = 0,
+        .CP0_LLAddr_shift = 0,
+        .SYNCI_Step = 32,
+        .CCRes = 2,
+        .CP0_Status_rw_bitmask = 0x30D8FFFF,
+        .CP0_PageGrain = (1 << CP0PG_IEC) | (1 << CP0PG_XIE) |
+                         (1U << CP0PG_RIE),
+        .CP0_PageGrain_rw_bitmask = 0,
+        .CP1_fcr0 = (1 << FCR0_F64) | (1 << FCR0_L) | (1 << FCR0_W) |
+                    (1 << FCR0_D) | (1 << FCR0_S) | (0x00 << FCR0_PRID) |
+                    (0x0 << FCR0_REV),
+        .SEGBITS = 42,
+        /* The architectural limit is 59, but we have hardcoded 36 bit
+           in some places...
+        .PABITS = 59, */ /* the architectural limit */
+        .PABITS = 36,
+        .insn_flags = CPU_MIPS64R6,
+        .mmu_type = MMU_TYPE_R4000,
+    },
+    {
         .name = "Loongson-2E",
         .CP0_PRid = 0x6302,
         /*64KB I-cache and d-cache. 4 way with 32 bit cache line size*/
@@ -561,7 +602,8 @@ static const mips_def_t mips_defs[] =
                        (2 << CP0C1_DS) | (4 << CP0C1_DL) | (3 << CP0C1_DA) |
                        (1 << CP0C1_PC) | (1 << CP0C1_WR) | (1 << CP0C1_EP),
         .CP0_Config2 = MIPS_CONFIG2,
-        .CP0_Config3 = MIPS_CONFIG3 | (1 << CP0C3_LPA),
+        .CP0_Config3 = MIPS_CONFIG3 | (1U << CP0C3_M) | (1 << CP0C3_DSP2P) |
+                       (1 << CP0C3_DSPP) | (1 << CP0C3_LPA),
         .CP0_LLAddr_rw_bitmask = 0,
         .CP0_LLAddr_shift = 0,
         .SYNCI_Step = 32,
@@ -625,6 +667,8 @@ static void r4k_mmu_init (CPUMIPSState *env, const mips_def_t *def)
     env->tlb->helper_tlbwr = r4k_helper_tlbwr;
     env->tlb->helper_tlbp = r4k_helper_tlbp;
     env->tlb->helper_tlbr = r4k_helper_tlbr;
+    env->tlb->helper_tlbinv = r4k_helper_tlbinv;
+    env->tlb->helper_tlbinvf = r4k_helper_tlbinvf;
 }
 
 static void mmu_init (CPUMIPSState *env, const mips_def_t *def)
@@ -688,3 +732,36 @@ static void mvp_init (CPUMIPSState *env, const mips_def_t *def)
                              (0x0 << CP0MVPC1_PCX) | (0x0 << CP0MVPC1_PCP2) |
                              (0x1 << CP0MVPC1_PCP1);
 }
+
+static void msa_reset(CPUMIPSState *env)
+{
+#ifdef CONFIG_USER_ONLY
+    /* MSA access enabled */
+    env->CP0_Config5 |= 1 << CP0C5_MSAEn;
+    env->CP0_Status |= (1 << CP0St_CU1) | (1 << CP0St_FR);
+#endif
+
+    /* MSA CSR:
+       - non-signaling floating point exception mode off (NX bit is 0)
+       - Cause, Enables, and Flags are all 0
+       - round to nearest / ties to even (RM bits are 0) */
+    env->active_tc.msacsr = 0;
+
+    /* tininess detected after rounding.*/
+    set_float_detect_tininess(float_tininess_after_rounding,
+                              &env->active_tc.msa_fp_status);
+
+    /* clear float_status exception flags */
+    set_float_exception_flags(0, &env->active_tc.msa_fp_status);
+
+    /* set float_status rounding mode */
+    set_float_rounding_mode(float_round_nearest_even,
+                            &env->active_tc.msa_fp_status);
+
+    /* set float_status flush modes */
+    set_flush_to_zero(0, &env->active_tc.msa_fp_status);
+    set_flush_inputs_to_zero(0, &env->active_tc.msa_fp_status);
+
+    /* clear float_status nan mode */
+    set_default_nan_mode(0, &env->active_tc.msa_fp_status);
+}
diff --git a/target-openrisc/cpu.c b/target-openrisc/cpu.c
index 08e724c12..39bedc108 100644
--- a/target-openrisc/cpu.c
+++ b/target-openrisc/cpu.c
@@ -165,6 +165,7 @@ static void openrisc_cpu_class_init(ObjectClass *oc, void *data)
     cc->class_by_name = openrisc_cpu_class_by_name;
     cc->has_work = openrisc_cpu_has_work;
     cc->do_interrupt = openrisc_cpu_do_interrupt;
+    cc->cpu_exec_interrupt = openrisc_cpu_exec_interrupt;
     cc->dump_state = openrisc_cpu_dump_state;
     cc->set_pc = openrisc_cpu_set_pc;
     cc->gdb_read_register = openrisc_cpu_gdb_read_register;
diff --git a/target-openrisc/cpu.h b/target-openrisc/cpu.h
index 4512f459b..69b96c666 100644
--- a/target-openrisc/cpu.h
+++ b/target-openrisc/cpu.h
@@ -348,6 +348,7 @@ OpenRISCCPU *cpu_openrisc_init(const char *cpu_model);
 void cpu_openrisc_list(FILE *f, fprintf_function cpu_fprintf);
 int cpu_openrisc_exec(CPUOpenRISCState *s);
 void openrisc_cpu_do_interrupt(CPUState *cpu);
+bool openrisc_cpu_exec_interrupt(CPUState *cpu, int int_req);
 void openrisc_cpu_dump_state(CPUState *cpu, FILE *f,
                              fprintf_function cpu_fprintf, int flags);
 hwaddr openrisc_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
diff --git a/target-openrisc/interrupt.c b/target-openrisc/interrupt.c
index 3de567eee..e480cfd1b 100644
--- a/target-openrisc/interrupt.c
+++ b/target-openrisc/interrupt.c
@@ -63,3 +63,23 @@ void openrisc_cpu_do_interrupt(CPUState *cs)
 
     cs->exception_index = -1;
 }
+
+bool openrisc_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    OpenRISCCPU *cpu = OPENRISC_CPU(cs);
+    CPUOpenRISCState *env = &cpu->env;
+    int idx = -1;
+
+    if ((interrupt_request & CPU_INTERRUPT_HARD) && (env->sr & SR_IEE)) {
+        idx = EXCP_INT;
+    }
+    if ((interrupt_request & CPU_INTERRUPT_TIMER) && (env->sr & SR_TEE)) {
+        idx = EXCP_TICK;
+    }
+    if (idx >= 0) {
+        cs->exception_index = idx;
+        openrisc_cpu_do_interrupt(cs);
+        return true;
+    }
+    return false;
+}
diff --git a/target-openrisc/translate.c b/target-openrisc/translate.c
index 55ff935d5..407bd9762 100644
--- a/target-openrisc/translate.c
+++ b/target-openrisc/translate.c
@@ -31,6 +31,9 @@
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
 
+#include "trace-tcg.h"
+
+
 #define OPENRISC_DISAS
 
 #ifdef OPENRISC_DISAS
diff --git a/target-ppc/cpu-models.c b/target-ppc/cpu-models.c
index 52ac6ec15..3f18996bb 100644
--- a/target-ppc/cpu-models.c
+++ b/target-ppc/cpu-models.c
@@ -309,6 +309,9 @@
 #endif
     POWERPC_DEF("440-Xilinx",    CPU_POWERPC_440_XILINX,             440x5,
                 "PowerPC 440 Xilinx 5")
+
+    POWERPC_DEF("440-Xilinx-w-dfpu",    CPU_POWERPC_440_XILINX, 440x5wDFPU,
+                "PowerPC 440 Xilinx 5 With a Double Prec. FPU")
 #if defined(TODO)
     POWERPC_DEF("440A5",         CPU_POWERPC_440A5,                  440x5,
                 "PowerPC 440 A5")
diff --git a/target-ppc/cpu-qom.h b/target-ppc/cpu-qom.h
index 0fee36f06..6967a8028 100644
--- a/target-ppc/cpu-qom.h
+++ b/target-ppc/cpu-qom.h
@@ -113,6 +113,7 @@ PowerPCCPUClass *ppc_cpu_class_by_pvr(uint32_t pvr);
 PowerPCCPUClass *ppc_cpu_class_by_pvr_mask(uint32_t pvr);
 
 void ppc_cpu_do_interrupt(CPUState *cpu);
+bool ppc_cpu_exec_interrupt(CPUState *cpu, int int_req);
 void ppc_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
                         int flags);
 void ppc_cpu_dump_statistics(CPUState *cpu, FILE *f,
@@ -127,6 +128,7 @@ int ppc64_cpu_write_elf64_qemunote(WriteCoreDumpFunction f,
 int ppc64_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs,
                                int cpuid, void *opaque);
 #ifndef CONFIG_USER_ONLY
+void ppc_cpu_do_system_reset(CPUState *cs);
 extern const struct VMStateDescription vmstate_ppc_cpu;
 
 typedef struct PPCTimebase {
diff --git a/target-ppc/cpu.h b/target-ppc/cpu.h
index b64c65295..068fcb24a 100644
--- a/target-ppc/cpu.h
+++ b/target-ppc/cpu.h
@@ -924,7 +924,8 @@ struct ppc_segment_page_sizes {
 /* The whole PowerPC CPU context */
 #define NB_MMU_MODES 3
 
-#define PPC_CPU_OPCODES_LEN 0x40
+#define PPC_CPU_OPCODES_LEN          0x40
+#define PPC_CPU_INDIRECT_OPCODES_LEN 0x20
 
 struct CPUPPCState {
     /* First are the most commonly used resources
@@ -1147,7 +1148,6 @@ int cpu_ppc_exec (CPUPPCState *s);
    is returned if the signal was handled by the virtual CPU.  */
 int cpu_ppc_signal_handler (int host_signum, void *pinfo,
                             void *puc);
-void ppc_hw_interrupt (CPUPPCState *env);
 #if defined(CONFIG_USER_ONLY)
 int ppc_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw,
                              int mmu_idx);
@@ -2008,13 +2008,16 @@ enum {
     PPC2_ALTIVEC_207   = 0x0000000000004000ULL,
     /* PowerISA 2.07 Book3s specification                                    */
     PPC2_ISA207S       = 0x0000000000008000ULL,
+    /* Double precision floating point conversion for signed integer 64      */
+    PPC2_FP_CVT_S64    = 0x0000000000010000ULL,
 
 #define PPC_TCG_INSNS2 (PPC2_BOOKE206 | PPC2_VSX | PPC2_PRCNTL | PPC2_DBRX | \
                         PPC2_ISA205 | PPC2_VSX207 | PPC2_PERM_ISA206 | \
                         PPC2_DIVE_ISA206 | PPC2_ATOMIC_ISA206 | \
                         PPC2_FP_CVT_ISA206 | PPC2_FP_TST_ISA206 | \
                         PPC2_BCTAR_ISA207 | PPC2_LSQ_ISA207 | \
-                        PPC2_ALTIVEC_207 | PPC2_ISA207S | PPC2_DFP)
+                        PPC2_ALTIVEC_207 | PPC2_ISA207S | PPC2_DFP | \
+                        PPC2_FP_CVT_S64)
 };
 
 /*****************************************************************************/
diff --git a/target-ppc/excp_helper.c b/target-ppc/excp_helper.c
index be7159013..b80347506 100644
--- a/target-ppc/excp_helper.c
+++ b/target-ppc/excp_helper.c
@@ -48,7 +48,7 @@ void ppc_cpu_do_interrupt(CPUState *cs)
     env->error_code = 0;
 }
 
-void ppc_hw_interrupt(CPUPPCState *env)
+static void ppc_hw_interrupt(CPUPPCState *env)
 {
     CPUState *cs = CPU(ppc_env_get_cpu(env));
 
@@ -692,7 +692,7 @@ void ppc_cpu_do_interrupt(CPUState *cs)
     powerpc_excp(cpu, env->excp_model, cs->exception_index);
 }
 
-void ppc_hw_interrupt(CPUPPCState *env)
+static void ppc_hw_interrupt(CPUPPCState *env)
 {
     PowerPCCPU *cpu = ppc_env_get_cpu(env);
     int hdice;
@@ -810,8 +810,31 @@ void ppc_hw_interrupt(CPUPPCState *env)
         }
     }
 }
+
+void ppc_cpu_do_system_reset(CPUState *cs)
+{
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *env = &cpu->env;
+
+    powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_RESET);
+}
 #endif /* !CONFIG_USER_ONLY */
 
+bool ppc_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *env = &cpu->env;
+
+    if (interrupt_request & CPU_INTERRUPT_HARD) {
+        ppc_hw_interrupt(env);
+        if (env->pending_interrupts == 0) {
+            cs->interrupt_request &= ~CPU_INTERRUPT_HARD;
+        }
+        return true;
+    }
+    return false;
+}
+
 #if defined(DEBUG_OP)
 static void cpu_dump_rfi(target_ulong RA, target_ulong msr)
 {
diff --git a/target-ppc/fpu_helper.c b/target-ppc/fpu_helper.c
index da93d1215..7f74466f3 100644
--- a/target-ppc/fpu_helper.c
+++ b/target-ppc/fpu_helper.c
@@ -649,14 +649,10 @@ FPU_FCTI(fctiw, int32, 0x80000000U)
 FPU_FCTI(fctiwz, int32_round_to_zero, 0x80000000U)
 FPU_FCTI(fctiwu, uint32, 0x00000000U)
 FPU_FCTI(fctiwuz, uint32_round_to_zero, 0x00000000U)
-#if defined(TARGET_PPC64)
 FPU_FCTI(fctid, int64, 0x8000000000000000ULL)
 FPU_FCTI(fctidz, int64_round_to_zero, 0x8000000000000000ULL)
 FPU_FCTI(fctidu, uint64, 0x0000000000000000ULL)
 FPU_FCTI(fctiduz, uint64_round_to_zero, 0x0000000000000000ULL)
-#endif
-
-#if defined(TARGET_PPC64)
 
 #define FPU_FCFI(op, cvtr, is_single)                      \
 uint64_t helper_##op(CPUPPCState *env, uint64_t arg)       \
@@ -678,8 +674,6 @@ FPU_FCFI(fcfids, int64_to_float32, 1)
 FPU_FCFI(fcfidu, uint64_to_float64, 0)
 FPU_FCFI(fcfidus, uint64_to_float32, 1)
 
-#endif
-
 static inline uint64_t do_fri(CPUPPCState *env, uint64_t arg,
                               int rounding_mode)
 {
diff --git a/target-ppc/helper.h b/target-ppc/helper.h
index 509eae52f..210fd97f6 100644
--- a/target-ppc/helper.h
+++ b/target-ppc/helper.h
@@ -28,7 +28,6 @@ DEF_HELPER_2(icbi, void, env, tl)
 DEF_HELPER_5(lscbx, tl, env, tl, i32, i32, i32)
 
 #if defined(TARGET_PPC64)
-DEF_HELPER_3(mulldo, i64, env, i64, i64)
 DEF_HELPER_4(divdeu, i64, env, i64, i64, i32)
 DEF_HELPER_4(divde, i64, env, i64, i64, i32)
 #endif
@@ -67,7 +66,6 @@ DEF_HELPER_2(fctiw, i64, env, i64)
 DEF_HELPER_2(fctiwu, i64, env, i64)
 DEF_HELPER_2(fctiwz, i64, env, i64)
 DEF_HELPER_2(fctiwuz, i64, env, i64)
-#if defined(TARGET_PPC64)
 DEF_HELPER_2(fcfid, i64, env, i64)
 DEF_HELPER_2(fcfidu, i64, env, i64)
 DEF_HELPER_2(fcfids, i64, env, i64)
@@ -76,7 +74,6 @@ DEF_HELPER_2(fctid, i64, env, i64)
 DEF_HELPER_2(fctidu, i64, env, i64)
 DEF_HELPER_2(fctidz, i64, env, i64)
 DEF_HELPER_2(fctiduz, i64, env, i64)
-#endif
 DEF_HELPER_2(frsp, i64, env, i64)
 DEF_HELPER_2(frin, i64, env, i64)
 DEF_HELPER_2(friz, i64, env, i64)
diff --git a/target-ppc/int_helper.c b/target-ppc/int_helper.c
index f6e884670..4c2b71c70 100644
--- a/target-ppc/int_helper.c
+++ b/target-ppc/int_helper.c
@@ -24,23 +24,6 @@
 #include "helper_regs.h"
 /*****************************************************************************/
 /* Fixed point operations helpers */
-#if defined(TARGET_PPC64)
-
-uint64_t helper_mulldo(CPUPPCState *env, uint64_t arg1, uint64_t arg2)
-{
-    int64_t th;
-    uint64_t tl;
-
-    muls64(&tl, (uint64_t *)&th, arg1, arg2);
-    /* If th != 0 && th != -1, then we had an overflow */
-    if (likely((uint64_t)(th + 1) <= 1)) {
-        env->ov = 0;
-    } else {
-        env->so = env->ov = 1;
-    }
-    return (int64_t)tl;
-}
-#endif
 
 target_ulong helper_divweu(CPUPPCState *env, target_ulong ra, target_ulong rb,
                            uint32_t oe)
@@ -238,7 +221,7 @@ target_ulong helper_srad(CPUPPCState *env, target_ulong value,
         if (likely((uint64_t)shift != 0)) {
             shift &= 0x3f;
             ret = (int64_t)value >> shift;
-            if (likely(ret >= 0 || (value & ((1 << shift) - 1)) == 0)) {
+            if (likely(ret >= 0 || (value & ((1ULL << shift) - 1)) == 0)) {
                 env->ca = 0;
             } else {
                 env->ca = 1;
@@ -725,7 +708,7 @@ static inline void vcmpbfp_internal(CPUPPCState *env, ppc_avr_t *r,
         int le_rel = float32_compare_quiet(a->f[i], b->f[i], &env->vec_status);
         if (le_rel == float_relation_unordered) {
             r->u32[i] = 0xc0000000;
-            /* ALL_IN does not need to be updated here.  */
+            all_in = 1;
         } else {
             float32 bneg = float32_chs(b->f[i]);
             int ge_rel = float32_compare_quiet(a->f[i], bneg, &env->vec_status);
@@ -1569,13 +1552,6 @@ void helper_vlogefp(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *b)
     }
 }
 
-#if defined(HOST_WORDS_BIGENDIAN)
-#define LEFT 0
-#define RIGHT 1
-#else
-#define LEFT 1
-#define RIGHT 0
-#endif
 /* The specification says that the results are undefined if all of the
  * shift counts are not identical.  We check to make sure that they are
  * to conform to what real hardware appears to do.  */
@@ -1605,11 +1581,9 @@ void helper_vlogefp(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *b)
             }                                                           \
         }                                                               \
     }
-VSHIFT(l, LEFT)
-VSHIFT(r, RIGHT)
+VSHIFT(l, 1)
+VSHIFT(r, 0)
 #undef VSHIFT
-#undef LEFT
-#undef RIGHT
 
 #define VSL(suffix, element, mask)                                      \
     void helper_vsl##suffix(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)   \
@@ -2303,25 +2277,25 @@ uint32_t helper_bcdadd(ppc_avr_t *r,  ppc_avr_t *a, ppc_avr_t *b, uint32_t ps)
         if (sgna == sgnb) {
             result.u8[BCD_DIG_BYTE(0)] = bcd_preferred_sgn(sgna, ps);
             zero = bcd_add_mag(&result, a, b, &invalid, &overflow);
-            cr = (sgna > 0) ? 4 : 8;
+            cr = (sgna > 0) ? 1 << CRF_GT : 1 << CRF_LT;
         } else if (bcd_cmp_mag(a, b) > 0) {
             result.u8[BCD_DIG_BYTE(0)] = bcd_preferred_sgn(sgna, ps);
             zero = bcd_sub_mag(&result, a, b, &invalid, &overflow);
-            cr = (sgna > 0) ? 4 : 8;
+            cr = (sgna > 0) ? 1 << CRF_GT : 1 << CRF_LT;
         } else {
             result.u8[BCD_DIG_BYTE(0)] = bcd_preferred_sgn(sgnb, ps);
             zero = bcd_sub_mag(&result, b, a, &invalid, &overflow);
-            cr = (sgnb > 0) ? 4 : 8;
+            cr = (sgnb > 0) ? 1 << CRF_GT : 1 << CRF_LT;
         }
     }
 
     if (unlikely(invalid)) {
         result.u64[HI_IDX] = result.u64[LO_IDX] = -1;
-        cr = 1;
+        cr = 1 << CRF_SO;
     } else if (overflow) {
-        cr |= 1;
+        cr |= 1 << CRF_SO;
     } else if (zero) {
-        cr = 2;
+        cr = 1 << CRF_EQ;
     }
 
     *r = result;
@@ -2369,7 +2343,7 @@ void helper_vcipherlast(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)
     int i;
 
     VECTOR_FOR_INORDER_I(i, u8) {
-        r->AVRB(i) = b->AVRB(i) ^ (AES_Te4[a->AVRB(AES_shifts[i])] & 0xFF);
+        r->AVRB(i) = b->AVRB(i) ^ (AES_sbox[a->AVRB(AES_shifts[i])]);
     }
 }
 
@@ -2398,7 +2372,7 @@ void helper_vncipherlast(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)
     int i;
 
     VECTOR_FOR_INORDER_I(i, u8) {
-        r->AVRB(i) = b->AVRB(i) ^ (AES_Td4[a->AVRB(AES_ishifts[i])] & 0xFF);
+        r->AVRB(i) = b->AVRB(i) ^ (AES_isbox[a->AVRB(AES_ishifts[i])]);
     }
 }
 
@@ -2573,6 +2547,7 @@ target_ulong helper_dlmzb(CPUPPCState *env, target_ulong high,
         }
         i++;
     }
+    i = 8;
     if (update_Rc) {
         env->crf[0] = 0x2;
     }
diff --git a/target-ppc/kvm.c b/target-ppc/kvm.c
index 42718f77a..6843fa0b9 100644
--- a/target-ppc/kvm.c
+++ b/target-ppc/kvm.c
@@ -38,6 +38,7 @@
 #include "hw/ppc/ppc.h"
 #include "sysemu/watchdog.h"
 #include "trace.h"
+#include "exec/gdbstub.h"
 
 //#define DEBUG_KVM
 
@@ -72,6 +73,8 @@ static int cap_papr;
 static int cap_htab_fd;
 static int cap_fixup_hcalls;
 
+static uint32_t debug_inst_opcode;
+
 /* XXX We have a race condition where we actually have a level triggered
  *     interrupt, but the infrastructure can't expose that yet, so the guest
  *     takes but ignores it, goes to sleep and never gets notified that there's
@@ -410,6 +413,38 @@ unsigned long kvm_arch_vcpu_id(CPUState *cpu)
     return ppc_get_vcpu_dt_id(POWERPC_CPU(cpu));
 }
 
+/* e500 supports 2 h/w breakpoint and 2 watchpoint.
+ * book3s supports only 1 watchpoint, so array size
+ * of 4 is sufficient for now.
+ */
+#define MAX_HW_BKPTS 4
+
+static struct HWBreakpoint {
+    target_ulong addr;
+    int type;
+} hw_debug_points[MAX_HW_BKPTS];
+
+static CPUWatchpoint hw_watchpoint;
+
+/* Default there is no breakpoint and watchpoint supported */
+static int max_hw_breakpoint;
+static int max_hw_watchpoint;
+static int nb_hw_breakpoint;
+static int nb_hw_watchpoint;
+
+static void kvmppc_hw_debug_points_init(CPUPPCState *cenv)
+{
+    if (cenv->excp_model == POWERPC_EXCP_BOOKE) {
+        max_hw_breakpoint = 2;
+        max_hw_watchpoint = 2;
+    }
+
+    if ((max_hw_breakpoint + max_hw_watchpoint) > MAX_HW_BKPTS) {
+        fprintf(stderr, "Error initializing h/w breakpoints\n");
+        return;
+    }
+}
+
 int kvm_arch_init_vcpu(CPUState *cs)
 {
     PowerPCCPU *cpu = POWERPC_CPU(cs);
@@ -436,6 +471,9 @@ int kvm_arch_init_vcpu(CPUState *cs)
         break;
     }
 
+    kvm_get_one_reg(cs, KVM_REG_PPC_DEBUG_INST, &debug_inst_opcode);
+    kvmppc_hw_debug_points_init(cenv);
+
     return ret;
 }
 
@@ -899,6 +937,11 @@ int kvm_arch_put_registers(CPUState *cs, int level)
     return ret;
 }
 
+static void kvm_sync_excp(CPUPPCState *env, int vector, int ivor)
+{
+     env->excp_vectors[vector] = env->spr[ivor] + env->spr[SPR_BOOKE_IVPR];
+}
+
 int kvm_arch_get_registers(CPUState *cs)
 {
     PowerPCCPU *cpu = POWERPC_CPU(cs);
@@ -981,35 +1024,57 @@ int kvm_arch_get_registers(CPUState *cs)
 
         if (sregs.u.e.features & KVM_SREGS_E_IVOR) {
             env->spr[SPR_BOOKE_IVOR0] = sregs.u.e.ivor_low[0];
+            kvm_sync_excp(env, POWERPC_EXCP_CRITICAL,  SPR_BOOKE_IVOR0);
             env->spr[SPR_BOOKE_IVOR1] = sregs.u.e.ivor_low[1];
+            kvm_sync_excp(env, POWERPC_EXCP_MCHECK,  SPR_BOOKE_IVOR1);
             env->spr[SPR_BOOKE_IVOR2] = sregs.u.e.ivor_low[2];
+            kvm_sync_excp(env, POWERPC_EXCP_DSI,  SPR_BOOKE_IVOR2);
             env->spr[SPR_BOOKE_IVOR3] = sregs.u.e.ivor_low[3];
+            kvm_sync_excp(env, POWERPC_EXCP_ISI,  SPR_BOOKE_IVOR3);
             env->spr[SPR_BOOKE_IVOR4] = sregs.u.e.ivor_low[4];
+            kvm_sync_excp(env, POWERPC_EXCP_EXTERNAL,  SPR_BOOKE_IVOR4);
             env->spr[SPR_BOOKE_IVOR5] = sregs.u.e.ivor_low[5];
+            kvm_sync_excp(env, POWERPC_EXCP_ALIGN,  SPR_BOOKE_IVOR5);
             env->spr[SPR_BOOKE_IVOR6] = sregs.u.e.ivor_low[6];
+            kvm_sync_excp(env, POWERPC_EXCP_PROGRAM,  SPR_BOOKE_IVOR6);
             env->spr[SPR_BOOKE_IVOR7] = sregs.u.e.ivor_low[7];
+            kvm_sync_excp(env, POWERPC_EXCP_FPU,  SPR_BOOKE_IVOR7);
             env->spr[SPR_BOOKE_IVOR8] = sregs.u.e.ivor_low[8];
+            kvm_sync_excp(env, POWERPC_EXCP_SYSCALL,  SPR_BOOKE_IVOR8);
             env->spr[SPR_BOOKE_IVOR9] = sregs.u.e.ivor_low[9];
+            kvm_sync_excp(env, POWERPC_EXCP_APU,  SPR_BOOKE_IVOR9);
             env->spr[SPR_BOOKE_IVOR10] = sregs.u.e.ivor_low[10];
+            kvm_sync_excp(env, POWERPC_EXCP_DECR,  SPR_BOOKE_IVOR10);
             env->spr[SPR_BOOKE_IVOR11] = sregs.u.e.ivor_low[11];
+            kvm_sync_excp(env, POWERPC_EXCP_FIT,  SPR_BOOKE_IVOR11);
             env->spr[SPR_BOOKE_IVOR12] = sregs.u.e.ivor_low[12];
+            kvm_sync_excp(env, POWERPC_EXCP_WDT,  SPR_BOOKE_IVOR12);
             env->spr[SPR_BOOKE_IVOR13] = sregs.u.e.ivor_low[13];
+            kvm_sync_excp(env, POWERPC_EXCP_DTLB,  SPR_BOOKE_IVOR13);
             env->spr[SPR_BOOKE_IVOR14] = sregs.u.e.ivor_low[14];
+            kvm_sync_excp(env, POWERPC_EXCP_ITLB,  SPR_BOOKE_IVOR14);
             env->spr[SPR_BOOKE_IVOR15] = sregs.u.e.ivor_low[15];
+            kvm_sync_excp(env, POWERPC_EXCP_DEBUG,  SPR_BOOKE_IVOR15);
 
             if (sregs.u.e.features & KVM_SREGS_E_SPE) {
                 env->spr[SPR_BOOKE_IVOR32] = sregs.u.e.ivor_high[0];
+                kvm_sync_excp(env, POWERPC_EXCP_SPEU,  SPR_BOOKE_IVOR32);
                 env->spr[SPR_BOOKE_IVOR33] = sregs.u.e.ivor_high[1];
+                kvm_sync_excp(env, POWERPC_EXCP_EFPDI,  SPR_BOOKE_IVOR33);
                 env->spr[SPR_BOOKE_IVOR34] = sregs.u.e.ivor_high[2];
+                kvm_sync_excp(env, POWERPC_EXCP_EFPRI,  SPR_BOOKE_IVOR34);
             }
 
             if (sregs.u.e.features & KVM_SREGS_E_PM) {
                 env->spr[SPR_BOOKE_IVOR35] = sregs.u.e.ivor_high[3];
+                kvm_sync_excp(env, POWERPC_EXCP_EPERFM,  SPR_BOOKE_IVOR35);
             }
 
             if (sregs.u.e.features & KVM_SREGS_E_PC) {
                 env->spr[SPR_BOOKE_IVOR36] = sregs.u.e.ivor_high[4];
+                kvm_sync_excp(env, POWERPC_EXCP_DOORI,  SPR_BOOKE_IVOR36);
                 env->spr[SPR_BOOKE_IVOR37] = sregs.u.e.ivor_high[5];
+                kvm_sync_excp(env, POWERPC_EXCP_DOORCI, SPR_BOOKE_IVOR37);
             }
         }
 
@@ -1244,6 +1309,259 @@ static int kvmppc_handle_dcr_write(CPUPPCState *env, uint32_t dcrn, uint32_t dat
     return 0;
 }
 
+int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
+{
+    /* Mixed endian case is not handled */
+    uint32_t sc = debug_inst_opcode;
+
+    if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn,
+                            sizeof(sc), 0) ||
+        cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&sc, sizeof(sc), 1)) {
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
+{
+    uint32_t sc;
+
+    if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&sc, sizeof(sc), 0) ||
+        sc != debug_inst_opcode ||
+        cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn,
+                            sizeof(sc), 1)) {
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+static int find_hw_breakpoint(target_ulong addr, int type)
+{
+    int n;
+
+    assert((nb_hw_breakpoint + nb_hw_watchpoint)
+           <= ARRAY_SIZE(hw_debug_points));
+
+    for (n = 0; n < nb_hw_breakpoint + nb_hw_watchpoint; n++) {
+        if (hw_debug_points[n].addr == addr &&
+             hw_debug_points[n].type == type) {
+            return n;
+        }
+    }
+
+    return -1;
+}
+
+static int find_hw_watchpoint(target_ulong addr, int *flag)
+{
+    int n;
+
+    n = find_hw_breakpoint(addr, GDB_WATCHPOINT_ACCESS);
+    if (n >= 0) {
+        *flag = BP_MEM_ACCESS;
+        return n;
+    }
+
+    n = find_hw_breakpoint(addr, GDB_WATCHPOINT_WRITE);
+    if (n >= 0) {
+        *flag = BP_MEM_WRITE;
+        return n;
+    }
+
+    n = find_hw_breakpoint(addr, GDB_WATCHPOINT_READ);
+    if (n >= 0) {
+        *flag = BP_MEM_READ;
+        return n;
+    }
+
+    return -1;
+}
+
+int kvm_arch_insert_hw_breakpoint(target_ulong addr,
+                                  target_ulong len, int type)
+{
+    if ((nb_hw_breakpoint + nb_hw_watchpoint) >= ARRAY_SIZE(hw_debug_points)) {
+        return -ENOBUFS;
+    }
+
+    hw_debug_points[nb_hw_breakpoint + nb_hw_watchpoint].addr = addr;
+    hw_debug_points[nb_hw_breakpoint + nb_hw_watchpoint].type = type;
+
+    switch (type) {
+    case GDB_BREAKPOINT_HW:
+        if (nb_hw_breakpoint >= max_hw_breakpoint) {
+            return -ENOBUFS;
+        }
+
+        if (find_hw_breakpoint(addr, type) >= 0) {
+            return -EEXIST;
+        }
+
+        nb_hw_breakpoint++;
+        break;
+
+    case GDB_WATCHPOINT_WRITE:
+    case GDB_WATCHPOINT_READ:
+    case GDB_WATCHPOINT_ACCESS:
+        if (nb_hw_watchpoint >= max_hw_watchpoint) {
+            return -ENOBUFS;
+        }
+
+        if (find_hw_breakpoint(addr, type) >= 0) {
+            return -EEXIST;
+        }
+
+        nb_hw_watchpoint++;
+        break;
+
+    default:
+        return -ENOSYS;
+    }
+
+    return 0;
+}
+
+int kvm_arch_remove_hw_breakpoint(target_ulong addr,
+                                  target_ulong len, int type)
+{
+    int n;
+
+    n = find_hw_breakpoint(addr, type);
+    if (n < 0) {
+        return -ENOENT;
+    }
+
+    switch (type) {
+    case GDB_BREAKPOINT_HW:
+        nb_hw_breakpoint--;
+        break;
+
+    case GDB_WATCHPOINT_WRITE:
+    case GDB_WATCHPOINT_READ:
+    case GDB_WATCHPOINT_ACCESS:
+        nb_hw_watchpoint--;
+        break;
+
+    default:
+        return -ENOSYS;
+    }
+    hw_debug_points[n] = hw_debug_points[nb_hw_breakpoint + nb_hw_watchpoint];
+
+    return 0;
+}
+
+void kvm_arch_remove_all_hw_breakpoints(void)
+{
+    nb_hw_breakpoint = nb_hw_watchpoint = 0;
+}
+
+void kvm_arch_update_guest_debug(CPUState *cs, struct kvm_guest_debug *dbg)
+{
+    int n;
+
+    /* Software Breakpoint updates */
+    if (kvm_sw_breakpoints_active(cs)) {
+        dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
+    }
+
+    assert((nb_hw_breakpoint + nb_hw_watchpoint)
+           <= ARRAY_SIZE(hw_debug_points));
+    assert((nb_hw_breakpoint + nb_hw_watchpoint) <= ARRAY_SIZE(dbg->arch.bp));
+
+    if (nb_hw_breakpoint + nb_hw_watchpoint > 0) {
+        dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP;
+        memset(dbg->arch.bp, 0, sizeof(dbg->arch.bp));
+        for (n = 0; n < nb_hw_breakpoint + nb_hw_watchpoint; n++) {
+            switch (hw_debug_points[n].type) {
+            case GDB_BREAKPOINT_HW:
+                dbg->arch.bp[n].type = KVMPPC_DEBUG_BREAKPOINT;
+                break;
+            case GDB_WATCHPOINT_WRITE:
+                dbg->arch.bp[n].type = KVMPPC_DEBUG_WATCH_WRITE;
+                break;
+            case GDB_WATCHPOINT_READ:
+                dbg->arch.bp[n].type = KVMPPC_DEBUG_WATCH_READ;
+                break;
+            case GDB_WATCHPOINT_ACCESS:
+                dbg->arch.bp[n].type = KVMPPC_DEBUG_WATCH_WRITE |
+                                        KVMPPC_DEBUG_WATCH_READ;
+                break;
+            default:
+                cpu_abort(cs, "Unsupported breakpoint type\n");
+            }
+            dbg->arch.bp[n].addr = hw_debug_points[n].addr;
+        }
+    }
+}
+
+static int kvm_handle_debug(PowerPCCPU *cpu, struct kvm_run *run)
+{
+    CPUState *cs = CPU(cpu);
+    CPUPPCState *env = &cpu->env;
+    struct kvm_debug_exit_arch *arch_info = &run->debug.arch;
+    int handle = 0;
+    int n;
+    int flag = 0;
+
+    if (cs->singlestep_enabled) {
+        handle = 1;
+    } else if (arch_info->status) {
+        if (nb_hw_breakpoint + nb_hw_watchpoint > 0) {
+            if (arch_info->status & KVMPPC_DEBUG_BREAKPOINT) {
+                n = find_hw_breakpoint(arch_info->address, GDB_BREAKPOINT_HW);
+                if (n >= 0) {
+                    handle = 1;
+                }
+            } else if (arch_info->status & (KVMPPC_DEBUG_WATCH_READ |
+                                            KVMPPC_DEBUG_WATCH_WRITE)) {
+                n = find_hw_watchpoint(arch_info->address,  &flag);
+                if (n >= 0) {
+                    handle = 1;
+                    cs->watchpoint_hit = &hw_watchpoint;
+                    hw_watchpoint.vaddr = hw_debug_points[n].addr;
+                    hw_watchpoint.flags = flag;
+                }
+            }
+        }
+    } else if (kvm_find_sw_breakpoint(cs, arch_info->address)) {
+        handle = 1;
+    } else {
+        /* QEMU is not able to handle debug exception, so inject
+         * program exception to guest;
+         * Yes program exception NOT debug exception !!
+         * When QEMU is using debug resources then debug exception must
+         * be always set. To achieve this we set MSR_DE and also set
+         * MSRP_DEP so guest cannot change MSR_DE.
+         * When emulating debug resource for guest we want guest
+         * to control MSR_DE (enable/disable debug interrupt on need).
+         * Supporting both configurations are NOT possible.
+         * So the result is that we cannot share debug resources
+         * between QEMU and Guest on BOOKE architecture.
+         * In the current design QEMU gets the priority over guest,
+         * this means that if QEMU is using debug resources then guest
+         * cannot use them;
+         * For software breakpoint QEMU uses a privileged instruction;
+         * So there cannot be any reason that we are here for guest
+         * set debug exception, only possibility is guest executed a
+         * privileged / illegal instruction and that's why we are
+         * injecting a program interrupt.
+         */
+
+        cpu_synchronize_state(cs);
+        /* env->nip is PC, so increment this by 4 to use
+         * ppc_cpu_do_interrupt(), which set srr0 = env->nip - 4.
+         */
+        env->nip += 4;
+        cs->exception_index = POWERPC_EXCP_PROGRAM;
+        env->error_code = POWERPC_EXCP_INVAL;
+        ppc_cpu_do_interrupt(cs);
+    }
+
+    return handle;
+}
+
 int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
 {
     PowerPCCPU *cpu = POWERPC_CPU(cs);
@@ -1284,6 +1602,16 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
         ret = 0;
         break;
 
+    case KVM_EXIT_DEBUG:
+        DPRINTF("handle debug exception\n");
+        if (kvm_handle_debug(cpu, run)) {
+            ret = EXCP_DEBUG;
+            break;
+        }
+        /* re-enter, this exception was guest-internal */
+        ret = 0;
+        break;
+
     default:
         fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
         ret = -1;
@@ -1369,7 +1697,7 @@ static int read_cpuinfo(const char *field, char *value, int len)
     }
 
     do {
-        if(!fgets(line, sizeof(line), f)) {
+        if (!fgets(line, sizeof(line), f)) {
             break;
         }
         if (!strncmp(line, field, field_len)) {
@@ -1404,6 +1732,17 @@ uint32_t kvmppc_get_tbfreq(void)
     return retval;
 }
 
+bool kvmppc_get_host_serial(char **value)
+{
+    return g_file_get_contents("/proc/device-tree/system-id", value, NULL,
+                               NULL);
+}
+
+bool kvmppc_get_host_model(char **value)
+{
+    return g_file_get_contents("/proc/device-tree/model", value, NULL, NULL);
+}
+
 /* Try to find a device tree node for a CPU with clock-frequency property */
 static int kvmppc_find_cpu_dt(char *buf, int buf_len)
 {
@@ -1443,7 +1782,7 @@ static int kvmppc_find_cpu_dt(char *buf, int buf_len)
  * format) */
 static uint64_t kvmppc_read_int_cpu_dt(const char *propname)
 {
-    char buf[PATH_MAX];
+    char buf[PATH_MAX], *tmp;
     union {
         uint32_t v32;
         uint64_t v64;
@@ -1455,10 +1794,10 @@ static uint64_t kvmppc_read_int_cpu_dt(const char *propname)
         return -1;
     }
 
-    strncat(buf, "/", sizeof(buf) - strlen(buf));
-    strncat(buf, propname, sizeof(buf) - strlen(buf));
+    tmp = g_strdup_printf("%s/%s", buf, propname);
 
-    f = fopen(buf, "rb");
+    f = fopen(tmp, "rb");
+    g_free(tmp);
     if (!f) {
         return -1;
     }
@@ -1496,7 +1835,7 @@ static int kvmppc_get_pvinfo(CPUPPCState *env, struct kvm_ppc_pvinfo *pvinfo)
      PowerPCCPU *cpu = ppc_env_get_cpu(env);
      CPUState *cs = CPU(cpu);
 
-    if (kvm_check_extension(cs->kvm_state, KVM_CAP_PPC_GET_PVINFO) &&
+    if (kvm_vm_check_extension(cs->kvm_state, KVM_CAP_PPC_GET_PVINFO) &&
         !kvm_vm_ioctl(cs->kvm_state, KVM_PPC_GET_PVINFO, pvinfo)) {
         return 0;
     }
@@ -1965,34 +2304,6 @@ void kvm_arch_init_irq_routing(KVMState *s)
 {
 }
 
-int kvm_arch_insert_sw_breakpoint(CPUState *cpu, struct kvm_sw_breakpoint *bp)
-{
-    return -EINVAL;
-}
-
-int kvm_arch_remove_sw_breakpoint(CPUState *cpu, struct kvm_sw_breakpoint *bp)
-{
-    return -EINVAL;
-}
-
-int kvm_arch_insert_hw_breakpoint(target_ulong addr, target_ulong len, int type)
-{
-    return -EINVAL;
-}
-
-int kvm_arch_remove_hw_breakpoint(target_ulong addr, target_ulong len, int type)
-{
-    return -EINVAL;
-}
-
-void kvm_arch_remove_all_hw_breakpoints(void)
-{
-}
-
-void kvm_arch_update_guest_debug(CPUState *cpu, struct kvm_guest_debug *dbg)
-{
-}
-
 struct kvm_get_htab_buf {
     struct kvm_get_htab_header header;
     /*
diff --git a/target-ppc/kvm_ppc.h b/target-ppc/kvm_ppc.h
index d9516e73e..2e0224c6a 100644
--- a/target-ppc/kvm_ppc.h
+++ b/target-ppc/kvm_ppc.h
@@ -19,6 +19,8 @@ uint32_t kvmppc_get_tbfreq(void);
 uint64_t kvmppc_get_clockfreq(void);
 uint32_t kvmppc_get_vmx(void);
 uint32_t kvmppc_get_dfp(void);
+bool kvmppc_get_host_model(char **buf);
+bool kvmppc_get_host_serial(char **buf);
 int kvmppc_get_hasidle(CPUPPCState *env);
 int kvmppc_get_hypercall(CPUPPCState *env, uint8_t *buf, int buf_len);
 int kvmppc_set_interrupt(PowerPCCPU *cpu, int irq, int level);
@@ -60,6 +62,16 @@ static inline uint32_t kvmppc_get_tbfreq(void)
     return 0;
 }
 
+static inline bool kvmppc_get_host_model(char **buf)
+{
+    return false;
+}
+
+static inline bool kvmppc_get_host_serial(char **buf)
+{
+    return false;
+}
+
 static inline uint64_t kvmppc_get_clockfreq(void)
 {
     return 0;
diff --git a/target-ppc/translate.c b/target-ppc/translate.c
index b23933f7b..d381632c8 100644
--- a/target-ppc/translate.c
+++ b/target-ppc/translate.c
@@ -27,6 +27,9 @@
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
 
+#include "trace-tcg.h"
+
+
 #define CPU_SINGLE_STEP 0x1
 #define CPU_BRANCH_STEP 0x2
 #define GDBSTUB_SINGLE_STEP 0x4
@@ -186,6 +189,7 @@ typedef struct DisasContext {
     uint32_t opcode;
     uint32_t exception;
     /* Routine used to access memory */
+    bool pr, hv;
     int mem_idx;
     int access_type;
     /* Translation flags */
@@ -640,20 +644,6 @@ static opc_handler_t invalid_handler = {
     .handler = gen_invalid,
 };
 
-#if defined(TARGET_PPC64)
-/* NOTE: as this time, the only use of is_user_mode() is in 64 bit code.  And */
-/*       so the function is wrapped in the standard 64-bit ifdef in order to  */
-/*       avoid compiler warnings in 32-bit implementations.                   */
-static bool is_user_mode(DisasContext *ctx)
-{
-#if defined(CONFIG_USER_ONLY)
-    return true;
-#else
-    return ctx->mem_idx == 0;
-#endif
-}
-#endif
-
 /***                           Integer comparison                          ***/
 
 static inline void gen_op_cmp(TCGv arg0, TCGv arg1, int s, int crf)
@@ -781,7 +771,7 @@ static void gen_isel(DisasContext *ctx)
     l1 = gen_new_label();
     l2 = gen_new_label();
 
-    mask = 1 << (3 - (bi & 0x03));
+    mask = 0x08 >> (bi & 0x03);
     t0 = tcg_temp_new_i32();
     tcg_gen_andi_i32(t0, cpu_crf[bi >> 2], mask);
     tcg_gen_brcondi_i32(TCG_COND_EQ, t0, 0, l1);
@@ -1125,9 +1115,19 @@ static void gen_mulhwu(DisasContext *ctx)
 /* mullw  mullw. */
 static void gen_mullw(DisasContext *ctx)
 {
-    tcg_gen_mul_tl(cpu_gpr[rD(ctx->opcode)], cpu_gpr[rA(ctx->opcode)],
-                   cpu_gpr[rB(ctx->opcode)]);
-    tcg_gen_ext32s_tl(cpu_gpr[rD(ctx->opcode)], cpu_gpr[rD(ctx->opcode)]);
+#if defined(TARGET_PPC64)
+    TCGv_i64 t0, t1;
+    t0 = tcg_temp_new_i64();
+    t1 = tcg_temp_new_i64();
+    tcg_gen_ext32s_tl(t0, cpu_gpr[rA(ctx->opcode)]);
+    tcg_gen_ext32s_tl(t1, cpu_gpr[rB(ctx->opcode)]);
+    tcg_gen_mul_i64(cpu_gpr[rD(ctx->opcode)], t0, t1);
+    tcg_temp_free(t0);
+    tcg_temp_free(t1);
+#else
+    tcg_gen_mul_i32(cpu_gpr[rD(ctx->opcode)], cpu_gpr[rA(ctx->opcode)],
+                    cpu_gpr[rB(ctx->opcode)]);
+#endif
     if (unlikely(Rc(ctx->opcode) != 0))
         gen_set_Rc0(ctx, cpu_gpr[rD(ctx->opcode)]);
 }
@@ -1141,7 +1141,11 @@ static void gen_mullwo(DisasContext *ctx)
     tcg_gen_trunc_tl_i32(t0, cpu_gpr[rA(ctx->opcode)]);
     tcg_gen_trunc_tl_i32(t1, cpu_gpr[rB(ctx->opcode)]);
     tcg_gen_muls2_i32(t0, t1, t0, t1);
-    tcg_gen_ext_i32_tl(cpu_gpr[rD(ctx->opcode)], t0);
+#if defined(TARGET_PPC64)
+    tcg_gen_concat_i32_i64(cpu_gpr[rD(ctx->opcode)], t0, t1);
+#else
+    tcg_gen_mov_i32(cpu_gpr[rD(ctx->opcode)], t0);
+#endif
 
     tcg_gen_sari_i32(t0, t0, 31);
     tcg_gen_setcond_i32(TCG_COND_NE, t0, t0, t1);
@@ -1198,8 +1202,20 @@ static void gen_mulld(DisasContext *ctx)
 /* mulldo  mulldo. */
 static void gen_mulldo(DisasContext *ctx)
 {
-    gen_helper_mulldo(cpu_gpr[rD(ctx->opcode)], cpu_env,
-                      cpu_gpr[rA(ctx->opcode)], cpu_gpr[rB(ctx->opcode)]);
+    TCGv_i64 t0 = tcg_temp_new_i64();
+    TCGv_i64 t1 = tcg_temp_new_i64();
+
+    tcg_gen_muls2_i64(t0, t1, cpu_gpr[rA(ctx->opcode)],
+                      cpu_gpr[rB(ctx->opcode)]);
+    tcg_gen_mov_i64(cpu_gpr[rD(ctx->opcode)], t0);
+
+    tcg_gen_sari_i64(t0, t0, 63);
+    tcg_gen_setcond_i64(TCG_COND_NE, cpu_ov, t0, t1);
+    tcg_gen_or_tl(cpu_so, cpu_so, cpu_ov);
+
+    tcg_temp_free_i64(t0);
+    tcg_temp_free_i64(t1);
+
     if (unlikely(Rc(ctx->opcode) != 0)) {
         gen_set_Rc0(ctx, cpu_gpr[rD(ctx->opcode)]);
     }
@@ -1427,25 +1443,25 @@ static void gen_or(DisasContext *ctx)
             break;
 #if !defined(CONFIG_USER_ONLY)
         case 31:
-            if (ctx->mem_idx > 0) {
+            if (!ctx->pr) {
                 /* Set process priority to very low */
                 prio = 1;
             }
             break;
         case 5:
-            if (ctx->mem_idx > 0) {
+            if (!ctx->pr) {
                 /* Set process priority to medium-hight */
                 prio = 5;
             }
             break;
         case 3:
-            if (ctx->mem_idx > 0) {
+            if (!ctx->pr) {
                 /* Set process priority to high */
                 prio = 6;
             }
             break;
         case 7:
-            if (ctx->mem_idx > 1) {
+            if (ctx->hv) {
                 /* Set process priority to very high */
                 prio = 7;
             }
@@ -1613,18 +1629,17 @@ static void gen_rlwimi(DisasContext *ctx)
     mb = MB(ctx->opcode);
     me = ME(ctx->opcode);
     sh = SH(ctx->opcode);
-    if (likely(sh == 0 && mb == 0 && me == 31)) {
-        tcg_gen_ext32u_tl(cpu_gpr[rA(ctx->opcode)], cpu_gpr[rS(ctx->opcode)]);
+    if (likely(sh == (31-me) && mb <= me)) {
+        tcg_gen_deposit_tl(cpu_gpr[rA(ctx->opcode)], cpu_gpr[rA(ctx->opcode)],
+                           cpu_gpr[rS(ctx->opcode)], sh, me - mb + 1);
     } else {
         target_ulong mask;
         TCGv t1;
         TCGv t0 = tcg_temp_new();
 #if defined(TARGET_PPC64)
-        TCGv_i32 t2 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(t2, cpu_gpr[rS(ctx->opcode)]);
-        tcg_gen_rotli_i32(t2, t2, sh);
-        tcg_gen_extu_i32_i64(t0, t2);
-        tcg_temp_free_i32(t2);
+        tcg_gen_deposit_i64(t0, cpu_gpr[rS(ctx->opcode)],
+            cpu_gpr[rS(ctx->opcode)], 32, 32);
+        tcg_gen_rotli_i64(t0, t0, sh);
 #else
         tcg_gen_rotli_i32(t0, cpu_gpr[rS(ctx->opcode)], sh);
 #endif
@@ -1669,14 +1684,18 @@ static void gen_rlwinm(DisasContext *ctx)
         tcg_gen_shri_tl(t0, t0, mb);
         tcg_gen_ext32u_tl(cpu_gpr[rA(ctx->opcode)], t0);
         tcg_temp_free(t0);
+    } else if (likely(mb == 0 && me == 31)) {
+        TCGv_i32 t0 = tcg_temp_new_i32();
+        tcg_gen_trunc_tl_i32(t0, cpu_gpr[rS(ctx->opcode)]);
+        tcg_gen_rotli_i32(t0, t0, sh);
+        tcg_gen_extu_i32_tl(cpu_gpr[rA(ctx->opcode)], t0);
+        tcg_temp_free_i32(t0);
     } else {
         TCGv t0 = tcg_temp_new();
 #if defined(TARGET_PPC64)
-        TCGv_i32 t1 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(t1, cpu_gpr[rS(ctx->opcode)]);
-        tcg_gen_rotli_i32(t1, t1, sh);
-        tcg_gen_extu_i32_i64(t0, t1);
-        tcg_temp_free_i32(t1);
+        tcg_gen_deposit_i64(t0, cpu_gpr[rS(ctx->opcode)],
+            cpu_gpr[rS(ctx->opcode)], 32, 32);
+        tcg_gen_rotli_i64(t0, t0, sh);
 #else
         tcg_gen_rotli_i32(t0, cpu_gpr[rS(ctx->opcode)], sh);
 #endif
@@ -1695,37 +1714,49 @@ static void gen_rlwinm(DisasContext *ctx)
 static void gen_rlwnm(DisasContext *ctx)
 {
     uint32_t mb, me;
-    TCGv t0;
+    mb = MB(ctx->opcode);
+    me = ME(ctx->opcode);
+
+    if (likely(mb == 0 && me == 31)) {
+        TCGv_i32 t0, t1;
+        t0 = tcg_temp_new_i32();
+        t1 = tcg_temp_new_i32();
+        tcg_gen_trunc_tl_i32(t0, cpu_gpr[rB(ctx->opcode)]);
+        tcg_gen_trunc_tl_i32(t1, cpu_gpr[rS(ctx->opcode)]);
+        tcg_gen_andi_i32(t0, t0, 0x1f);
+        tcg_gen_rotl_i32(t1, t1, t0);
+        tcg_gen_extu_i32_tl(cpu_gpr[rA(ctx->opcode)], t1);
+        tcg_temp_free_i32(t0);
+        tcg_temp_free_i32(t1);
+    } else {
+        TCGv t0;
 #if defined(TARGET_PPC64)
-    TCGv_i32 t1, t2;
+        TCGv t1;
 #endif
 
-    mb = MB(ctx->opcode);
-    me = ME(ctx->opcode);
-    t0 = tcg_temp_new();
-    tcg_gen_andi_tl(t0, cpu_gpr[rB(ctx->opcode)], 0x1f);
+        t0 = tcg_temp_new();
+        tcg_gen_andi_tl(t0, cpu_gpr[rB(ctx->opcode)], 0x1f);
 #if defined(TARGET_PPC64)
-    t1 = tcg_temp_new_i32();
-    t2 = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(t1, cpu_gpr[rS(ctx->opcode)]);
-    tcg_gen_trunc_i64_i32(t2, t0);
-    tcg_gen_rotl_i32(t1, t1, t2);
-    tcg_gen_extu_i32_i64(t0, t1);
-    tcg_temp_free_i32(t1);
-    tcg_temp_free_i32(t2);
+        t1 = tcg_temp_new_i64();
+        tcg_gen_deposit_i64(t1, cpu_gpr[rS(ctx->opcode)],
+                            cpu_gpr[rS(ctx->opcode)], 32, 32);
+        tcg_gen_rotl_i64(t0, t1, t0);
+        tcg_temp_free_i64(t1);
 #else
-    tcg_gen_rotl_i32(t0, cpu_gpr[rS(ctx->opcode)], t0);
+        tcg_gen_rotl_i32(t0, cpu_gpr[rS(ctx->opcode)], t0);
 #endif
-    if (unlikely(mb != 0 || me != 31)) {
+        if (unlikely(mb != 0 || me != 31)) {
 #if defined(TARGET_PPC64)
-        mb += 32;
-        me += 32;
+            mb += 32;
+            me += 32;
 #endif
-        tcg_gen_andi_tl(cpu_gpr[rA(ctx->opcode)], t0, MASK(mb, me));
-    } else {
-        tcg_gen_mov_tl(cpu_gpr[rA(ctx->opcode)], t0);
+            tcg_gen_andi_tl(cpu_gpr[rA(ctx->opcode)], t0, MASK(mb, me));
+        } else {
+            tcg_gen_andi_tl(t0, t0, MASK(32, 63));
+            tcg_gen_mov_tl(cpu_gpr[rA(ctx->opcode)], t0);
+        }
+        tcg_temp_free(t0);
     }
-    tcg_temp_free(t0);
     if (unlikely(Rc(ctx->opcode) != 0))
         gen_set_Rc0(ctx, cpu_gpr[rA(ctx->opcode)]);
 }
@@ -1921,7 +1952,7 @@ static void gen_srawi(DisasContext *ctx)
     TCGv dst = cpu_gpr[rA(ctx->opcode)];
     TCGv src = cpu_gpr[rS(ctx->opcode)];
     if (sh == 0) {
-        tcg_gen_mov_tl(dst, src);
+        tcg_gen_ext32s_tl(dst, src);
         tcg_gen_movi_tl(cpu_ca, 0);
     } else {
         TCGv t0;
@@ -2243,9 +2274,8 @@ GEN_FLOAT_B(ctiwz, 0x0F, 0x00, 0, PPC_FLOAT);
 GEN_FLOAT_B(ctiwuz, 0x0F, 0x04, 0, PPC2_FP_CVT_ISA206);
 /* frsp */
 GEN_FLOAT_B(rsp, 0x0C, 0x00, 1, PPC_FLOAT);
-#if defined(TARGET_PPC64)
 /* fcfid */
-GEN_FLOAT_B(cfid, 0x0E, 0x1A, 1, PPC_64B);
+GEN_FLOAT_B(cfid, 0x0E, 0x1A, 1, PPC2_FP_CVT_S64);
 /* fcfids */
 GEN_FLOAT_B(cfids, 0x0E, 0x1A, 0, PPC2_FP_CVT_ISA206);
 /* fcfidu */
@@ -2253,14 +2283,13 @@ GEN_FLOAT_B(cfidu, 0x0E, 0x1E, 0, PPC2_FP_CVT_ISA206);
 /* fcfidus */
 GEN_FLOAT_B(cfidus, 0x0E, 0x1E, 0, PPC2_FP_CVT_ISA206);
 /* fctid */
-GEN_FLOAT_B(ctid, 0x0E, 0x19, 0, PPC_64B);
+GEN_FLOAT_B(ctid, 0x0E, 0x19, 0, PPC2_FP_CVT_S64);
 /* fctidu */
 GEN_FLOAT_B(ctidu, 0x0E, 0x1D, 0, PPC2_FP_CVT_ISA206);
 /* fctidz */
-GEN_FLOAT_B(ctidz, 0x0F, 0x19, 0, PPC_64B);
+GEN_FLOAT_B(ctidz, 0x0F, 0x19, 0, PPC2_FP_CVT_S64);
 /* fctidu */
 GEN_FLOAT_B(ctiduz, 0x0F, 0x1D, 0, PPC2_FP_CVT_ISA206);
-#endif
 
 /* frin */
 GEN_FLOAT_B(rin, 0x08, 0x0C, 1, PPC_FLOAT_EXT);
@@ -2859,7 +2888,7 @@ static void gen_lq(DisasContext *ctx)
     bool legal_in_user_mode = (ctx->insns_flags2 & PPC2_LSQ_ISA207) != 0;
     bool le_is_supported = (ctx->insns_flags2 & PPC2_LSQ_ISA207) != 0;
 
-    if (!legal_in_user_mode && is_user_mode(ctx)) {
+    if (!legal_in_user_mode && ctx->pr) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -2982,7 +3011,7 @@ static void gen_std(DisasContext *ctx)
         bool legal_in_user_mode = (ctx->insns_flags2 & PPC2_LSQ_ISA207) != 0;
         bool le_is_supported = (ctx->insns_flags2 & PPC2_LSQ_ISA207) != 0;
 
-        if (!legal_in_user_mode && is_user_mode(ctx)) {
+        if (!legal_in_user_mode && ctx->pr) {
             gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
             return;
         }
@@ -3845,7 +3874,7 @@ static inline void gen_bcond(DisasContext *ctx, int type)
     if ((bo & 0x10) == 0) {
         /* Test CR */
         uint32_t bi = BI(ctx->opcode);
-        uint32_t mask = 1 << (3 - (bi & 0x03));
+        uint32_t mask = 0x08 >> (bi & 0x03);
         TCGv_i32 temp = tcg_temp_new_i32();
 
         if (bo & 0x8) {
@@ -3927,7 +3956,7 @@ static void glue(gen_, name)(DisasContext *ctx)
     else                                                                      \
         tcg_gen_mov_i32(t1, cpu_crf[crbB(ctx->opcode) >> 2]);                 \
     tcg_op(t0, t0, t1);                                                       \
-    bitmask = 1 << (3 - (crbD(ctx->opcode) & 0x03));                          \
+    bitmask = 0x08 >> (crbD(ctx->opcode) & 0x03);                             \
     tcg_gen_andi_i32(t0, t0, bitmask);                                        \
     tcg_gen_andi_i32(t1, cpu_crf[crbD(ctx->opcode) >> 2], ~bitmask);          \
     tcg_gen_or_i32(cpu_crf[crbD(ctx->opcode) >> 2], t0, t1);                  \
@@ -3960,14 +3989,14 @@ static void gen_mcrf(DisasContext *ctx)
 
 /***                           System linkage                              ***/
 
-/* rfi (mem_idx only) */
+/* rfi (supervisor only) */
 static void gen_rfi(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
     /* Restore CPU state */
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -3984,7 +4013,7 @@ static void gen_rfid(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
     /* Restore CPU state */
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -4000,7 +4029,7 @@ static void gen_hrfid(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
     /* Restore CPU state */
-    if (unlikely(ctx->mem_idx <= 1)) {
+    if (unlikely(!ctx->hv)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -4169,7 +4198,7 @@ static void gen_mfmsr(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
@@ -4193,9 +4222,9 @@ static inline void gen_op_mfspr(DisasContext *ctx)
     uint32_t sprn = SPR(ctx->opcode);
 
 #if !defined(CONFIG_USER_ONLY)
-    if (ctx->mem_idx == 2)
+    if (ctx->hv)
         read_cb = ctx->spr_cb[sprn].hea_read;
-    else if (ctx->mem_idx)
+    else if (!ctx->pr)
         read_cb = ctx->spr_cb[sprn].oea_read;
     else
 #endif
@@ -4273,7 +4302,7 @@ static void gen_mtmsrd(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
@@ -4304,7 +4333,7 @@ static void gen_mtmsr(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
@@ -4344,9 +4373,9 @@ static void gen_mtspr(DisasContext *ctx)
     uint32_t sprn = SPR(ctx->opcode);
 
 #if !defined(CONFIG_USER_ONLY)
-    if (ctx->mem_idx == 2)
+    if (ctx->hv)
         write_cb = ctx->spr_cb[sprn].hea_write;
-    else if (ctx->mem_idx)
+    else if (!ctx->pr)
         write_cb = ctx->spr_cb[sprn].oea_write;
     else
 #endif
@@ -4393,7 +4422,7 @@ static void gen_dcbi(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
     TCGv EA, val;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -4530,7 +4559,7 @@ static void gen_mfsr(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
     TCGv t0;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
@@ -4547,7 +4576,7 @@ static void gen_mfsrin(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
     TCGv t0;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
@@ -4566,7 +4595,7 @@ static void gen_mtsr(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
     TCGv t0;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
@@ -4583,7 +4612,7 @@ static void gen_mtsrin(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
     TCGv t0;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
@@ -4605,7 +4634,7 @@ static void gen_mfsr_64b(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
     TCGv t0;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
@@ -4622,7 +4651,7 @@ static void gen_mfsrin_64b(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
     TCGv t0;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
@@ -4641,7 +4670,7 @@ static void gen_mtsr_64b(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
     TCGv t0;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
@@ -4658,7 +4687,7 @@ static void gen_mtsrin_64b(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
     TCGv t0;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
@@ -4676,7 +4705,7 @@ static void gen_slbmte(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
@@ -4690,7 +4719,7 @@ static void gen_slbmfee(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
@@ -4704,7 +4733,7 @@ static void gen_slbmfev(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
@@ -4715,7 +4744,7 @@ static void gen_slbmfev(DisasContext *ctx)
 #endif /* defined(TARGET_PPC64) */
 
 /***                      Lookaside buffer management                      ***/
-/* Optional & mem_idx only: */
+/* Optional & supervisor only: */
 
 /* tlbia */
 static void gen_tlbia(DisasContext *ctx)
@@ -4723,7 +4752,7 @@ static void gen_tlbia(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -4737,7 +4766,7 @@ static void gen_tlbiel(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -4751,7 +4780,7 @@ static void gen_tlbie(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -4772,7 +4801,7 @@ static void gen_tlbsync(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -4790,7 +4819,7 @@ static void gen_slbia(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -4804,7 +4833,7 @@ static void gen_slbie(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -5512,7 +5541,7 @@ static void gen_mfrom(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -5528,7 +5557,7 @@ static void gen_tlbld_6xx(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -5542,7 +5571,7 @@ static void gen_tlbli_6xx(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -5558,7 +5587,7 @@ static void gen_tlbld_74xx(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -5572,7 +5601,7 @@ static void gen_tlbli_74xx(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -5595,7 +5624,7 @@ static void gen_cli(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -5616,7 +5645,7 @@ static void gen_mfsri(DisasContext *ctx)
     int ra = rA(ctx->opcode);
     int rd = rD(ctx->opcode);
     TCGv t0;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -5637,7 +5666,7 @@ static void gen_rac(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
     TCGv t0;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -5653,7 +5682,7 @@ static void gen_rfsvc(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -5815,7 +5844,7 @@ static void gen_tlbiva(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
     TCGv t0;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6048,7 +6077,7 @@ static void gen_mfdcr(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
     TCGv dcrn;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
@@ -6067,7 +6096,7 @@ static void gen_mtdcr(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
     TCGv dcrn;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
@@ -6086,7 +6115,7 @@ static void gen_mfdcrx(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
@@ -6105,7 +6134,7 @@ static void gen_mtdcrx(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
@@ -6143,7 +6172,7 @@ static void gen_dccci(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6158,7 +6187,7 @@ static void gen_dcread(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
     TCGv EA, val;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6188,7 +6217,7 @@ static void gen_iccci(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6202,7 +6231,7 @@ static void gen_icread(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6210,13 +6239,13 @@ static void gen_icread(DisasContext *ctx)
 #endif
 }
 
-/* rfci (mem_idx only) */
+/* rfci (supervisor only) */
 static void gen_rfci_40x(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6231,7 +6260,7 @@ static void gen_rfci(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6249,7 +6278,7 @@ static void gen_rfdi(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6265,7 +6294,7 @@ static void gen_rfmci(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6283,7 +6312,7 @@ static void gen_tlbre_40x(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6310,7 +6339,7 @@ static void gen_tlbsx_40x(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
     TCGv t0;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6334,7 +6363,7 @@ static void gen_tlbwe_40x(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6362,7 +6391,7 @@ static void gen_tlbre_440(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6391,7 +6420,7 @@ static void gen_tlbsx_440(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
     TCGv t0;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6415,7 +6444,7 @@ static void gen_tlbwe_440(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6445,7 +6474,7 @@ static void gen_tlbre_booke206(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6461,7 +6490,7 @@ static void gen_tlbsx_booke206(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
     TCGv t0;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6485,7 +6514,7 @@ static void gen_tlbwe_booke206(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6500,7 +6529,7 @@ static void gen_tlbivax_booke206(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
     TCGv t0;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6519,7 +6548,7 @@ static void gen_tlbilx_booke206(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
     TCGv t0;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6554,7 +6583,7 @@ static void gen_wrtee(DisasContext *ctx)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
     TCGv t0;
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6576,7 +6605,7 @@ static void gen_wrteei(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(!ctx->mem_idx)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6629,7 +6658,7 @@ static void gen_msgclr(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(ctx->mem_idx == 0)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6643,7 +6672,7 @@ static void gen_msgsnd(DisasContext *ctx)
 #if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
 #else
-    if (unlikely(ctx->mem_idx == 0)) {
+    if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
@@ -6819,7 +6848,7 @@ static void gen_mtvscr(DisasContext *ctx)
         gen_exception(ctx, POWERPC_EXCP_VPU);
         return;
     }
-    p = gen_avr_ptr(rD(ctx->opcode));
+    p = gen_avr_ptr(rB(ctx->opcode));
     gen_helper_mtvscr(cpu_env, p);
     tcg_temp_free_ptr(p);
 }
@@ -7209,10 +7238,10 @@ GEN_VXFORM_NOA_ENV(vrefp, 5, 4);
 GEN_VXFORM_NOA_ENV(vrsqrtefp, 5, 5);
 GEN_VXFORM_NOA_ENV(vexptefp, 5, 6);
 GEN_VXFORM_NOA_ENV(vlogefp, 5, 7);
-GEN_VXFORM_NOA_ENV(vrfim, 5, 8);
-GEN_VXFORM_NOA_ENV(vrfin, 5, 9);
+GEN_VXFORM_NOA_ENV(vrfim, 5, 11);
+GEN_VXFORM_NOA_ENV(vrfin, 5, 8);
 GEN_VXFORM_NOA_ENV(vrfip, 5, 10);
-GEN_VXFORM_NOA_ENV(vrfiz, 5, 11);
+GEN_VXFORM_NOA_ENV(vrfiz, 5, 9);
 
 #define GEN_VXFORM_SIMM(name, opc2, opc3)                               \
 static void glue(gen_, name)(DisasContext *ctx)                                 \
@@ -8177,7 +8206,7 @@ static inline TCGv_ptr gen_fprp_ptr(int reg)
 }
 
 #if defined(TARGET_PPC64)
-static void gen_set_cr6_from_fpscr(DisasContext *ctx)
+static void gen_set_cr1_from_fpscr(DisasContext *ctx)
 {
     TCGv_i32 tmp = tcg_temp_new_i32();
     tcg_gen_trunc_tl_i32(tmp, cpu_fpscr);
@@ -8185,7 +8214,7 @@ static void gen_set_cr6_from_fpscr(DisasContext *ctx)
     tcg_temp_free_i32(tmp);
 }
 #else
-static void gen_set_cr6_from_fpscr(DisasContext *ctx)
+static void gen_set_cr1_from_fpscr(DisasContext *ctx)
 {
         tcg_gen_shri_tl(cpu_crf[1], cpu_fpscr, 28);
 }
@@ -8205,7 +8234,7 @@ static void gen_##name(DisasContext *ctx)        \
     rb = gen_fprp_ptr(rB(ctx->opcode));          \
     gen_helper_##name(cpu_env, rd, ra, rb);      \
     if (unlikely(Rc(ctx->opcode) != 0)) {        \
-        gen_set_cr6_from_fpscr(ctx);             \
+        gen_set_cr1_from_fpscr(ctx);             \
     }                                            \
     tcg_temp_free_ptr(rd);                       \
     tcg_temp_free_ptr(ra);                       \
@@ -8263,7 +8292,7 @@ static void gen_##name(DisasContext *ctx)             \
     u32_2 = tcg_const_i32(u32f2(ctx->opcode));        \
     gen_helper_##name(cpu_env, rt, rb, u32_1, u32_2); \
     if (unlikely(Rc(ctx->opcode) != 0)) {             \
-        gen_set_cr6_from_fpscr(ctx);                  \
+        gen_set_cr1_from_fpscr(ctx);                  \
     }                                                 \
     tcg_temp_free_ptr(rt);                            \
     tcg_temp_free_ptr(rb);                            \
@@ -8287,7 +8316,7 @@ static void gen_##name(DisasContext *ctx)        \
     i32 = tcg_const_i32(i32fld(ctx->opcode));    \
     gen_helper_##name(cpu_env, rt, ra, rb, i32); \
     if (unlikely(Rc(ctx->opcode) != 0)) {        \
-        gen_set_cr6_from_fpscr(ctx);             \
+        gen_set_cr1_from_fpscr(ctx);             \
     }                                            \
     tcg_temp_free_ptr(rt);                       \
     tcg_temp_free_ptr(rb);                       \
@@ -8308,7 +8337,7 @@ static void gen_##name(DisasContext *ctx)        \
     rb = gen_fprp_ptr(rB(ctx->opcode));          \
     gen_helper_##name(cpu_env, rt, rb);          \
     if (unlikely(Rc(ctx->opcode) != 0)) {        \
-        gen_set_cr6_from_fpscr(ctx);             \
+        gen_set_cr1_from_fpscr(ctx);             \
     }                                            \
     tcg_temp_free_ptr(rt);                       \
     tcg_temp_free_ptr(rb);                       \
@@ -8329,7 +8358,7 @@ static void gen_##name(DisasContext *ctx)          \
     i32 = tcg_const_i32(i32fld(ctx->opcode));      \
     gen_helper_##name(cpu_env, rt, rs, i32);       \
     if (unlikely(Rc(ctx->opcode) != 0)) {          \
-        gen_set_cr6_from_fpscr(ctx);               \
+        gen_set_cr1_from_fpscr(ctx);               \
     }                                              \
     tcg_temp_free_ptr(rt);                         \
     tcg_temp_free_ptr(rs);                         \
@@ -10047,16 +10076,14 @@ GEN_HANDLER_E(fctiwu, 0x3F, 0x0E, 0x04, 0, PPC_NONE, PPC2_FP_CVT_ISA206),
 GEN_FLOAT_B(ctiwz, 0x0F, 0x00, 0, PPC_FLOAT),
 GEN_HANDLER_E(fctiwuz, 0x3F, 0x0F, 0x04, 0, PPC_NONE, PPC2_FP_CVT_ISA206),
 GEN_FLOAT_B(rsp, 0x0C, 0x00, 1, PPC_FLOAT),
-#if defined(TARGET_PPC64)
-GEN_FLOAT_B(cfid, 0x0E, 0x1A, 1, PPC_64B),
+GEN_HANDLER_E(fcfid, 0x3F, 0x0E, 0x1A, 0x001F0000, PPC_NONE, PPC2_FP_CVT_S64),
 GEN_HANDLER_E(fcfids, 0x3B, 0x0E, 0x1A, 0, PPC_NONE, PPC2_FP_CVT_ISA206),
 GEN_HANDLER_E(fcfidu, 0x3F, 0x0E, 0x1E, 0, PPC_NONE, PPC2_FP_CVT_ISA206),
 GEN_HANDLER_E(fcfidus, 0x3B, 0x0E, 0x1E, 0, PPC_NONE, PPC2_FP_CVT_ISA206),
-GEN_FLOAT_B(ctid, 0x0E, 0x19, 0, PPC_64B),
+GEN_HANDLER_E(fctid, 0x3F, 0x0E, 0x19, 0x001F0000, PPC_NONE, PPC2_FP_CVT_S64),
 GEN_HANDLER_E(fctidu, 0x3F, 0x0E, 0x1D, 0, PPC_NONE, PPC2_FP_CVT_ISA206),
-GEN_FLOAT_B(ctidz, 0x0F, 0x19, 0, PPC_64B),
+GEN_HANDLER_E(fctidz, 0x3F, 0x0F, 0x19, 0x001F0000, PPC_NONE, PPC2_FP_CVT_S64),
 GEN_HANDLER_E(fctiduz, 0x3F, 0x0F, 0x1D, 0, PPC_NONE, PPC2_FP_CVT_ISA206),
-#endif
 GEN_FLOAT_B(rin, 0x08, 0x0C, 1, PPC_FLOAT_EXT),
 GEN_FLOAT_B(riz, 0x08, 0x0D, 1, PPC_FLOAT_EXT),
 GEN_FLOAT_B(rip, 0x08, 0x0E, 1, PPC_FLOAT_EXT),
@@ -10447,10 +10474,10 @@ GEN_VXFORM_NOA(vrefp, 5, 4),
 GEN_VXFORM_NOA(vrsqrtefp, 5, 5),
 GEN_VXFORM_NOA(vexptefp, 5, 6),
 GEN_VXFORM_NOA(vlogefp, 5, 7),
-GEN_VXFORM_NOA(vrfim, 5, 8),
-GEN_VXFORM_NOA(vrfin, 5, 9),
+GEN_VXFORM_NOA(vrfim, 5, 11),
+GEN_VXFORM_NOA(vrfin, 5, 8),
 GEN_VXFORM_NOA(vrfip, 5, 10),
-GEN_VXFORM_NOA(vrfiz, 5, 11),
+GEN_VXFORM_NOA(vrfiz, 5, 9),
 
 #undef GEN_VXFORM_UIMM
 #define GEN_VXFORM_UIMM(name, opc2, opc3)                               \
@@ -11258,6 +11285,8 @@ static inline void gen_intermediate_code_internal(PowerPCCPU *cpu,
     ctx.tb = tb;
     ctx.exception = POWERPC_EXCP_NONE;
     ctx.spr_cb = env->spr_cb;
+    ctx.pr = msr_pr;
+    ctx.hv = !msr_pr && msr_hv;
     ctx.mem_idx = env->mmu_idx;
     ctx.insns_flags = env->insns_flags;
     ctx.insns_flags2 = env->insns_flags2;
diff --git a/target-ppc/translate_init.c b/target-ppc/translate_init.c
index 48177ed0a..1fece7b97 100644
--- a/target-ppc/translate_init.c
+++ b/target-ppc/translate_init.c
@@ -2786,7 +2786,7 @@ static void init_excp_BookE (CPUPPCState *env)
     env->excp_vectors[POWERPC_EXCP_DTLB]     = 0x00000000;
     env->excp_vectors[POWERPC_EXCP_ITLB]     = 0x00000000;
     env->excp_vectors[POWERPC_EXCP_DEBUG]    = 0x00000000;
-    env->ivor_mask = 0x0000FFE0UL;
+    env->ivor_mask = 0x0000FFF0UL;
     env->ivpr_mask = 0xFFFF0000UL;
     /* Hardware reset vector */
     env->hreset_vector = 0xFFFFFFFCUL;
@@ -3923,6 +3923,44 @@ POWERPC_FAMILY(440x5)(ObjectClass *oc, void *data)
                  POWERPC_FLAG_DE | POWERPC_FLAG_BUS_CLK;
 }
 
+POWERPC_FAMILY(440x5wDFPU)(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    PowerPCCPUClass *pcc = POWERPC_CPU_CLASS(oc);
+
+    dc->desc = "PowerPC 440x5 with double precision FPU";
+    pcc->init_proc = init_proc_440x5;
+    pcc->check_pow = check_pow_nocheck;
+    pcc->insns_flags = PPC_INSNS_BASE | PPC_STRING |
+                       PPC_FLOAT | PPC_FLOAT_FSQRT |
+                       PPC_FLOAT_STFIWX |
+                       PPC_DCR | PPC_WRTEE | PPC_RFMCI |
+                       PPC_CACHE | PPC_CACHE_ICBI |
+                       PPC_CACHE_DCBZ | PPC_CACHE_DCBA |
+                       PPC_MEM_TLBSYNC | PPC_MFTB |
+                       PPC_BOOKE | PPC_4xx_COMMON | PPC_405_MAC |
+                       PPC_440_SPEC;
+    pcc->insns_flags2 = PPC2_FP_CVT_S64;
+    pcc->msr_mask = (1ull << MSR_POW) |
+                    (1ull << MSR_CE) |
+                    (1ull << MSR_EE) |
+                    (1ull << MSR_PR) |
+                    (1ull << MSR_FP) |
+                    (1ull << MSR_ME) |
+                    (1ull << MSR_FE0) |
+                    (1ull << MSR_DWE) |
+                    (1ull << MSR_DE) |
+                    (1ull << MSR_FE1) |
+                    (1ull << MSR_IR) |
+                    (1ull << MSR_DR);
+    pcc->mmu_model = POWERPC_MMU_BOOKE;
+    pcc->excp_model = POWERPC_EXCP_BOOKE;
+    pcc->bus_model = PPC_FLAGS_INPUT_BookE;
+    pcc->bfd_mach = bfd_mach_ppc_403;
+    pcc->flags = POWERPC_FLAG_CE | POWERPC_FLAG_DWE |
+                 POWERPC_FLAG_DE | POWERPC_FLAG_BUS_CLK;
+}
+
 static void init_proc_460 (CPUPPCState *env)
 {
     /* Time base */
@@ -4336,32 +4374,6 @@ static void init_proc_G2LE (CPUPPCState *env)
                  SPR_NOACCESS, SPR_NOACCESS,
                  &spr_read_generic, &spr_write_generic,
                  0x00000000);
-    /* Breakpoints */
-    /* XXX : not implemented */
-    spr_register(env, SPR_DABR, "DABR",
-                 SPR_NOACCESS, SPR_NOACCESS,
-                 &spr_read_generic, &spr_write_generic,
-                 0x00000000);
-    /* XXX : not implemented */
-    spr_register(env, SPR_DABR2, "DABR2",
-                 SPR_NOACCESS, SPR_NOACCESS,
-                 &spr_read_generic, &spr_write_generic,
-                 0x00000000);
-    /* XXX : not implemented */
-    spr_register(env, SPR_IABR2, "IABR2",
-                 SPR_NOACCESS, SPR_NOACCESS,
-                 &spr_read_generic, &spr_write_generic,
-                 0x00000000);
-    /* XXX : not implemented */
-    spr_register(env, SPR_IBCR, "IBCR",
-                 SPR_NOACCESS, SPR_NOACCESS,
-                 &spr_read_generic, &spr_write_generic,
-                 0x00000000);
-    /* XXX : not implemented */
-    spr_register(env, SPR_DBCR, "DBCR",
-                 SPR_NOACCESS, SPR_NOACCESS,
-                 &spr_read_generic, &spr_write_generic,
-                 0x00000000);
 
     /* Memory management */
     gen_low_BATs(env);
@@ -4590,6 +4602,32 @@ static void init_proc_e300 (CPUPPCState *env)
                  SPR_NOACCESS, SPR_NOACCESS,
                  &spr_read_generic, &spr_write_generic,
                  0x00000000);
+    /* Breakpoints */
+    /* XXX : not implemented */
+    spr_register(env, SPR_DABR, "DABR",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    /* XXX : not implemented */
+    spr_register(env, SPR_DABR2, "DABR2",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    /* XXX : not implemented */
+    spr_register(env, SPR_IABR2, "IABR2",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    /* XXX : not implemented */
+    spr_register(env, SPR_IBCR, "IBCR",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    /* XXX : not implemented */
+    spr_register(env, SPR_DBCR, "DBCR",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
     /* Memory management */
     gen_low_BATs(env);
     gen_high_BATs(env);
@@ -5010,7 +5048,8 @@ POWERPC_FAMILY(e5500)(ObjectClass *oc, void *data)
                        PPC_FLOAT_STFIWX | PPC_WAIT |
                        PPC_MEM_TLBSYNC | PPC_TLBIVAX | PPC_MEM_SYNC |
                        PPC_64B | PPC_POPCNTB | PPC_POPCNTWD;
-    pcc->insns_flags2 = PPC2_BOOKE206 | PPC2_PRCNTL | PPC2_PERM_ISA206;
+    pcc->insns_flags2 = PPC2_BOOKE206 | PPC2_PRCNTL | PPC2_PERM_ISA206 | \
+                        PPC2_FP_CVT_S64;
     pcc->msr_mask = (1ull << MSR_CM) |
                     (1ull << MSR_GS) |
                     (1ull << MSR_UCLE) |
@@ -7906,6 +7945,7 @@ POWERPC_FAMILY(970)(ObjectClass *oc, void *data)
                        PPC_MEM_TLBIE | PPC_MEM_TLBSYNC |
                        PPC_64B | PPC_ALTIVEC |
                        PPC_SEGMENT_64B | PPC_SLBI;
+    pcc->insns_flags2 = PPC2_FP_CVT_S64;
     pcc->msr_mask = (1ull << MSR_SF) |
                     (1ull << MSR_VR) |
                     (1ull << MSR_POW) |
@@ -7958,6 +7998,7 @@ POWERPC_FAMILY(POWER5P)(ObjectClass *oc, void *data)
                        PPC_MEM_TLBIE | PPC_MEM_TLBSYNC |
                        PPC_64B |
                        PPC_SEGMENT_64B | PPC_SLBI;
+    pcc->insns_flags2 = PPC2_FP_CVT_S64;
     pcc->msr_mask = (1ull << MSR_SF) |
                     (1ull << MSR_VR) |
                     (1ull << MSR_POW) |
@@ -8044,7 +8085,7 @@ static void powerpc_set_compat(Object *obj, Visitor *v,
 
 static PropertyInfo powerpc_compat_propinfo = {
     .name = "str",
-    .legacy_name = "powerpc-server-compat",
+    .description = "compatibility mode, power6/power7/power8",
     .get = powerpc_get_compat,
     .set = powerpc_set_compat,
 };
@@ -8100,7 +8141,7 @@ POWERPC_FAMILY(POWER7)(ObjectClass *oc, void *data)
     pcc->insns_flags2 = PPC2_VSX | PPC2_DFP | PPC2_DBRX | PPC2_ISA205 |
                         PPC2_PERM_ISA206 | PPC2_DIVE_ISA206 |
                         PPC2_ATOMIC_ISA206 | PPC2_FP_CVT_ISA206 |
-                        PPC2_FP_TST_ISA206;
+                        PPC2_FP_TST_ISA206 | PPC2_FP_CVT_S64;
     pcc->msr_mask = (1ull << MSR_SF) |
                     (1ull << MSR_VR) |
                     (1ull << MSR_VSX) |
@@ -8178,7 +8219,7 @@ POWERPC_FAMILY(POWER8)(ObjectClass *oc, void *data)
                         PPC2_ATOMIC_ISA206 | PPC2_FP_CVT_ISA206 |
                         PPC2_FP_TST_ISA206 | PPC2_BCTAR_ISA207 |
                         PPC2_LSQ_ISA207 | PPC2_ALTIVEC_207 |
-                        PPC2_ISA205 | PPC2_ISA207S;
+                        PPC2_ISA205 | PPC2_ISA207S | PPC2_FP_CVT_S64;
     pcc->msr_mask = (1ull << MSR_SF) |
                     (1ull << MSR_TM) |
                     (1ull << MSR_VR) |
@@ -8432,14 +8473,16 @@ enum {
     PPC_INDIRECT = 1, /* Indirect opcode table */
 };
 
+#define PPC_OPCODE_MASK 0x3
+
 static inline int is_indirect_opcode (void *handler)
 {
-    return ((uintptr_t)handler & 0x03) == PPC_INDIRECT;
+    return ((uintptr_t)handler & PPC_OPCODE_MASK) == PPC_INDIRECT;
 }
 
 static inline opc_handler_t **ind_table(void *handler)
 {
-    return (opc_handler_t **)((uintptr_t)handler & ~3);
+    return (opc_handler_t **)((uintptr_t)handler & ~PPC_OPCODE_MASK);
 }
 
 /* Instruction table creation */
@@ -8456,8 +8499,8 @@ static int create_new_table (opc_handler_t **table, unsigned char idx)
 {
     opc_handler_t **tmp;
 
-    tmp = g_new(opc_handler_t *, 0x20);
-    fill_new_table(tmp, 0x20);
+    tmp = g_new(opc_handler_t *, PPC_CPU_INDIRECT_OPCODES_LEN);
+    fill_new_table(tmp, PPC_CPU_INDIRECT_OPCODES_LEN);
     table[idx] = (opc_handler_t *)((uintptr_t)tmp | PPC_INDIRECT);
 
     return 0;
@@ -8584,7 +8627,8 @@ static int test_opcode_table (opc_handler_t **table, int len)
             table[i] = &invalid_handler;
         if (table[i] != &invalid_handler) {
             if (is_indirect_opcode(table[i])) {
-                tmp = test_opcode_table(ind_table(table[i]), 0x20);
+                tmp = test_opcode_table(ind_table(table[i]),
+                    PPC_CPU_INDIRECT_OPCODES_LEN);
                 if (tmp == 0) {
                     free(table[i]);
                     table[i] = &invalid_handler;
@@ -8602,7 +8646,7 @@ static int test_opcode_table (opc_handler_t **table, int len)
 
 static void fix_opcode_tables (opc_handler_t **ppc_opcodes)
 {
-    if (test_opcode_table(ppc_opcodes, 0x40) == 0)
+    if (test_opcode_table(ppc_opcodes, PPC_CPU_OPCODES_LEN) == 0)
         printf("*** WARNING: no opcode defined !\n");
 }
 
@@ -8613,7 +8657,7 @@ static void create_ppc_opcodes(PowerPCCPU *cpu, Error **errp)
     CPUPPCState *env = &cpu->env;
     opcode_t *opc;
 
-    fill_new_table(env->opcodes, 0x40);
+    fill_new_table(env->opcodes, PPC_CPU_OPCODES_LEN);
     for (opc = opcodes; opc < &opcodes[ARRAY_SIZE(opcodes)]; opc++) {
         if (((opc->handler.type & pcc->insns_flags) != 0) ||
             ((opc->handler.type2 & pcc->insns_flags2) != 0)) {
@@ -8639,12 +8683,12 @@ static void dump_ppc_insns (CPUPPCState *env)
 
     printf("Instructions set:\n");
     /* opc1 is 6 bits long */
-    for (opc1 = 0x00; opc1 < 0x40; opc1++) {
+    for (opc1 = 0x00; opc1 < PPC_CPU_OPCODES_LEN; opc1++) {
         table = env->opcodes;
         handler = table[opc1];
         if (is_indirect_opcode(handler)) {
             /* opc2 is 5 bits long */
-            for (opc2 = 0; opc2 < 0x20; opc2++) {
+            for (opc2 = 0; opc2 < PPC_CPU_INDIRECT_OPCODES_LEN; opc2++) {
                 table = env->opcodes;
                 handler = env->opcodes[opc1];
                 table = ind_table(handler);
@@ -8652,7 +8696,8 @@ static void dump_ppc_insns (CPUPPCState *env)
                 if (is_indirect_opcode(handler)) {
                     table = ind_table(handler);
                     /* opc3 is 5 bits long */
-                    for (opc3 = 0; opc3 < 0x20; opc3++) {
+                    for (opc3 = 0; opc3 < PPC_CPU_INDIRECT_OPCODES_LEN;
+                            opc3++) {
                         handler = table[opc3];
                         if (handler->handler != &gen_invalid) {
                             /* Special hack to properly dump SPE insns */
@@ -9087,11 +9132,24 @@ static void ppc_cpu_unrealizefn(DeviceState *dev, Error **errp)
 {
     PowerPCCPU *cpu = POWERPC_CPU(dev);
     CPUPPCState *env = &cpu->env;
-    int i;
+    opc_handler_t **table;
+    int i, j;
 
     for (i = 0; i < PPC_CPU_OPCODES_LEN; i++) {
-        if (env->opcodes[i] != &invalid_handler) {
-            g_free(env->opcodes[i]);
+        if (env->opcodes[i] == &invalid_handler) {
+            continue;
+        }
+        if (is_indirect_opcode(env->opcodes[i])) {
+            table = ind_table(env->opcodes[i]);
+            for (j = 0; j < PPC_CPU_INDIRECT_OPCODES_LEN; j++) {
+                if (table[j] != &invalid_handler &&
+                        is_indirect_opcode(table[j])) {
+                    g_free((opc_handler_t *)((uintptr_t)table[j] &
+                        ~PPC_INDIRECT));
+                }
+            }
+            g_free((opc_handler_t *)((uintptr_t)env->opcodes[i] &
+                ~PPC_INDIRECT));
         }
     }
 }
@@ -9137,7 +9195,7 @@ int ppc_set_compat(PowerPCCPU *cpu, uint32_t cpu_version)
         break;
     }
 
-    if (kvm_enabled() && kvmppc_set_compat(cpu, cpu->max_compat) < 0) {
+    if (kvm_enabled() && kvmppc_set_compat(cpu, cpu->cpu_version) < 0) {
         error_report("Unable to set compatibility mode in KVM");
         ret = -1;
     }
@@ -9456,6 +9514,14 @@ static bool ppc_cpu_has_work(CPUState *cs)
     return msr_ee && (cs->interrupt_request & CPU_INTERRUPT_HARD);
 }
 
+static void ppc_cpu_exec_enter(CPUState *cs)
+{
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *env = &cpu->env;
+
+    env->reserve_addr = -1;
+}
+
 /* CPUClass::reset() */
 static void ppc_cpu_reset(CPUState *s)
 {
@@ -9623,6 +9689,7 @@ static void ppc_cpu_class_init(ObjectClass *oc, void *data)
     cc->class_by_name = ppc_cpu_class_by_name;
     cc->has_work = ppc_cpu_has_work;
     cc->do_interrupt = ppc_cpu_do_interrupt;
+    cc->cpu_exec_interrupt = ppc_cpu_exec_interrupt;
     cc->dump_state = ppc_cpu_dump_state;
     cc->dump_statistics = ppc_cpu_dump_statistics;
     cc->set_pc = ppc_cpu_set_pc;
@@ -9638,6 +9705,7 @@ static void ppc_cpu_class_init(ObjectClass *oc, void *data)
     cc->write_elf64_qemunote = ppc64_cpu_write_elf64_qemunote;
 #endif
 #endif
+    cc->cpu_exec_enter = ppc_cpu_exec_enter;
 
     cc->gdb_num_core_regs = 71;
 
diff --git a/target-s390x/Makefile.objs b/target-s390x/Makefile.objs
index f8731463f..2c5749447 100644
--- a/target-s390x/Makefile.objs
+++ b/target-s390x/Makefile.objs
@@ -1,5 +1,5 @@
 obj-y += translate.o helper.o cpu.o interrupt.o
 obj-y += int_helper.o fpu_helper.o cc_helper.o mem_helper.o misc_helper.o
 obj-y += gdbstub.o
-obj-$(CONFIG_SOFTMMU) += ioinst.o arch_dump.o
+obj-$(CONFIG_SOFTMMU) += machine.o ioinst.o arch_dump.o
 obj-$(CONFIG_KVM) += kvm.o
diff --git a/target-s390x/cpu-qom.h b/target-s390x/cpu-qom.h
index f9c96d13a..8b376df1b 100644
--- a/target-s390x/cpu-qom.h
+++ b/target-s390x/cpu-qom.h
@@ -77,7 +77,12 @@ static inline S390CPU *s390_env_get_cpu(CPUS390XState *env)
 
 #define ENV_OFFSET offsetof(S390CPU, env)
 
+#ifndef CONFIG_USER_ONLY
+extern const struct VMStateDescription vmstate_s390_cpu;
+#endif
+
 void s390_cpu_do_interrupt(CPUState *cpu);
+bool s390_cpu_exec_interrupt(CPUState *cpu, int int_req);
 void s390_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
                          int flags);
 int s390_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs,
@@ -89,5 +94,6 @@ hwaddr s390_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
 hwaddr s390_cpu_get_phys_addr_debug(CPUState *cpu, vaddr addr);
 int s390_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
 int s390_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+void s390_cpu_gdb_init(CPUState *cs);
 
 #endif
diff --git a/target-s390x/cpu.c b/target-s390x/cpu.c
index c3082b73c..d2f6312e0 100644
--- a/target-s390x/cpu.c
+++ b/target-s390x/cpu.c
@@ -26,7 +26,9 @@
 #include "cpu.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
+#include "qemu/error-report.h"
 #include "hw/hw.h"
+#include "trace.h"
 #ifndef CONFIG_USER_ONLY
 #include "sysemu/arch_init.h"
 #endif
@@ -81,7 +83,7 @@ static void s390_cpu_load_normal(CPUState *s)
     S390CPU *cpu = S390_CPU(s);
     cpu->env.psw.addr = ldl_phys(s->as, 4) & PSW_MASK_ESA_ADDR;
     cpu->env.psw.mask = PSW_MASK_32 | PSW_MASK_64;
-    s390_add_running_cpu(cpu);
+    s390_cpu_set_state(CPU_STATE_OPERATING, cpu);
 }
 #endif
 
@@ -93,11 +95,8 @@ static void s390_cpu_reset(CPUState *s)
     CPUS390XState *env = &cpu->env;
 
     env->pfault_token = -1UL;
-    s390_del_running_cpu(cpu);
     scc->parent_reset(s);
-#if !defined(CONFIG_USER_ONLY)
-    s->halted = 1;
-#endif
+    s390_cpu_set_state(CPU_STATE_STOPPED, cpu);
     tlb_flush(s, 1);
 }
 
@@ -118,14 +117,10 @@ static void s390_cpu_initial_reset(CPUState *s)
 
     env->pfault_token = -1UL;
 
-#if defined(CONFIG_KVM)
     /* Reset state inside the kernel that we cannot access yet from QEMU. */
     if (kvm_enabled()) {
-        if (kvm_vcpu_ioctl(s, KVM_S390_INITIAL_RESET, NULL)) {
-            perror("Initial CPU reset failed");
-        }
+        kvm_s390_reset_vcpu(cpu);
     }
-#endif
 }
 
 /* CPUClass:reset() */
@@ -135,9 +130,8 @@ static void s390_cpu_full_reset(CPUState *s)
     S390CPUClass *scc = S390_CPU_GET_CLASS(cpu);
     CPUS390XState *env = &cpu->env;
 
-    s390_del_running_cpu(cpu);
-
     scc->parent_reset(s);
+    s390_cpu_set_state(CPU_STATE_STOPPED, cpu);
 
     memset(env, 0, offsetof(CPUS390XState, cpu_num));
 
@@ -147,16 +141,10 @@ static void s390_cpu_full_reset(CPUState *s)
 
     env->pfault_token = -1UL;
 
-    /* set halted to 1 to make sure we can add the cpu in
-     * s390_ipl_cpu code, where CPUState::halted is set back to 0
-     * after incrementing the cpu counter */
-#if !defined(CONFIG_USER_ONLY)
-    s->halted = 1;
-
+    /* Reset state inside the kernel that we cannot access yet from QEMU. */
     if (kvm_enabled()) {
         kvm_s390_reset_vcpu(cpu);
     }
-#endif
     tlb_flush(s, 1);
 }
 
@@ -165,7 +153,7 @@ static void s390_cpu_machine_reset_cb(void *opaque)
 {
     S390CPU *cpu = opaque;
 
-    cpu_reset(CPU(cpu));
+    run_on_cpu(CPU(cpu), s390_do_cpu_full_reset, CPU(cpu));
 }
 #endif
 
@@ -174,8 +162,13 @@ static void s390_cpu_realizefn(DeviceState *dev, Error **errp)
     CPUState *cs = CPU(dev);
     S390CPUClass *scc = S390_CPU_GET_CLASS(dev);
 
+    s390_cpu_gdb_init(cs);
     qemu_init_vcpu(cs);
+#if !defined(CONFIG_USER_ONLY)
+    run_on_cpu(cs, s390_do_cpu_full_reset, cs);
+#else
     cpu_reset(cs);
+#endif
 
     scc->parent_realize(dev, errp);
 }
@@ -201,10 +194,7 @@ static void s390_cpu_initfn(Object *obj)
     env->tod_basetime = 0;
     env->tod_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, s390x_tod_timer, cpu);
     env->cpu_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, s390x_cpu_timer, cpu);
-    /* set CPUState::halted state to 1 to avoid decrementing the running
-     * cpu counter in s390_cpu_reset to a negative number at
-     * initial ipl */
-    cs->halted = 1;
+    s390_cpu_set_state(CPU_STATE_STOPPED, cpu);
 #endif
     env->cpu_num = cpu_num++;
     env->ext_index = -1;
@@ -224,10 +214,83 @@ static void s390_cpu_finalize(Object *obj)
 #endif
 }
 
-static const VMStateDescription vmstate_s390_cpu = {
-    .name = "cpu",
-    .unmigratable = 1,
-};
+#if !defined(CONFIG_USER_ONLY)
+static bool disabled_wait(CPUState *cpu)
+{
+    return cpu->halted && !(S390_CPU(cpu)->env.psw.mask &
+                            (PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK));
+}
+
+static unsigned s390_count_running_cpus(void)
+{
+    CPUState *cpu;
+    int nr_running = 0;
+
+    CPU_FOREACH(cpu) {
+        uint8_t state = S390_CPU(cpu)->env.cpu_state;
+        if (state == CPU_STATE_OPERATING ||
+            state == CPU_STATE_LOAD) {
+            if (!disabled_wait(cpu)) {
+                nr_running++;
+            }
+        }
+    }
+
+    return nr_running;
+}
+
+unsigned int s390_cpu_halt(S390CPU *cpu)
+{
+    CPUState *cs = CPU(cpu);
+    trace_cpu_halt(cs->cpu_index);
+
+    if (!cs->halted) {
+        cs->halted = 1;
+        cs->exception_index = EXCP_HLT;
+    }
+
+    return s390_count_running_cpus();
+}
+
+void s390_cpu_unhalt(S390CPU *cpu)
+{
+    CPUState *cs = CPU(cpu);
+    trace_cpu_unhalt(cs->cpu_index);
+
+    if (cs->halted) {
+        cs->halted = 0;
+        cs->exception_index = -1;
+    }
+}
+
+unsigned int s390_cpu_set_state(uint8_t cpu_state, S390CPU *cpu)
+ {
+    trace_cpu_set_state(CPU(cpu)->cpu_index, cpu_state);
+
+    switch (cpu_state) {
+    case CPU_STATE_STOPPED:
+    case CPU_STATE_CHECK_STOP:
+        /* halt the cpu for common infrastructure */
+        s390_cpu_halt(cpu);
+        break;
+    case CPU_STATE_OPERATING:
+    case CPU_STATE_LOAD:
+        /* unhalt the cpu for common infrastructure */
+        s390_cpu_unhalt(cpu);
+        break;
+    default:
+        error_report("Requested CPU state is not a valid S390 CPU state: %u",
+                     cpu_state);
+        exit(1);
+    }
+    if (kvm_enabled() && cpu->env.cpu_state != cpu_state) {
+        kvm_s390_set_cpu_state(cpu, cpu_state);
+    }
+    cpu->env.cpu_state = cpu_state;
+
+    return s390_count_running_cpus();
+}
+#endif
 
 static void s390_cpu_class_init(ObjectClass *oc, void *data)
 {
@@ -255,11 +318,13 @@ static void s390_cpu_class_init(ObjectClass *oc, void *data)
     cc->handle_mmu_fault = s390_cpu_handle_mmu_fault;
 #else
     cc->get_phys_page_debug = s390_cpu_get_phys_page_debug;
+    cc->vmsd = &vmstate_s390_cpu;
     cc->write_elf64_note = s390_cpu_write_elf64_note;
     cc->write_elf64_qemunote = s390_cpu_write_elf64_qemunote;
+    cc->cpu_exec_interrupt = s390_cpu_exec_interrupt;
 #endif
-    dc->vmsd = &vmstate_s390_cpu;
-    cc->gdb_num_core_regs = S390_NUM_REGS;
+    cc->gdb_num_core_regs = S390_NUM_CORE_REGS;
+    cc->gdb_core_xml_file = "s390x-core64.xml";
 }
 
 static const TypeInfo s390_cpu_type_info = {
diff --git a/target-s390x/cpu.h b/target-s390x/cpu.h
index b13761d92..fe2f95d08 100644
--- a/target-s390x/cpu.h
+++ b/target-s390x/cpu.h
@@ -141,6 +141,20 @@ typedef struct CPUS390XState {
     QEMUTimer *tod_timer;
 
     QEMUTimer *cpu_timer;
+
+    /*
+     * The cpu state represents the logical state of a cpu. In contrast to other
+     * architectures, there is a difference between a halt and a stop on s390.
+     * If all cpus are either stopped (including check stop) or in the disabled
+     * wait state, the vm can be shut down.
+     */
+#define CPU_STATE_UNINITIALIZED        0x00
+#define CPU_STATE_STOPPED              0x01
+#define CPU_STATE_CHECK_STOP           0x02
+#define CPU_STATE_OPERATING            0x03
+#define CPU_STATE_LOAD                 0x04
+    uint8_t cpu_state;
+
 } CPUS390XState;
 
 #include "cpu-qom.h"
@@ -353,6 +367,21 @@ static inline hwaddr decode_basedisp_s(CPUS390XState *env, uint32_t ipb)
 /* Base/displacement are at the same locations. */
 #define decode_basedisp_rs decode_basedisp_s
 
+/* helper functions for run_on_cpu() */
+static inline void s390_do_cpu_reset(void *arg)
+{
+    CPUState *cs = arg;
+    S390CPUClass *scc = S390_CPU_GET_CLASS(cs);
+
+    scc->cpu_reset(cs);
+}
+static inline void s390_do_cpu_full_reset(void *arg)
+{
+    CPUState *cs = arg;
+
+    cpu_reset(cs);
+}
+
 void s390x_tod_timer(void *opaque);
 void s390x_cpu_timer(void *opaque);
 
@@ -360,16 +389,12 @@ int s390_virtio_hypercall(CPUS390XState *env);
 void s390_virtio_irq(int config_change, uint64_t token);
 
 #ifdef CONFIG_KVM
-void kvm_s390_reset_vcpu(S390CPU *cpu);
 void kvm_s390_virtio_irq(int config_change, uint64_t token);
 void kvm_s390_service_interrupt(uint32_t parm);
 void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq);
 void kvm_s390_floating_interrupt(struct kvm_s390_irq *irq);
 int kvm_s390_inject_flic(struct kvm_s390_irq *irq);
 #else
-static inline void kvm_s390_reset_vcpu(S390CPU *cpu)
-{
-}
 static inline void kvm_s390_virtio_irq(int config_change, uint64_t token)
 {
 }
@@ -378,8 +403,9 @@ static inline void kvm_s390_service_interrupt(uint32_t parm)
 }
 #endif
 S390CPU *s390_cpu_addr2state(uint16_t cpu_addr);
-void s390_add_running_cpu(S390CPU *cpu);
-unsigned s390_del_running_cpu(S390CPU *cpu);
+unsigned int s390_cpu_halt(S390CPU *cpu);
+void s390_cpu_unhalt(S390CPU *cpu);
+unsigned int s390_cpu_set_state(uint8_t cpu_state, S390CPU *cpu);
 
 /* service interrupts are floating therefore we must not pass an cpustate */
 void s390_sclp_extint(uint32_t parm);
@@ -388,11 +414,16 @@ void s390_sclp_extint(uint32_t parm);
 extern const hwaddr virtio_size;
 
 #else
-static inline void s390_add_running_cpu(S390CPU *cpu)
+static inline unsigned int s390_cpu_halt(S390CPU *cpu)
+{
+    return 0;
+}
+
+static inline void s390_cpu_unhalt(S390CPU *cpu)
 {
 }
 
-static inline unsigned s390_del_running_cpu(S390CPU *cpu)
+static inline unsigned int s390_cpu_set_state(uint8_t cpu_state, S390CPU *cpu)
 {
     return 0;
 }
@@ -551,44 +582,8 @@ void s390_cpu_list(FILE *f, fprintf_function cpu_fprintf);
 #define S390_R13_REGNUM 15
 #define S390_R14_REGNUM 16
 #define S390_R15_REGNUM 17
-/* Access Registers.  */
-#define S390_A0_REGNUM 18
-#define S390_A1_REGNUM 19
-#define S390_A2_REGNUM 20
-#define S390_A3_REGNUM 21
-#define S390_A4_REGNUM 22
-#define S390_A5_REGNUM 23
-#define S390_A6_REGNUM 24
-#define S390_A7_REGNUM 25
-#define S390_A8_REGNUM 26
-#define S390_A9_REGNUM 27
-#define S390_A10_REGNUM 28
-#define S390_A11_REGNUM 29
-#define S390_A12_REGNUM 30
-#define S390_A13_REGNUM 31
-#define S390_A14_REGNUM 32
-#define S390_A15_REGNUM 33
-/* Floating Point Control Word.  */
-#define S390_FPC_REGNUM 34
-/* Floating Point Registers.  */
-#define S390_F0_REGNUM 35
-#define S390_F1_REGNUM 36
-#define S390_F2_REGNUM 37
-#define S390_F3_REGNUM 38
-#define S390_F4_REGNUM 39
-#define S390_F5_REGNUM 40
-#define S390_F6_REGNUM 41
-#define S390_F7_REGNUM 42
-#define S390_F8_REGNUM 43
-#define S390_F9_REGNUM 44
-#define S390_F10_REGNUM 45
-#define S390_F11_REGNUM 46
-#define S390_F12_REGNUM 47
-#define S390_F13_REGNUM 48
-#define S390_F14_REGNUM 49
-#define S390_F15_REGNUM 50
-/* Total.  */
-#define S390_NUM_REGS 51
+/* Total Core Registers. */
+#define S390_NUM_CORE_REGS 18
 
 /* CC optimization */
 
@@ -1045,6 +1040,10 @@ static inline void cpu_inject_crw_mchk(S390CPU *cpu)
     cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HARD);
 }
 
+/* from s390-virtio-ccw */
+#define MEM_SECTION_SIZE             0x10000000UL
+#define MAX_AVAIL_SLOTS              32
+
 /* fpu_helper.c */
 uint32_t set_cc_nz_f32(float32 v);
 uint32_t set_cc_nz_f64(float64 v);
@@ -1067,7 +1066,10 @@ void kvm_s390_enable_css_support(S390CPU *cpu);
 int kvm_s390_assign_subch_ioeventfd(EventNotifier *notifier, uint32_t sch,
                                     int vq, bool assign);
 int kvm_s390_cpu_restart(S390CPU *cpu);
+int kvm_s390_get_memslot_count(KVMState *s);
 void kvm_s390_clear_cmma_callback(void *opaque);
+int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state);
+void kvm_s390_reset_vcpu(S390CPU *cpu);
 #else
 static inline void kvm_s390_io_interrupt(uint16_t subchannel_id,
                                         uint16_t subchannel_nr,
@@ -1094,6 +1096,17 @@ static inline int kvm_s390_cpu_restart(S390CPU *cpu)
 static inline void kvm_s390_clear_cmma_callback(void *opaque)
 {
 }
+static inline int kvm_s390_get_memslot_count(KVMState *s)
+{
+  return MAX_AVAIL_SLOTS;
+}
+static inline int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state)
+{
+    return -ENOSYS;
+}
+static inline void kvm_s390_reset_vcpu(S390CPU *cpu)
+{
+}
 #endif
 
 static inline void cmma_reset(S390CPU *cpu)
@@ -1112,6 +1125,15 @@ static inline int s390_cpu_restart(S390CPU *cpu)
     return -ENOSYS;
 }
 
+static inline int s390_get_memslot_count(KVMState *s)
+{
+    if (kvm_enabled()) {
+        return kvm_s390_get_memslot_count(s);
+    } else {
+        return MAX_AVAIL_SLOTS;
+    }
+}
+
 void s390_io_interrupt(uint16_t subchannel_id, uint16_t subchannel_nr,
                        uint32_t io_int_parm, uint32_t io_int_word);
 void s390_crw_mchk(void);
diff --git a/target-s390x/gdbstub.c b/target-s390x/gdbstub.c
index a129742e2..8945f0271 100644
--- a/target-s390x/gdbstub.c
+++ b/target-s390x/gdbstub.c
@@ -31,21 +31,18 @@ int s390_cpu_gdb_read_register(CPUState *cs, uint8_t *mem_buf, int n)
 
     switch (n) {
     case S390_PSWM_REGNUM:
-        cc_op = calc_cc(env, env->cc_op, env->cc_src, env->cc_dst, env->cc_vr);
-        val = deposit64(env->psw.mask, 44, 2, cc_op);
-        return gdb_get_regl(mem_buf, val);
+        if (tcg_enabled()) {
+            cc_op = calc_cc(env, env->cc_op, env->cc_src, env->cc_dst,
+                            env->cc_vr);
+            val = deposit64(env->psw.mask, 44, 2, cc_op);
+            return gdb_get_regl(mem_buf, val);
+        }
+        return gdb_get_regl(mem_buf, env->psw.mask);
     case S390_PSWA_REGNUM:
         return gdb_get_regl(mem_buf, env->psw.addr);
     case S390_R0_REGNUM ... S390_R15_REGNUM:
-        return gdb_get_regl(mem_buf, env->regs[n-S390_R0_REGNUM]);
-    case S390_A0_REGNUM ... S390_A15_REGNUM:
-        return gdb_get_reg32(mem_buf, env->aregs[n-S390_A0_REGNUM]);
-    case S390_FPC_REGNUM:
-        return gdb_get_reg32(mem_buf, env->fpc);
-    case S390_F0_REGNUM ... S390_F15_REGNUM:
-        return gdb_get_reg64(mem_buf, env->fregs[n-S390_F0_REGNUM].ll);
+        return gdb_get_regl(mem_buf, env->regs[n - S390_R0_REGNUM]);
     }
-
     return 0;
 }
 
@@ -53,36 +50,94 @@ int s390_cpu_gdb_write_register(CPUState *cs, uint8_t *mem_buf, int n)
 {
     S390CPU *cpu = S390_CPU(cs);
     CPUS390XState *env = &cpu->env;
-    target_ulong tmpl;
-    uint32_t tmp32;
-    int r = 8;
-    tmpl = ldtul_p(mem_buf);
-    tmp32 = ldl_p(mem_buf);
+    target_ulong tmpl = ldtul_p(mem_buf);
 
     switch (n) {
     case S390_PSWM_REGNUM:
         env->psw.mask = tmpl;
-        env->cc_op = extract64(tmpl, 44, 2);
+        if (tcg_enabled()) {
+            env->cc_op = extract64(tmpl, 44, 2);
+        }
         break;
     case S390_PSWA_REGNUM:
         env->psw.addr = tmpl;
         break;
     case S390_R0_REGNUM ... S390_R15_REGNUM:
-        env->regs[n-S390_R0_REGNUM] = tmpl;
+        env->regs[n - S390_R0_REGNUM] = tmpl;
         break;
+    default:
+        return 0;
+    }
+    return 8;
+}
+
+/* the values represent the positions in s390-acr.xml */
+#define S390_A0_REGNUM 0
+#define S390_A15_REGNUM 15
+/* total number of registers in s390-acr.xml */
+#define S390_NUM_AC_REGS 16
+
+static int cpu_read_ac_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    switch (n) {
     case S390_A0_REGNUM ... S390_A15_REGNUM:
-        env->aregs[n-S390_A0_REGNUM] = tmp32;
-        r = 4;
-        break;
+        return gdb_get_reg32(mem_buf, env->aregs[n]);
+    default:
+        return 0;
+    }
+}
+
+static int cpu_write_ac_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    switch (n) {
+    case S390_A0_REGNUM ... S390_A15_REGNUM:
+        env->aregs[n] = ldl_p(mem_buf);
+        return 4;
+    default:
+        return 0;
+    }
+}
+
+/* the values represent the positions in s390-fpr.xml */
+#define S390_FPC_REGNUM 0
+#define S390_F0_REGNUM 1
+#define S390_F15_REGNUM 16
+/* total number of registers in s390-fpr.xml */
+#define S390_NUM_FP_REGS 17
+
+static int cpu_read_fp_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    switch (n) {
     case S390_FPC_REGNUM:
-        env->fpc = tmp32;
-        r = 4;
-        break;
+        return gdb_get_reg32(mem_buf, env->fpc);
     case S390_F0_REGNUM ... S390_F15_REGNUM:
-        env->fregs[n-S390_F0_REGNUM].ll = tmpl;
-        break;
+        return gdb_get_reg64(mem_buf, env->fregs[n - S390_F0_REGNUM].ll);
     default:
         return 0;
     }
-    return r;
+}
+
+static int cpu_write_fp_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    switch (n) {
+    case S390_FPC_REGNUM:
+        env->fpc = ldl_p(mem_buf);
+        return 4;
+    case S390_F0_REGNUM ... S390_F15_REGNUM:
+        env->fregs[n - S390_F0_REGNUM].ll = ldtul_p(mem_buf);
+        return 8;
+    default:
+        return 0;
+    }
+}
+
+void s390_cpu_gdb_init(CPUState *cs)
+{
+    gdb_register_coprocessor(cs, cpu_read_ac_reg,
+                             cpu_write_ac_reg,
+                             S390_NUM_AC_REGS, "s390-acr.xml", 0);
+
+    gdb_register_coprocessor(cs, cpu_read_fp_reg,
+                             cpu_write_fp_reg,
+                             S390_NUM_FP_REGS, "s390-fpr.xml", 0);
 }
diff --git a/target-s390x/helper.c b/target-s390x/helper.c
index 67ab1065a..09aec7b42 100644
--- a/target-s390x/helper.c
+++ b/target-s390x/helper.c
@@ -504,23 +504,18 @@ hwaddr s390_cpu_get_phys_addr_debug(CPUState *cs, vaddr vaddr)
 
 void load_psw(CPUS390XState *env, uint64_t mask, uint64_t addr)
 {
+    env->psw.addr = addr;
+    env->psw.mask = mask;
+    env->cc_op = (mask >> 44) & 3;
+
     if (mask & PSW_MASK_WAIT) {
         S390CPU *cpu = s390_env_get_cpu(env);
-        CPUState *cs = CPU(cpu);
-        if (!(mask & (PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK))) {
-            if (s390_del_running_cpu(cpu) == 0) {
+        if (s390_cpu_halt(cpu) == 0) {
 #ifndef CONFIG_USER_ONLY
-                qemu_system_shutdown_request();
+            qemu_system_shutdown_request();
 #endif
-            }
         }
-        cs->halted = 1;
-        cs->exception_index = EXCP_HLT;
     }
-
-    env->psw.addr = addr;
-    env->psw.mask = mask;
-    env->cc_op = (mask >> 44) & 3;
 }
 
 static uint64_t get_psw_mask(CPUS390XState *env)
@@ -818,7 +813,7 @@ void s390_cpu_do_interrupt(CPUState *cs)
     qemu_log_mask(CPU_LOG_INT, "%s: %d at pc=%" PRIx64 "\n",
                   __func__, cs->exception_index, env->psw.addr);
 
-    s390_add_running_cpu(cpu);
+    s390_cpu_set_state(CPU_STATE_OPERATING, cpu);
     /* handle machine checks */
     if ((env->psw.mask & PSW_MASK_MCHECK) &&
         (cs->exception_index == -1)) {
@@ -876,4 +871,17 @@ void s390_cpu_do_interrupt(CPUState *cs)
     }
 }
 
+bool s390_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    if (interrupt_request & CPU_INTERRUPT_HARD) {
+        S390CPU *cpu = S390_CPU(cs);
+        CPUS390XState *env = &cpu->env;
+
+        if (env->psw.mask & PSW_MASK_EXT) {
+            s390_cpu_do_interrupt(cs);
+            return true;
+        }
+    }
+    return false;
+}
 #endif /* CONFIG_USER_ONLY */
diff --git a/target-s390x/insn-data.def b/target-s390x/insn-data.def
index b42ebb6a1..4d2feb697 100644
--- a/target-s390x/insn-data.def
+++ b/target-s390x/insn-data.def
@@ -744,9 +744,9 @@
 /* SERVICE CALL LOGICAL PROCESSOR (PV hypercall) */
     C(0xb220, SERVC,   RRE,   Z,   r1_o, r2_o, 0, 0, servc, 0)
 /* SET ADDRESSING MODE */
-    /* We only do 64-bit, so accept this as a no-op.
-       Let SAM24 and SAM31 signal illegal instruction.  */
-    C(0x010e, SAM64,   E,     Z,   0, 0, 0, 0, 0, 0)
+    D(0x010c, SAM24,   E,     Z,   0, 0, 0, 0, sam, 0, 0)
+    D(0x010d, SAM31,   E,     Z,   0, 0, 0, 0, sam, 0, 1)
+    D(0x010e, SAM64,   E,     Z,   0, 0, 0, 0, sam, 0, 3)
 /* SET ADDRESS SPACE CONTROL FAST */
     C(0xb279, SACF,    S,     Z,   0, a2, 0, 0, sacf, 0)
 /* SET CLOCK */
diff --git a/target-s390x/interrupt.c b/target-s390x/interrupt.c
index 23a9114f5..1404d0afd 100644
--- a/target-s390x/interrupt.c
+++ b/target-s390x/interrupt.c
@@ -22,9 +22,7 @@ void s390_sclp_extint(uint32_t parm)
         kvm_s390_service_interrupt(parm);
     } else {
         S390CPU *dummy_cpu = s390_cpu_addr2state(0);
-        CPUS390XState *env = &dummy_cpu->env;
 
-        env->psw.addr += 4;
         cpu_inject_ext(dummy_cpu, EXT_SERVICE, parm, 0);
     }
 }
diff --git a/target-s390x/ioinst.h b/target-s390x/ioinst.h
index 5bbc67d15..29f6423df 100644
--- a/target-s390x/ioinst.h
+++ b/target-s390x/ioinst.h
@@ -156,6 +156,16 @@ typedef struct ORB {
 #define ORB_CTRL1_MASK_ORBX 0x01
 #define ORB_CTRL1_MASK_INVALID 0x3e
 
+/* channel command word (type 0) */
+typedef struct CCW0 {
+        uint8_t cmd_code;
+        uint8_t cda0;
+        uint16_t cda1;
+        uint8_t flags;
+        uint8_t reserved;
+        uint16_t count;
+} QEMU_PACKED CCW0;
+
 /* channel command word (type 1) */
 typedef struct CCW1 {
     uint8_t cmd_code;
diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
index a32d91aa0..2c638ab7b 100644
--- a/target-s390x/kvm.c
+++ b/target-s390x/kvm.c
@@ -106,7 +106,7 @@ const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
 static int cap_sync_regs;
 static int cap_async_pf;
 
-static void *legacy_s390_alloc(size_t size);
+static void *legacy_s390_alloc(size_t size, uint64_t *align);
 
 static int kvm_s390_check_clear_cmma(KVMState *s)
 {
@@ -181,9 +181,10 @@ unsigned long kvm_arch_vcpu_id(CPUState *cpu)
     return cpu->cpu_index;
 }
 
-int kvm_arch_init_vcpu(CPUState *cpu)
+int kvm_arch_init_vcpu(CPUState *cs)
 {
-    /* nothing todo yet */
+    S390CPU *cpu = S390_CPU(cs);
+    kvm_s390_set_cpu_state(cpu, cpu->env.cpu_state);
     return 0;
 }
 
@@ -197,7 +198,7 @@ void kvm_s390_reset_vcpu(S390CPU *cpu)
      * Before this ioctl cpu_synchronize_state() is called in common kvm
      * code (kvm-all) */
     if (kvm_vcpu_ioctl(cs, KVM_S390_INITIAL_RESET, NULL)) {
-        perror("Can't reset vcpu\n");
+        error_report("Initial CPU reset failed on CPU %i\n", cs->cpu_index);
     }
 }
 
@@ -403,7 +404,7 @@ int kvm_arch_get_registers(CPUState *cs)
  * to grow. We also have to use MAP parameters that avoid
  * read-only mapping of guest pages.
  */
-static void *legacy_s390_alloc(size_t size)
+static void *legacy_s390_alloc(size_t size, uint64_t *align)
 {
     void *mem;
 
@@ -826,18 +827,18 @@ static int handle_b9(S390CPU *cpu, struct kvm_run *run, uint8_t ipa1)
     return r;
 }
 
-static int handle_eb(S390CPU *cpu, struct kvm_run *run, uint8_t ipa1)
+static int handle_eb(S390CPU *cpu, struct kvm_run *run, uint8_t ipbl)
 {
     int r = 0;
 
-    switch (ipa1) {
+    switch (ipbl) {
     case PRIV_EB_SQBS:
         /* just inject exception */
         r = -1;
         break;
     default:
         r = -1;
-        DPRINTF("KVM: unhandled PRIV: 0xeb%x\n", ipa1);
+        DPRINTF("KVM: unhandled PRIV: 0xeb%x\n", ipbl);
         break;
     }
 
@@ -916,23 +917,30 @@ static int handle_diag(S390CPU *cpu, struct kvm_run *run, uint32_t ipb)
     return r;
 }
 
-static int kvm_s390_cpu_start(S390CPU *cpu)
+static void sigp_cpu_start(void *arg)
 {
-    s390_add_running_cpu(cpu);
-    qemu_cpu_kick(CPU(cpu));
+    CPUState *cs = arg;
+    S390CPU *cpu = S390_CPU(cs);
+
+    s390_cpu_set_state(CPU_STATE_OPERATING, cpu);
     DPRINTF("DONE: KVM cpu start: %p\n", &cpu->env);
-    return 0;
 }
 
-int kvm_s390_cpu_restart(S390CPU *cpu)
+static void sigp_cpu_restart(void *arg)
 {
+    CPUState *cs = arg;
+    S390CPU *cpu = S390_CPU(cs);
     struct kvm_s390_irq irq = {
         .type = KVM_S390_RESTART,
     };
 
     kvm_s390_vcpu_interrupt(cpu, &irq);
-    s390_add_running_cpu(cpu);
-    qemu_cpu_kick(CPU(cpu));
+    s390_cpu_set_state(CPU_STATE_OPERATING, cpu);
+}
+
+int kvm_s390_cpu_restart(S390CPU *cpu)
+{
+    run_on_cpu(CPU(cpu), sigp_cpu_restart, CPU(cpu));
     DPRINTF("DONE: KVM cpu restart: %p\n", &cpu->env);
     return 0;
 }
@@ -944,6 +952,7 @@ static void sigp_initial_cpu_reset(void *arg)
 
     cpu_synchronize_state(cpu);
     scc->initial_cpu_reset(cpu);
+    cpu_synchronize_post_reset(cpu);
 }
 
 static void sigp_cpu_reset(void *arg)
@@ -953,6 +962,7 @@ static void sigp_cpu_reset(void *arg)
 
     cpu_synchronize_state(cpu);
     scc->cpu_reset(cpu);
+    cpu_synchronize_post_reset(cpu);
 }
 
 #define SIGP_ORDER_MASK 0x000000ff
@@ -980,10 +990,12 @@ static int handle_sigp(S390CPU *cpu, struct kvm_run *run, uint8_t ipa1)
 
     switch (order_code) {
     case SIGP_START:
-        cc = kvm_s390_cpu_start(target_cpu);
+        run_on_cpu(CPU(target_cpu), sigp_cpu_start, CPU(target_cpu));
+        cc = 0;
         break;
     case SIGP_RESTART:
-        cc = kvm_s390_cpu_restart(target_cpu);
+        run_on_cpu(CPU(target_cpu), sigp_cpu_restart, CPU(target_cpu));
+        cc = 0;
         break;
     case SIGP_SET_ARCH:
         *statusreg &= 0xffffffff00000000UL;
@@ -1027,7 +1039,7 @@ static int handle_instruction(S390CPU *cpu, struct kvm_run *run)
         r = handle_b9(cpu, run, ipa1);
         break;
     case IPA0_EB:
-        r = handle_eb(cpu, run, ipa1);
+        r = handle_eb(cpu, run, run->s390_sieic.ipb & 0xff);
         break;
     case IPA0_DIAG:
         r = handle_diag(cpu, run, run->s390_sieic.ipb);
@@ -1065,7 +1077,7 @@ static void unmanageable_intercept(S390CPU *cpu, const char *str, int pswoffset)
     error_report("Unmanageable %s! CPU%i new PSW: 0x%016lx:%016lx",
                  str, cs->cpu_index, ldq_phys(cs->as, cpu->env.psa + pswoffset),
                  ldq_phys(cs->as, cpu->env.psa + pswoffset + 8));
-    s390_del_running_cpu(cpu);
+    s390_cpu_halt(cpu);
     guest_panicked();
 }
 
@@ -1094,7 +1106,8 @@ static int handle_intercept(S390CPU *cpu)
             break;
         case ICPT_WAITPSW:
             /* disabled wait, since enabled wait is handled in kernel */
-            if (s390_del_running_cpu(cpu) == 0) {
+            cpu_synchronize_state(cs);
+            if (s390_cpu_halt(cpu) == 0) {
                 if (is_special_wait_psw(cs)) {
                     qemu_system_shutdown_request();
                 } else {
@@ -1104,7 +1117,7 @@ static int handle_intercept(S390CPU *cpu)
             r = EXCP_HALTED;
             break;
         case ICPT_CPU_STOP:
-            if (s390_del_running_cpu(cpu) == 0) {
+            if (s390_cpu_set_state(CPU_STATE_STOPPED, cpu) == 0) {
                 qemu_system_shutdown_request();
             }
             r = EXCP_HALTED;
@@ -1259,7 +1272,7 @@ void kvm_s390_crw_mchk(void)
     struct kvm_s390_irq irq = {
         .type = KVM_S390_MCHK,
         .u.mchk.cr14 = 1 << 28,
-        .u.mchk.mcic = 0x00400f1d40330000,
+        .u.mchk.mcic = 0x00400f1d40330000ULL,
     };
     kvm_s390_floating_interrupt(&irq);
 }
@@ -1306,3 +1319,46 @@ int kvm_s390_assign_subch_ioeventfd(EventNotifier *notifier, uint32_t sch,
     }
     return kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
 }
+
+int kvm_s390_get_memslot_count(KVMState *s)
+{
+    return kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS);
+}
+
+int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state)
+{
+    struct kvm_mp_state mp_state = {};
+    int ret;
+
+    /* the kvm part might not have been initialized yet */
+    if (CPU(cpu)->kvm_state == NULL) {
+        return 0;
+    }
+
+    switch (cpu_state) {
+    case CPU_STATE_STOPPED:
+        mp_state.mp_state = KVM_MP_STATE_STOPPED;
+        break;
+    case CPU_STATE_CHECK_STOP:
+        mp_state.mp_state = KVM_MP_STATE_CHECK_STOP;
+        break;
+    case CPU_STATE_OPERATING:
+        mp_state.mp_state = KVM_MP_STATE_OPERATING;
+        break;
+    case CPU_STATE_LOAD:
+        mp_state.mp_state = KVM_MP_STATE_LOAD;
+        break;
+    default:
+        error_report("Requested CPU state is not a valid S390 CPU state: %u",
+                     cpu_state);
+        exit(1);
+    }
+
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MP_STATE, &mp_state);
+    if (ret) {
+        trace_kvm_failed_cpu_state_set(CPU(cpu)->cpu_index, cpu_state,
+                                       strerror(-ret));
+    }
+
+    return ret;
+}
diff --git a/target-s390x/machine.c b/target-s390x/machine.c
new file mode 100644
index 000000000..fbcb0d086
--- /dev/null
+++ b/target-s390x/machine.c
@@ -0,0 +1,76 @@
+/*
+ * S390x machine definitions and functions
+ *
+ * Copyright IBM Corp. 2014
+ *
+ * Authors:
+ *   Thomas Huth <thuth@linux.vnet.ibm.com>
+ *   Christian Borntraeger <borntraeger@de.ibm.com>
+ *   Jason J. Herne <jjherne@us.ibm.com>
+ *
+ * This work is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation; either version 2 of the License,
+ * or (at your option) any later version.
+ */
+
+#include "hw/hw.h"
+#include "cpu.h"
+#include "sysemu/kvm.h"
+
+static int cpu_post_load(void *opaque, int version_id)
+{
+    S390CPU *cpu = opaque;
+
+    /*
+     * As the cpu state is pushed to kvm via kvm_set_mp_state rather
+     * than via cpu_synchronize_state, we need update kvm here.
+     */
+    if (kvm_enabled()) {
+        kvm_s390_set_cpu_state(cpu, cpu->env.cpu_state);
+    }
+
+    return 0;
+}
+
+const VMStateDescription vmstate_s390_cpu = {
+    .name = "cpu",
+    .post_load = cpu_post_load,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_UINT64(env.fregs[0].ll, S390CPU),
+        VMSTATE_UINT64(env.fregs[1].ll, S390CPU),
+        VMSTATE_UINT64(env.fregs[2].ll, S390CPU),
+        VMSTATE_UINT64(env.fregs[3].ll, S390CPU),
+        VMSTATE_UINT64(env.fregs[4].ll, S390CPU),
+        VMSTATE_UINT64(env.fregs[5].ll, S390CPU),
+        VMSTATE_UINT64(env.fregs[6].ll, S390CPU),
+        VMSTATE_UINT64(env.fregs[7].ll, S390CPU),
+        VMSTATE_UINT64(env.fregs[8].ll, S390CPU),
+        VMSTATE_UINT64(env.fregs[9].ll, S390CPU),
+        VMSTATE_UINT64(env.fregs[10].ll, S390CPU),
+        VMSTATE_UINT64(env.fregs[11].ll, S390CPU),
+        VMSTATE_UINT64(env.fregs[12].ll, S390CPU),
+        VMSTATE_UINT64(env.fregs[13].ll, S390CPU),
+        VMSTATE_UINT64(env.fregs[14].ll, S390CPU),
+        VMSTATE_UINT64(env.fregs[15].ll, S390CPU),
+        VMSTATE_UINT64_ARRAY(env.regs, S390CPU, 16),
+        VMSTATE_UINT64(env.psw.mask, S390CPU),
+        VMSTATE_UINT64(env.psw.addr, S390CPU),
+        VMSTATE_UINT64(env.psa, S390CPU),
+        VMSTATE_UINT32(env.fpc, S390CPU),
+        VMSTATE_UINT32(env.todpr, S390CPU),
+        VMSTATE_UINT64(env.pfault_token, S390CPU),
+        VMSTATE_UINT64(env.pfault_compare, S390CPU),
+        VMSTATE_UINT64(env.pfault_select, S390CPU),
+        VMSTATE_UINT64(env.cputm, S390CPU),
+        VMSTATE_UINT64(env.ckc, S390CPU),
+        VMSTATE_UINT64(env.gbea, S390CPU),
+        VMSTATE_UINT64(env.pp, S390CPU),
+        VMSTATE_UINT32_ARRAY(env.aregs, S390CPU, 16),
+        VMSTATE_UINT64_ARRAY(env.cregs, S390CPU, 16),
+        VMSTATE_UINT8(env.cpu_state, S390CPU),
+        VMSTATE_END_OF_LIST()
+     },
+};
diff --git a/target-s390x/misc_helper.c b/target-s390x/misc_helper.c
index 0b625826e..ef9758a96 100644
--- a/target-s390x/misc_helper.c
+++ b/target-s390x/misc_helper.c
@@ -114,33 +114,16 @@ uint32_t HELPER(servc)(CPUS390XState *env, uint64_t r1, uint64_t r2)
 }
 
 #ifndef CONFIG_USER_ONLY
-static void cpu_reset_all(void)
-{
-    CPUState *cs;
-    S390CPUClass *scc;
-
-    CPU_FOREACH(cs) {
-        scc = S390_CPU_GET_CLASS(cs);
-        scc->cpu_reset(cs);
-    }
-}
-
-static void cpu_full_reset_all(void)
-{
-    CPUState *cpu;
-
-    CPU_FOREACH(cpu) {
-        cpu_reset(cpu);
-    }
-}
-
 static int modified_clear_reset(S390CPU *cpu)
 {
     S390CPUClass *scc = S390_CPU_GET_CLASS(cpu);
+    CPUState *t;
 
     pause_all_vcpus();
     cpu_synchronize_all_states();
-    cpu_full_reset_all();
+    CPU_FOREACH(t) {
+        run_on_cpu(t, s390_do_cpu_full_reset, t);
+    }
     cmma_reset(cpu);
     io_subsystem_reset();
     scc->load_normal(CPU(cpu));
@@ -152,10 +135,13 @@ static int modified_clear_reset(S390CPU *cpu)
 static int load_normal_reset(S390CPU *cpu)
 {
     S390CPUClass *scc = S390_CPU_GET_CLASS(cpu);
+    CPUState *t;
 
     pause_all_vcpus();
     cpu_synchronize_all_states();
-    cpu_reset_all();
+    CPU_FOREACH(t) {
+        run_on_cpu(t, s390_do_cpu_reset, t);
+    }
     cmma_reset(cpu);
     io_subsystem_reset();
     scc->initial_cpu_reset(CPU(cpu));
diff --git a/target-s390x/translate.c b/target-s390x/translate.c
index e2a1d05f1..dbf1993d4 100644
--- a/target-s390x/translate.c
+++ b/target-s390x/translate.c
@@ -42,6 +42,8 @@ static TCGv_ptr cpu_env;
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
 
+#include "trace-tcg.h"
+
 
 /* Information that (most) every instruction needs to manipulate.  */
 typedef struct DisasContext DisasContext;
@@ -2923,6 +2925,18 @@ static ExitStatus op_sacf(DisasContext *s, DisasOps *o)
     /* Addressing mode has changed, so end the block.  */
     return EXIT_PC_STALE;
 }
+
+static ExitStatus op_sam(DisasContext *s, DisasOps *o)
+{
+    int sam = s->insn->data;
+    TCGv_i64 tsam = tcg_const_i64(sam);
+
+    /* Overwrite PSW_MASK_64 and PSW_MASK_32 */
+    tcg_gen_deposit_i64(psw_mask, psw_mask, tsam, 31, 2);
+
+    tcg_temp_free_i64(tsam);
+    return EXIT_PC_STALE;
+}
 #endif
 
 static ExitStatus op_sar(DisasContext *s, DisasOps *o)
diff --git a/target-sh4/cpu-qom.h b/target-sh4/cpu-qom.h
index c04e78631..6341238aa 100644
--- a/target-sh4/cpu-qom.h
+++ b/target-sh4/cpu-qom.h
@@ -84,6 +84,7 @@ static inline SuperHCPU *sh_env_get_cpu(CPUSH4State *env)
 #define ENV_OFFSET offsetof(SuperHCPU, env)
 
 void superh_cpu_do_interrupt(CPUState *cpu);
+bool superh_cpu_exec_interrupt(CPUState *cpu, int int_req);
 void superh_cpu_dump_state(CPUState *cpu, FILE *f,
                            fprintf_function cpu_fprintf, int flags);
 hwaddr superh_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
diff --git a/target-sh4/cpu.c b/target-sh4/cpu.c
index e7f05212d..d187a2bdb 100644
--- a/target-sh4/cpu.c
+++ b/target-sh4/cpu.c
@@ -276,6 +276,7 @@ static void superh_cpu_class_init(ObjectClass *oc, void *data)
     cc->class_by_name = superh_cpu_class_by_name;
     cc->has_work = superh_cpu_has_work;
     cc->do_interrupt = superh_cpu_do_interrupt;
+    cc->cpu_exec_interrupt = superh_cpu_exec_interrupt;
     cc->dump_state = superh_cpu_dump_state;
     cc->set_pc = superh_cpu_set_pc;
     cc->synchronize_from_tb = superh_cpu_synchronize_from_tb;
diff --git a/target-sh4/helper.c b/target-sh4/helper.c
index 9ebdd5c9b..58113601e 100644
--- a/target-sh4/helper.c
+++ b/target-sh4/helper.c
@@ -863,3 +863,12 @@ int cpu_sh4_is_cached(CPUSH4State * env, target_ulong addr)
 }
 
 #endif
+
+bool superh_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    if (interrupt_request & CPU_INTERRUPT_HARD) {
+        superh_cpu_do_interrupt(cs);
+        return true;
+    }
+    return false;
+}
diff --git a/target-sh4/translate.c b/target-sh4/translate.c
index 812681814..3088edc6a 100644
--- a/target-sh4/translate.c
+++ b/target-sh4/translate.c
@@ -28,6 +28,9 @@
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
 
+#include "trace-tcg.h"
+
+
 typedef struct DisasContext {
     struct TranslationBlock *tb;
     target_ulong pc;
diff --git a/target-sparc/cpu.c b/target-sparc/cpu.c
index 3a0ee504e..aa7626c1b 100644
--- a/target-sparc/cpu.c
+++ b/target-sparc/cpu.c
@@ -70,6 +70,26 @@ static void sparc_cpu_reset(CPUState *s)
     env->cache_control = 0;
 }
 
+static bool sparc_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    if (interrupt_request & CPU_INTERRUPT_HARD) {
+        SPARCCPU *cpu = SPARC_CPU(cs);
+        CPUSPARCState *env = &cpu->env;
+
+        if (cpu_interrupts_enabled(env) && env->interrupt_index > 0) {
+            int pil = env->interrupt_index & 0xf;
+            int type = env->interrupt_index & 0xf0;
+
+            if (type != TT_EXTINT || cpu_pil_allowed(env, pil)) {
+                cs->exception_index = env->interrupt_index;
+                sparc_cpu_do_interrupt(cs);
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
 static int cpu_sparc_register(SPARCCPU *cpu, const char *cpu_model)
 {
     CPUClass *cc = CPU_GET_CLASS(cpu);
@@ -813,6 +833,7 @@ static void sparc_cpu_class_init(ObjectClass *oc, void *data)
     cc->parse_features = sparc_cpu_parse_features;
     cc->has_work = sparc_cpu_has_work;
     cc->do_interrupt = sparc_cpu_do_interrupt;
+    cc->cpu_exec_interrupt = sparc_cpu_exec_interrupt;
     cc->dump_state = sparc_cpu_dump_state;
 #if !defined(TARGET_SPARC64) && !defined(CONFIG_USER_ONLY)
     cc->memory_rw_debug = sparc_cpu_memory_rw_debug;
diff --git a/target-sparc/ldst_helper.c b/target-sparc/ldst_helper.c
index 03bd9f970..1a62e193b 100644
--- a/target-sparc/ldst_helper.c
+++ b/target-sparc/ldst_helper.c
@@ -2154,7 +2154,6 @@ void helper_stf_asi(CPUSPARCState *env, target_ulong addr, int asi, int size,
     unsigned int i;
     target_ulong val;
 
-    helper_check_align(env, addr, 3);
     addr = asi_address_mask(env, asi, addr);
 
     switch (asi) {
@@ -2192,7 +2191,21 @@ void helper_stf_asi(CPUSPARCState *env, target_ulong addr, int asi, int size,
         }
 
         return;
+    case 0xd2: /* 16-bit floating point load primary */
+    case 0xd3: /* 16-bit floating point load secondary */
+    case 0xda: /* 16-bit floating point load primary, LE */
+    case 0xdb: /* 16-bit floating point load secondary, LE */
+        helper_check_align(env, addr, 1);
+        /* Fall through */
+    case 0xd0: /* 8-bit floating point load primary */
+    case 0xd1: /* 8-bit floating point load secondary */
+    case 0xd8: /* 8-bit floating point load primary, LE */
+    case 0xd9: /* 8-bit floating point load secondary, LE */
+        val = env->fpr[rd / 2].l.lower;
+        helper_st_asi(env, addr, val, asi & 0x8d, ((asi & 2) >> 1) + 1);
+        return;
     default:
+        helper_check_align(env, addr, 3);
         break;
     }
 
diff --git a/target-sparc/translate.c b/target-sparc/translate.c
index 1ab07a18a..78c4e21cf 100644
--- a/target-sparc/translate.c
+++ b/target-sparc/translate.c
@@ -32,6 +32,9 @@
 
 #include "exec/helper-gen.h"
 
+#include "trace-tcg.h"
+
+
 #define DEBUG_DISAS
 
 #define DYNAMIC_PC  1 /* dynamic pc value */
diff --git a/target-tricore/Makefile.objs b/target-tricore/Makefile.objs
new file mode 100644
index 000000000..21e820d8f
--- /dev/null
+++ b/target-tricore/Makefile.objs
@@ -0,0 +1 @@
+obj-y += translate.o helper.o cpu.o op_helper.o
diff --git a/target-tricore/cpu-qom.h b/target-tricore/cpu-qom.h
new file mode 100644
index 000000000..66c966474
--- /dev/null
+++ b/target-tricore/cpu-qom.h
@@ -0,0 +1,70 @@
+/*
+ *  Copyright (c) 2012-2014 Bastian Koppelmann C-Lab/University Paderborn
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef QEMU_TRICORE_CPU_QOM_H
+#define QEMU_TRICORE_CPU_QOM_H
+
+#include "qom/cpu.h"
+
+
+#define TYPE_TRICORE_CPU "tricore-cpu"
+
+#define TRICORE_CPU_CLASS(klass) \
+    OBJECT_CLASS_CHECK(TriCoreCPUClass, (klass), TYPE_TRICORE_CPU)
+#define TRICORE_CPU(obj) \
+    OBJECT_CHECK(TriCoreCPU, (obj), TYPE_TRICORE_CPU)
+#define TRICORE_CPU_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(TriCoreCPUClass, (obj), TYPE_TRICORE_CPU)
+
+typedef struct TriCoreCPUClass {
+    /*< private >*/
+    CPUClass parent_class;
+    /*< public >*/
+
+    DeviceRealize parent_realize;
+    void (*parent_reset)(CPUState *cpu);
+} TriCoreCPUClass;
+
+/**
+ * TriCoreCPU:
+ * @env: #CPUTriCoreState
+ *
+ * A TriCore CPU.
+ */
+typedef struct TriCoreCPU {
+    /*< private >*/
+    CPUState parent_obj;
+    /*< public >*/
+
+    CPUTriCoreState env;
+} TriCoreCPU;
+
+static inline TriCoreCPU *tricore_env_get_cpu(CPUTriCoreState *env)
+{
+    return TRICORE_CPU(container_of(env, TriCoreCPU, env));
+}
+
+#define ENV_GET_CPU(e) CPU(tricore_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(TriCoreCPU, env)
+
+hwaddr tricore_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
+void tricore_cpu_dump_state(CPUState *cpu, FILE *f,
+                            fprintf_function cpu_fprintf, int flags);
+
+
+#endif /*QEMU_TRICORE_CPU_QOM_H */
diff --git a/target-tricore/cpu.c b/target-tricore/cpu.c
new file mode 100644
index 000000000..7bf041afb
--- /dev/null
+++ b/target-tricore/cpu.c
@@ -0,0 +1,191 @@
+/*
+ *  TriCore emulation for qemu: main translation routines.
+ *
+ *  Copyright (c) 2012-2014 Bastian Koppelmann C-Lab/University Paderborn
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cpu.h"
+#include "qemu-common.h"
+
+static inline void set_feature(CPUTriCoreState *env, int feature)
+{
+    env->features |= 1ULL << feature;
+}
+
+static void tricore_cpu_set_pc(CPUState *cs, vaddr value)
+{
+    TriCoreCPU *cpu = TRICORE_CPU(cs);
+    CPUTriCoreState *env = &cpu->env;
+
+    env->PC = value & ~(target_ulong)1;
+}
+
+static void tricore_cpu_synchronize_from_tb(CPUState *cs,
+                                            TranslationBlock *tb)
+{
+    TriCoreCPU *cpu = TRICORE_CPU(cs);
+    CPUTriCoreState *env = &cpu->env;
+
+    env->PC = tb->pc;
+}
+
+static void tricore_cpu_reset(CPUState *s)
+{
+    TriCoreCPU *cpu = TRICORE_CPU(s);
+    TriCoreCPUClass *tcc = TRICORE_CPU_GET_CLASS(cpu);
+    CPUTriCoreState *env = &cpu->env;
+
+    tcc->parent_reset(s);
+
+    tlb_flush(s, 1);
+
+    cpu_state_reset(env);
+}
+
+static bool tricore_cpu_has_work(CPUState *cs)
+{
+    return true;
+}
+
+static void tricore_cpu_realizefn(DeviceState *dev, Error **errp)
+{
+    CPUState *cs = CPU(dev);
+    TriCoreCPUClass *tcc = TRICORE_CPU_GET_CLASS(dev);
+
+    cpu_reset(cs);
+    qemu_init_vcpu(cs);
+
+    tcc->parent_realize(dev, errp);
+}
+
+
+static void tricore_cpu_initfn(Object *obj)
+{
+    CPUState *cs = CPU(obj);
+    TriCoreCPU *cpu = TRICORE_CPU(obj);
+    CPUTriCoreState *env = &cpu->env;
+
+    cs->env_ptr = env;
+    cpu_exec_init(env);
+
+    if (tcg_enabled()) {
+        tricore_tcg_init();
+    }
+}
+
+static ObjectClass *tricore_cpu_class_by_name(const char *cpu_model)
+{
+    ObjectClass *oc;
+    char *typename;
+
+    if (!cpu_model) {
+        return NULL;
+    }
+
+    typename = g_strdup_printf("%s-" TYPE_TRICORE_CPU, cpu_model);
+    oc = object_class_by_name(typename);
+    g_free(typename);
+    if (!oc || !object_class_dynamic_cast(oc, TYPE_TRICORE_CPU) ||
+        object_class_is_abstract(oc)) {
+        return NULL;
+    }
+    return oc;
+}
+
+static void tc1796_initfn(Object *obj)
+{
+    TriCoreCPU *cpu = TRICORE_CPU(obj);
+
+    set_feature(&cpu->env, TRICORE_FEATURE_13);
+}
+
+static void aurix_initfn(Object *obj)
+{
+    TriCoreCPU *cpu = TRICORE_CPU(obj);
+
+    set_feature(&cpu->env, TRICORE_FEATURE_16);
+}
+
+typedef struct TriCoreCPUInfo {
+    const char *name;
+    void (*initfn)(Object *obj);
+    void (*class_init)(ObjectClass *oc, void *data);
+} TriCoreCPUInfo;
+
+static const TriCoreCPUInfo tricore_cpus[] = {
+    { .name = "tc1796",      .initfn = tc1796_initfn },
+    { .name = "aurix",       .initfn = aurix_initfn },
+    { .name = NULL }
+};
+
+static void tricore_cpu_class_init(ObjectClass *c, void *data)
+{
+    TriCoreCPUClass *mcc = TRICORE_CPU_CLASS(c);
+    CPUClass *cc = CPU_CLASS(c);
+    DeviceClass *dc = DEVICE_CLASS(c);
+
+    mcc->parent_realize = dc->realize;
+    dc->realize = tricore_cpu_realizefn;
+
+    mcc->parent_reset = cc->reset;
+    cc->reset = tricore_cpu_reset;
+    cc->class_by_name = tricore_cpu_class_by_name;
+    cc->has_work = tricore_cpu_has_work;
+
+    cc->dump_state = tricore_cpu_dump_state;
+    cc->set_pc = tricore_cpu_set_pc;
+    cc->synchronize_from_tb = tricore_cpu_synchronize_from_tb;
+
+}
+
+static void cpu_register(const TriCoreCPUInfo *info)
+{
+    TypeInfo type_info = {
+        .parent = TYPE_TRICORE_CPU,
+        .instance_size = sizeof(TriCoreCPU),
+        .instance_init = info->initfn,
+        .class_size = sizeof(TriCoreCPUClass),
+        .class_init = info->class_init,
+    };
+
+    type_info.name = g_strdup_printf("%s-" TYPE_TRICORE_CPU, info->name);
+    type_register(&type_info);
+    g_free((void *)type_info.name);
+}
+
+static const TypeInfo tricore_cpu_type_info = {
+    .name = TYPE_TRICORE_CPU,
+    .parent = TYPE_CPU,
+    .instance_size = sizeof(TriCoreCPU),
+    .instance_init = tricore_cpu_initfn,
+    .abstract = true,
+    .class_size = sizeof(TriCoreCPUClass),
+    .class_init = tricore_cpu_class_init,
+};
+
+static void tricore_cpu_register_types(void)
+{
+    const TriCoreCPUInfo *info = tricore_cpus;
+
+    type_register_static(&tricore_cpu_type_info);
+
+    while (info->name) {
+        cpu_register(info);
+        info++;
+    }
+}
+
+type_init(tricore_cpu_register_types)
diff --git a/target-tricore/cpu.h b/target-tricore/cpu.h
new file mode 100644
index 000000000..7555b70a4
--- /dev/null
+++ b/target-tricore/cpu.h
@@ -0,0 +1,403 @@
+/*
+ *  TriCore emulation for qemu: main CPU struct.
+ *
+ *  Copyright (c) 2012-2014 Bastian Koppelmann C-Lab/University Paderborn
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+#if !defined(__TRICORE_CPU_H__)
+#define __TRICORE_CPU_H__
+
+#include "tricore-defs.h"
+#include "config.h"
+#include "qemu-common.h"
+#include "exec/cpu-defs.h"
+#include "fpu/softfloat.h"
+
+#define ELF_MACHINE     EM_TRICORE
+
+#define CPUArchState struct CPUTriCoreState
+
+struct CPUTriCoreState;
+
+struct tricore_boot_info;
+
+#define NB_MMU_MODES 3
+
+typedef struct tricore_def_t tricore_def_t;
+
+typedef struct CPUTriCoreState CPUTriCoreState;
+struct CPUTriCoreState {
+    /* GPR Register */
+    uint32_t gpr_a[16];
+    uint32_t gpr_d[16];
+    /* CSFR Register */
+    uint32_t PCXI;
+/* Frequently accessed PSW_USB bits are stored separately for efficiency.
+       This contains all the other bits.  Use psw_{read,write} to access
+       the whole PSW.  */
+    uint32_t PSW;
+
+    /* PSW flag cache for faster execution
+    */
+    uint32_t PSW_USB_C;
+    uint32_t PSW_USB_V;   /* Only if bit 31 set, then flag is set  */
+    uint32_t PSW_USB_SV;  /* Only if bit 31 set, then flag is set  */
+    uint32_t PSW_USB_AV;  /* Only if bit 31 set, then flag is set. */
+    uint32_t PSW_USB_SAV; /* Only if bit 31 set, then flag is set. */
+
+    uint32_t PC;
+    uint32_t SYSCON;
+    uint32_t CPU_ID;
+    uint32_t BIV;
+    uint32_t BTV;
+    uint32_t ISP;
+    uint32_t ICR;
+    uint32_t FCX;
+    uint32_t LCX;
+    uint32_t COMPAT;
+
+    /* Mem Protection Register */
+    uint32_t DPR0_0L;
+    uint32_t DPR0_0U;
+    uint32_t DPR0_1L;
+    uint32_t DPR0_1U;
+    uint32_t DPR0_2L;
+    uint32_t DPR0_2U;
+    uint32_t DPR0_3L;
+    uint32_t DPR0_3U;
+
+    uint32_t DPR1_0L;
+    uint32_t DPR1_0U;
+    uint32_t DPR1_1L;
+    uint32_t DPR1_1U;
+    uint32_t DPR1_2L;
+    uint32_t DPR1_2U;
+    uint32_t DPR1_3L;
+    uint32_t DPR1_3U;
+
+    uint32_t DPR2_0L;
+    uint32_t DPR2_0U;
+    uint32_t DPR2_1L;
+    uint32_t DPR2_1U;
+    uint32_t DPR2_2L;
+    uint32_t DPR2_2U;
+    uint32_t DPR2_3L;
+    uint32_t DPR2_3U;
+
+    uint32_t DPR3_0L;
+    uint32_t DPR3_0U;
+    uint32_t DPR3_1L;
+    uint32_t DPR3_1U;
+    uint32_t DPR3_2L;
+    uint32_t DPR3_2U;
+    uint32_t DPR3_3L;
+    uint32_t DPR3_3U;
+
+    uint32_t CPR0_0L;
+    uint32_t CPR0_0U;
+    uint32_t CPR0_1L;
+    uint32_t CPR0_1U;
+    uint32_t CPR0_2L;
+    uint32_t CPR0_2U;
+    uint32_t CPR0_3L;
+    uint32_t CPR0_3U;
+
+    uint32_t CPR1_0L;
+    uint32_t CPR1_0U;
+    uint32_t CPR1_1L;
+    uint32_t CPR1_1U;
+    uint32_t CPR1_2L;
+    uint32_t CPR1_2U;
+    uint32_t CPR1_3L;
+    uint32_t CPR1_3U;
+
+    uint32_t CPR2_0L;
+    uint32_t CPR2_0U;
+    uint32_t CPR2_1L;
+    uint32_t CPR2_1U;
+    uint32_t CPR2_2L;
+    uint32_t CPR2_2U;
+    uint32_t CPR2_3L;
+    uint32_t CPR2_3U;
+
+    uint32_t CPR3_0L;
+    uint32_t CPR3_0U;
+    uint32_t CPR3_1L;
+    uint32_t CPR3_1U;
+    uint32_t CPR3_2L;
+    uint32_t CPR3_2U;
+    uint32_t CPR3_3L;
+    uint32_t CPR3_3U;
+
+    uint32_t DPM0;
+    uint32_t DPM1;
+    uint32_t DPM2;
+    uint32_t DPM3;
+
+    uint32_t CPM0;
+    uint32_t CPM1;
+    uint32_t CPM2;
+    uint32_t CPM3;
+
+    /* Memory Management Registers */
+    uint32_t MMU_CON;
+    uint32_t MMU_ASI;
+    uint32_t MMU_TVA;
+    uint32_t MMU_TPA;
+    uint32_t MMU_TPX;
+    uint32_t MMU_TFA;
+    /* {1.3.1 only */
+    uint32_t BMACON;
+    uint32_t SMACON;
+    uint32_t DIEAR;
+    uint32_t DIETR;
+    uint32_t CCDIER;
+    uint32_t MIECON;
+    uint32_t PIEAR;
+    uint32_t PIETR;
+    uint32_t CCPIER;
+    /*} */
+    /* Debug Registers */
+    uint32_t DBGSR;
+    uint32_t EXEVT;
+    uint32_t CREVT;
+    uint32_t SWEVT;
+    uint32_t TR0EVT;
+    uint32_t TR1EVT;
+    uint32_t DMS;
+    uint32_t DCX;
+    uint32_t DBGTCR;
+    uint32_t CCTRL;
+    uint32_t CCNT;
+    uint32_t ICNT;
+    uint32_t M1CNT;
+    uint32_t M2CNT;
+    uint32_t M3CNT;
+    /* Floating Point Registers */
+    /* XXX: */
+
+    /* QEMU */
+    int error_code;
+    uint32_t hflags;    /* CPU State */
+
+    CPU_COMMON
+
+    /* Internal CPU feature flags.  */
+    uint64_t features;
+
+    const tricore_def_t *cpu_model;
+    void *irq[8];
+    struct QEMUTimer *timer; /* Internal timer */
+};
+
+#define MASK_PCXI_PCPN 0xff000000
+#define MASK_PCXI_PIE  0x00800000
+#define MASK_PCXI_UL   0x00400000
+#define MASK_PCXI_PCXS 0x000f0000
+#define MASK_PCXI_PCXO 0x0000ffff
+
+#define MASK_PSW_USB 0xff000000
+#define MASK_USB_C   0x80000000
+#define MASK_USB_V   0x40000000
+#define MASK_USB_SV  0x20000000
+#define MASK_USB_AV  0x10000000
+#define MASK_USB_SAV 0x08000000
+#define MASK_PSW_PRS 0x00003000
+#define MASK_PSW_IO  0x00000c00
+#define MASK_PSW_IS  0x00000200
+#define MASK_PSW_GW  0x00000100
+#define MASK_PSW_CDE 0x00000080
+#define MASK_PSW_CDC 0x0000007f
+
+#define MASK_SYSCON_PRO_TEN 0x2
+#define MASK_SYSCON_FCD_SF  0x1
+
+#define MASK_CPUID_MOD     0xffff0000
+#define MASK_CPUID_MOD_32B 0x0000ff00
+#define MASK_CPUID_REV     0x000000ff
+
+#define MASK_ICR_PIPN 0x00ff0000
+#define MASK_ICR_IE   0x00000100
+#define MASK_ICR_CCPN 0x000000ff
+
+#define MASK_FCX_FCXS 0x000f0000
+#define MASK_FCX_FCXO 0x0000ffff
+
+#define MASK_LCX_LCXS 0x000f0000
+#define MASK_LCX_LCX0 0x0000ffff
+
+#define TRICORE_HFLAG_UM0     0x00002 /* user mode-0 flag          */
+#define TRICORE_HFLAG_UM1     0x00001 /* user mode-1 flag          */
+#define TRICORE_HFLAG_SM      0x00000 /* kernel mode flag          */
+
+enum tricore_features {
+    TRICORE_FEATURE_13,
+    TRICORE_FEATURE_131,
+    TRICORE_FEATURE_16,
+};
+
+static inline int tricore_feature(CPUTriCoreState *env, int feature)
+{
+    return (env->features & (1ULL << feature)) != 0;
+}
+
+/* TriCore Traps Classes*/
+enum {
+    TRAPC_NONE     = -1,
+    TRAPC_MMU      = 0,
+    TRAPC_PROT     = 1,
+    TRAPC_INSN_ERR = 2,
+    TRAPC_CTX_MNG  = 3,
+    TRAPC_SYSBUS   = 4,
+    TRAPC_ASSERT   = 5,
+    TRAPC_SYSCALL  = 6,
+    TRAPC_NMI      = 7,
+};
+
+/* Class 0 TIN */
+enum {
+    TIN0_VAF = 0,
+    TIN0_VAP = 1,
+};
+
+/* Class 1 TIN */
+enum {
+    TIN1_PRIV = 1,
+    TIN1_MPR  = 2,
+    TIN1_MPW  = 3,
+    TIN1_MPX  = 4,
+    TIN1_MPP  = 5,
+    TIN1_MPN  = 6,
+    TIN1_GRWP = 7,
+};
+
+/* Class 2 TIN */
+enum {
+    TIN2_IOPC = 1,
+    TIN2_UOPC = 2,
+    TIN2_OPD  = 3,
+    TIN2_ALN  = 4,
+    TIN2_MEM  = 5,
+};
+
+/* Class 3 TIN */
+enum {
+    TIN3_FCD  = 1,
+    TIN3_CDO  = 2,
+    TIN3_CDU  = 3,
+    TIN3_FCU  = 4,
+    TIN3_CSU  = 5,
+    TIN3_CTYP = 6,
+    TIN3_NEST = 7,
+};
+
+/* Class 4 TIN */
+enum {
+    TIN4_PSE = 1,
+    TIN4_DSE = 2,
+    TIN4_DAE = 3,
+    TIN4_CAE = 4,
+    TIN4_PIE = 5,
+    TIN4_DIE = 6,
+};
+
+/* Class 5 TIN */
+enum {
+    TIN5_OVF  = 1,
+    TIN5_SOVF = 1,
+};
+
+/* Class 6 TIN
+ *
+ * Is always TIN6_SYS
+ */
+
+/* Class 7 TIN */
+enum {
+    TIN7_NMI = 0,
+};
+
+uint32_t psw_read(CPUTriCoreState *env);
+void psw_write(CPUTriCoreState *env, uint32_t val);
+
+#include "cpu-qom.h"
+#define MMU_USER_IDX 2
+
+void tricore_cpu_list(FILE *f, fprintf_function cpu_fprintf);
+
+#define cpu_exec cpu_tricore_exec
+#define cpu_signal_handler cpu_tricore_signal_handler
+#define cpu_list tricore_cpu_list
+
+static inline int cpu_mmu_index(CPUTriCoreState *env)
+{
+    return 0;
+}
+
+
+
+#include "exec/cpu-all.h"
+
+enum {
+    /* 1 bit to define user level / supervisor access */
+    ACCESS_USER  = 0x00,
+    ACCESS_SUPER = 0x01,
+    /* 1 bit to indicate direction */
+    ACCESS_STORE = 0x02,
+    /* Type of instruction that generated the access */
+    ACCESS_CODE  = 0x10, /* Code fetch access                */
+    ACCESS_INT   = 0x20, /* Integer load/store access        */
+    ACCESS_FLOAT = 0x30, /* floating point load/store access */
+};
+
+void cpu_state_reset(CPUTriCoreState *s);
+int cpu_tricore_exec(CPUTriCoreState *s);
+void tricore_tcg_init(void);
+int cpu_tricore_signal_handler(int host_signum, void *pinfo, void *puc);
+
+static inline void cpu_get_tb_cpu_state(CPUTriCoreState *env, target_ulong *pc,
+                                        target_ulong *cs_base, int *flags)
+{
+    *pc = env->PC;
+    *cs_base = 0;
+    *flags = 0;
+}
+
+TriCoreCPU *cpu_tricore_init(const char *cpu_model);
+
+static inline CPUTriCoreState *cpu_init(const char *cpu_model)
+{
+    TriCoreCPU *cpu = cpu_tricore_init(cpu_model);
+    if (cpu == NULL) {
+        return NULL;
+    }
+    return &cpu->env;
+
+}
+
+
+/* helpers.c */
+int cpu_tricore_handle_mmu_fault(CPUState *cpu, target_ulong address,
+                                 int rw, int mmu_idx);
+#define cpu_handle_mmu_fault cpu_tricore_handle_mmu_fault
+
+#include "exec/exec-all.h"
+
+static inline void cpu_pc_from_tb(CPUTriCoreState *env, TranslationBlock *tb)
+{
+    env->PC = tb->pc;
+}
+
+#endif /*__TRICORE_CPU_H__ */
diff --git a/target-tricore/helper.c b/target-tricore/helper.c
new file mode 100644
index 000000000..f52504c9f
--- /dev/null
+++ b/target-tricore/helper.c
@@ -0,0 +1,140 @@
+/*
+ *  Copyright (c) 2012-2014 Bastian Koppelmann C-Lab/University Paderborn
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stdarg.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <inttypes.h>
+#include <signal.h>
+
+#include "cpu.h"
+
+enum {
+    TLBRET_DIRTY = -4,
+    TLBRET_INVALID = -3,
+    TLBRET_NOMATCH = -2,
+    TLBRET_BADADDR = -1,
+    TLBRET_MATCH = 0
+};
+
+#if defined(CONFIG_SOFTMMU)
+static int get_physical_address(CPUTriCoreState *env, hwaddr *physical,
+                                int *prot, target_ulong address,
+                                int rw, int access_type)
+{
+    int ret = TLBRET_MATCH;
+
+    *physical = address & 0xFFFFFFFF;
+    *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
+
+    return ret;
+}
+#endif
+
+/* TODO: Add exeption support*/
+static void raise_mmu_exception(CPUTriCoreState *env, target_ulong address,
+                                int rw, int tlb_error)
+{
+}
+
+int cpu_tricore_handle_mmu_fault(CPUState *cs, target_ulong address,
+                                 int rw, int mmu_idx)
+{
+    TriCoreCPU *cpu = TRICORE_CPU(cs);
+    CPUTriCoreState *env = &cpu->env;
+    hwaddr physical;
+    int prot;
+    int access_type;
+    int ret = 0;
+
+    rw &= 1;
+    access_type = ACCESS_INT;
+    ret = get_physical_address(env, &physical, &prot,
+                               address, rw, access_type);
+    qemu_log("%s address=" TARGET_FMT_lx " ret %d physical " TARGET_FMT_plx
+             " prot %d\n", __func__, address, ret, physical, prot);
+
+    if (ret == TLBRET_MATCH) {
+        tlb_set_page(cs, address & TARGET_PAGE_MASK,
+                     physical & TARGET_PAGE_MASK, prot | PAGE_EXEC,
+                     mmu_idx, TARGET_PAGE_SIZE);
+        ret = 0;
+    } else if (ret < 0) {
+        raise_mmu_exception(env, address, rw, ret);
+        ret = 1;
+    }
+
+    return ret;
+}
+
+TriCoreCPU *cpu_tricore_init(const char *cpu_model)
+{
+    return TRICORE_CPU(cpu_generic_init(TYPE_TRICORE_CPU, cpu_model));
+}
+
+static void tricore_cpu_list_entry(gpointer data, gpointer user_data)
+{
+    ObjectClass *oc = data;
+    CPUListState *s = user_data;
+    const char *typename;
+    char *name;
+
+    typename = object_class_get_name(oc);
+    name = g_strndup(typename, strlen(typename) - strlen("-" TYPE_TRICORE_CPU));
+    (*s->cpu_fprintf)(s->file, "  %s\n",
+                      name);
+    g_free(name);
+}
+
+void tricore_cpu_list(FILE *f, fprintf_function cpu_fprintf)
+{
+    CPUListState s = {
+        .file = f,
+        .cpu_fprintf = cpu_fprintf,
+    };
+    GSList *list;
+
+    list = object_class_get_list(TYPE_TRICORE_CPU, false);
+    (*cpu_fprintf)(f, "Available CPUs:\n");
+    g_slist_foreach(list, tricore_cpu_list_entry, &s);
+    g_slist_free(list);
+}
+
+uint32_t psw_read(CPUTriCoreState *env)
+{
+    /* clear all USB bits */
+    env->PSW &= 0xffffff;
+    /* now set them from the cache */
+    env->PSW |= ((env->PSW_USB_C != 0) << 31);
+    env->PSW |= ((env->PSW_USB_V   & (1 << 31))  >> 1);
+    env->PSW |= ((env->PSW_USB_SV  & (1 << 31))  >> 2);
+    env->PSW |= ((env->PSW_USB_AV  & (1 << 31))  >> 3);
+    env->PSW |= ((env->PSW_USB_SAV & (1 << 31))  >> 4);
+
+    return env->PSW;
+}
+
+void psw_write(CPUTriCoreState *env, uint32_t val)
+{
+    env->PSW_USB_C = (val & MASK_USB_C);
+    env->PSW_USB_V = (val & MASK_USB_V << 1);
+    env->PSW_USB_SV = (val & MASK_USB_SV << 2);
+    env->PSW_USB_AV = ((val & MASK_USB_AV) << 3);
+    env->PSW_USB_SAV = ((val & MASK_USB_SAV) << 4);
+    env->PSW = val;
+}
diff --git a/target-tricore/helper.h b/target-tricore/helper.h
new file mode 100644
index 000000000..b3fc33cea
--- /dev/null
+++ b/target-tricore/helper.h
@@ -0,0 +1,32 @@
+/*
+ *  Copyright (c) 2012-2014 Bastian Koppelmann C-Lab/University Paderborn
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* Arithmetic */
+DEF_HELPER_3(add_ssov, i32, env, i32, i32)
+DEF_HELPER_3(sub_ssov, i32, env, i32, i32)
+/* CSA */
+DEF_HELPER_2(call, void, env, i32)
+DEF_HELPER_1(ret, void, env)
+DEF_HELPER_2(bisr, void, env, i32)
+DEF_HELPER_1(rfe, void, env)
+DEF_HELPER_2(ldlcx, void, env, i32)
+DEF_HELPER_2(lducx, void, env, i32)
+DEF_HELPER_2(stlcx, void, env, i32)
+DEF_HELPER_2(stucx, void, env, i32)
+/* Address mode helper */
+DEF_HELPER_1(br_update, i32, i32)
+DEF_HELPER_2(circ_update, i32, i32, i32)
diff --git a/target-tricore/op_helper.c b/target-tricore/op_helper.c
new file mode 100644
index 000000000..a36988aa1
--- /dev/null
+++ b/target-tricore/op_helper.c
@@ -0,0 +1,468 @@
+/*
+ *  Copyright (c) 2012-2014 Bastian Koppelmann C-Lab/University Paderborn
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+#include <stdlib.h>
+#include "cpu.h"
+#include "qemu/host-utils.h"
+#include "exec/helper-proto.h"
+#include "exec/cpu_ldst.h"
+
+/* Addressing mode helper */
+
+static uint16_t reverse16(uint16_t val)
+{
+    uint8_t high = (uint8_t)(val >> 8);
+    uint8_t low  = (uint8_t)(val & 0xff);
+
+    uint16_t rh, rl;
+
+    rl = (uint16_t)((high * 0x0202020202ULL & 0x010884422010ULL) % 1023);
+    rh = (uint16_t)((low * 0x0202020202ULL & 0x010884422010ULL) % 1023);
+
+    return (rh << 8) | rl;
+}
+
+uint32_t helper_br_update(uint32_t reg)
+{
+    uint32_t index = reg & 0xffff;
+    uint32_t incr  = reg >> 16;
+    uint32_t new_index = reverse16(reverse16(index) + reverse16(incr));
+    return reg - index + new_index;
+}
+
+uint32_t helper_circ_update(uint32_t reg, uint32_t off)
+{
+    uint32_t index = reg & 0xffff;
+    uint32_t length = reg >> 16;
+    int32_t new_index = index + off;
+    if (new_index < 0) {
+        new_index += length;
+    } else {
+        new_index %= length;
+    }
+    return reg - index + new_index;
+}
+
+#define SSOV(env, ret, arg, len) do {               \
+    int64_t max_pos = INT##len ##_MAX;              \
+    int64_t max_neg = INT##len ##_MIN;              \
+    if (arg > max_pos) {                            \
+        env->PSW_USB_V = (1 << 31);                 \
+        env->PSW_USB_SV = (1 << 31);                \
+        ret = (target_ulong)max_pos;                \
+    } else {                                        \
+        if (arg < max_neg) {                        \
+            env->PSW_USB_V = (1 << 31);             \
+            env->PSW_USB_SV = (1 << 31);            \
+            ret = (target_ulong)max_neg;            \
+        } else {                                    \
+            env->PSW_USB_V = 0;                     \
+            ret = (target_ulong)arg;                \
+        }                                           \
+    }                                               \
+    env->PSW_USB_AV = arg ^ arg * 2u;               \
+    env->PSW_USB_SAV |= env->PSW_USB_AV;            \
+} while (0)
+
+target_ulong helper_add_ssov(CPUTriCoreState *env, target_ulong r1,
+                             target_ulong r2)
+{
+    target_ulong ret;
+    int64_t t1 = sextract64(r1, 0, 32);
+    int64_t t2 = sextract64(r2, 0, 32);
+    int64_t result = t1 + t2;
+    SSOV(env, ret, result, 32);
+    return ret;
+}
+
+target_ulong helper_sub_ssov(CPUTriCoreState *env, target_ulong r1,
+                             target_ulong r2)
+{
+    target_ulong ret;
+    int64_t t1 = sextract64(r1, 0, 32);
+    int64_t t2 = sextract64(r2, 0, 32);
+    int64_t result = t1 - t2;
+    SSOV(env, ret, result, 32);
+    return ret;
+}
+
+/* context save area (CSA) related helpers */
+
+static int cdc_increment(target_ulong *psw)
+{
+    if ((*psw & MASK_PSW_CDC) == 0x7f) {
+        return 0;
+    }
+
+    (*psw)++;
+    /* check for overflow */
+    int lo = clo32((*psw & MASK_PSW_CDC) << (32 - 7));
+    int mask = (1u << (7 - lo)) - 1;
+    int count = *psw & mask;
+    if (count == 0) {
+        (*psw)--;
+        return 1;
+    }
+    return 0;
+}
+
+static int cdc_decrement(target_ulong *psw)
+{
+    if ((*psw & MASK_PSW_CDC) == 0x7f) {
+        return 0;
+    }
+    /* check for underflow */
+    int lo = clo32((*psw & MASK_PSW_CDC) << (32 - 7));
+    int mask = (1u << (7 - lo)) - 1;
+    int count = *psw & mask;
+    if (count == 0) {
+        return 1;
+    }
+    (*psw)--;
+    return 0;
+}
+
+static bool cdc_zero(target_ulong *psw)
+{
+    int cdc = *psw & MASK_PSW_CDC;
+    /* Returns TRUE if PSW.CDC.COUNT == 0 or if PSW.CDC ==
+       7'b1111111, otherwise returns FALSE. */
+    if (cdc == 0x7f) {
+        return true;
+    }
+    /* find CDC.COUNT */
+    int lo = clo32((*psw & MASK_PSW_CDC) << (32 - 7));
+    int mask = (1u << (7 - lo)) - 1;
+    int count = *psw & mask;
+    return count == 0;
+}
+
+static void save_context_upper(CPUTriCoreState *env, int ea)
+{
+    cpu_stl_data(env, ea, env->PCXI);
+    cpu_stl_data(env, ea+4, env->PSW);
+    cpu_stl_data(env, ea+8, env->gpr_a[10]);
+    cpu_stl_data(env, ea+12, env->gpr_a[11]);
+    cpu_stl_data(env, ea+16, env->gpr_d[8]);
+    cpu_stl_data(env, ea+20, env->gpr_d[9]);
+    cpu_stl_data(env, ea+24, env->gpr_d[10]);
+    cpu_stl_data(env, ea+28, env->gpr_d[11]);
+    cpu_stl_data(env, ea+32, env->gpr_a[12]);
+    cpu_stl_data(env, ea+36, env->gpr_a[13]);
+    cpu_stl_data(env, ea+40, env->gpr_a[14]);
+    cpu_stl_data(env, ea+44, env->gpr_a[15]);
+    cpu_stl_data(env, ea+48, env->gpr_d[12]);
+    cpu_stl_data(env, ea+52, env->gpr_d[13]);
+    cpu_stl_data(env, ea+56, env->gpr_d[14]);
+    cpu_stl_data(env, ea+60, env->gpr_d[15]);
+}
+
+static void save_context_lower(CPUTriCoreState *env, int ea)
+{
+    cpu_stl_data(env, ea, env->PCXI);
+    cpu_stl_data(env, ea+4, env->gpr_a[11]);
+    cpu_stl_data(env, ea+8, env->gpr_a[2]);
+    cpu_stl_data(env, ea+12, env->gpr_a[3]);
+    cpu_stl_data(env, ea+16, env->gpr_d[0]);
+    cpu_stl_data(env, ea+20, env->gpr_d[1]);
+    cpu_stl_data(env, ea+24, env->gpr_d[2]);
+    cpu_stl_data(env, ea+28, env->gpr_d[3]);
+    cpu_stl_data(env, ea+32, env->gpr_a[4]);
+    cpu_stl_data(env, ea+36, env->gpr_a[5]);
+    cpu_stl_data(env, ea+40, env->gpr_a[6]);
+    cpu_stl_data(env, ea+44, env->gpr_a[7]);
+    cpu_stl_data(env, ea+48, env->gpr_d[4]);
+    cpu_stl_data(env, ea+52, env->gpr_d[5]);
+    cpu_stl_data(env, ea+56, env->gpr_d[6]);
+    cpu_stl_data(env, ea+60, env->gpr_d[7]);
+}
+
+static void restore_context_upper(CPUTriCoreState *env, int ea,
+                                  target_ulong *new_PCXI, target_ulong *new_PSW)
+{
+    *new_PCXI = cpu_ldl_data(env, ea);
+    *new_PSW = cpu_ldl_data(env, ea+4);
+    env->gpr_a[10] = cpu_ldl_data(env, ea+8);
+    env->gpr_a[11] = cpu_ldl_data(env, ea+12);
+    env->gpr_d[8]  = cpu_ldl_data(env, ea+16);
+    env->gpr_d[9]  = cpu_ldl_data(env, ea+20);
+    env->gpr_d[10] = cpu_ldl_data(env, ea+24);
+    env->gpr_d[11] = cpu_ldl_data(env, ea+28);
+    env->gpr_a[12] = cpu_ldl_data(env, ea+32);
+    env->gpr_a[13] = cpu_ldl_data(env, ea+36);
+    env->gpr_a[14] = cpu_ldl_data(env, ea+40);
+    env->gpr_a[15] = cpu_ldl_data(env, ea+44);
+    env->gpr_d[12] = cpu_ldl_data(env, ea+48);
+    env->gpr_d[13] = cpu_ldl_data(env, ea+52);
+    env->gpr_d[14] = cpu_ldl_data(env, ea+56);
+    env->gpr_d[15] = cpu_ldl_data(env, ea+60);
+}
+
+static void restore_context_lower(CPUTriCoreState *env, int ea,
+                                  target_ulong *ra, target_ulong *pcxi)
+{
+    *pcxi = cpu_ldl_data(env, ea);
+    *ra = cpu_ldl_data(env, ea+4);
+    env->gpr_a[2] = cpu_ldl_data(env, ea+8);
+    env->gpr_a[3] = cpu_ldl_data(env, ea+12);
+    env->gpr_d[0] = cpu_ldl_data(env, ea+16);
+    env->gpr_d[1] = cpu_ldl_data(env, ea+20);
+    env->gpr_d[2] = cpu_ldl_data(env, ea+24);
+    env->gpr_d[3] = cpu_ldl_data(env, ea+28);
+    env->gpr_a[4] = cpu_ldl_data(env, ea+32);
+    env->gpr_a[5] = cpu_ldl_data(env, ea+36);
+    env->gpr_a[6] = cpu_ldl_data(env, ea+40);
+    env->gpr_a[7] = cpu_ldl_data(env, ea+44);
+    env->gpr_d[4] = cpu_ldl_data(env, ea+48);
+    env->gpr_d[5] = cpu_ldl_data(env, ea+52);
+    env->gpr_d[6] = cpu_ldl_data(env, ea+56);
+    env->gpr_d[7] = cpu_ldl_data(env, ea+60);
+}
+
+void helper_call(CPUTriCoreState *env, uint32_t next_pc)
+{
+    target_ulong tmp_FCX;
+    target_ulong ea;
+    target_ulong new_FCX;
+    target_ulong psw;
+
+    psw = psw_read(env);
+    /* if (FCX == 0) trap(FCU); */
+    if (env->FCX == 0) {
+        /* FCU trap */
+    }
+    /* if (PSW.CDE) then if (cdc_increment()) then trap(CDO); */
+    if (psw & MASK_PSW_CDE) {
+        if (cdc_increment(&psw)) {
+            /* CDO trap */
+        }
+    }
+    /* PSW.CDE = 1;*/
+    psw |= MASK_PSW_CDE;
+    /* tmp_FCX = FCX; */
+    tmp_FCX = env->FCX;
+    /* EA = {FCX.FCXS, 6'b0, FCX.FCXO, 6'b0}; */
+    ea = ((env->FCX & MASK_FCX_FCXS) << 12) +
+         ((env->FCX & MASK_FCX_FCXO) << 6);
+    /* new_FCX = M(EA, word); */
+    new_FCX = cpu_ldl_data(env, ea);
+    /* M(EA, 16 * word) = {PCXI, PSW, A[10], A[11], D[8], D[9], D[10], D[11],
+                           A[12], A[13], A[14], A[15], D[12], D[13], D[14],
+                           D[15]}; */
+    save_context_upper(env, ea);
+
+    /* PCXI.PCPN = ICR.CCPN; */
+    env->PCXI = (env->PCXI & 0xffffff) +
+                ((env->ICR & MASK_ICR_CCPN) << 24);
+    /* PCXI.PIE = ICR.IE; */
+    env->PCXI = ((env->PCXI & ~MASK_PCXI_PIE) +
+                ((env->ICR & MASK_ICR_IE) << 15));
+    /* PCXI.UL = 1; */
+    env->PCXI |= MASK_PCXI_UL;
+
+    /* PCXI[19: 0] = FCX[19: 0]; */
+    env->PCXI = (env->PCXI & 0xfff00000) + (env->FCX & 0xfffff);
+    /* FCX[19: 0] = new_FCX[19: 0]; */
+    env->FCX = (env->FCX & 0xfff00000) + (new_FCX & 0xfffff);
+    /* A[11] = next_pc[31: 0]; */
+    env->gpr_a[11] = next_pc;
+
+    /* if (tmp_FCX == LCX) trap(FCD);*/
+    if (tmp_FCX == env->LCX) {
+        /* FCD trap */
+    }
+    psw_write(env, psw);
+}
+
+void helper_ret(CPUTriCoreState *env)
+{
+    target_ulong ea;
+    target_ulong new_PCXI;
+    target_ulong new_PSW, psw;
+
+    psw = psw_read(env);
+     /* if (PSW.CDE) then if (cdc_decrement()) then trap(CDU);*/
+    if (env->PSW & MASK_PSW_CDE) {
+        if (cdc_decrement(&(env->PSW))) {
+            /* CDU trap */
+        }
+    }
+    /*   if (PCXI[19: 0] == 0) then trap(CSU); */
+    if ((env->PCXI & 0xfffff) == 0) {
+        /* CSU trap */
+    }
+    /* if (PCXI.UL == 0) then trap(CTYP); */
+    if ((env->PCXI & MASK_PCXI_UL) == 0) {
+        /* CTYP trap */
+    }
+    /* PC = {A11 [31: 1], 1’b0}; */
+    env->PC = env->gpr_a[11] & 0xfffffffe;
+
+    /* EA = {PCXI.PCXS, 6'b0, PCXI.PCXO, 6'b0}; */
+    ea = ((env->PCXI & MASK_PCXI_PCXS) << 12) +
+         ((env->PCXI & MASK_PCXI_PCXO) << 6);
+    /* {new_PCXI, new_PSW, A[10], A[11], D[8], D[9], D[10], D[11], A[12],
+        A[13], A[14], A[15], D[12], D[13], D[14], D[15]} = M(EA, 16 * word); */
+    restore_context_upper(env, ea, &new_PCXI, &new_PSW);
+    /* M(EA, word) = FCX; */
+    cpu_stl_data(env, ea, env->FCX);
+    /* FCX[19: 0] = PCXI[19: 0]; */
+    env->FCX = (env->FCX & 0xfff00000) + (env->PCXI & 0x000fffff);
+    /* PCXI = new_PCXI; */
+    env->PCXI = new_PCXI;
+
+    if (tricore_feature(env, TRICORE_FEATURE_13)) {
+        /* PSW = new_PSW */
+        psw_write(env, new_PSW);
+    } else {
+        /* PSW = {new_PSW[31:26], PSW[25:24], new_PSW[23:0]}; */
+        psw_write(env, (new_PSW & ~(0x3000000)) + (psw & (0x3000000)));
+    }
+}
+
+void helper_bisr(CPUTriCoreState *env, uint32_t const9)
+{
+    target_ulong tmp_FCX;
+    target_ulong ea;
+    target_ulong new_FCX;
+
+    if (env->FCX == 0) {
+        /* FCU trap */
+    }
+
+    tmp_FCX = env->FCX;
+    ea = ((env->FCX & 0xf0000) << 12) + ((env->FCX & 0xffff) << 6);
+
+    /* new_FCX = M(EA, word); */
+    new_FCX = cpu_ldl_data(env, ea);
+    /* M(EA, 16 * word) = {PCXI, A[11], A[2], A[3], D[0], D[1], D[2], D[3], A[4]
+                           , A[5], A[6], A[7], D[4], D[5], D[6], D[7]}; */
+    save_context_lower(env, ea);
+
+
+    /* PCXI.PCPN = ICR.CCPN */
+    env->PCXI = (env->PCXI & 0xffffff) +
+                 ((env->ICR & MASK_ICR_CCPN) << 24);
+    /* PCXI.PIE  = ICR.IE */
+    env->PCXI = ((env->PCXI & ~MASK_PCXI_PIE) +
+                 ((env->ICR & MASK_ICR_IE) << 15));
+    /* PCXI.UL = 0 */
+    env->PCXI &= ~(MASK_PCXI_UL);
+    /* PCXI[19: 0] = FCX[19: 0] */
+    env->PCXI = (env->PCXI & 0xfff00000) + (env->FCX & 0xfffff);
+    /* FXC[19: 0] = new_FCX[19: 0] */
+    env->FCX = (env->FCX & 0xfff00000) + (new_FCX & 0xfffff);
+    /* ICR.IE = 1 */
+    env->ICR |= MASK_ICR_IE;
+
+    env->ICR |= const9; /* ICR.CCPN = const9[7: 0];*/
+
+    if (tmp_FCX == env->LCX) {
+        /* FCD trap */
+    }
+}
+
+void helper_rfe(CPUTriCoreState *env)
+{
+    target_ulong ea;
+    target_ulong new_PCXI;
+    target_ulong new_PSW;
+    /* if (PCXI[19: 0] == 0) then trap(CSU); */
+    if ((env->PCXI & 0xfffff) == 0) {
+        /* raise csu trap */
+    }
+    /* if (PCXI.UL == 0) then trap(CTYP); */
+    if ((env->PCXI & MASK_PCXI_UL) == 0) {
+        /* raise CTYP trap */
+    }
+    /* if (!cdc_zero() AND PSW.CDE) then trap(NEST); */
+    if (!cdc_zero(&(env->PSW)) && (env->PSW & MASK_PSW_CDE)) {
+        /* raise MNG trap */
+    }
+    /* ICR.IE = PCXI.PIE; */
+    env->ICR = (env->ICR & ~MASK_ICR_IE) + ((env->PCXI & MASK_PCXI_PIE) >> 15);
+    /* ICR.CCPN = PCXI.PCPN; */
+    env->ICR = (env->ICR & ~MASK_ICR_CCPN) +
+               ((env->PCXI & MASK_PCXI_PCPN) >> 24);
+    /*EA = {PCXI.PCXS, 6'b0, PCXI.PCXO, 6'b0};*/
+    ea = ((env->PCXI & MASK_PCXI_PCXS) << 12) +
+         ((env->PCXI & MASK_PCXI_PCXO) << 6);
+    /*{new_PCXI, PSW, A[10], A[11], D[8], D[9], D[10], D[11], A[12],
+      A[13], A[14], A[15], D[12], D[13], D[14], D[15]} = M(EA, 16 * word); */
+    restore_context_upper(env, ea, &new_PCXI, &new_PSW);
+    /* M(EA, word) = FCX;*/
+    cpu_stl_data(env, ea, env->FCX);
+    /* FCX[19: 0] = PCXI[19: 0]; */
+    env->FCX = (env->FCX & 0xfff00000) + (env->PCXI & 0x000fffff);
+    /* PCXI = new_PCXI; */
+    env->PCXI = new_PCXI;
+    /* write psw */
+    psw_write(env, new_PSW);
+}
+
+void helper_ldlcx(CPUTriCoreState *env, uint32_t ea)
+{
+    uint32_t dummy;
+    /* insn doesn't load PCXI and RA */
+    restore_context_lower(env, ea, &dummy, &dummy);
+}
+
+void helper_lducx(CPUTriCoreState *env, uint32_t ea)
+{
+    uint32_t dummy;
+    /* insn doesn't load PCXI and PSW */
+    restore_context_upper(env, ea, &dummy, &dummy);
+}
+
+void helper_stlcx(CPUTriCoreState *env, uint32_t ea)
+{
+    save_context_lower(env, ea);
+}
+
+void helper_stucx(CPUTriCoreState *env, uint32_t ea)
+{
+    save_context_upper(env, ea);
+}
+
+static inline void QEMU_NORETURN do_raise_exception_err(CPUTriCoreState *env,
+                                                        uint32_t exception,
+                                                        int error_code,
+                                                        uintptr_t pc)
+{
+    CPUState *cs = CPU(tricore_env_get_cpu(env));
+    cs->exception_index = exception;
+    env->error_code = error_code;
+
+    if (pc) {
+        /* now we have a real cpu fault */
+        cpu_restore_state(cs, pc);
+    }
+
+    cpu_loop_exit(cs);
+}
+
+void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
+              uintptr_t retaddr)
+{
+    int ret;
+    ret = cpu_tricore_handle_mmu_fault(cs, addr, is_write, mmu_idx);
+    if (ret) {
+        TriCoreCPU *cpu = TRICORE_CPU(cs);
+        CPUTriCoreState *env = &cpu->env;
+        do_raise_exception_err(env, cs->exception_index,
+                               env->error_code, retaddr);
+    }
+}
diff --git a/target-tricore/translate.c b/target-tricore/translate.c
new file mode 100644
index 000000000..d5a95969b
--- /dev/null
+++ b/target-tricore/translate.c
@@ -0,0 +1,2568 @@
+/*
+ *  TriCore emulation for qemu: main translation routines.
+ *
+ *  Copyright (c) 2013-2014 Bastian Koppelmann C-Lab/University Paderborn
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "cpu.h"
+#include "disas/disas.h"
+#include "tcg-op.h"
+#include "exec/cpu_ldst.h"
+
+#include "exec/helper-proto.h"
+#include "exec/helper-gen.h"
+
+#include "tricore-opcodes.h"
+
+/*
+ * TCG registers
+ */
+static TCGv cpu_PC;
+static TCGv cpu_PCXI;
+static TCGv cpu_PSW;
+static TCGv cpu_ICR;
+/* GPR registers */
+static TCGv cpu_gpr_a[16];
+static TCGv cpu_gpr_d[16];
+/* PSW Flag cache */
+static TCGv cpu_PSW_C;
+static TCGv cpu_PSW_V;
+static TCGv cpu_PSW_SV;
+static TCGv cpu_PSW_AV;
+static TCGv cpu_PSW_SAV;
+/* CPU env */
+static TCGv_ptr cpu_env;
+
+#include "exec/gen-icount.h"
+
+static const char *regnames_a[] = {
+      "a0"  , "a1"  , "a2"  , "a3" , "a4"  , "a5" ,
+      "a6"  , "a7"  , "a8"  , "a9" , "sp" , "a11" ,
+      "a12" , "a13" , "a14" , "a15",
+    };
+
+static const char *regnames_d[] = {
+      "d0"  , "d1"  , "d2"  , "d3" , "d4"  , "d5"  ,
+      "d6"  , "d7"  , "d8"  , "d9" , "d10" , "d11" ,
+      "d12" , "d13" , "d14" , "d15",
+    };
+
+typedef struct DisasContext {
+    struct TranslationBlock *tb;
+    target_ulong pc, saved_pc, next_pc;
+    uint32_t opcode;
+    int singlestep_enabled;
+    /* Routine used to access memory */
+    int mem_idx;
+    uint32_t hflags, saved_hflags;
+    int bstate;
+} DisasContext;
+
+enum {
+
+    BS_NONE   = 0,
+    BS_STOP   = 1,
+    BS_BRANCH = 2,
+    BS_EXCP   = 3,
+};
+
+void tricore_cpu_dump_state(CPUState *cs, FILE *f,
+                            fprintf_function cpu_fprintf, int flags)
+{
+    TriCoreCPU *cpu = TRICORE_CPU(cs);
+    CPUTriCoreState *env = &cpu->env;
+    int i;
+
+    cpu_fprintf(f, "PC=%08x\n", env->PC);
+    for (i = 0; i < 16; ++i) {
+        if ((i & 3) == 0) {
+            cpu_fprintf(f, "GPR A%02d:", i);
+        }
+        cpu_fprintf(f, " %s " TARGET_FMT_lx, regnames_a[i], env->gpr_a[i]);
+    }
+    for (i = 0; i < 16; ++i) {
+        if ((i & 3) == 0) {
+            cpu_fprintf(f, "GPR D%02d:", i);
+        }
+        cpu_fprintf(f, " %s " TARGET_FMT_lx, regnames_d[i], env->gpr_d[i]);
+    }
+
+}
+
+/*
+ * Functions to generate micro-ops
+ */
+
+/* Makros for generating helpers */
+
+#define gen_helper_1arg(name, arg) do {                           \
+    TCGv_i32 helper_tmp = tcg_const_i32(arg);                     \
+    gen_helper_##name(cpu_env, helper_tmp);                       \
+    tcg_temp_free_i32(helper_tmp);                                \
+    } while (0)
+
+#define EA_ABS_FORMAT(con) (((con & 0x3C000) << 14) + (con & 0x3FFF))
+#define EA_B_ABSOLUT(con) (((offset & 0xf00000) << 8) | \
+                           ((offset & 0x0fffff) << 1))
+
+/* Functions for load/save to/from memory */
+
+static inline void gen_offset_ld(DisasContext *ctx, TCGv r1, TCGv r2,
+                                 int16_t con, TCGMemOp mop)
+{
+    TCGv temp = tcg_temp_new();
+    tcg_gen_addi_tl(temp, r2, con);
+    tcg_gen_qemu_ld_tl(r1, temp, ctx->mem_idx, mop);
+    tcg_temp_free(temp);
+}
+
+static inline void gen_offset_st(DisasContext *ctx, TCGv r1, TCGv r2,
+                                 int16_t con, TCGMemOp mop)
+{
+    TCGv temp = tcg_temp_new();
+    tcg_gen_addi_tl(temp, r2, con);
+    tcg_gen_qemu_st_tl(r1, temp, ctx->mem_idx, mop);
+    tcg_temp_free(temp);
+}
+
+static void gen_st_2regs_64(TCGv rh, TCGv rl, TCGv address, DisasContext *ctx)
+{
+    TCGv_i64 temp = tcg_temp_new_i64();
+
+    tcg_gen_concat_i32_i64(temp, rl, rh);
+    tcg_gen_qemu_st_i64(temp, address, ctx->mem_idx, MO_LEQ);
+
+    tcg_temp_free_i64(temp);
+}
+
+static void gen_offset_st_2regs(TCGv rh, TCGv rl, TCGv base, int16_t con,
+                                DisasContext *ctx)
+{
+    TCGv temp = tcg_temp_new();
+    tcg_gen_addi_tl(temp, base, con);
+    gen_st_2regs_64(rh, rl, temp, ctx);
+    tcg_temp_free(temp);
+}
+
+static void gen_ld_2regs_64(TCGv rh, TCGv rl, TCGv address, DisasContext *ctx)
+{
+    TCGv_i64 temp = tcg_temp_new_i64();
+
+    tcg_gen_qemu_ld_i64(temp, address, ctx->mem_idx, MO_LEQ);
+    /* write back to two 32 bit regs */
+    tcg_gen_extr_i64_i32(rl, rh, temp);
+
+    tcg_temp_free_i64(temp);
+}
+
+static void gen_offset_ld_2regs(TCGv rh, TCGv rl, TCGv base, int16_t con,
+                                DisasContext *ctx)
+{
+    TCGv temp = tcg_temp_new();
+    tcg_gen_addi_tl(temp, base, con);
+    gen_ld_2regs_64(rh, rl, temp, ctx);
+    tcg_temp_free(temp);
+}
+
+static void gen_st_preincr(DisasContext *ctx, TCGv r1, TCGv r2, int16_t off,
+                           TCGMemOp mop)
+{
+    TCGv temp = tcg_temp_new();
+    tcg_gen_addi_tl(temp, r2, off);
+    tcg_gen_qemu_st_tl(r1, temp, ctx->mem_idx, mop);
+    tcg_gen_mov_tl(r2, temp);
+    tcg_temp_free(temp);
+}
+
+static void gen_ld_preincr(DisasContext *ctx, TCGv r1, TCGv r2, int16_t off,
+                           TCGMemOp mop)
+{
+    TCGv temp = tcg_temp_new();
+    tcg_gen_addi_tl(temp, r2, off);
+    tcg_gen_qemu_ld_tl(r1, temp, ctx->mem_idx, mop);
+    tcg_gen_mov_tl(r2, temp);
+    tcg_temp_free(temp);
+}
+
+/* M(EA, word) = (M(EA, word) & ~E[a][63:32]) | (E[a][31:0] & E[a][63:32]); */
+static void gen_ldmst(DisasContext *ctx, int ereg, TCGv ea)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+
+    /* temp = (M(EA, word) */
+    tcg_gen_qemu_ld_tl(temp, ea, ctx->mem_idx, MO_LEUL);
+    /* temp = temp & ~E[a][63:32]) */
+    tcg_gen_andc_tl(temp, temp, cpu_gpr_d[ereg+1]);
+    /* temp2 = (E[a][31:0] & E[a][63:32]); */
+    tcg_gen_and_tl(temp2, cpu_gpr_d[ereg], cpu_gpr_d[ereg+1]);
+    /* temp = temp | temp2; */
+    tcg_gen_or_tl(temp, temp, temp2);
+    /* M(EA, word) = temp; */
+    tcg_gen_qemu_st_tl(temp, ea, ctx->mem_idx, MO_LEUL);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+/* tmp = M(EA, word);
+   M(EA, word) = D[a];
+   D[a] = tmp[31:0];*/
+static void gen_swap(DisasContext *ctx, int reg, TCGv ea)
+{
+    TCGv temp = tcg_temp_new();
+
+    tcg_gen_qemu_ld_tl(temp, ea, ctx->mem_idx, MO_LEUL);
+    tcg_gen_qemu_st_tl(cpu_gpr_d[reg], ea, ctx->mem_idx, MO_LEUL);
+    tcg_gen_mov_tl(cpu_gpr_d[reg], temp);
+
+    tcg_temp_free(temp);
+}
+
+/* Functions for arithmetic instructions  */
+
+static inline void gen_add_d(TCGv ret, TCGv r1, TCGv r2)
+{
+    TCGv t0 = tcg_temp_new_i32();
+    TCGv result = tcg_temp_new_i32();
+    /* Addition and set V/SV bits */
+    tcg_gen_add_tl(result, r1, r2);
+    /* calc V bit */
+    tcg_gen_xor_tl(cpu_PSW_V, result, r1);
+    tcg_gen_xor_tl(t0, r1, r2);
+    tcg_gen_andc_tl(cpu_PSW_V, cpu_PSW_V, t0);
+    /* Calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV/SAV bits */
+    tcg_gen_add_tl(cpu_PSW_AV, result, result);
+    tcg_gen_xor_tl(cpu_PSW_AV, result, cpu_PSW_AV);
+    /* calc SAV */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+    /* write back result */
+    tcg_gen_mov_tl(ret, result);
+
+    tcg_temp_free(result);
+    tcg_temp_free(t0);
+}
+
+static inline void gen_addi_d(TCGv ret, TCGv r1, target_ulong r2)
+{
+    TCGv temp = tcg_const_i32(r2);
+    gen_add_d(ret, r1, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void gen_cond_add(TCGCond cond, TCGv r1, TCGv r2, TCGv r3,
+                                TCGv r4)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+    TCGv result = tcg_temp_new();
+    TCGv mask = tcg_temp_new();
+    TCGv t0 = tcg_const_i32(0);
+
+    /* create mask for sticky bits */
+    tcg_gen_setcond_tl(cond, mask, r4, t0);
+    tcg_gen_shli_tl(mask, mask, 31);
+
+    tcg_gen_add_tl(result, r1, r2);
+    /* Calc PSW_V */
+    tcg_gen_xor_tl(temp, result, r1);
+    tcg_gen_xor_tl(temp2, r1, r2);
+    tcg_gen_andc_tl(temp, temp, temp2);
+    tcg_gen_movcond_tl(cond, cpu_PSW_V, r4, t0, temp, cpu_PSW_V);
+    /* Set PSW_SV */
+    tcg_gen_and_tl(temp, temp, mask);
+    tcg_gen_or_tl(cpu_PSW_SV, temp, cpu_PSW_SV);
+    /* calc AV bit */
+    tcg_gen_add_tl(temp, result, result);
+    tcg_gen_xor_tl(temp, temp, result);
+    tcg_gen_movcond_tl(cond, cpu_PSW_AV, r4, t0, temp, cpu_PSW_AV);
+    /* calc SAV bit */
+    tcg_gen_and_tl(temp, temp, mask);
+    tcg_gen_or_tl(cpu_PSW_SAV, temp, cpu_PSW_SAV);
+    /* write back result */
+    tcg_gen_movcond_tl(cond, r3, r4, t0, result, r3);
+
+    tcg_temp_free(t0);
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free(result);
+    tcg_temp_free(mask);
+}
+
+static inline void gen_condi_add(TCGCond cond, TCGv r1, int32_t r2,
+                                 TCGv r3, TCGv r4)
+{
+    TCGv temp = tcg_const_i32(r2);
+    gen_cond_add(cond, r1, temp, r3, r4);
+    tcg_temp_free(temp);
+}
+
+static inline void gen_sub_d(TCGv ret, TCGv r1, TCGv r2)
+{
+    TCGv temp = tcg_temp_new_i32();
+    TCGv result = tcg_temp_new_i32();
+
+    tcg_gen_sub_tl(result, r1, r2);
+    /* calc V bit */
+    tcg_gen_xor_tl(cpu_PSW_V, result, r1);
+    tcg_gen_xor_tl(temp, r1, r2);
+    tcg_gen_and_tl(cpu_PSW_V, cpu_PSW_V, temp);
+    /* calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV bit */
+    tcg_gen_add_tl(cpu_PSW_AV, result, result);
+    tcg_gen_xor_tl(cpu_PSW_AV, result, cpu_PSW_AV);
+    /* calc SAV bit */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+    /* write back result */
+    tcg_gen_mov_tl(ret, result);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(result);
+}
+
+static inline void gen_mul_i32s(TCGv ret, TCGv r1, TCGv r2)
+{
+    TCGv high = tcg_temp_new();
+    TCGv low = tcg_temp_new();
+
+    tcg_gen_muls2_tl(low, high, r1, r2);
+    tcg_gen_mov_tl(ret, low);
+    /* calc V bit */
+    tcg_gen_sari_tl(low, low, 31);
+    tcg_gen_setcond_tl(TCG_COND_NE, cpu_PSW_V, high, low);
+    tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
+    /* calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV bit */
+    tcg_gen_add_tl(cpu_PSW_AV, ret, ret);
+    tcg_gen_xor_tl(cpu_PSW_AV, ret, cpu_PSW_AV);
+    /* calc SAV bit */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+
+    tcg_temp_free(high);
+    tcg_temp_free(low);
+}
+
+static void gen_saturate(TCGv ret, TCGv arg, int32_t up, int32_t low)
+{
+    TCGv sat_neg = tcg_const_i32(low);
+    TCGv temp = tcg_const_i32(up);
+
+    /* sat_neg = (arg < low ) ? low : arg; */
+    tcg_gen_movcond_tl(TCG_COND_LT, sat_neg, arg, sat_neg, sat_neg, arg);
+
+    /* ret = (sat_neg > up ) ? up  : sat_neg; */
+    tcg_gen_movcond_tl(TCG_COND_GT, ret, sat_neg, temp, temp, sat_neg);
+
+    tcg_temp_free(sat_neg);
+    tcg_temp_free(temp);
+}
+
+static void gen_saturate_u(TCGv ret, TCGv arg, int32_t up)
+{
+    TCGv temp = tcg_const_i32(up);
+    /* sat_neg = (arg > up ) ? up : arg; */
+    tcg_gen_movcond_tl(TCG_COND_GTU, ret, arg, temp, temp, arg);
+    tcg_temp_free(temp);
+}
+
+static void gen_shi(TCGv ret, TCGv r1, int32_t shift_count)
+{
+    if (shift_count == -32) {
+        tcg_gen_movi_tl(ret, 0);
+    } else if (shift_count >= 0) {
+        tcg_gen_shli_tl(ret, r1, shift_count);
+    } else {
+        tcg_gen_shri_tl(ret, r1, -shift_count);
+    }
+}
+
+static void gen_shaci(TCGv ret, TCGv r1, int32_t shift_count)
+{
+    uint32_t msk, msk_start;
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+    TCGv t_0 = tcg_const_i32(0);
+
+    if (shift_count == 0) {
+        /* Clear PSW.C and PSW.V */
+        tcg_gen_movi_tl(cpu_PSW_C, 0);
+        tcg_gen_mov_tl(cpu_PSW_V, cpu_PSW_C);
+        tcg_gen_mov_tl(ret, r1);
+    } else if (shift_count == -32) {
+        /* set PSW.C */
+        tcg_gen_mov_tl(cpu_PSW_C, r1);
+        /* fill ret completly with sign bit */
+        tcg_gen_sari_tl(ret, r1, 31);
+        /* clear PSW.V */
+        tcg_gen_movi_tl(cpu_PSW_V, 0);
+    } else if (shift_count > 0) {
+        TCGv t_max = tcg_const_i32(0x7FFFFFFF >> shift_count);
+        TCGv t_min = tcg_const_i32(((int32_t) -0x80000000) >> shift_count);
+
+        /* calc carry */
+        msk_start = 32 - shift_count;
+        msk = ((1 << shift_count) - 1) << msk_start;
+        tcg_gen_andi_tl(cpu_PSW_C, r1, msk);
+        /* calc v/sv bits */
+        tcg_gen_setcond_tl(TCG_COND_GT, temp, r1, t_max);
+        tcg_gen_setcond_tl(TCG_COND_LT, temp2, r1, t_min);
+        tcg_gen_or_tl(cpu_PSW_V, temp, temp2);
+        tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
+        /* calc sv */
+        tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_V, cpu_PSW_SV);
+        /* do shift */
+        tcg_gen_shli_tl(ret, r1, shift_count);
+
+        tcg_temp_free(t_max);
+        tcg_temp_free(t_min);
+    } else {
+        /* clear PSW.V */
+        tcg_gen_movi_tl(cpu_PSW_V, 0);
+        /* calc carry */
+        msk = (1 << -shift_count) - 1;
+        tcg_gen_andi_tl(cpu_PSW_C, r1, msk);
+        /* do shift */
+        tcg_gen_sari_tl(ret, r1, -shift_count);
+    }
+    /* calc av overflow bit */
+    tcg_gen_add_tl(cpu_PSW_AV, ret, ret);
+    tcg_gen_xor_tl(cpu_PSW_AV, ret, cpu_PSW_AV);
+    /* calc sav overflow bit */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free(t_0);
+}
+
+static inline void gen_adds(TCGv ret, TCGv r1, TCGv r2)
+{
+    gen_helper_add_ssov(ret, cpu_env, r1, r2);
+}
+
+static inline void gen_subs(TCGv ret, TCGv r1, TCGv r2)
+{
+    gen_helper_sub_ssov(ret, cpu_env, r1, r2);
+}
+
+static inline void gen_bit_2op(TCGv ret, TCGv r1, TCGv r2,
+                               int pos1, int pos2,
+                               void(*op1)(TCGv, TCGv, TCGv),
+                               void(*op2)(TCGv, TCGv, TCGv))
+{
+    TCGv temp1, temp2;
+
+    temp1 = tcg_temp_new();
+    temp2 = tcg_temp_new();
+
+    tcg_gen_shri_tl(temp2, r2, pos2);
+    tcg_gen_shri_tl(temp1, r1, pos1);
+
+    (*op1)(temp1, temp1, temp2);
+    (*op2)(temp1 , ret, temp1);
+
+    tcg_gen_deposit_tl(ret, ret, temp1, 0, 1);
+
+    tcg_temp_free(temp1);
+    tcg_temp_free(temp2);
+}
+
+/* ret = r1[pos1] op1 r2[pos2]; */
+static inline void gen_bit_1op(TCGv ret, TCGv r1, TCGv r2,
+                               int pos1, int pos2,
+                               void(*op1)(TCGv, TCGv, TCGv))
+{
+    TCGv temp1, temp2;
+
+    temp1 = tcg_temp_new();
+    temp2 = tcg_temp_new();
+
+    tcg_gen_shri_tl(temp2, r2, pos2);
+    tcg_gen_shri_tl(temp1, r1, pos1);
+
+    (*op1)(ret, temp1, temp2);
+
+    tcg_gen_andi_tl(ret, ret, 0x1);
+
+    tcg_temp_free(temp1);
+    tcg_temp_free(temp2);
+}
+
+/* helpers for generating program flow micro-ops */
+
+static inline void gen_save_pc(target_ulong pc)
+{
+    tcg_gen_movi_tl(cpu_PC, pc);
+}
+
+static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest)
+{
+    TranslationBlock *tb;
+    tb = ctx->tb;
+    if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) &&
+        likely(!ctx->singlestep_enabled)) {
+        tcg_gen_goto_tb(n);
+        gen_save_pc(dest);
+        tcg_gen_exit_tb((uintptr_t)tb + n);
+    } else {
+        gen_save_pc(dest);
+        if (ctx->singlestep_enabled) {
+            /* raise exception debug */
+        }
+        tcg_gen_exit_tb(0);
+    }
+}
+
+static inline void gen_branch_cond(DisasContext *ctx, TCGCond cond, TCGv r1,
+                                   TCGv r2, int16_t address)
+{
+    int jumpLabel;
+    jumpLabel = gen_new_label();
+    tcg_gen_brcond_tl(cond, r1, r2, jumpLabel);
+
+    gen_goto_tb(ctx, 1, ctx->next_pc);
+
+    gen_set_label(jumpLabel);
+    gen_goto_tb(ctx, 0, ctx->pc + address * 2);
+}
+
+static inline void gen_branch_condi(DisasContext *ctx, TCGCond cond, TCGv r1,
+                                    int r2, int16_t address)
+{
+    TCGv temp = tcg_const_i32(r2);
+    gen_branch_cond(ctx, cond, r1, temp, address);
+    tcg_temp_free(temp);
+}
+
+static void gen_loop(DisasContext *ctx, int r1, int32_t offset)
+{
+    int l1;
+    l1 = gen_new_label();
+
+    tcg_gen_subi_tl(cpu_gpr_a[r1], cpu_gpr_a[r1], 1);
+    tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_gpr_a[r1], -1, l1);
+    gen_goto_tb(ctx, 1, ctx->pc + offset);
+    gen_set_label(l1);
+    gen_goto_tb(ctx, 0, ctx->next_pc);
+}
+
+static void gen_compute_branch(DisasContext *ctx, uint32_t opc, int r1,
+                               int r2 , int32_t constant , int32_t offset)
+{
+    TCGv temp;
+
+    switch (opc) {
+/* SB-format jumps */
+    case OPC1_16_SB_J:
+    case OPC1_32_B_J:
+        gen_goto_tb(ctx, 0, ctx->pc + offset * 2);
+        break;
+    case OPC1_32_B_CALL:
+    case OPC1_16_SB_CALL:
+        gen_helper_1arg(call, ctx->next_pc);
+        gen_goto_tb(ctx, 0, ctx->pc + offset * 2);
+        break;
+    case OPC1_16_SB_JZ:
+        gen_branch_condi(ctx, TCG_COND_EQ, cpu_gpr_d[15], 0, offset);
+        break;
+    case OPC1_16_SB_JNZ:
+        gen_branch_condi(ctx, TCG_COND_NE, cpu_gpr_d[15], 0, offset);
+        break;
+/* SBC-format jumps */
+    case OPC1_16_SBC_JEQ:
+        gen_branch_condi(ctx, TCG_COND_EQ, cpu_gpr_d[15], constant, offset);
+        break;
+    case OPC1_16_SBC_JNE:
+        gen_branch_condi(ctx, TCG_COND_NE, cpu_gpr_d[15], constant, offset);
+        break;
+/* SBRN-format jumps */
+    case OPC1_16_SBRN_JZ_T:
+        temp = tcg_temp_new();
+        tcg_gen_andi_tl(temp, cpu_gpr_d[15], 0x1u << constant);
+        gen_branch_condi(ctx, TCG_COND_EQ, temp, 0, offset);
+        tcg_temp_free(temp);
+        break;
+    case OPC1_16_SBRN_JNZ_T:
+        temp = tcg_temp_new();
+        tcg_gen_andi_tl(temp, cpu_gpr_d[15], 0x1u << constant);
+        gen_branch_condi(ctx, TCG_COND_NE, temp, 0, offset);
+        tcg_temp_free(temp);
+        break;
+/* SBR-format jumps */
+    case OPC1_16_SBR_JEQ:
+        gen_branch_cond(ctx, TCG_COND_EQ, cpu_gpr_d[r1], cpu_gpr_d[15],
+                        offset);
+        break;
+    case OPC1_16_SBR_JNE:
+        gen_branch_cond(ctx, TCG_COND_NE, cpu_gpr_d[r1], cpu_gpr_d[15],
+                        offset);
+        break;
+    case OPC1_16_SBR_JNZ:
+        gen_branch_condi(ctx, TCG_COND_NE, cpu_gpr_d[r1], 0, offset);
+        break;
+    case OPC1_16_SBR_JNZ_A:
+        gen_branch_condi(ctx, TCG_COND_NE, cpu_gpr_a[r1], 0, offset);
+        break;
+    case OPC1_16_SBR_JGEZ:
+        gen_branch_condi(ctx, TCG_COND_GE, cpu_gpr_d[r1], 0, offset);
+        break;
+    case OPC1_16_SBR_JGTZ:
+        gen_branch_condi(ctx, TCG_COND_GT, cpu_gpr_d[r1], 0, offset);
+        break;
+    case OPC1_16_SBR_JLEZ:
+        gen_branch_condi(ctx, TCG_COND_LE, cpu_gpr_d[r1], 0, offset);
+        break;
+    case OPC1_16_SBR_JLTZ:
+        gen_branch_condi(ctx, TCG_COND_LT, cpu_gpr_d[r1], 0, offset);
+        break;
+    case OPC1_16_SBR_JZ:
+        gen_branch_condi(ctx, TCG_COND_EQ, cpu_gpr_d[r1], 0, offset);
+        break;
+    case OPC1_16_SBR_JZ_A:
+        gen_branch_condi(ctx, TCG_COND_EQ, cpu_gpr_a[r1], 0, offset);
+        break;
+    case OPC1_16_SBR_LOOP:
+        gen_loop(ctx, r1, offset * 2 - 32);
+        break;
+/* SR-format jumps */
+    case OPC1_16_SR_JI:
+        tcg_gen_andi_tl(cpu_PC, cpu_gpr_a[r1], 0xfffffffe);
+        tcg_gen_exit_tb(0);
+        break;
+    case OPC2_16_SR_RET:
+        gen_helper_ret(cpu_env);
+        tcg_gen_exit_tb(0);
+        break;
+/* B-format */
+    case OPC1_32_B_CALLA:
+        gen_helper_1arg(call, ctx->next_pc);
+        gen_goto_tb(ctx, 0, EA_B_ABSOLUT(offset));
+        break;
+    case OPC1_32_B_JLA:
+        tcg_gen_movi_tl(cpu_gpr_a[11], ctx->next_pc);
+    case OPC1_32_B_JA:
+        gen_goto_tb(ctx, 0, EA_B_ABSOLUT(offset));
+        break;
+    case OPC1_32_B_JL:
+        tcg_gen_movi_tl(cpu_gpr_a[11], ctx->next_pc);
+        gen_goto_tb(ctx, 0, ctx->pc + offset * 2);
+        break;
+    default:
+        printf("Branch Error at %x\n", ctx->pc);
+    }
+    ctx->bstate = BS_BRANCH;
+}
+
+
+/*
+ * Functions for decoding instructions
+ */
+
+static void decode_src_opc(DisasContext *ctx, int op1)
+{
+    int r1;
+    int32_t const4;
+    TCGv temp, temp2;
+
+    r1 = MASK_OP_SRC_S1D(ctx->opcode);
+    const4 = MASK_OP_SRC_CONST4_SEXT(ctx->opcode);
+
+    switch (op1) {
+    case OPC1_16_SRC_ADD:
+        gen_addi_d(cpu_gpr_d[r1], cpu_gpr_d[r1], const4);
+        break;
+    case OPC1_16_SRC_ADD_A15:
+        gen_addi_d(cpu_gpr_d[r1], cpu_gpr_d[15], const4);
+        break;
+    case OPC1_16_SRC_ADD_15A:
+        gen_addi_d(cpu_gpr_d[15], cpu_gpr_d[r1], const4);
+        break;
+    case OPC1_16_SRC_ADD_A:
+        tcg_gen_addi_tl(cpu_gpr_a[r1], cpu_gpr_a[r1], const4);
+        break;
+    case OPC1_16_SRC_CADD:
+        gen_condi_add(TCG_COND_NE, cpu_gpr_d[r1], const4, cpu_gpr_d[r1],
+                      cpu_gpr_d[15]);
+        break;
+    case OPC1_16_SRC_CADDN:
+        gen_condi_add(TCG_COND_EQ, cpu_gpr_d[r1], const4, cpu_gpr_d[r1],
+                      cpu_gpr_d[15]);
+        break;
+    case OPC1_16_SRC_CMOV:
+        temp = tcg_const_tl(0);
+        temp2 = tcg_const_tl(const4);
+        tcg_gen_movcond_tl(TCG_COND_NE, cpu_gpr_d[r1], cpu_gpr_d[15], temp,
+                           temp2, cpu_gpr_d[r1]);
+        tcg_temp_free(temp);
+        tcg_temp_free(temp2);
+        break;
+    case OPC1_16_SRC_CMOVN:
+        temp = tcg_const_tl(0);
+        temp2 = tcg_const_tl(const4);
+        tcg_gen_movcond_tl(TCG_COND_EQ, cpu_gpr_d[r1], cpu_gpr_d[15], temp,
+                           temp2, cpu_gpr_d[r1]);
+        tcg_temp_free(temp);
+        tcg_temp_free(temp2);
+        break;
+    case OPC1_16_SRC_EQ:
+        tcg_gen_setcondi_tl(TCG_COND_EQ, cpu_gpr_d[15], cpu_gpr_d[r1],
+                            const4);
+        break;
+    case OPC1_16_SRC_LT:
+        tcg_gen_setcondi_tl(TCG_COND_LT, cpu_gpr_d[15], cpu_gpr_d[r1],
+                            const4);
+        break;
+    case OPC1_16_SRC_MOV:
+        tcg_gen_movi_tl(cpu_gpr_d[r1], const4);
+        break;
+    case OPC1_16_SRC_MOV_A:
+        const4 = MASK_OP_SRC_CONST4(ctx->opcode);
+        tcg_gen_movi_tl(cpu_gpr_a[r1], const4);
+        break;
+    case OPC1_16_SRC_SH:
+        gen_shi(cpu_gpr_d[r1], cpu_gpr_d[r1], const4);
+        break;
+    case OPC1_16_SRC_SHA:
+        gen_shaci(cpu_gpr_d[r1], cpu_gpr_d[r1], const4);
+        break;
+    }
+}
+
+static void decode_srr_opc(DisasContext *ctx, int op1)
+{
+    int r1, r2;
+    TCGv temp;
+
+    r1 = MASK_OP_SRR_S1D(ctx->opcode);
+    r2 = MASK_OP_SRR_S2(ctx->opcode);
+
+    switch (op1) {
+    case OPC1_16_SRR_ADD:
+        gen_add_d(cpu_gpr_d[r1], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC1_16_SRR_ADD_A15:
+        gen_add_d(cpu_gpr_d[r1], cpu_gpr_d[15], cpu_gpr_d[r2]);
+        break;
+    case OPC1_16_SRR_ADD_15A:
+        gen_add_d(cpu_gpr_d[15], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC1_16_SRR_ADD_A:
+        tcg_gen_add_tl(cpu_gpr_a[r1], cpu_gpr_a[r1], cpu_gpr_a[r2]);
+        break;
+    case OPC1_16_SRR_ADDS:
+        gen_adds(cpu_gpr_d[r1], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC1_16_SRR_AND:
+        tcg_gen_and_tl(cpu_gpr_d[r1], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC1_16_SRR_CMOV:
+        temp = tcg_const_tl(0);
+        tcg_gen_movcond_tl(TCG_COND_NE, cpu_gpr_d[r1], cpu_gpr_d[15], temp,
+                           cpu_gpr_d[r2], cpu_gpr_d[r1]);
+        tcg_temp_free(temp);
+        break;
+    case OPC1_16_SRR_CMOVN:
+        temp = tcg_const_tl(0);
+        tcg_gen_movcond_tl(TCG_COND_EQ, cpu_gpr_d[r1], cpu_gpr_d[15], temp,
+                           cpu_gpr_d[r2], cpu_gpr_d[r1]);
+        tcg_temp_free(temp);
+        break;
+    case OPC1_16_SRR_EQ:
+        tcg_gen_setcond_tl(TCG_COND_EQ, cpu_gpr_d[15], cpu_gpr_d[r1],
+                           cpu_gpr_d[r2]);
+        break;
+    case OPC1_16_SRR_LT:
+        tcg_gen_setcond_tl(TCG_COND_LT, cpu_gpr_d[15], cpu_gpr_d[r1],
+                           cpu_gpr_d[r2]);
+        break;
+    case OPC1_16_SRR_MOV:
+        tcg_gen_mov_tl(cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC1_16_SRR_MOV_A:
+        tcg_gen_mov_tl(cpu_gpr_a[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC1_16_SRR_MOV_AA:
+        tcg_gen_mov_tl(cpu_gpr_a[r1], cpu_gpr_a[r2]);
+        break;
+    case OPC1_16_SRR_MOV_D:
+        tcg_gen_mov_tl(cpu_gpr_d[r1], cpu_gpr_a[r2]);
+        break;
+    case OPC1_16_SRR_MUL:
+        gen_mul_i32s(cpu_gpr_d[r1], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC1_16_SRR_OR:
+        tcg_gen_or_tl(cpu_gpr_d[r1], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC1_16_SRR_SUB:
+        gen_sub_d(cpu_gpr_d[r1], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC1_16_SRR_SUB_A15B:
+        gen_sub_d(cpu_gpr_d[r1], cpu_gpr_d[15], cpu_gpr_d[r2]);
+        break;
+    case OPC1_16_SRR_SUB_15AB:
+        gen_sub_d(cpu_gpr_d[15], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC1_16_SRR_SUBS:
+        gen_subs(cpu_gpr_d[r1], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC1_16_SRR_XOR:
+        tcg_gen_xor_tl(cpu_gpr_d[r1], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    }
+}
+
+static void decode_ssr_opc(DisasContext *ctx, int op1)
+{
+    int r1, r2;
+
+    r1 = MASK_OP_SSR_S1(ctx->opcode);
+    r2 = MASK_OP_SSR_S2(ctx->opcode);
+
+    switch (op1) {
+    case OPC1_16_SSR_ST_A:
+        tcg_gen_qemu_st_tl(cpu_gpr_a[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LEUL);
+        break;
+    case OPC1_16_SSR_ST_A_POSTINC:
+        tcg_gen_qemu_st_tl(cpu_gpr_a[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LEUL);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], 4);
+        break;
+    case OPC1_16_SSR_ST_B:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_UB);
+        break;
+    case OPC1_16_SSR_ST_B_POSTINC:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_UB);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], 1);
+        break;
+    case OPC1_16_SSR_ST_H:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LEUW);
+        break;
+    case OPC1_16_SSR_ST_H_POSTINC:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LEUW);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], 2);
+        break;
+    case OPC1_16_SSR_ST_W:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LEUL);
+        break;
+    case OPC1_16_SSR_ST_W_POSTINC:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LEUL);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], 4);
+        break;
+    }
+}
+
+static void decode_sc_opc(DisasContext *ctx, int op1)
+{
+    int32_t const16;
+
+    const16 = MASK_OP_SC_CONST8(ctx->opcode);
+
+    switch (op1) {
+    case OPC1_16_SC_AND:
+        tcg_gen_andi_tl(cpu_gpr_d[15], cpu_gpr_d[15], const16);
+        break;
+    case OPC1_16_SC_BISR:
+        gen_helper_1arg(bisr, const16 & 0xff);
+        break;
+    case OPC1_16_SC_LD_A:
+        gen_offset_ld(ctx, cpu_gpr_a[15], cpu_gpr_a[10], const16 * 4, MO_LESL);
+        break;
+    case OPC1_16_SC_LD_W:
+        gen_offset_ld(ctx, cpu_gpr_d[15], cpu_gpr_a[10], const16 * 4, MO_LESL);
+        break;
+    case OPC1_16_SC_MOV:
+        tcg_gen_movi_tl(cpu_gpr_d[15], const16);
+        break;
+    case OPC1_16_SC_OR:
+        tcg_gen_ori_tl(cpu_gpr_d[15], cpu_gpr_d[15], const16);
+        break;
+    case OPC1_16_SC_ST_A:
+        gen_offset_st(ctx, cpu_gpr_a[15], cpu_gpr_a[10], const16 * 4, MO_LESL);
+        break;
+    case OPC1_16_SC_ST_W:
+        gen_offset_st(ctx, cpu_gpr_d[15], cpu_gpr_a[10], const16 * 4, MO_LESL);
+        break;
+    case OPC1_16_SC_SUB_A:
+        tcg_gen_subi_tl(cpu_gpr_a[10], cpu_gpr_a[10], const16);
+        break;
+    }
+}
+
+static void decode_slr_opc(DisasContext *ctx, int op1)
+{
+    int r1, r2;
+
+    r1 = MASK_OP_SLR_D(ctx->opcode);
+    r2 = MASK_OP_SLR_S2(ctx->opcode);
+
+    switch (op1) {
+/* SLR-format */
+    case OPC1_16_SLR_LD_A:
+        tcg_gen_qemu_ld_tl(cpu_gpr_a[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LESL);
+        break;
+    case OPC1_16_SLR_LD_A_POSTINC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_a[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LESL);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], 4);
+        break;
+    case OPC1_16_SLR_LD_BU:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_UB);
+        break;
+    case OPC1_16_SLR_LD_BU_POSTINC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_UB);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], 1);
+        break;
+    case OPC1_16_SLR_LD_H:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LESW);
+        break;
+    case OPC1_16_SLR_LD_H_POSTINC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LESW);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], 2);
+        break;
+    case OPC1_16_SLR_LD_W:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LESW);
+        break;
+    case OPC1_16_SLR_LD_W_POSTINC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LESW);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], 4);
+        break;
+    }
+}
+
+static void decode_sro_opc(DisasContext *ctx, int op1)
+{
+    int r2;
+    int32_t address;
+
+    r2 = MASK_OP_SRO_S2(ctx->opcode);
+    address = MASK_OP_SRO_OFF4(ctx->opcode);
+
+/* SRO-format */
+    switch (op1) {
+    case OPC1_16_SRO_LD_A:
+        gen_offset_ld(ctx, cpu_gpr_a[15], cpu_gpr_a[r2], address * 4, MO_LESL);
+        break;
+    case OPC1_16_SRO_LD_BU:
+        gen_offset_ld(ctx, cpu_gpr_d[15], cpu_gpr_a[r2], address, MO_UB);
+        break;
+    case OPC1_16_SRO_LD_H:
+        gen_offset_ld(ctx, cpu_gpr_d[15], cpu_gpr_a[r2], address, MO_LESW);
+        break;
+    case OPC1_16_SRO_LD_W:
+        gen_offset_ld(ctx, cpu_gpr_d[15], cpu_gpr_a[r2], address * 4, MO_LESL);
+        break;
+    case OPC1_16_SRO_ST_A:
+        gen_offset_st(ctx, cpu_gpr_a[15], cpu_gpr_a[r2], address * 4, MO_LESL);
+        break;
+    case OPC1_16_SRO_ST_B:
+        gen_offset_st(ctx, cpu_gpr_d[15], cpu_gpr_a[r2], address, MO_UB);
+        break;
+    case OPC1_16_SRO_ST_H:
+        gen_offset_st(ctx, cpu_gpr_d[15], cpu_gpr_a[r2], address * 2, MO_LESW);
+        break;
+    case OPC1_16_SRO_ST_W:
+        gen_offset_st(ctx, cpu_gpr_d[15], cpu_gpr_a[r2], address * 4, MO_LESL);
+        break;
+    }
+}
+
+static void decode_sr_system(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    op2 = MASK_OP_SR_OP2(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_16_SR_NOP:
+        break;
+    case OPC2_16_SR_RET:
+        gen_compute_branch(ctx, op2, 0, 0, 0, 0);
+        break;
+    case OPC2_16_SR_RFE:
+        gen_helper_rfe(cpu_env);
+        tcg_gen_exit_tb(0);
+        ctx->bstate = BS_BRANCH;
+        break;
+    case OPC2_16_SR_DEBUG:
+        /* raise EXCP_DEBUG */
+        break;
+    }
+}
+
+static void decode_sr_accu(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    uint32_t r1;
+    TCGv temp;
+
+    r1 = MASK_OP_SR_S1D(ctx->opcode);
+    op2 = MASK_OP_SR_OP2(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_16_SR_RSUB:
+        /* overflow only if r1 = -0x80000000 */
+        temp = tcg_const_i32(-0x80000000);
+        /* calc V bit */
+        tcg_gen_setcond_tl(TCG_COND_EQ, cpu_PSW_V, cpu_gpr_d[r1], temp);
+        tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
+        /* calc SV bit */
+        tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+        /* sub */
+        tcg_gen_neg_tl(cpu_gpr_d[r1], cpu_gpr_d[r1]);
+        /* calc av */
+        tcg_gen_add_tl(cpu_PSW_AV, cpu_gpr_d[r1], cpu_gpr_d[r1]);
+        tcg_gen_xor_tl(cpu_PSW_AV, cpu_gpr_d[r1], cpu_PSW_AV);
+        /* calc sav */
+        tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+        tcg_temp_free(temp);
+        break;
+    case OPC2_16_SR_SAT_B:
+        gen_saturate(cpu_gpr_d[r1], cpu_gpr_d[r1], 0x7f, -0x80);
+        break;
+    case OPC2_16_SR_SAT_BU:
+        gen_saturate_u(cpu_gpr_d[r1], cpu_gpr_d[r1], 0xff);
+        break;
+    case OPC2_16_SR_SAT_H:
+        gen_saturate(cpu_gpr_d[r1], cpu_gpr_d[r1], 0x7fff, -0x8000);
+        break;
+    case OPC2_16_SR_SAT_HU:
+        gen_saturate_u(cpu_gpr_d[r1], cpu_gpr_d[r1], 0xffff);
+        break;
+    }
+}
+
+static void decode_16Bit_opc(CPUTriCoreState *env, DisasContext *ctx)
+{
+    int op1;
+    int r1, r2;
+    int32_t const16;
+    int32_t address;
+    TCGv temp;
+
+    op1 = MASK_OP_MAJOR(ctx->opcode);
+
+    /* handle ADDSC.A opcode only being 6 bit long */
+    if (unlikely((op1 & 0x3f) == OPC1_16_SRRS_ADDSC_A)) {
+        op1 = OPC1_16_SRRS_ADDSC_A;
+    }
+
+    switch (op1) {
+    case OPC1_16_SRC_ADD:
+    case OPC1_16_SRC_ADD_A15:
+    case OPC1_16_SRC_ADD_15A:
+    case OPC1_16_SRC_ADD_A:
+    case OPC1_16_SRC_CADD:
+    case OPC1_16_SRC_CADDN:
+    case OPC1_16_SRC_CMOV:
+    case OPC1_16_SRC_CMOVN:
+    case OPC1_16_SRC_EQ:
+    case OPC1_16_SRC_LT:
+    case OPC1_16_SRC_MOV:
+    case OPC1_16_SRC_MOV_A:
+    case OPC1_16_SRC_SH:
+    case OPC1_16_SRC_SHA:
+        decode_src_opc(ctx, op1);
+        break;
+/* SRR-format */
+    case OPC1_16_SRR_ADD:
+    case OPC1_16_SRR_ADD_A15:
+    case OPC1_16_SRR_ADD_15A:
+    case OPC1_16_SRR_ADD_A:
+    case OPC1_16_SRR_ADDS:
+    case OPC1_16_SRR_AND:
+    case OPC1_16_SRR_CMOV:
+    case OPC1_16_SRR_CMOVN:
+    case OPC1_16_SRR_EQ:
+    case OPC1_16_SRR_LT:
+    case OPC1_16_SRR_MOV:
+    case OPC1_16_SRR_MOV_A:
+    case OPC1_16_SRR_MOV_AA:
+    case OPC1_16_SRR_MOV_D:
+    case OPC1_16_SRR_MUL:
+    case OPC1_16_SRR_OR:
+    case OPC1_16_SRR_SUB:
+    case OPC1_16_SRR_SUB_A15B:
+    case OPC1_16_SRR_SUB_15AB:
+    case OPC1_16_SRR_SUBS:
+    case OPC1_16_SRR_XOR:
+        decode_srr_opc(ctx, op1);
+        break;
+/* SSR-format */
+    case OPC1_16_SSR_ST_A:
+    case OPC1_16_SSR_ST_A_POSTINC:
+    case OPC1_16_SSR_ST_B:
+    case OPC1_16_SSR_ST_B_POSTINC:
+    case OPC1_16_SSR_ST_H:
+    case OPC1_16_SSR_ST_H_POSTINC:
+    case OPC1_16_SSR_ST_W:
+    case OPC1_16_SSR_ST_W_POSTINC:
+        decode_ssr_opc(ctx, op1);
+        break;
+/* SRRS-format */
+    case OPC1_16_SRRS_ADDSC_A:
+        r2 = MASK_OP_SRRS_S2(ctx->opcode);
+        r1 = MASK_OP_SRRS_S1D(ctx->opcode);
+        const16 = MASK_OP_SRRS_N(ctx->opcode);
+        temp = tcg_temp_new();
+        tcg_gen_shli_tl(temp, cpu_gpr_d[15], const16);
+        tcg_gen_add_tl(cpu_gpr_a[r1], cpu_gpr_a[r2], temp);
+        tcg_temp_free(temp);
+        break;
+/* SLRO-format */
+    case OPC1_16_SLRO_LD_A:
+        r1 = MASK_OP_SLRO_D(ctx->opcode);
+        const16 = MASK_OP_SLRO_OFF4(ctx->opcode);
+        gen_offset_ld(ctx, cpu_gpr_a[r1], cpu_gpr_a[15], const16 * 4, MO_LESL);
+        break;
+    case OPC1_16_SLRO_LD_BU:
+        r1 = MASK_OP_SLRO_D(ctx->opcode);
+        const16 = MASK_OP_SLRO_OFF4(ctx->opcode);
+        gen_offset_ld(ctx, cpu_gpr_d[r1], cpu_gpr_a[15], const16, MO_UB);
+        break;
+    case OPC1_16_SLRO_LD_H:
+        r1 = MASK_OP_SLRO_D(ctx->opcode);
+        const16 = MASK_OP_SLRO_OFF4(ctx->opcode);
+        gen_offset_ld(ctx, cpu_gpr_d[r1], cpu_gpr_a[15], const16 * 2, MO_LESW);
+        break;
+    case OPC1_16_SLRO_LD_W:
+        r1 = MASK_OP_SLRO_D(ctx->opcode);
+        const16 = MASK_OP_SLRO_OFF4(ctx->opcode);
+        gen_offset_ld(ctx, cpu_gpr_d[r1], cpu_gpr_a[15], const16 * 4, MO_LESL);
+        break;
+/* SB-format */
+    case OPC1_16_SB_CALL:
+    case OPC1_16_SB_J:
+    case OPC1_16_SB_JNZ:
+    case OPC1_16_SB_JZ:
+        address = MASK_OP_SB_DISP8_SEXT(ctx->opcode);
+        gen_compute_branch(ctx, op1, 0, 0, 0, address);
+        break;
+/* SBC-format */
+    case OPC1_16_SBC_JEQ:
+    case OPC1_16_SBC_JNE:
+        address = MASK_OP_SBC_DISP4(ctx->opcode);
+        const16 = MASK_OP_SBC_CONST4_SEXT(ctx->opcode);
+        gen_compute_branch(ctx, op1, 0, 0, const16, address);
+        break;
+/* SBRN-format */
+    case OPC1_16_SBRN_JNZ_T:
+    case OPC1_16_SBRN_JZ_T:
+        address = MASK_OP_SBRN_DISP4(ctx->opcode);
+        const16 = MASK_OP_SBRN_N(ctx->opcode);
+        gen_compute_branch(ctx, op1, 0, 0, const16, address);
+        break;
+/* SBR-format */
+    case OPC1_16_SBR_JEQ:
+    case OPC1_16_SBR_JGEZ:
+    case OPC1_16_SBR_JGTZ:
+    case OPC1_16_SBR_JLEZ:
+    case OPC1_16_SBR_JLTZ:
+    case OPC1_16_SBR_JNE:
+    case OPC1_16_SBR_JNZ:
+    case OPC1_16_SBR_JNZ_A:
+    case OPC1_16_SBR_JZ:
+    case OPC1_16_SBR_JZ_A:
+    case OPC1_16_SBR_LOOP:
+        r1 = MASK_OP_SBR_S2(ctx->opcode);
+        address = MASK_OP_SBR_DISP4(ctx->opcode);
+        gen_compute_branch(ctx, op1, r1, 0, 0, address);
+        break;
+/* SC-format */
+    case OPC1_16_SC_AND:
+    case OPC1_16_SC_BISR:
+    case OPC1_16_SC_LD_A:
+    case OPC1_16_SC_LD_W:
+    case OPC1_16_SC_MOV:
+    case OPC1_16_SC_OR:
+    case OPC1_16_SC_ST_A:
+    case OPC1_16_SC_ST_W:
+    case OPC1_16_SC_SUB_A:
+        decode_sc_opc(ctx, op1);
+        break;
+/* SLR-format */
+    case OPC1_16_SLR_LD_A:
+    case OPC1_16_SLR_LD_A_POSTINC:
+    case OPC1_16_SLR_LD_BU:
+    case OPC1_16_SLR_LD_BU_POSTINC:
+    case OPC1_16_SLR_LD_H:
+    case OPC1_16_SLR_LD_H_POSTINC:
+    case OPC1_16_SLR_LD_W:
+    case OPC1_16_SLR_LD_W_POSTINC:
+        decode_slr_opc(ctx, op1);
+        break;
+/* SRO-format */
+    case OPC1_16_SRO_LD_A:
+    case OPC1_16_SRO_LD_BU:
+    case OPC1_16_SRO_LD_H:
+    case OPC1_16_SRO_LD_W:
+    case OPC1_16_SRO_ST_A:
+    case OPC1_16_SRO_ST_B:
+    case OPC1_16_SRO_ST_H:
+    case OPC1_16_SRO_ST_W:
+        decode_sro_opc(ctx, op1);
+        break;
+/* SSRO-format */
+    case OPC1_16_SSRO_ST_A:
+        r1 = MASK_OP_SSRO_S1(ctx->opcode);
+        const16 = MASK_OP_SSRO_OFF4(ctx->opcode);
+        gen_offset_st(ctx, cpu_gpr_a[r1], cpu_gpr_a[15], const16 * 4, MO_LESL);
+        break;
+    case OPC1_16_SSRO_ST_B:
+        r1 = MASK_OP_SSRO_S1(ctx->opcode);
+        const16 = MASK_OP_SSRO_OFF4(ctx->opcode);
+        gen_offset_st(ctx, cpu_gpr_d[r1], cpu_gpr_a[15], const16, MO_UB);
+        break;
+    case OPC1_16_SSRO_ST_H:
+        r1 = MASK_OP_SSRO_S1(ctx->opcode);
+        const16 = MASK_OP_SSRO_OFF4(ctx->opcode);
+        gen_offset_st(ctx, cpu_gpr_d[r1], cpu_gpr_a[15], const16 * 2, MO_LESW);
+        break;
+    case OPC1_16_SSRO_ST_W:
+        r1 = MASK_OP_SSRO_S1(ctx->opcode);
+        const16 = MASK_OP_SSRO_OFF4(ctx->opcode);
+        gen_offset_st(ctx, cpu_gpr_d[r1], cpu_gpr_a[15], const16 * 4, MO_LESL);
+        break;
+/* SR-format */
+    case OPCM_16_SR_SYSTEM:
+        decode_sr_system(env, ctx);
+        break;
+    case OPCM_16_SR_ACCU:
+        decode_sr_accu(env, ctx);
+        break;
+    case OPC1_16_SR_JI:
+        r1 = MASK_OP_SR_S1D(ctx->opcode);
+        gen_compute_branch(ctx, op1, r1, 0, 0, 0);
+        break;
+    case OPC1_16_SR_NOT:
+        r1 = MASK_OP_SR_S1D(ctx->opcode);
+        tcg_gen_not_tl(cpu_gpr_d[r1], cpu_gpr_d[r1]);
+        break;
+    }
+}
+
+/*
+ * 32 bit instructions
+ */
+
+/* ABS-format */
+static void decode_abs_ldw(CPUTriCoreState *env, DisasContext *ctx)
+{
+    int32_t op2;
+    int32_t r1;
+    uint32_t address;
+    TCGv temp;
+
+    r1 = MASK_OP_ABS_S1D(ctx->opcode);
+    address = MASK_OP_ABS_OFF18(ctx->opcode);
+    op2 = MASK_OP_ABS_OP2(ctx->opcode);
+
+    temp = tcg_const_i32(EA_ABS_FORMAT(address));
+
+    switch (op2) {
+    case OPC2_32_ABS_LD_A:
+        tcg_gen_qemu_ld_tl(cpu_gpr_a[r1], temp, ctx->mem_idx, MO_LESL);
+        break;
+    case OPC2_32_ABS_LD_D:
+        gen_ld_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], temp, ctx);
+        break;
+    case OPC2_32_ABS_LD_DA:
+        gen_ld_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], temp, ctx);
+        break;
+    case OPC2_32_ABS_LD_W:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp, ctx->mem_idx, MO_LESL);
+        break;
+    }
+
+    tcg_temp_free(temp);
+}
+
+static void decode_abs_ldb(CPUTriCoreState *env, DisasContext *ctx)
+{
+    int32_t op2;
+    int32_t r1;
+    uint32_t address;
+    TCGv temp;
+
+    r1 = MASK_OP_ABS_S1D(ctx->opcode);
+    address = MASK_OP_ABS_OFF18(ctx->opcode);
+    op2 = MASK_OP_ABS_OP2(ctx->opcode);
+
+    temp = tcg_const_i32(EA_ABS_FORMAT(address));
+
+    switch (op2) {
+    case OPC2_32_ABS_LD_B:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp, ctx->mem_idx, MO_SB);
+        break;
+    case OPC2_32_ABS_LD_BU:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp, ctx->mem_idx, MO_UB);
+        break;
+    case OPC2_32_ABS_LD_H:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp, ctx->mem_idx, MO_LESW);
+        break;
+    case OPC2_32_ABS_LD_HU:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp, ctx->mem_idx, MO_LEUW);
+        break;
+    }
+
+    tcg_temp_free(temp);
+}
+
+static void decode_abs_ldst_swap(CPUTriCoreState *env, DisasContext *ctx)
+{
+    int32_t op2;
+    int32_t r1;
+    uint32_t address;
+    TCGv temp;
+
+    r1 = MASK_OP_ABS_S1D(ctx->opcode);
+    address = MASK_OP_ABS_OFF18(ctx->opcode);
+    op2 = MASK_OP_ABS_OP2(ctx->opcode);
+
+    temp = tcg_const_i32(EA_ABS_FORMAT(address));
+
+    switch (op2) {
+    case OPC2_32_ABS_LDMST:
+        gen_ldmst(ctx, r1, temp);
+        break;
+    case OPC2_32_ABS_SWAP_W:
+        gen_swap(ctx, r1, temp);
+        break;
+    }
+
+    tcg_temp_free(temp);
+}
+
+static void decode_abs_ldst_context(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int32_t off18;
+
+    off18 = MASK_OP_ABS_OFF18(ctx->opcode);
+    op2   = MASK_OP_ABS_OP2(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_ABS_LDLCX:
+        gen_helper_1arg(ldlcx, EA_ABS_FORMAT(off18));
+        break;
+    case OPC2_32_ABS_LDUCX:
+        gen_helper_1arg(lducx, EA_ABS_FORMAT(off18));
+        break;
+    case OPC2_32_ABS_STLCX:
+        gen_helper_1arg(stlcx, EA_ABS_FORMAT(off18));
+        break;
+    case OPC2_32_ABS_STUCX:
+        gen_helper_1arg(stucx, EA_ABS_FORMAT(off18));
+        break;
+    }
+}
+
+static void decode_abs_store(CPUTriCoreState *env, DisasContext *ctx)
+{
+    int32_t op2;
+    int32_t r1;
+    uint32_t address;
+    TCGv temp;
+
+    r1 = MASK_OP_ABS_S1D(ctx->opcode);
+    address = MASK_OP_ABS_OFF18(ctx->opcode);
+    op2 = MASK_OP_ABS_OP2(ctx->opcode);
+
+    temp = tcg_const_i32(EA_ABS_FORMAT(address));
+
+    switch (op2) {
+    case OPC2_32_ABS_ST_A:
+        tcg_gen_qemu_st_tl(cpu_gpr_a[r1], temp, ctx->mem_idx, MO_LESL);
+        break;
+    case OPC2_32_ABS_ST_D:
+        gen_st_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], temp, ctx);
+        break;
+    case OPC2_32_ABS_ST_DA:
+        gen_st_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], temp, ctx);
+        break;
+    case OPC2_32_ABS_ST_W:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], temp, ctx->mem_idx, MO_LESL);
+        break;
+
+    }
+    tcg_temp_free(temp);
+}
+
+static void decode_abs_storeb_h(CPUTriCoreState *env, DisasContext *ctx)
+{
+    int32_t op2;
+    int32_t r1;
+    uint32_t address;
+    TCGv temp;
+
+    r1 = MASK_OP_ABS_S1D(ctx->opcode);
+    address = MASK_OP_ABS_OFF18(ctx->opcode);
+    op2 = MASK_OP_ABS_OP2(ctx->opcode);
+
+    temp = tcg_const_i32(EA_ABS_FORMAT(address));
+
+    switch (op2) {
+    case OPC2_32_ABS_ST_B:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], temp, ctx->mem_idx, MO_UB);
+        break;
+    case OPC2_32_ABS_ST_H:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], temp, ctx->mem_idx, MO_LEUW);
+        break;
+    }
+    tcg_temp_free(temp);
+}
+
+/* Bit-format */
+
+static void decode_bit_andacc(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r2, r3;
+    int pos1, pos2;
+
+    r1 = MASK_OP_BIT_S1(ctx->opcode);
+    r2 = MASK_OP_BIT_S2(ctx->opcode);
+    r3 = MASK_OP_BIT_D(ctx->opcode);
+    pos1 = MASK_OP_BIT_POS1(ctx->opcode);
+    pos2 = MASK_OP_BIT_POS2(ctx->opcode);
+    op2 = MASK_OP_BIT_OP2(ctx->opcode);
+
+
+    switch (op2) {
+    case OPC2_32_BIT_AND_AND_T:
+        gen_bit_2op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_and_tl, &tcg_gen_and_tl);
+        break;
+    case OPC2_32_BIT_AND_ANDN_T:
+        gen_bit_2op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_andc_tl, &tcg_gen_and_tl);
+        break;
+    case OPC2_32_BIT_AND_NOR_T:
+        if (TCG_TARGET_HAS_andc_i32) {
+            gen_bit_2op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                        pos1, pos2, &tcg_gen_or_tl, &tcg_gen_andc_tl);
+        } else {
+            gen_bit_2op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                        pos1, pos2, &tcg_gen_nor_tl, &tcg_gen_and_tl);
+        }
+        break;
+    case OPC2_32_BIT_AND_OR_T:
+        gen_bit_2op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_or_tl, &tcg_gen_and_tl);
+        break;
+    }
+}
+
+static void decode_bit_logical_t(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r2, r3;
+    int pos1, pos2;
+    r1 = MASK_OP_BIT_S1(ctx->opcode);
+    r2 = MASK_OP_BIT_S2(ctx->opcode);
+    r3 = MASK_OP_BIT_D(ctx->opcode);
+    pos1 = MASK_OP_BIT_POS1(ctx->opcode);
+    pos2 = MASK_OP_BIT_POS2(ctx->opcode);
+    op2 = MASK_OP_BIT_OP2(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_BIT_AND_T:
+        gen_bit_1op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_and_tl);
+        break;
+    case OPC2_32_BIT_ANDN_T:
+        gen_bit_1op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_andc_tl);
+        break;
+    case OPC2_32_BIT_NOR_T:
+        gen_bit_1op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_nor_tl);
+        break;
+    case OPC2_32_BIT_OR_T:
+        gen_bit_1op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_or_tl);
+        break;
+    }
+}
+
+static void decode_bit_insert(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r2, r3;
+    int pos1, pos2;
+    TCGv temp;
+    op2 = MASK_OP_BIT_OP2(ctx->opcode);
+    r1 = MASK_OP_BIT_S1(ctx->opcode);
+    r2 = MASK_OP_BIT_S2(ctx->opcode);
+    r3 = MASK_OP_BIT_D(ctx->opcode);
+    pos1 = MASK_OP_BIT_POS1(ctx->opcode);
+    pos2 = MASK_OP_BIT_POS2(ctx->opcode);
+
+    temp = tcg_temp_new();
+
+    tcg_gen_shri_tl(temp, cpu_gpr_d[r2], pos2);
+    if (op2 == OPC2_32_BIT_INSN_T) {
+        tcg_gen_not_tl(temp, temp);
+    }
+    tcg_gen_deposit_tl(cpu_gpr_d[r3], cpu_gpr_d[r1], temp, pos1, 1);
+    tcg_temp_free(temp);
+}
+
+static void decode_bit_logical_t2(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+
+    int r1, r2, r3;
+    int pos1, pos2;
+
+    op2 = MASK_OP_BIT_OP2(ctx->opcode);
+    r1 = MASK_OP_BIT_S1(ctx->opcode);
+    r2 = MASK_OP_BIT_S2(ctx->opcode);
+    r3 = MASK_OP_BIT_D(ctx->opcode);
+    pos1 = MASK_OP_BIT_POS1(ctx->opcode);
+    pos2 = MASK_OP_BIT_POS2(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_BIT_NAND_T:
+        gen_bit_1op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_nand_tl);
+        break;
+    case OPC2_32_BIT_ORN_T:
+        gen_bit_1op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_orc_tl);
+        break;
+    case OPC2_32_BIT_XNOR_T:
+        gen_bit_1op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_eqv_tl);
+        break;
+    case OPC2_32_BIT_XOR_T:
+        gen_bit_1op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_xor_tl);
+        break;
+    }
+}
+
+static void decode_bit_orand(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+
+    int r1, r2, r3;
+    int pos1, pos2;
+
+    op2 = MASK_OP_BIT_OP2(ctx->opcode);
+    r1 = MASK_OP_BIT_S1(ctx->opcode);
+    r2 = MASK_OP_BIT_S2(ctx->opcode);
+    r3 = MASK_OP_BIT_D(ctx->opcode);
+    pos1 = MASK_OP_BIT_POS1(ctx->opcode);
+    pos2 = MASK_OP_BIT_POS2(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_BIT_OR_AND_T:
+        gen_bit_2op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_and_tl, &tcg_gen_or_tl);
+        break;
+    case OPC2_32_BIT_OR_ANDN_T:
+        gen_bit_2op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_andc_tl, &tcg_gen_or_tl);
+        break;
+    case OPC2_32_BIT_OR_NOR_T:
+        if (TCG_TARGET_HAS_orc_i32) {
+            gen_bit_2op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                        pos1, pos2, &tcg_gen_or_tl, &tcg_gen_orc_tl);
+        } else {
+            gen_bit_2op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                        pos1, pos2, &tcg_gen_nor_tl, &tcg_gen_or_tl);
+        }
+        break;
+    case OPC2_32_BIT_OR_OR_T:
+        gen_bit_2op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_or_tl, &tcg_gen_or_tl);
+        break;
+    }
+}
+
+static void decode_bit_sh_logic1(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r2, r3;
+    int pos1, pos2;
+    TCGv temp;
+
+    op2 = MASK_OP_BIT_OP2(ctx->opcode);
+    r1 = MASK_OP_BIT_S1(ctx->opcode);
+    r2 = MASK_OP_BIT_S2(ctx->opcode);
+    r3 = MASK_OP_BIT_D(ctx->opcode);
+    pos1 = MASK_OP_BIT_POS1(ctx->opcode);
+    pos2 = MASK_OP_BIT_POS2(ctx->opcode);
+
+    temp = tcg_temp_new();
+
+    switch (op2) {
+    case OPC2_32_BIT_SH_AND_T:
+        gen_bit_1op(temp, cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_and_tl);
+        break;
+    case OPC2_32_BIT_SH_ANDN_T:
+        gen_bit_1op(temp, cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_andc_tl);
+        break;
+    case OPC2_32_BIT_SH_NOR_T:
+        gen_bit_1op(temp, cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_nor_tl);
+        break;
+    case OPC2_32_BIT_SH_OR_T:
+        gen_bit_1op(temp, cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_or_tl);
+        break;
+    }
+    tcg_gen_shli_tl(cpu_gpr_d[r3], cpu_gpr_d[r3], 1);
+    tcg_gen_add_tl(cpu_gpr_d[r3], cpu_gpr_d[r3], temp);
+    tcg_temp_free(temp);
+}
+
+static void decode_bit_sh_logic2(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r2, r3;
+    int pos1, pos2;
+    TCGv temp;
+
+    op2 = MASK_OP_BIT_OP2(ctx->opcode);
+    r1 = MASK_OP_BIT_S1(ctx->opcode);
+    r2 = MASK_OP_BIT_S2(ctx->opcode);
+    r3 = MASK_OP_BIT_D(ctx->opcode);
+    pos1 = MASK_OP_BIT_POS1(ctx->opcode);
+    pos2 = MASK_OP_BIT_POS2(ctx->opcode);
+
+    temp = tcg_temp_new();
+
+    switch (op2) {
+    case OPC2_32_BIT_SH_NAND_T:
+        gen_bit_1op(temp, cpu_gpr_d[r1] , cpu_gpr_d[r2] ,
+                    pos1, pos2, &tcg_gen_nand_tl);
+        break;
+    case OPC2_32_BIT_SH_ORN_T:
+        gen_bit_1op(temp, cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_orc_tl);
+        break;
+    case OPC2_32_BIT_SH_XNOR_T:
+        gen_bit_1op(temp, cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_eqv_tl);
+        break;
+    case OPC2_32_BIT_SH_XOR_T:
+        gen_bit_1op(temp, cpu_gpr_d[r1], cpu_gpr_d[r2],
+                    pos1, pos2, &tcg_gen_xor_tl);
+        break;
+    }
+    tcg_gen_shli_tl(cpu_gpr_d[r3], cpu_gpr_d[r3], 1);
+    tcg_gen_add_tl(cpu_gpr_d[r3], cpu_gpr_d[r3], temp);
+    tcg_temp_free(temp);
+}
+
+/* BO-format */
+
+
+static void decode_bo_addrmode_post_pre_base(CPUTriCoreState *env,
+                                             DisasContext *ctx)
+{
+    uint32_t op2;
+    uint32_t off10;
+    int32_t r1, r2;
+    TCGv temp;
+
+    r1 = MASK_OP_BO_S1D(ctx->opcode);
+    r2  = MASK_OP_BO_S2(ctx->opcode);
+    off10 = MASK_OP_BO_OFF10_SEXT(ctx->opcode);
+    op2 = MASK_OP_BO_OP2(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_BO_CACHEA_WI_SHORTOFF:
+    case OPC2_32_BO_CACHEA_W_SHORTOFF:
+    case OPC2_32_BO_CACHEA_I_SHORTOFF:
+        /* instruction to access the cache */
+        break;
+    case OPC2_32_BO_CACHEA_WI_POSTINC:
+    case OPC2_32_BO_CACHEA_W_POSTINC:
+    case OPC2_32_BO_CACHEA_I_POSTINC:
+        /* instruction to access the cache, but we still need to handle
+           the addressing mode */
+        tcg_gen_addi_tl(cpu_gpr_d[r2], cpu_gpr_d[r2], off10);
+        break;
+    case OPC2_32_BO_CACHEA_WI_PREINC:
+    case OPC2_32_BO_CACHEA_W_PREINC:
+    case OPC2_32_BO_CACHEA_I_PREINC:
+        /* instruction to access the cache, but we still need to handle
+           the addressing mode */
+        tcg_gen_addi_tl(cpu_gpr_d[r2], cpu_gpr_d[r2], off10);
+        break;
+    case OPC2_32_BO_CACHEI_WI_SHORTOFF:
+    case OPC2_32_BO_CACHEI_W_SHORTOFF:
+        /* TODO: Raise illegal opcode trap,
+                 if tricore_feature(TRICORE_FEATURE_13) */
+        break;
+    case OPC2_32_BO_CACHEI_W_POSTINC:
+    case OPC2_32_BO_CACHEI_WI_POSTINC:
+        if (!tricore_feature(env, TRICORE_FEATURE_13)) {
+            tcg_gen_addi_tl(cpu_gpr_d[r2], cpu_gpr_d[r2], off10);
+        } /* TODO: else raise illegal opcode trap */
+        break;
+    case OPC2_32_BO_CACHEI_W_PREINC:
+    case OPC2_32_BO_CACHEI_WI_PREINC:
+        if (!tricore_feature(env, TRICORE_FEATURE_13)) {
+            tcg_gen_addi_tl(cpu_gpr_d[r2], cpu_gpr_d[r2], off10);
+        } /* TODO: else raise illegal opcode trap */
+        break;
+    case OPC2_32_BO_ST_A_SHORTOFF:
+        gen_offset_st(ctx, cpu_gpr_a[r1], cpu_gpr_a[r2], off10, MO_LESL);
+        break;
+    case OPC2_32_BO_ST_A_POSTINC:
+        tcg_gen_qemu_st_tl(cpu_gpr_a[r1], cpu_gpr_a[r2], ctx->mem_idx,
+                           MO_LESL);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_ST_A_PREINC:
+        gen_st_preincr(ctx, cpu_gpr_a[r1], cpu_gpr_a[r2], off10, MO_LESL);
+        break;
+    case OPC2_32_BO_ST_B_SHORTOFF:
+        gen_offset_st(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_UB);
+        break;
+    case OPC2_32_BO_ST_B_POSTINC:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx,
+                           MO_UB);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_ST_B_PREINC:
+        gen_st_preincr(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_UB);
+        break;
+    case OPC2_32_BO_ST_D_SHORTOFF:
+        gen_offset_st_2regs(cpu_gpr_d[r1+1], cpu_gpr_d[r1], cpu_gpr_a[r2],
+                            off10, ctx);
+        break;
+    case OPC2_32_BO_ST_D_POSTINC:
+        gen_st_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], cpu_gpr_a[r2], ctx);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_ST_D_PREINC:
+        temp = tcg_temp_new();
+        tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
+        gen_st_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], temp, ctx);
+        tcg_gen_mov_tl(cpu_gpr_a[r2], temp);
+        tcg_temp_free(temp);
+        break;
+    case OPC2_32_BO_ST_DA_SHORTOFF:
+        gen_offset_st_2regs(cpu_gpr_a[r1+1], cpu_gpr_a[r1], cpu_gpr_a[r2],
+                            off10, ctx);
+        break;
+    case OPC2_32_BO_ST_DA_POSTINC:
+        gen_st_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], cpu_gpr_a[r2], ctx);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_ST_DA_PREINC:
+        temp = tcg_temp_new();
+        tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
+        gen_st_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], temp, ctx);
+        tcg_gen_mov_tl(cpu_gpr_a[r2], temp);
+        tcg_temp_free(temp);
+        break;
+    case OPC2_32_BO_ST_H_SHORTOFF:
+        gen_offset_st(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_LEUW);
+        break;
+    case OPC2_32_BO_ST_H_POSTINC:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx,
+                           MO_LEUW);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_ST_H_PREINC:
+        gen_st_preincr(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_LEUW);
+        break;
+    case OPC2_32_BO_ST_Q_SHORTOFF:
+        temp = tcg_temp_new();
+        tcg_gen_shri_tl(temp, cpu_gpr_d[r1], 16);
+        gen_offset_st(ctx, temp, cpu_gpr_a[r2], off10, MO_LEUW);
+        tcg_temp_free(temp);
+        break;
+    case OPC2_32_BO_ST_Q_POSTINC:
+        temp = tcg_temp_new();
+        tcg_gen_shri_tl(temp, cpu_gpr_d[r1], 16);
+        tcg_gen_qemu_st_tl(temp, cpu_gpr_a[r2], ctx->mem_idx,
+                           MO_LEUW);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        tcg_temp_free(temp);
+        break;
+    case OPC2_32_BO_ST_Q_PREINC:
+        temp = tcg_temp_new();
+        tcg_gen_shri_tl(temp, cpu_gpr_d[r1], 16);
+        gen_st_preincr(ctx, temp, cpu_gpr_a[r2], off10, MO_LEUW);
+        tcg_temp_free(temp);
+        break;
+    case OPC2_32_BO_ST_W_SHORTOFF:
+        gen_offset_st(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_LEUL);
+        break;
+    case OPC2_32_BO_ST_W_POSTINC:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx,
+                           MO_LEUL);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_ST_W_PREINC:
+        gen_st_preincr(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_LEUL);
+        break;
+    }
+}
+
+static void decode_bo_addrmode_bitreverse_circular(CPUTriCoreState *env,
+                                                   DisasContext *ctx)
+{
+    uint32_t op2;
+    uint32_t off10;
+    int32_t r1, r2;
+    TCGv temp, temp2, temp3;
+
+    r1 = MASK_OP_BO_S1D(ctx->opcode);
+    r2  = MASK_OP_BO_S2(ctx->opcode);
+    off10 = MASK_OP_BO_OFF10_SEXT(ctx->opcode);
+    op2 = MASK_OP_BO_OP2(ctx->opcode);
+
+    temp = tcg_temp_new();
+    temp2 = tcg_temp_new();
+    temp3 = tcg_const_i32(off10);
+
+    tcg_gen_ext16u_tl(temp, cpu_gpr_a[r2+1]);
+    tcg_gen_add_tl(temp2, cpu_gpr_a[r2], temp);
+
+    switch (op2) {
+    case OPC2_32_BO_CACHEA_WI_BR:
+    case OPC2_32_BO_CACHEA_W_BR:
+    case OPC2_32_BO_CACHEA_I_BR:
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_CACHEA_WI_CIRC:
+    case OPC2_32_BO_CACHEA_W_CIRC:
+    case OPC2_32_BO_CACHEA_I_CIRC:
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    case OPC2_32_BO_ST_A_BR:
+        tcg_gen_qemu_st_tl(cpu_gpr_a[r1], temp2, ctx->mem_idx, MO_LEUL);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_ST_A_CIRC:
+        tcg_gen_qemu_st_tl(cpu_gpr_a[r1], temp2, ctx->mem_idx, MO_LEUL);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    case OPC2_32_BO_ST_B_BR:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_UB);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_ST_B_CIRC:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_UB);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    case OPC2_32_BO_ST_D_BR:
+        gen_st_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], temp2, ctx);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_ST_D_CIRC:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUL);
+        tcg_gen_shri_tl(temp2, cpu_gpr_a[r2+1], 16);
+        tcg_gen_addi_tl(temp, temp, 4);
+        tcg_gen_rem_tl(temp, temp, temp2);
+        tcg_gen_add_tl(temp2, cpu_gpr_a[r2], temp);
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1+1], temp2, ctx->mem_idx, MO_LEUL);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    case OPC2_32_BO_ST_DA_BR:
+        gen_st_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], temp2, ctx);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_ST_DA_CIRC:
+        tcg_gen_qemu_st_tl(cpu_gpr_a[r1], temp2, ctx->mem_idx, MO_LEUL);
+        tcg_gen_shri_tl(temp2, cpu_gpr_a[r2+1], 16);
+        tcg_gen_addi_tl(temp, temp, 4);
+        tcg_gen_rem_tl(temp, temp, temp2);
+        tcg_gen_add_tl(temp2, cpu_gpr_a[r2], temp);
+        tcg_gen_qemu_st_tl(cpu_gpr_a[r1+1], temp2, ctx->mem_idx, MO_LEUL);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    case OPC2_32_BO_ST_H_BR:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUW);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_ST_H_CIRC:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUW);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    case OPC2_32_BO_ST_Q_BR:
+        tcg_gen_shri_tl(temp, cpu_gpr_d[r1], 16);
+        tcg_gen_qemu_st_tl(temp, temp2, ctx->mem_idx, MO_LEUW);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_ST_Q_CIRC:
+        tcg_gen_shri_tl(temp, cpu_gpr_d[r1], 16);
+        tcg_gen_qemu_st_tl(temp, temp2, ctx->mem_idx, MO_LEUW);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    case OPC2_32_BO_ST_W_BR:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUL);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_ST_W_CIRC:
+        tcg_gen_qemu_st_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUL);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    }
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free(temp3);
+}
+
+static void decode_bo_addrmode_ld_post_pre_base(CPUTriCoreState *env,
+                                                DisasContext *ctx)
+{
+    uint32_t op2;
+    uint32_t off10;
+    int32_t r1, r2;
+    TCGv temp;
+
+    r1 = MASK_OP_BO_S1D(ctx->opcode);
+    r2  = MASK_OP_BO_S2(ctx->opcode);
+    off10 = MASK_OP_BO_OFF10_SEXT(ctx->opcode);
+    op2 = MASK_OP_BO_OP2(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_BO_LD_A_SHORTOFF:
+        gen_offset_ld(ctx, cpu_gpr_a[r1], cpu_gpr_a[r2], off10, MO_LEUL);
+        break;
+    case OPC2_32_BO_LD_A_POSTINC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_a[r1], cpu_gpr_a[r2], ctx->mem_idx,
+                           MO_LEUL);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_LD_A_PREINC:
+        gen_ld_preincr(ctx, cpu_gpr_a[r1], cpu_gpr_a[r2], off10, MO_LEUL);
+        break;
+    case OPC2_32_BO_LD_B_SHORTOFF:
+        gen_offset_ld(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_SB);
+        break;
+    case OPC2_32_BO_LD_B_POSTINC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx,
+                           MO_SB);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_LD_B_PREINC:
+        gen_ld_preincr(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_SB);
+        break;
+    case OPC2_32_BO_LD_BU_SHORTOFF:
+        gen_offset_ld(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_UB);
+        break;
+    case OPC2_32_BO_LD_BU_POSTINC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx,
+                           MO_UB);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_LD_BU_PREINC:
+        gen_ld_preincr(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_SB);
+        break;
+    case OPC2_32_BO_LD_D_SHORTOFF:
+        gen_offset_ld_2regs(cpu_gpr_d[r1+1], cpu_gpr_d[r1], cpu_gpr_a[r2],
+                            off10, ctx);
+        break;
+    case OPC2_32_BO_LD_D_POSTINC:
+        gen_ld_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], cpu_gpr_a[r2], ctx);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_LD_D_PREINC:
+        temp = tcg_temp_new();
+        tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
+        gen_ld_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], temp, ctx);
+        tcg_gen_mov_tl(cpu_gpr_a[r2], temp);
+        tcg_temp_free(temp);
+        break;
+    case OPC2_32_BO_LD_DA_SHORTOFF:
+        gen_offset_ld_2regs(cpu_gpr_a[r1+1], cpu_gpr_a[r1], cpu_gpr_a[r2],
+                            off10, ctx);
+        break;
+    case OPC2_32_BO_LD_DA_POSTINC:
+        gen_ld_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], cpu_gpr_a[r2], ctx);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_LD_DA_PREINC:
+        temp = tcg_temp_new();
+        tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
+        gen_ld_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], temp, ctx);
+        tcg_gen_mov_tl(cpu_gpr_a[r2], temp);
+        tcg_temp_free(temp);
+        break;
+    case OPC2_32_BO_LD_H_SHORTOFF:
+        gen_offset_ld(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_LESW);
+        break;
+    case OPC2_32_BO_LD_H_POSTINC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx,
+                           MO_LESW);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_LD_H_PREINC:
+        gen_ld_preincr(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_LESW);
+        break;
+    case OPC2_32_BO_LD_HU_SHORTOFF:
+        gen_offset_ld(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_LEUW);
+        break;
+    case OPC2_32_BO_LD_HU_POSTINC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx,
+                           MO_LEUW);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_LD_HU_PREINC:
+        gen_ld_preincr(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_LEUW);
+        break;
+    case OPC2_32_BO_LD_Q_SHORTOFF:
+        gen_offset_ld(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_LEUW);
+        tcg_gen_shli_tl(cpu_gpr_d[r1], cpu_gpr_d[r1], 16);
+        break;
+    case OPC2_32_BO_LD_Q_POSTINC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx,
+                           MO_LEUW);
+        tcg_gen_shli_tl(cpu_gpr_d[r1], cpu_gpr_d[r1], 16);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_LD_Q_PREINC:
+        gen_ld_preincr(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_LEUW);
+        tcg_gen_shli_tl(cpu_gpr_d[r1], cpu_gpr_d[r1], 16);
+        break;
+    case OPC2_32_BO_LD_W_SHORTOFF:
+        gen_offset_ld(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_LEUL);
+        break;
+    case OPC2_32_BO_LD_W_POSTINC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx,
+                           MO_LEUL);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_LD_W_PREINC:
+        gen_ld_preincr(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_LEUL);
+        break;
+    }
+}
+
+static void decode_bo_addrmode_ld_bitreverse_circular(CPUTriCoreState *env,
+                                                DisasContext *ctx)
+{
+    uint32_t op2;
+    uint32_t off10;
+    int r1, r2;
+
+    TCGv temp, temp2, temp3;
+
+    r1 = MASK_OP_BO_S1D(ctx->opcode);
+    r2 = MASK_OP_BO_S2(ctx->opcode);
+    off10 = MASK_OP_BO_OFF10_SEXT(ctx->opcode);
+    op2 = MASK_OP_BO_OP2(ctx->opcode);
+
+    temp = tcg_temp_new();
+    temp2 = tcg_temp_new();
+    temp3 = tcg_const_i32(off10);
+
+    tcg_gen_ext16u_tl(temp, cpu_gpr_a[r2+1]);
+    tcg_gen_add_tl(temp2, cpu_gpr_a[r2], temp);
+
+
+    switch (op2) {
+    case OPC2_32_BO_LD_A_BR:
+        tcg_gen_qemu_ld_tl(cpu_gpr_a[r1], temp2, ctx->mem_idx, MO_LEUL);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_LD_A_CIRC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_a[r1], temp2, ctx->mem_idx, MO_LEUL);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    case OPC2_32_BO_LD_B_BR:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_SB);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_LD_B_CIRC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_SB);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    case OPC2_32_BO_LD_BU_BR:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_UB);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_LD_BU_CIRC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_UB);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    case OPC2_32_BO_LD_D_BR:
+        gen_ld_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], temp2, ctx);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_LD_D_CIRC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUL);
+        tcg_gen_shri_tl(temp2, cpu_gpr_a[r2+1], 16);
+        tcg_gen_addi_tl(temp, temp, 4);
+        tcg_gen_rem_tl(temp, temp, temp2);
+        tcg_gen_add_tl(temp2, cpu_gpr_a[r2], temp);
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1+1], temp2, ctx->mem_idx, MO_LEUL);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    case OPC2_32_BO_LD_DA_BR:
+        gen_ld_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], temp2, ctx);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_LD_DA_CIRC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_a[r1], temp2, ctx->mem_idx, MO_LEUL);
+        tcg_gen_shri_tl(temp2, cpu_gpr_a[r2+1], 16);
+        tcg_gen_addi_tl(temp, temp, 4);
+        tcg_gen_rem_tl(temp, temp, temp2);
+        tcg_gen_add_tl(temp2, cpu_gpr_a[r2], temp);
+        tcg_gen_qemu_ld_tl(cpu_gpr_a[r1+1], temp2, ctx->mem_idx, MO_LEUL);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    case OPC2_32_BO_LD_H_BR:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LESW);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_LD_H_CIRC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LESW);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    case OPC2_32_BO_LD_HU_BR:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUW);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_LD_HU_CIRC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUW);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    case OPC2_32_BO_LD_Q_BR:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUW);
+        tcg_gen_shli_tl(cpu_gpr_d[r1], cpu_gpr_d[r1], 16);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_LD_Q_CIRC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUW);
+        tcg_gen_shli_tl(cpu_gpr_d[r1], cpu_gpr_d[r1], 16);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    case OPC2_32_BO_LD_W_BR:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUL);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_LD_W_CIRC:
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUL);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    }
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free(temp3);
+}
+
+static void decode_bo_addrmode_stctx_post_pre_base(CPUTriCoreState *env,
+                                                   DisasContext *ctx)
+{
+    uint32_t op2;
+    uint32_t off10;
+    int r1, r2;
+
+    TCGv temp, temp2;
+
+    r1 = MASK_OP_BO_S1D(ctx->opcode);
+    r2 = MASK_OP_BO_S2(ctx->opcode);
+    off10 = MASK_OP_BO_OFF10_SEXT(ctx->opcode);
+    op2 = MASK_OP_BO_OP2(ctx->opcode);
+
+
+    temp = tcg_temp_new();
+    temp2 = tcg_temp_new();
+
+    switch (op2) {
+    case OPC2_32_BO_LDLCX_SHORTOFF:
+        tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
+        gen_helper_ldlcx(cpu_env, temp);
+        break;
+    case OPC2_32_BO_LDMST_SHORTOFF:
+        tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
+        gen_ldmst(ctx, r1, temp);
+        break;
+    case OPC2_32_BO_LDMST_POSTINC:
+        gen_ldmst(ctx, r1, cpu_gpr_a[r2]);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_LDMST_PREINC:
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        gen_ldmst(ctx, r1, cpu_gpr_a[r2]);
+        break;
+    case OPC2_32_BO_LDUCX_SHORTOFF:
+        tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
+        gen_helper_lducx(cpu_env, temp);
+        break;
+    case OPC2_32_BO_LEA_SHORTOFF:
+        tcg_gen_addi_tl(cpu_gpr_a[r1], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_STLCX_SHORTOFF:
+        tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
+        gen_helper_stlcx(cpu_env, temp);
+        break;
+    case OPC2_32_BO_STUCX_SHORTOFF:
+        tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
+        gen_helper_stucx(cpu_env, temp);
+        break;
+    case OPC2_32_BO_SWAP_W_SHORTOFF:
+        tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
+        gen_swap(ctx, r1, temp);
+        break;
+    case OPC2_32_BO_SWAP_W_POSTINC:
+        gen_swap(ctx, r1, cpu_gpr_a[r2]);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_SWAP_W_PREINC:
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        gen_swap(ctx, r1, cpu_gpr_a[r2]);
+        break;
+    }
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+static void decode_bo_addrmode_ldmst_bitreverse_circular(CPUTriCoreState *env,
+                                                         DisasContext *ctx)
+{
+    uint32_t op2;
+    uint32_t off10;
+    int r1, r2;
+
+    TCGv temp, temp2, temp3;
+
+    r1 = MASK_OP_BO_S1D(ctx->opcode);
+    r2 = MASK_OP_BO_S2(ctx->opcode);
+    off10 = MASK_OP_BO_OFF10_SEXT(ctx->opcode);
+    op2 = MASK_OP_BO_OP2(ctx->opcode);
+
+    temp = tcg_temp_new();
+    temp2 = tcg_temp_new();
+    temp3 = tcg_const_i32(off10);
+
+    tcg_gen_ext16u_tl(temp, cpu_gpr_a[r2+1]);
+    tcg_gen_add_tl(temp2, cpu_gpr_a[r2], temp);
+
+    switch (op2) {
+    case OPC2_32_BO_LDMST_BR:
+        gen_ldmst(ctx, r1, temp2);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_LDMST_CIRC:
+        gen_ldmst(ctx, r1, temp2);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    case OPC2_32_BO_SWAP_W_BR:
+        gen_swap(ctx, r1, temp2);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_SWAP_W_CIRC:
+        gen_swap(ctx, r1, temp2);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    }
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free(temp3);
+}
+
+static void decode_32Bit_opc(CPUTriCoreState *env, DisasContext *ctx)
+{
+    int op1;
+    int32_t r1;
+    int32_t address;
+    int8_t b;
+    int32_t bpos;
+    TCGv temp, temp2;
+
+    op1 = MASK_OP_MAJOR(ctx->opcode);
+
+    switch (op1) {
+/* ABS-format */
+    case OPCM_32_ABS_LDW:
+        decode_abs_ldw(env, ctx);
+        break;
+    case OPCM_32_ABS_LDB:
+        decode_abs_ldb(env, ctx);
+        break;
+    case OPCM_32_ABS_LDMST_SWAP:
+        decode_abs_ldst_swap(env, ctx);
+        break;
+    case OPCM_32_ABS_LDST_CONTEXT:
+        decode_abs_ldst_context(env, ctx);
+        break;
+    case OPCM_32_ABS_STORE:
+        decode_abs_store(env, ctx);
+        break;
+    case OPCM_32_ABS_STOREB_H:
+        decode_abs_storeb_h(env, ctx);
+        break;
+    case OPC1_32_ABS_STOREQ:
+        address = MASK_OP_ABS_OFF18(ctx->opcode);
+        r1 = MASK_OP_ABS_S1D(ctx->opcode);
+        temp = tcg_const_i32(EA_ABS_FORMAT(address));
+        temp2 = tcg_temp_new();
+
+        tcg_gen_shri_tl(temp2, cpu_gpr_d[r1], 16);
+        tcg_gen_qemu_st_tl(temp2, temp, ctx->mem_idx, MO_LEUW);
+
+        tcg_temp_free(temp2);
+        tcg_temp_free(temp);
+        break;
+    case OPC1_32_ABS_LD_Q:
+        address = MASK_OP_ABS_OFF18(ctx->opcode);
+        r1 = MASK_OP_ABS_S1D(ctx->opcode);
+        temp = tcg_const_i32(EA_ABS_FORMAT(address));
+
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp, ctx->mem_idx, MO_LEUW);
+        tcg_gen_shli_tl(cpu_gpr_d[r1], cpu_gpr_d[r1], 16);
+
+        tcg_temp_free(temp);
+        break;
+    case OPC1_32_ABS_LEA:
+        address = MASK_OP_ABS_OFF18(ctx->opcode);
+        r1 = MASK_OP_ABS_S1D(ctx->opcode);
+        tcg_gen_movi_tl(cpu_gpr_a[r1], EA_ABS_FORMAT(address));
+        break;
+/* ABSB-format */
+    case OPC1_32_ABSB_ST_T:
+        address = MASK_OP_ABS_OFF18(ctx->opcode);
+        b = MASK_OP_ABSB_B(ctx->opcode);
+        bpos = MASK_OP_ABSB_BPOS(ctx->opcode);
+
+        temp = tcg_const_i32(EA_ABS_FORMAT(address));
+        temp2 = tcg_temp_new();
+
+        tcg_gen_qemu_ld_tl(temp2, temp, ctx->mem_idx, MO_UB);
+        tcg_gen_andi_tl(temp2, temp2, ~(0x1u << bpos));
+        tcg_gen_ori_tl(temp2, temp2, (b << bpos));
+        tcg_gen_qemu_st_tl(temp2, temp, ctx->mem_idx, MO_UB);
+
+        tcg_temp_free(temp);
+        tcg_temp_free(temp2);
+        break;
+/* B-format */
+    case OPC1_32_B_CALL:
+    case OPC1_32_B_CALLA:
+    case OPC1_32_B_J:
+    case OPC1_32_B_JA:
+    case OPC1_32_B_JL:
+    case OPC1_32_B_JLA:
+        address = MASK_OP_B_DISP24(ctx->opcode);
+        gen_compute_branch(ctx, op1, 0, 0, 0, address);
+        break;
+/* Bit-format */
+    case OPCM_32_BIT_ANDACC:
+        decode_bit_andacc(env, ctx);
+        break;
+    case OPCM_32_BIT_LOGICAL_T1:
+        decode_bit_logical_t(env, ctx);
+        break;
+    case OPCM_32_BIT_INSERT:
+        decode_bit_insert(env, ctx);
+        break;
+    case OPCM_32_BIT_LOGICAL_T2:
+        decode_bit_logical_t2(env, ctx);
+        break;
+    case OPCM_32_BIT_ORAND:
+        decode_bit_orand(env, ctx);
+        break;
+    case OPCM_32_BIT_SH_LOGIC1:
+        decode_bit_sh_logic1(env, ctx);
+        break;
+    case OPCM_32_BIT_SH_LOGIC2:
+        decode_bit_sh_logic2(env, ctx);
+        break;
+    /* BO Format */
+    case OPCM_32_BO_ADDRMODE_POST_PRE_BASE:
+        decode_bo_addrmode_post_pre_base(env, ctx);
+        break;
+    case OPCM_32_BO_ADDRMODE_BITREVERSE_CIRCULAR:
+        decode_bo_addrmode_bitreverse_circular(env, ctx);
+        break;
+    case OPCM_32_BO_ADDRMODE_LD_POST_PRE_BASE:
+        decode_bo_addrmode_ld_post_pre_base(env, ctx);
+        break;
+    case OPCM_32_BO_ADDRMODE_LD_BITREVERSE_CIRCULAR:
+        decode_bo_addrmode_ld_bitreverse_circular(env, ctx);
+        break;
+    case OPCM_32_BO_ADDRMODE_STCTX_POST_PRE_BASE:
+        decode_bo_addrmode_stctx_post_pre_base(env, ctx);
+        break;
+    case OPCM_32_BO_ADDRMODE_LDMST_BITREVERSE_CIRCULAR:
+        decode_bo_addrmode_ldmst_bitreverse_circular(env, ctx);
+        break;
+    }
+}
+
+static void decode_opc(CPUTriCoreState *env, DisasContext *ctx, int *is_branch)
+{
+    /* 16-Bit Instruction */
+    if ((ctx->opcode & 0x1) == 0) {
+        ctx->next_pc = ctx->pc + 2;
+        decode_16Bit_opc(env, ctx);
+    /* 32-Bit Instruction */
+    } else {
+        ctx->next_pc = ctx->pc + 4;
+        decode_32Bit_opc(env, ctx);
+    }
+}
+
+static inline void
+gen_intermediate_code_internal(TriCoreCPU *cpu, struct TranslationBlock *tb,
+                              int search_pc)
+{
+    CPUState *cs = CPU(cpu);
+    CPUTriCoreState *env = &cpu->env;
+    DisasContext ctx;
+    target_ulong pc_start;
+    int num_insns;
+    uint16_t *gen_opc_end;
+
+    if (search_pc) {
+        qemu_log("search pc %d\n", search_pc);
+    }
+
+    num_insns = 0;
+    pc_start = tb->pc;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
+    ctx.pc = pc_start;
+    ctx.saved_pc = -1;
+    ctx.tb = tb;
+    ctx.singlestep_enabled = cs->singlestep_enabled;
+    ctx.bstate = BS_NONE;
+    ctx.mem_idx = cpu_mmu_index(env);
+
+    tcg_clear_temp_count();
+    gen_tb_start();
+    while (ctx.bstate == BS_NONE) {
+        ctx.opcode = cpu_ldl_code(env, ctx.pc);
+        decode_opc(env, &ctx, 0);
+
+        num_insns++;
+
+        if (tcg_ctx.gen_opc_ptr >= gen_opc_end) {
+            gen_save_pc(ctx.next_pc);
+            tcg_gen_exit_tb(0);
+            break;
+        }
+        if (singlestep) {
+            gen_save_pc(ctx.next_pc);
+            tcg_gen_exit_tb(0);
+            break;
+        }
+        ctx.pc = ctx.next_pc;
+    }
+
+    gen_tb_end(tb, num_insns);
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
+    if (search_pc) {
+        printf("done_generating search pc\n");
+    } else {
+        tb->size = ctx.pc - pc_start;
+        tb->icount = num_insns;
+    }
+    if (tcg_check_temp_count()) {
+        printf("LEAK at %08x\n", env->PC);
+    }
+
+#ifdef DEBUG_DISAS
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+        qemu_log("IN: %s\n", lookup_symbol(pc_start));
+        log_target_disas(env, pc_start, ctx.pc - pc_start, 0);
+        qemu_log("\n");
+    }
+#endif
+}
+
+void
+gen_intermediate_code(CPUTriCoreState *env, struct TranslationBlock *tb)
+{
+    gen_intermediate_code_internal(tricore_env_get_cpu(env), tb, false);
+}
+
+void
+gen_intermediate_code_pc(CPUTriCoreState *env, struct TranslationBlock *tb)
+{
+    gen_intermediate_code_internal(tricore_env_get_cpu(env), tb, true);
+}
+
+void
+restore_state_to_opc(CPUTriCoreState *env, TranslationBlock *tb, int pc_pos)
+{
+    env->PC = tcg_ctx.gen_opc_pc[pc_pos];
+}
+/*
+ *
+ * Initialization
+ *
+ */
+
+void cpu_state_reset(CPUTriCoreState *env)
+{
+    /* Reset Regs to Default Value */
+    env->PSW = 0xb80;
+}
+
+static void tricore_tcg_init_csfr(void)
+{
+    cpu_PCXI = tcg_global_mem_new(TCG_AREG0,
+                          offsetof(CPUTriCoreState, PCXI), "PCXI");
+    cpu_PSW = tcg_global_mem_new(TCG_AREG0,
+                          offsetof(CPUTriCoreState, PSW), "PSW");
+    cpu_PC = tcg_global_mem_new(TCG_AREG0,
+                          offsetof(CPUTriCoreState, PC), "PC");
+    cpu_ICR = tcg_global_mem_new(TCG_AREG0,
+                          offsetof(CPUTriCoreState, ICR), "ICR");
+}
+
+void tricore_tcg_init(void)
+{
+    int i;
+    static int inited;
+    if (inited) {
+        return;
+    }
+    cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+    /* reg init */
+    for (i = 0 ; i < 16 ; i++) {
+        cpu_gpr_a[i] = tcg_global_mem_new(TCG_AREG0,
+                                          offsetof(CPUTriCoreState, gpr_a[i]),
+                                          regnames_a[i]);
+    }
+    for (i = 0 ; i < 16 ; i++) {
+        cpu_gpr_d[i] = tcg_global_mem_new(TCG_AREG0,
+                                  offsetof(CPUTriCoreState, gpr_d[i]),
+                                           regnames_d[i]);
+    }
+    tricore_tcg_init_csfr();
+    /* init PSW flag cache */
+    cpu_PSW_C = tcg_global_mem_new(TCG_AREG0,
+                                   offsetof(CPUTriCoreState, PSW_USB_C),
+                                   "PSW_C");
+    cpu_PSW_V = tcg_global_mem_new(TCG_AREG0,
+                                   offsetof(CPUTriCoreState, PSW_USB_V),
+                                   "PSW_V");
+    cpu_PSW_SV = tcg_global_mem_new(TCG_AREG0,
+                                    offsetof(CPUTriCoreState, PSW_USB_SV),
+                                    "PSW_SV");
+    cpu_PSW_AV = tcg_global_mem_new(TCG_AREG0,
+                                    offsetof(CPUTriCoreState, PSW_USB_AV),
+                                    "PSW_AV");
+    cpu_PSW_SAV = tcg_global_mem_new(TCG_AREG0,
+                                     offsetof(CPUTriCoreState, PSW_USB_SAV),
+                                     "PSW_SAV");
+}
diff --git a/target-tricore/tricore-defs.h b/target-tricore/tricore-defs.h
new file mode 100644
index 000000000..4350b0304
--- /dev/null
+++ b/target-tricore/tricore-defs.h
@@ -0,0 +1,28 @@
+/*
+ *  Copyright (c) 2012-2014 Bastian Koppelmann C-Lab/University Paderborn
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#if !defined(__QEMU_TRICORE_DEFS_H__)
+#define __QEMU_TRICORE_DEFS_H__
+
+#define TARGET_PAGE_BITS 14
+#define TARGET_LONG_BITS 32
+#define TARGET_PHYS_ADDR_SPACE_BITS 32
+#define TARGET_VIRT_ADDR_SPACE_BITS 32
+
+#define TRICORE_TLB_MAX 128
+
+#endif /* __QEMU_TRICORE_DEFS_H__ */
diff --git a/target-tricore/tricore-opcodes.h b/target-tricore/tricore-opcodes.h
new file mode 100644
index 000000000..7e6f33bd6
--- /dev/null
+++ b/target-tricore/tricore-opcodes.h
@@ -0,0 +1,1408 @@
+/*
+ *  Copyright (c) 2012-2014 Bastian Koppelmann C-Lab/University Paderborn
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * Opcode Masks for Tricore
+ * Format MASK_OP_InstrFormatName_Field
+ */
+
+/* This creates a mask with bits start .. end set to 1 and applies it to op */
+#define MASK_BITS_SHIFT(op, start, end) (extract32(op, (start), \
+                                        (end) - (start) + 1))
+#define MASK_BITS_SHIFT_SEXT(op, start, end) (sextract32(op, (start),\
+                                             (end) - (start) + 1))
+
+/* new opcode masks */
+
+#define MASK_OP_MAJOR(op)      MASK_BITS_SHIFT(op, 0, 7)
+
+/* 16-Bit Formats */
+#define MASK_OP_SB_DISP8(op)   MASK_BITS_SHIFT(op, 8, 15)
+#define MASK_OP_SB_DISP8_SEXT(op) MASK_BITS_SHIFT_SEXT(op, 8, 15)
+
+#define MASK_OP_SBC_CONST4(op) MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_SBC_CONST4_SEXT(op) MASK_BITS_SHIFT_SEXT(op, 12, 15)
+#define MASK_OP_SBC_DISP4(op)  MASK_BITS_SHIFT(op, 8, 11)
+
+#define MASK_OP_SBR_S2(op)     MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_SBR_DISP4(op)  MASK_BITS_SHIFT(op, 8, 11)
+
+#define MASK_OP_SBRN_N(op)     MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_SBRN_DISP4(op) MASK_BITS_SHIFT(op, 8, 11)
+
+#define MASK_OP_SC_CONST8(op)  MASK_BITS_SHIFT(op, 8, 15)
+
+#define MASK_OP_SLR_S2(op)     MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_SLR_D(op)      MASK_BITS_SHIFT(op, 8, 11)
+
+#define MASK_OP_SLRO_OFF4(op)  MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_SLRO_D(op)     MASK_BITS_SHIFT(op, 8, 11)
+
+#define MASK_OP_SR_OP2(op)     MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_SR_S1D(op)     MASK_BITS_SHIFT(op, 8, 11)
+
+#define MASK_OP_SRC_CONST4(op) MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_SRC_CONST4_SEXT(op) MASK_BITS_SHIFT_SEXT(op, 12, 15)
+#define MASK_OP_SRC_S1D(op)    MASK_BITS_SHIFT(op, 8, 11)
+
+#define MASK_OP_SRO_S2(op)     MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_SRO_OFF4(op)   MASK_BITS_SHIFT(op, 8, 11)
+
+#define MASK_OP_SRR_S2(op)     MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_SRR_S1D(op)    MASK_BITS_SHIFT(op, 8, 11)
+
+#define MASK_OP_SRRS_S2(op)    MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_SRRS_S1D(op)   MASK_BITS_SHIFT(op, 8, 11)
+#define MASK_OP_SRRS_N(op)     MASK_BITS_SHIFT(op, 6, 7)
+
+#define MASK_OP_SSR_S2(op)     MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_SSR_S1(op)     MASK_BITS_SHIFT(op, 8, 11)
+
+#define MASK_OP_SSRO_OFF4(op)  MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_SSRO_S1(op)    MASK_BITS_SHIFT(op, 8, 11)
+
+/* 32-Bit Formats */
+
+/* ABS Format */
+#define MASK_OP_ABS_OFF18(op)  (MASK_BITS_SHIFT(op, 16, 21) +       \
+                               (MASK_BITS_SHIFT(op, 28, 31) << 6) + \
+                               (MASK_BITS_SHIFT(op, 22, 25) << 10) +\
+                               (MASK_BITS_SHIFT(op, 12, 15) << 14))
+#define MASK_OP_ABS_OP2(op)    MASK_BITS_SHIFT(op, 26, 27)
+#define MASK_OP_ABS_S1D(op)    MASK_BITS_SHIFT(op, 8, 11)
+
+/* ABSB Format */
+#define MASK_OP_ABSB_OFF18(op) MASK_OP_ABS_OFF18(op)
+#define MASK_OP_ABSB_OP2(op)   MASK_BITS_SHIFT(op, 26, 27)
+#define MASK_OP_ABSB_B(op)     MASK_BITS_SHIFT(op, 11, 11)
+#define MASK_OP_ABSB_BPOS(op)  MASK_BITS_SHIFT(op, 8, 10)
+
+/* B Format   */
+#define MASK_OP_B_DISP24(op)   (MASK_BITS_SHIFT(op, 16, 31) + \
+                               (MASK_BITS_SHIFT(op, 8, 15) << 16))
+/* BIT Format */
+#define MASK_OP_BIT_D(op)      MASK_BITS_SHIFT(op, 28, 31)
+#define MASK_OP_BIT_POS2(op)   MASK_BITS_SHIFT(op, 23, 27)
+#define MASK_OP_BIT_OP2(op)    MASK_BITS_SHIFT(op, 21, 22)
+#define MASK_OP_BIT_POS1(op)   MASK_BITS_SHIFT(op, 16, 20)
+#define MASK_OP_BIT_S2(op)     MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_BIT_S1(op)     MASK_BITS_SHIFT(op, 8, 11)
+
+/* BO Format */
+#define MASK_OP_BO_OFF10(op)   (MASK_BITS_SHIFT(op, 16, 21) + \
+                               (MASK_BITS_SHIFT(op, 28, 31) << 6))
+#define MASK_OP_BO_OFF10_SEXT(op)   (MASK_BITS_SHIFT_SEXT(op, 16, 21) + \
+                                    (MASK_BITS_SHIFT_SEXT(op, 28, 31) << 6))
+#define MASK_OP_BO_OP2(op)     MASK_BITS_SHIFT(op, 22, 27)
+#define MASK_OP_BO_S2(op)      MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_BO_S1D(op)     MASK_BITS_SHIFT(op, 8, 11)
+
+/* BOL Format */
+#define MASK_OP_BOL_OFF16(op)  ((MASK_BITS_SHIFT(op, 16, 21) +        \
+                               (MASK_BITS_SHIFT(op, 28, 31) << 6)) + \
+                               (MASK_BITS_SHIFT(op, 22, 27) >> 10))
+
+#define MASK_OP_BOL_S2(op)     MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_BOL_S1D(op)    MASK_BITS_SHIFT(op, 8, 11)
+
+/* BRC Format */
+#define MASK_OP_BRC_OP2(op)    MASK_BITS_SHIFT(op, 31, 31)
+#define MASK_OP_BRC_DISP15(op) MASK_BITS_SHIFT(op, 16, 30)
+#define MASK_OP_BRC_CONST4(op) MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_BRC_S1(op)     MASK_BITS_SHIFT(op, 8, 11)
+
+/* BRN Format */
+#define MASK_OP_BRN_OP2(op)    MASK_BITS_SHIFT(op, 31, 31)
+#define MASK_OP_BRN_DISP15(op) MASK_BITS_SHIFT(op, 16, 30)
+#define MASK_OP_BRN_N(op)      (MASK_BITS_SHIFT(op, 12, 15) + \
+                               (MASK_BITS_SHIFT(op, 7, 7) << 4))
+#define MASK_OP_BRN_S1(op)     MASK_BITS_SHIFT(op, 8, 11)
+/* BRR Format */
+#define MASK_OP_BRR_OP2(op)    MASK_BITS_SHIFT(op, 31, 31)
+#define MASK_OP_BRR_DISP15(op) MASK_BITS_SHIFT(op, 16, 30)
+#define MASK_OP_BRR_S2(op)     MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_BRR_S1(op)     MASK_BITS_SHIFT(op, 8, 11)
+
+/* META MASK for similar instr Formats */
+#define MASK_OP_META_D(op)     MASK_BITS_SHIFT(op, 28, 31)
+#define MASK_OP_META_S1(op)    MASK_BITS_SHIFT(op, 8, 11)
+
+/* RC Format */
+#define MASK_OP_RC_D(op)       MASK_OP_META_D(op)
+#define MASK_OP_RC_OP2(op)     MASK_BITS_SHIFT(op, 21, 27)
+#define MASK_OP_RC_CONST9(op)  MASK_BITS_SHIFT(op, 12, 20)
+#define MASK_OP_RC_S1(op)      MASK_OP_META_S1(op)
+
+/* RCPW Format */
+
+#define MASK_OP_RCPW_D(op)      MASK_OP_META_D(op)
+#define MASK_OP_RCPW_POS(op)    MASK_BITS_SHIFT(op, 23, 27)
+#define MASK_OP_RCPW_OP2(op)    MASK_BITS_SHIFT(op, 21, 22)
+#define MASK_OP_RCPW_WIDTH(op)  MASK_BITS_SHIFT(op, 16, 20)
+#define MASK_OP_RCPW_CONST4(op) MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_RCPW_S1(op)     MASK_OP_META_S1(op)
+
+/* RCR Format */
+
+#define MASK_OP_RCR_D(op)      MASK_OP_META_D(op)
+#define MASK_OP_RCR_S3(op)     MASK_BITS_SHIFT(op, 24, 27)
+#define MASK_OP_RCR_OP2(op)    MASK_BITS_SHIFT(op, 21, 23)
+#define MASK_OP_RCR_CONST9(op) MASK_BITS_SHIFT(op, 12, 20)
+#define MASK_OP_RCR_S1(op)     MASK_OP_META_S1(op)
+
+/* RCRR Format */
+
+#define MASK_OP_RCRR_D(op)      MASK_OP_META_D(op)
+#define MASK_OP_RCRR_S3(op)     MASK_BITS_SHIFT(op, 24, 27)
+#define MASK_OP_RCRR_OP2(op)    MASK_BITS_SHIFT(op, 21, 23)
+#define MASK_OP_RCRR_CONST4(op) MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_RCRR_S1(op)     MASK_OP_META_S1(op)
+
+/* RCRW Format */
+
+#define MASK_OP_RCRW_D(op)      MASK_OP_META_D(op)
+#define MASK_OP_RCRW_S3(op)     MASK_BITS_SHIFT(op, 24, 27)
+#define MASK_OP_RCRW_OP2(op)    MASK_BITS_SHIFT(op, 21, 23)
+#define MASK_OP_RCRW_WIDTH(op)  MASK_BITS_SHIFT(op, 16, 20)
+#define MASK_OP_RCRW_CONST4(op) MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_RCRW_S1(op)     MASK_OP_META_S1(op)
+
+/* RLC Format */
+
+#define MASK_OP_RLC_D(op)       MASK_OP_META_D(op)
+#define MASK_OP_RLC_CONST16(op) MASK_BITS_SHIFT(op, 12, 27)
+#define MASK_OP_RLC_S1(op)      MASK_OP_META_S1(op)
+
+/* RR  Format */
+#define MASK_OP_RR_D(op)        MASK_OP_META_D(op)
+#define MASK_OP_RR_OP2(op)      MASK_BITS_SHIFT(op, 20, 27)
+#define MASK_OP_RR_N(op)        MASK_BITS_SHIFT(op, 16, 17)
+#define MASK_OP_RR_S2(op)       MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_RR_S1(op)       MASK_OP_META_S1(op)
+
+/* RR1  Format */
+#define MASK_OP_RR1_D(op)       MASK_OP_META_D(op)
+#define MASK_OP_RR1_OP2(op)     MASK_BITS_SHIFT(op, 18, 27)
+#define MASK_OP_RR1_N(op)       MASK_BITS_SHIFT(op, 16, 17)
+#define MASK_OP_RR1_S2(op)      MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_RR1_S1(op)      MASK_OP_META_S1(op)
+
+/* RR2  Format */
+#define MASK_OP_RR2_D(op)       MASK_OP_META_D(op)
+#define MASK_OP_RR2_OP2(op)     MASK_BITS_SHIFT(op, 16, 27)
+#define MASK_OP_RR2_S2(op)      MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_RR2_S1(op)      MASK_OP_META_S1(op)
+
+/* RRPW  Format */
+#define MASK_OP_RRPW_D(op)      MASK_OP_META_D(op)
+#define MASK_OP_RRPW_POS(op)    MASK_BITS_SHIFT(op, 23, 27)
+#define MASK_OP_RRPW_OP2(op)    MASK_BITS_SHIFT(op, 21, 22)
+#define MASK_OP_RRPW_WIDTH(op)  MASK_BITS_SHIFT(op, 16, 20)
+#define MASK_OP_RRPW_S2(op)     MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_RRPW_S1(op)     MASK_OP_META_S1(op)
+
+/* RRR  Format */
+#define MASK_OP_RRR_D(op)       MASK_OP_META_D(op)
+#define MASK_OP_RRR_S3(op)      MASK_BITS_SHIFT(op, 24, 27)
+#define MASK_OP_RRR_OP2(op)     MASK_BITS_SHIFT(op, 20, 23)
+#define MASK_OP_RRR_N(op)       MASK_BITS_SHIFT(op, 16, 17)
+#define MASK_OP_RRR_S2(op)      MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_RRR_S1(op)      MASK_OP_META_S1(op)
+
+/* RRR1  Format */
+#define MASK_OP_RRR1_D(op)      MASK_OP_META_D(op)
+#define MASK_OP_RRR1_S3(op)     MASK_BITS_SHIFT(op, 24, 27)
+#define MASK_OP_RRR1_OP2(op)    MASK_BITS_SHIFT(op, 18, 23)
+#define MASK_OP_RRR1_N(op)      MASK_BITS_SHIFT(op, 16, 17)
+#define MASK_OP_RRR1_S2(op)     MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_RRR1_S1(op)     MASK_OP_META_S1(op)
+
+/* RRR2  Format */
+#define MASK_OP_RRR2_D(op)      MASK_OP_META_D(op)
+#define MASK_OP_RRR2_S3(op)     MASK_BITS_SHIFT(op, 24, 27)
+#define MASK_OP_RRR2_OP2(op)    MASK_BITS_SHIFT(op, 16, 23)
+#define MASK_OP_RRR2_S2(op)     MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_RRR2_S1(op)     MASK_OP_META_S1(op)
+
+/* RRRR  Format */
+#define MASK_OP_RRRR_D(op)      MASK_OP_META_D(op)
+#define MASK_OP_RRRR_S3(op)     MASK_BITS_SHIFT(op, 24, 27)
+#define MASK_OP_RRRR_OP2(op)    MASK_BITS_SHIFT(op, 21, 23)
+#define MASK_OP_RRRR_S2(op)     MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_RRRR_S1(op)     MASK_OP_META_S1(op)
+
+/* RRRW  Format */
+#define MASK_OP_RRRW_D(op)      MASK_OP_META_D(op)
+#define MASK_OP_RRRW_S3(op)     MASK_BITS_SHIFT(op, 24, 27)
+#define MASK_OP_RRRW_OP2(op)    MASK_BITS_SHIFT(op, 21, 23)
+#define MASK_OP_RRRW_WIDTH(op)  MASK_BITS_SHIFT(op, 16, 20)
+#define MASK_OP_RRRW_S2(op)     MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_RRRW_S1(op)     MASK_OP_META_S1(op)
+
+/* SYS Format */
+#define MASK_OP_SYS_OP2(op)     MASK_BITS_SHIFT(op, 22, 27)
+#define MASK_OP_SYS_S1D(op)     MASK_OP_META_S1(op)
+
+
+
+/*
+ * Tricore Opcodes Enums
+ *
+ * Format: OPC(1|2|M)_InstrLen_Name
+ * OPC1 = only op1 field is used
+ * OPC2 = op1 and op2 field used part of OPCM
+ * OPCM = op1 field used to group Instr
+ * InstrLen = 16|32
+ * Name = Name of Instr
+ */
+
+/* 16-Bit */
+enum {
+
+    OPCM_16_SR_SYSTEM                                = 0x00,
+    OPCM_16_SR_ACCU                                  = 0x32,
+
+    OPC1_16_SRC_ADD                                  = 0xc2,
+    OPC1_16_SRC_ADD_A15                              = 0x92,
+    OPC1_16_SRC_ADD_15A                              = 0x9a,
+    OPC1_16_SRR_ADD                                  = 0x42,
+    OPC1_16_SRR_ADD_A15                              = 0x12,
+    OPC1_16_SRR_ADD_15A                              = 0x1a,
+    OPC1_16_SRC_ADD_A                                = 0xb0,
+    OPC1_16_SRR_ADD_A                                = 0x30,
+    OPC1_16_SRR_ADDS                                 = 0x22,
+    OPC1_16_SRRS_ADDSC_A                             = 0x10,
+    OPC1_16_SC_AND                                   = 0x16,
+    OPC1_16_SRR_AND                                  = 0x26,
+    OPC1_16_SC_BISR                                  = 0xe0,
+    OPC1_16_SRC_CADD                                 = 0x8a,
+    OPC1_16_SRC_CADDN                                = 0xca,
+    OPC1_16_SB_CALL                                  = 0x5c,
+    OPC1_16_SRC_CMOV                                 = 0xaa,
+    OPC1_16_SRR_CMOV                                 = 0x2a,
+    OPC1_16_SRC_CMOVN                                = 0xea,
+    OPC1_16_SRR_CMOVN                                = 0x6a,
+    OPC1_16_SRC_EQ                                   = 0xba,
+    OPC1_16_SRR_EQ                                   = 0x3a,
+    OPC1_16_SB_J                                     = 0x3c,
+    OPC1_16_SBC_JEQ                                  = 0x1e,
+    OPC1_16_SBR_JEQ                                  = 0x3e,
+    OPC1_16_SBR_JGEZ                                 = 0xce,
+    OPC1_16_SBR_JGTZ                                 = 0x4e,
+    OPC1_16_SR_JI                                    = 0xdc,
+    OPC1_16_SBR_JLEZ                                 = 0x8e,
+    OPC1_16_SBR_JLTZ                                 = 0x0e,
+    OPC1_16_SBC_JNE                                  = 0x5e,
+    OPC1_16_SBR_JNE                                  = 0x7e,
+    OPC1_16_SB_JNZ                                   = 0xee,
+    OPC1_16_SBR_JNZ                                  = 0xf6,
+    OPC1_16_SBR_JNZ_A                                = 0x7c,
+    OPC1_16_SBRN_JNZ_T                               = 0xae,
+    OPC1_16_SB_JZ                                    = 0x6e,
+    OPC1_16_SBR_JZ                                   = 0x76,
+    OPC1_16_SBR_JZ_A                                 = 0xbc,
+    OPC1_16_SBRN_JZ_T                                = 0x2e,
+    OPC1_16_SC_LD_A                                  = 0xd8,
+    OPC1_16_SLR_LD_A                                 = 0xd4,
+    OPC1_16_SLR_LD_A_POSTINC                         = 0xc4,
+    OPC1_16_SLRO_LD_A                                = 0xc8,
+    OPC1_16_SRO_LD_A                                 = 0xcc,
+    OPC1_16_SLR_LD_BU                                = 0x14,
+    OPC1_16_SLR_LD_BU_POSTINC                        = 0x04,
+    OPC1_16_SLRO_LD_BU                               = 0x08,
+    OPC1_16_SRO_LD_BU                                = 0x0c,
+    OPC1_16_SLR_LD_H                                 = 0x94,
+    OPC1_16_SLR_LD_H_POSTINC                         = 0x84,
+    OPC1_16_SLRO_LD_H                                = 0x88,
+    OPC1_16_SRO_LD_H                                 = 0x8c,
+    OPC1_16_SC_LD_W                                  = 0x58,
+    OPC1_16_SLR_LD_W                                 = 0x54,
+    OPC1_16_SLR_LD_W_POSTINC                         = 0x44,
+    OPC1_16_SLRO_LD_W                                = 0x48,
+    OPC1_16_SRO_LD_W                                 = 0x4c,
+    OPC1_16_SBR_LOOP                                 = 0xfc,
+    OPC1_16_SRC_LT                                   = 0xfa,
+    OPC1_16_SRR_LT                                   = 0x7a,
+    OPC1_16_SC_MOV                                   = 0xda,
+    OPC1_16_SRC_MOV                                  = 0x82,
+    OPC1_16_SRR_MOV                                  = 0x02,
+    OPC1_16_SRC_MOV_E                                = 0xd2,/* 1.6 only */
+    OPC1_16_SRC_MOV_A                                = 0xa0,
+    OPC1_16_SRR_MOV_A                                = 0x60,
+    OPC1_16_SRR_MOV_AA                               = 0x40,
+    OPC1_16_SRR_MOV_D                                = 0x80,
+    OPC1_16_SRR_MUL                                  = 0xe2,
+    OPC1_16_SR_NOT                                   = 0x46,
+    OPC1_16_SC_OR                                    = 0x96,
+    OPC1_16_SRR_OR                                   = 0xa6,
+    OPC1_16_SRC_SH                                   = 0x06,
+    OPC1_16_SRC_SHA                                  = 0x86,
+    OPC1_16_SC_ST_A                                  = 0xf8,
+    OPC1_16_SRO_ST_A                                 = 0xec,
+    OPC1_16_SSR_ST_A                                 = 0xf4,
+    OPC1_16_SSR_ST_A_POSTINC                         = 0xe4,
+    OPC1_16_SSRO_ST_A                                = 0xe8,
+    OPC1_16_SRO_ST_B                                 = 0x2c,
+    OPC1_16_SSR_ST_B                                 = 0x34,
+    OPC1_16_SSR_ST_B_POSTINC                         = 0x24,
+    OPC1_16_SSRO_ST_B                                = 0x28,
+    OPC1_16_SRO_ST_H                                 = 0xac,
+    OPC1_16_SSR_ST_H                                 = 0xb4,
+    OPC1_16_SSR_ST_H_POSTINC                         = 0xa4,
+    OPC1_16_SSRO_ST_H                                = 0xa8,
+    OPC1_16_SC_ST_W                                  = 0x78,
+    OPC1_16_SRO_ST_W                                 = 0x6c,
+    OPC1_16_SSR_ST_W                                 = 0x74,
+    OPC1_16_SSR_ST_W_POSTINC                         = 0x64,
+    OPC1_16_SSRO_ST_W                                = 0x68,
+    OPC1_16_SRR_SUB                                  = 0xa2,
+    OPC1_16_SRR_SUB_A15B                             = 0x52,
+    OPC1_16_SRR_SUB_15AB                             = 0x5a,
+    OPC1_16_SC_SUB_A                                 = 0x20,
+    OPC1_16_SRR_SUBS                                 = 0x62,
+    OPC1_16_SRR_XOR                                  = 0xc6,
+
+};
+
+/*
+ * SR Format
+ */
+/* OPCM_16_SR_SYSTEM                                 */
+enum {
+
+    OPC2_16_SR_NOP                                   = 0x00,
+    OPC2_16_SR_RET                                   = 0x09,
+    OPC2_16_SR_RFE                                   = 0x08,
+    OPC2_16_SR_DEBUG                                 = 0x0a,
+};
+/* OPCM_16_SR_ACCU                                   */
+enum {
+    OPC2_16_SR_RSUB                                  = 0x05,
+    OPC2_16_SR_SAT_B                                 = 0x00,
+    OPC2_16_SR_SAT_BU                                = 0x01,
+    OPC2_16_SR_SAT_H                                 = 0x02,
+    OPC2_16_SR_SAT_HU                                = 0x03,
+
+};
+
+/* 32-Bit */
+
+enum {
+/* ABS Format 1, M */
+    OPCM_32_ABS_LDW                                  = 0x85,
+    OPCM_32_ABS_LDB                                  = 0x05,
+    OPCM_32_ABS_LDMST_SWAP                           = 0xe5,
+    OPCM_32_ABS_LDST_CONTEXT                         = 0x15,
+    OPCM_32_ABS_STORE                                = 0xa5,
+    OPCM_32_ABS_STOREB_H                             = 0x25,
+    OPC1_32_ABS_STOREQ                               = 0x65,
+    OPC1_32_ABS_LD_Q                                 = 0x45,
+    OPC1_32_ABS_LEA                                  = 0xc5,
+/* ABSB Format */
+    OPC1_32_ABSB_ST_T                                = 0xd5,
+/* B Format */
+    OPC1_32_B_CALL                                   = 0x6d,
+    OPC1_32_B_CALLA                                  = 0xed,
+    OPC1_32_B_J                                      = 0x1d,
+    OPC1_32_B_JA                                     = 0x9d,
+    OPC1_32_B_JL                                     = 0x5d,
+    OPC1_32_B_JLA                                    = 0xdd,
+/* Bit Format */
+    OPCM_32_BIT_ANDACC                               = 0x47,
+    OPCM_32_BIT_LOGICAL_T1                           = 0x87,
+    OPCM_32_BIT_INSERT                               = 0x67,
+    OPCM_32_BIT_LOGICAL_T2                           = 0x07,
+    OPCM_32_BIT_ORAND                                = 0xc7,
+    OPCM_32_BIT_SH_LOGIC1                            = 0x27,
+    OPCM_32_BIT_SH_LOGIC2                            = 0xa7,
+/* BO Format */
+    OPCM_32_BO_ADDRMODE_POST_PRE_BASE                = 0x89,
+    OPCM_32_BO_ADDRMODE_BITREVERSE_CIRCULAR          = 0xa9,
+    OPCM_32_BO_ADDRMODE_LD_POST_PRE_BASE             = 0x09,
+    OPCM_32_BO_ADDRMODE_LD_BITREVERSE_CIRCULAR       = 0x29,
+    OPCM_32_BO_ADDRMODE_STCTX_POST_PRE_BASE          = 0x49,
+    OPCM_32_BO_ADDRMODE_LDMST_BITREVERSE_CIRCULAR    = 0x69,
+/* BOL Format */
+    OPC1_32_BOL_LD_A_LONGOFF                         = 0x99,
+    OPC1_32_BOL_LD_W_LONFOFF                         = 0x19,
+    OPC1_32_BOL_LEA_LONGOFF                          = 0xd9,
+    OPC1_32_BOL_ST_W_LONGOFF                         = 0x59,
+    OPC1_32_BOL_ST_A_LONGOFF                         = 0xb5, /* 1.6 only */
+/* BRC Format */
+    OPCM_32_BRC_EQ_NEQ                               = 0xdf,
+    OPCM_32_BRC_GE                                   = 0xff,
+    OPCM_32_BRC_JLT                                  = 0xbf,
+    OPCM_32_BRC_JNE                                  = 0x9f,
+/* BRN Format */
+    OPCM_32_BRN_JTT                                  = 0x6f,
+/* BRR Format */
+    OPCM_32_BRR_EQ_NEQ                               = 0x5f,
+    OPCM_32_BRR_ADDR_EQ_NEQ                          = 0x7d,
+    OPCM_32_BRR_GE                                   = 0x7f,
+    OPCM_32_BRR_JLT                                  = 0x3f,
+    OPCM_32_BRR_JNE                                  = 0x1f,
+    OPCM_32_BRR_JNZ                                  = 0xbd,
+    OPCM_32_BRR_LOOP                                 = 0xfd,
+/* RC Format */
+    OPCM_32_RC_LOGICAL_SHIFT                         = 0x8f,
+    OPCM_32_RC_ACCUMULATOR                           = 0x8b,
+    OPCM_32_RC_SERVICEROUTINE                        = 0xad,
+    OPCM_32_RC_MUL                                   = 0x53,
+/* RCPW Format */
+    OPCM_32_RCPW_MASK_INSERT                         = 0xb7,
+/* RCR Format */
+    OPCM_32_RCR_COND_SELECT                          = 0xab,
+    OPCM_32_RCR_MADD                                 = 0x13,
+    OPCM_32_RCR_MSUB                                 = 0x33,
+/* RCRR Format */
+    OPC1_32_RCRR_INSERT                              = 0x97,
+/* RCRW Format */
+    OPCM_32_RCRW_MASK_INSERT                         = 0xd7,
+/* RLC Format */
+    OPC1_32_RLC_ADDI                                 = 0x1b,
+    OPC1_32_RLC_ADDIH                                = 0x9b,
+    OPC1_32_RLC_ADDIH_A                              = 0x11,
+    OPC1_32_RLC_MFCR                                 = 0x4d,
+    OPC1_32_RLC_MOV                                  = 0x3b,
+    OPC1_32_RLC_MOV_U                                = 0xbb,
+    OPC1_32_RLC_MOV_H                                = 0x7b,
+    OPC1_32_RLC_MOVH_A                               = 0x91,
+    OPC1_32_RLC_MTCR                                 = 0xcd,
+/* RR Format */
+    OPCM_32_RR_LOGICAL_SHIFT                         = 0x0f,
+    OPCM_32_RR_ACCUMULATOR                           = 0x0b,
+    OPCM_32_RR_ADRESS                                = 0x01,
+    OPCM_32_RR_FLOAT                                 = 0x4b,
+    OPCM_32_RR_IDIRECT                               = 0x2d,
+/* RR1 Format */
+    OPCM_32_RR1_MUL                                  = 0xb3,
+    OPCM_32_RR1_MULQ                                 = 0x93,
+/* RR2 Format */
+    OPCM_32_RR2_MUL                                  = 0x73,
+/* RRPW Format */
+    OPCM_32_RRPW_EXTRACT_INSERT                      = 0x37,
+    OPC1_32_RRPW_DEXTR                               = 0x77,
+/* RRR Format */
+    OPCM_32_RRR_COND_SELECT                          = 0x2b,
+    OPCM_32_RRR_FLOAT                                = 0x6b,
+/* RRR1 Format */
+    OPCM_32_RRR1_MADD                                = 0x83,
+    OPCM_32_RRR1_MADDQ_H                             = 0x43,
+    OPCM_32_RRR1_MADDSU_H                            = 0xc3,
+    OPCM_32_RRR1_MSUB_H                              = 0xa3,
+    OPCM_32_RRR1_MSUB_Q                              = 0x63,
+    OPCM_32_RRR1_MSUBADS_H                           = 0xe3,
+/* RRR2 Format */
+    OPCM_32_RRR2_MADD                                = 0x03,
+    OPCM_32_RRR2_MSUB                                = 0x23,
+/* RRRR Format */
+    OPCM_32_RRRR_EXTRACT_INSERT                      = 0x17,
+/* RRRW Format */
+    OPCM_32_RRRW_EXTRACT_INSERT                      = 0x57,
+/* SYS Format */
+    OPCM_32_SYS_INTERRUPTS                           = 0x0d,
+    OPC1_32_SYS_RSTV                                 = 0x2f,
+};
+
+
+
+/*
+ * ABS Format
+ */
+
+/* OPCM_32_ABS_LDW  */
+enum {
+
+    OPC2_32_ABS_LD_A                             = 0x02,
+    OPC2_32_ABS_LD_D                             = 0x01,
+    OPC2_32_ABS_LD_DA                            = 0x03,
+    OPC2_32_ABS_LD_W                             = 0x00,
+};
+
+/* OPCM_32_ABS_LDB */
+enum {
+    OPC2_32_ABS_LD_B                             = 0x00,
+    OPC2_32_ABS_LD_BU                            = 0x01,
+    OPC2_32_ABS_LD_H                             = 0x02,
+    OPC2_32_ABS_LD_HU                            = 0x03,
+};
+/* OPCM_32_ABS_LDMST_SWAP       */
+enum {
+    OPC2_32_ABS_LDMST                            = 0x01,
+    OPC2_32_ABS_SWAP_W                           = 0x00,
+};
+/* OPCM_32_ABS_LDST_CONTEXT     */
+enum {
+    OPC2_32_ABS_LDLCX                            = 0x02,
+    OPC2_32_ABS_LDUCX                            = 0x03,
+    OPC2_32_ABS_STLCX                            = 0x00,
+    OPC2_32_ABS_STUCX                            = 0x01,
+};
+/* OPCM_32_ABS_STORE            */
+enum {
+    OPC2_32_ABS_ST_A                             = 0x02,
+    OPC2_32_ABS_ST_D                             = 0x01,
+    OPC2_32_ABS_ST_DA                            = 0x03,
+    OPC2_32_ABS_ST_W                             = 0x00,
+};
+/* OPCM_32_ABS_STOREB_H */
+enum {
+    OPC2_32_ABS_ST_B                             = 0x00,
+    OPC2_32_ABS_ST_H                             = 0x02,
+};
+/*
+ * Bit Format
+ */
+/* OPCM_32_BIT_ANDACC              */
+enum {
+    OPC2_32_BIT_AND_AND_T                        = 0x00,
+    OPC2_32_BIT_AND_ANDN_T                       = 0x03,
+    OPC2_32_BIT_AND_NOR_T                        = 0x02,
+    OPC2_32_BIT_AND_OR_T                         = 0x01,
+};
+/* OPCM_32_BIT_LOGICAL_T                       */
+enum {
+    OPC2_32_BIT_AND_T                            = 0x00,
+    OPC2_32_BIT_ANDN_T                           = 0x03,
+    OPC2_32_BIT_NOR_T                            = 0x02,
+    OPC2_32_BIT_OR_T                             = 0x01,
+};
+/* OPCM_32_BIT_INSERT                   */
+enum {
+    OPC2_32_BIT_INS_T                            = 0x00,
+    OPC2_32_BIT_INSN_T                           = 0x01,
+};
+/* OPCM_32_BIT_LOGICAL_T2              */
+enum {
+    OPC2_32_BIT_NAND_T                           = 0x00,
+    OPC2_32_BIT_ORN_T                            = 0x01,
+    OPC2_32_BIT_XNOR_T                           = 0x02,
+    OPC2_32_BIT_XOR_T                            = 0x03,
+};
+/* OPCM_32_BIT_ORAND                    */
+enum {
+    OPC2_32_BIT_OR_AND_T                         = 0x00,
+    OPC2_32_BIT_OR_ANDN_T                        = 0x03,
+    OPC2_32_BIT_OR_NOR_T                         = 0x02,
+    OPC2_32_BIT_OR_OR_T                          = 0x01,
+};
+/*OPCM_32_BIT_SH_LOGIC1                 */
+enum {
+    OPC2_32_BIT_SH_AND_T                         = 0x00,
+    OPC2_32_BIT_SH_ANDN_T                        = 0x03,
+    OPC2_32_BIT_SH_NOR_T                         = 0x02,
+    OPC2_32_BIT_SH_OR_T                          = 0x01,
+};
+/* OPCM_32_BIT_SH_LOGIC2              */
+enum {
+    OPC2_32_BIT_SH_NAND_T                        = 0x00,
+    OPC2_32_BIT_SH_ORN_T                         = 0x01,
+    OPC2_32_BIT_SH_XNOR_T                        = 0x02,
+    OPC2_32_BIT_SH_XOR_T                         = 0x03,
+};
+/*
+ * BO Format
+ */
+/* OPCM_32_BO_ADDRMODE_POST_PRE_BASE     */
+enum {
+    OPC2_32_BO_CACHEA_I_SHORTOFF                 = 0x2e,
+    OPC2_32_BO_CACHEA_I_POSTINC                  = 0x0e,
+    OPC2_32_BO_CACHEA_I_PREINC                   = 0x1e,
+    OPC2_32_BO_CACHEA_W_SHORTOFF                 = 0x2c,
+    OPC2_32_BO_CACHEA_W_POSTINC                  = 0x0c,
+    OPC2_32_BO_CACHEA_W_PREINC                   = 0x1c,
+    OPC2_32_BO_CACHEA_WI_SHORTOFF                = 0x2d,
+    OPC2_32_BO_CACHEA_WI_POSTINC                 = 0x0d,
+    OPC2_32_BO_CACHEA_WI_PREINC                  = 0x1d,
+    /* 1.3.1 only */
+    OPC2_32_BO_CACHEI_W_SHORTOFF                 = 0x2b,
+    OPC2_32_BO_CACHEI_W_POSTINC                  = 0x0b,
+    OPC2_32_BO_CACHEI_W_PREINC                   = 0x1b,
+    OPC2_32_BO_CACHEI_WI_SHORTOFF                = 0x2f,
+    OPC2_32_BO_CACHEI_WI_POSTINC                 = 0x0f,
+    OPC2_32_BO_CACHEI_WI_PREINC                  = 0x1f,
+    /* end 1.3.1 only */
+    OPC2_32_BO_ST_A_SHORTOFF                     = 0x26,
+    OPC2_32_BO_ST_A_POSTINC                      = 0x06,
+    OPC2_32_BO_ST_A_PREINC                       = 0x16,
+    OPC2_32_BO_ST_B_SHORTOFF                     = 0x20,
+    OPC2_32_BO_ST_B_POSTINC                      = 0x00,
+    OPC2_32_BO_ST_B_PREINC                       = 0x10,
+    OPC2_32_BO_ST_D_SHORTOFF                     = 0x25,
+    OPC2_32_BO_ST_D_POSTINC                      = 0x05,
+    OPC2_32_BO_ST_D_PREINC                       = 0x15,
+    OPC2_32_BO_ST_DA_SHORTOFF                    = 0x27,
+    OPC2_32_BO_ST_DA_POSTINC                     = 0x07,
+    OPC2_32_BO_ST_DA_PREINC                      = 0x17,
+    OPC2_32_BO_ST_H_SHORTOFF                     = 0x22,
+    OPC2_32_BO_ST_H_POSTINC                      = 0x02,
+    OPC2_32_BO_ST_H_PREINC                       = 0x12,
+    OPC2_32_BO_ST_Q_SHORTOFF                     = 0x28,
+    OPC2_32_BO_ST_Q_POSTINC                      = 0x08,
+    OPC2_32_BO_ST_Q_PREINC                       = 0x18,
+    OPC2_32_BO_ST_W_SHORTOFF                     = 0x24,
+    OPC2_32_BO_ST_W_POSTINC                      = 0x04,
+    OPC2_32_BO_ST_W_PREINC                       = 0x14,
+};
+/* OPCM_32_BO_ADDRMODE_BITREVERSE_CIRCULAR   */
+enum {
+    OPC2_32_BO_CACHEA_I_BR                       = 0x0e,
+    OPC2_32_BO_CACHEA_I_CIRC                     = 0x1e,
+    OPC2_32_BO_CACHEA_W_BR                       = 0x0c,
+    OPC2_32_BO_CACHEA_W_CIRC                     = 0x1c,
+    OPC2_32_BO_CACHEA_WI_BR                      = 0x0d,
+    OPC2_32_BO_CACHEA_WI_CIRC                    = 0x1d,
+    OPC2_32_BO_ST_A_BR                           = 0x06,
+    OPC2_32_BO_ST_A_CIRC                         = 0x16,
+    OPC2_32_BO_ST_B_BR                           = 0x00,
+    OPC2_32_BO_ST_B_CIRC                         = 0x10,
+    OPC2_32_BO_ST_D_BR                           = 0x05,
+    OPC2_32_BO_ST_D_CIRC                         = 0x15,
+    OPC2_32_BO_ST_DA_BR                          = 0x07,
+    OPC2_32_BO_ST_DA_CIRC                        = 0x17,
+    OPC2_32_BO_ST_H_BR                           = 0x02,
+    OPC2_32_BO_ST_H_CIRC                         = 0x12,
+    OPC2_32_BO_ST_Q_BR                           = 0x08,
+    OPC2_32_BO_ST_Q_CIRC                         = 0x18,
+    OPC2_32_BO_ST_W_BR                           = 0x04,
+    OPC2_32_BO_ST_W_CIRC                         = 0x14,
+};
+/*    OPCM_32_BO_ADDRMODE_LD_POST_PRE_BASE   */
+enum {
+    OPC2_32_BO_LD_A_SHORTOFF                     = 0x26,
+    OPC2_32_BO_LD_A_POSTINC                      = 0x06,
+    OPC2_32_BO_LD_A_PREINC                       = 0x16,
+    OPC2_32_BO_LD_B_SHORTOFF                     = 0x20,
+    OPC2_32_BO_LD_B_POSTINC                      = 0x00,
+    OPC2_32_BO_LD_B_PREINC                       = 0x10,
+    OPC2_32_BO_LD_BU_SHORTOFF                    = 0x21,
+    OPC2_32_BO_LD_BU_POSTINC                     = 0x01,
+    OPC2_32_BO_LD_BU_PREINC                      = 0x11,
+    OPC2_32_BO_LD_D_SHORTOFF                     = 0x25,
+    OPC2_32_BO_LD_D_POSTINC                      = 0x05,
+    OPC2_32_BO_LD_D_PREINC                       = 0x15,
+    OPC2_32_BO_LD_DA_SHORTOFF                    = 0x27,
+    OPC2_32_BO_LD_DA_POSTINC                     = 0x07,
+    OPC2_32_BO_LD_DA_PREINC                      = 0x17,
+    OPC2_32_BO_LD_H_SHORTOFF                     = 0x22,
+    OPC2_32_BO_LD_H_POSTINC                      = 0x02,
+    OPC2_32_BO_LD_H_PREINC                       = 0x12,
+    OPC2_32_BO_LD_HU_SHORTOFF                    = 0x23,
+    OPC2_32_BO_LD_HU_POSTINC                     = 0x03,
+    OPC2_32_BO_LD_HU_PREINC                      = 0x13,
+    OPC2_32_BO_LD_Q_SHORTOFF                     = 0x28,
+    OPC2_32_BO_LD_Q_POSTINC                      = 0x08,
+    OPC2_32_BO_LD_Q_PREINC                       = 0x18,
+    OPC2_32_BO_LD_W_SHORTOFF                     = 0x24,
+    OPC2_32_BO_LD_W_POSTINC                      = 0x04,
+    OPC2_32_BO_LD_W_PREINC                       = 0x14,
+};
+/* OPCM_32_BO_ADDRMODE_LD_BITREVERSE_CIRCULAR  */
+enum {
+    OPC2_32_BO_LD_A_BR                           = 0x06,
+    OPC2_32_BO_LD_A_CIRC                         = 0x16,
+    OPC2_32_BO_LD_B_BR                           = 0x00,
+    OPC2_32_BO_LD_B_CIRC                         = 0x10,
+    OPC2_32_BO_LD_BU_BR                          = 0x01,
+    OPC2_32_BO_LD_BU_CIRC                        = 0x11,
+    OPC2_32_BO_LD_D_BR                           = 0x05,
+    OPC2_32_BO_LD_D_CIRC                         = 0x15,
+    OPC2_32_BO_LD_DA_BR                          = 0x07,
+    OPC2_32_BO_LD_DA_CIRC                        = 0x17,
+    OPC2_32_BO_LD_H_BR                           = 0x02,
+    OPC2_32_BO_LD_H_CIRC                         = 0x12,
+    OPC2_32_BO_LD_HU_BR                          = 0x03,
+    OPC2_32_BO_LD_HU_CIRC                        = 0x13,
+    OPC2_32_BO_LD_Q_BR                           = 0x08,
+    OPC2_32_BO_LD_Q_CIRC                         = 0x18,
+    OPC2_32_BO_LD_W_BR                           = 0x04,
+    OPC2_32_BO_LD_W_CIRC                         = 0x14,
+};
+/* OPCM_32_BO_ADDRMODE_STCTX_POST_PRE_BASE    */
+enum {
+    OPC2_32_BO_LDLCX_SHORTOFF                    = 0x24,
+    OPC2_32_BO_LDMST_SHORTOFF                    = 0x21,
+    OPC2_32_BO_LDMST_POSTINC                     = 0x01,
+    OPC2_32_BO_LDMST_PREINC                      = 0x11,
+    OPC2_32_BO_LDUCX_SHORTOFF                    = 0x25,
+    OPC2_32_BO_LEA_SHORTOFF                      = 0x28,
+    OPC2_32_BO_STLCX_SHORTOFF                    = 0x26,
+    OPC2_32_BO_STUCX_SHORTOFF                    = 0x27,
+    OPC2_32_BO_SWAP_W_SHORTOFF                   = 0x20,
+    OPC2_32_BO_SWAP_W_POSTINC                    = 0x00,
+    OPC2_32_BO_SWAP_W_PREINC                     = 0x10,
+};
+/*OPCM_32_BO_ADDRMODE_LDMST_BITREVERSE_CIRCULAR  */
+enum {
+    OPC2_32_BO_LDMST_BR                          = 0x01,
+    OPC2_32_BO_LDMST_CIRC                        = 0x11,
+    OPC2_32_BO_SWAP_W_BR                         = 0x00,
+    OPC2_32_BO_SWAP_W_CIRC                       = 0x10,
+};
+/*
+ * BRC Format
+ */
+/*OPCM_32_BRC_EQ_NEQ                             */
+enum {
+    OPC2_32_BRC_JEQ                              = 0x00,
+    OPC2_32_BRC_JNE                              = 0x01,
+};
+/* OPCM_32_BRC_GE                                   */
+enum {
+    OP2_BRC_JGE                                  = 0x00,
+    OPC_BRC_JGE_U                                = 0x01,
+};
+/* OPCM_32_BRC_JLT                                  */
+enum {
+    OPC2_32_BRC_JLT                              = 0x00,
+    OPC2_32_BRC_JLT_U                            = 0x01,
+};
+/* OPCM_32_BRC_JNE                                  */
+enum {
+    OPC2_32_BRC_JNED                             = 0x01,
+    OPC2_32_BRC_JNEI                             = 0x00,
+};
+/*
+ * BRN Format
+ */
+/* OPCM_32_BRN_JTT                                  */
+enum {
+    OPC2_32_BRN_JNZ_T                            = 0x01,
+    OPC2_32_BRN_JZ_T                             = 0x00,
+};
+/*
+ * BRR Format
+ */
+/* OPCM_32_BRR_EQ_NEQ                               */
+enum {
+    OPC2_32_BRR_JEQ                              = 0x00,
+    OPC2_32_BRR_JNE                              = 0x01,
+};
+/* OPCM_32_BRR_ADDR_EQ_NEQ                        */
+enum {
+    OPC2_32_BRR_JEQ_A                            = 0x00,
+    OPC2_32_BRR_JNE_A                            = 0x01,
+};
+/*OPCM_32_BRR_GE                                   */
+enum {
+    OPC2_32_BRR_JGE                              = 0x00,
+    OPC2_32_BRR_JGE_U                            = 0x01,
+};
+/* OPCM_32_BRR_JLT                                  */
+enum {
+    OPC2_32_BRR_JLT                              = 0x00,
+    OPC2_32_BRR_JLT_U                            = 0x01,
+};
+/* OPCM_32_BRR_JNE                                  */
+enum {
+    OPC2_32_BRR_JNED                             = 0x01,
+    OPC2_32_BRR_JNEI                             = 0x00,
+};
+/* OPCM_32_BRR_JNZ                                  */
+enum {
+    OPC2_32_BRR_JNZ_A                            = 0x01,
+    OPC2_32_BRR_JZ_A                             = 0x00,
+};
+/* OPCM_32_BRR_LOOP                                 */
+enum {
+    OPC2_32_BRR_LOOP                             = 0x00,
+    OPC2_32_BRR_LOOPU                            = 0x01,
+};
+/*
+ * RC Format
+ */
+/* OPCM_32_RC_LOGICAL_SHIFT                         */
+enum {
+    OPC2_32_RC_AND                               = 0x08,
+    OPC2_32_RC_ANDN                              = 0x0e,
+    OPC2_32_RC_NAND                              = 0x09,
+    OPC2_32_RC_NOR                               = 0x0b,
+    OPC2_32_RC_OR                                = 0x0a,
+    OPC2_32_RC_ORN                               = 0x0f,
+    OPC2_32_RC_SH                                = 0x00,
+    OPC2_32_RC_SH_H                              = 0x40,
+    OPC2_32_RC_SHA                               = 0x01,
+    OPC2_32_RC_SHA_H                             = 0x41,
+    OPC2_32_RC_SHAS                              = 0x02,
+    OPC2_32_RC_XNOR                              = 0x0d,
+    OPC2_32_RC_XOR                               = 0x0c,
+};
+/* OPCM_32_RC_ACCUMULATOR                           */
+enum {
+    OPC2_32_RC_ABSDIF                            = 0x0e,
+    OPC2_32_RC_ABSDIFS                           = 0x0f,
+    OPC2_32_RC_ADD                               = 0x00,
+    OPC2_32_RC_ADDC                              = 0x05,
+    OPC2_32_RC_ADDS                              = 0x02,
+    OPC2_32_RC_ADDS_U                            = 0x03,
+    OPC2_32_RC_ADDX                              = 0x04,
+    OPC2_32_RC_AND_EQ                            = 0x20,
+    OPC2_32_RC_AND_GE                            = 0x24,
+    OPC2_32_RC_AND_GE_U                          = 0x25,
+    OPC2_32_RC_AND_LT                            = 0x22,
+    OPC2_32_RC_AND_LT_U                          = 0x23,
+    OPC2_32_RC_AND_NE                            = 0x21,
+    OPC2_32_RC_EQ                                = 0x10,
+    OPC2_32_RC_EQANY_B                           = 0x56,
+    OPC2_32_RC_EQANY_H                           = 0x76,
+    OPC2_32_RC_GE                                = 0x14,
+    OPC2_32_RC_GE_U                              = 0x15,
+    OPC2_32_RC_LT                                = 0x12,
+    OPC2_32_RC_LT_U                              = 0x13,
+    OPC2_32_RC_MAX                               = 0x1a,
+    OPC2_32_RC_MAX_U                             = 0x1b,
+    OPC2_32_RC_MIN                               = 0x18,
+    OPC2_32_RC_MIN_U                             = 0x19,
+    OPC2_32_RC_NE                                = 0x11,
+    OPC2_32_RC_OR_EQ                             = 0x27,
+    OPC2_32_RC_OR_GE                             = 0x2b,
+    OPC2_32_RC_OR_GE_U                           = 0x2c,
+    OPC2_32_RC_OR_LT                             = 0x29,
+    OPC2_32_RC_OR_LT_U                           = 0x2a,
+    OPC2_32_RC_OR_NE                             = 0x28,
+    OPC2_32_RC_RSUB                              = 0x08,
+    OPC2_32_RC_RSUBS                             = 0x0a,
+    OPC2_32_RC_RSUBS_U                           = 0x0b,
+    OPC2_32_RC_SH_EQ                             = 0x37,
+    OPC2_32_RC_SH_GE                             = 0x3b,
+    OPC2_32_RC_SH_GE_U                           = 0x3c,
+    OPC2_32_RC_SH_LT                             = 0x39,
+    OPC2_32_RC_SH_LT_U                           = 0x3a,
+    OPC2_32_RC_SH_NE                             = 0x38,
+    OPC2_32_RC_XOR_EQ                            = 0x2f,
+    OPC2_32_RC_XOR_GE                            = 0x33,
+    OPC2_32_RC_XOR_GE_U                          = 0x34,
+    OPC2_32_RC_XOR_LT                            = 0x31,
+    OPC2_32_RC_XOR_LT_U                          = 0x32,
+    OPC2_32_RC_XOR_NE                            = 0x30,
+};
+/* OPCM_32_RC_SERVICEROUTINE                        */
+enum {
+    OPC2_32_RC_BISR                              = 0x00,
+    OPC2_32_RC_SYSCALL                           = 0x04,
+};
+/* OPCM_32_RC_MUL                                   */
+enum {
+    OPC2_32_RC_MUL_32                            = 0x01,
+    OPC2_32_RC_MUL_64                            = 0x03,
+    OPC2_32_RC_MULS_32                           = 0x05,
+    OPC2_32_RC_MUL_U_64                          = 0x02,
+    OPC2_32_RC_MULS_U_32                         = 0x04,
+};
+/*
+ * RCPW Format
+ */
+/* OPCM_32_RCPW_MASK_INSERT                         */
+enum {
+    OPC2_32_RCPW_IMASK                           = 0x01,
+    OPC2_32_RCPW_INSERT                          = 0x00,
+};
+/*
+ * RCR Format
+ */
+/* OPCM_32_RCR_COND_SELECT                          */
+enum {
+    OPC2_32_RCR_CADD                             = 0x00,
+    OPC2_32_RCR_CADDN                            = 0x01,
+    OPC2_32_RCR_SEL                              = 0x04,
+    OPC2_32_RCR_SELN                             = 0x05,
+};
+/* OPCM_32_RCR_MADD                                 */
+enum {
+    OPC2_32_RCR_MADD_32                          = 0x01,
+    OPC2_32_RCR_MADD_64                          = 0x03,
+    OPC2_32_RCR_MADDS_32                         = 0x05,
+    OPC2_32_RCR_MADDS_64                         = 0x07,
+    OPC2_32_RCR_MADD_U_64                        = 0x02,
+    OPC2_32_RCR_MADDS_U_32                       = 0x04,
+    OPC2_32_RCR_MADDS_U_64                       = 0x06,
+};
+/* OPCM_32_RCR_MSUB                                 */
+enum {
+    OPC2_32_RCR_MSUB_32                          = 0x01,
+    OPC2_32_RCR_MSUB_64                          = 0x03,
+    OPC2_32_RCR_MSUBS_32                         = 0x05,
+    OPC2_32_RCR_MSUBS_64                         = 0x07,
+    OPC2_32_RCR_MSUB_U_32                        = 0x02,
+    OPC2_32_RCR_MSUBS_U_32                       = 0x04,
+    OPC2_32_RCR_MSUBS_U_64                       = 0x06,
+};
+/*
+ * RCRW Format
+ */
+/* OPCM_32_RCRW_MASK_INSERT                         */
+enum {
+    OPC2_32_RCRW_IMASK                           = 0x01,
+    OPC2_32_RCRW_INSERT                          = 0x00,
+};
+
+/*
+ * RR Format
+ */
+/* OPCM_32_RR_LOGICAL_SHIFT                         */
+enum {
+    OPC2_32_RR_AND                               = 0x08,
+    OPC2_32_RR_ANDN                              = 0x0e,
+    OPC2_32_RR_CLO                               = 0x1c,
+    OPC2_32_RR_CLO_H                             = 0x7d,
+    OPC2_32_RR_CLS                               = 0x1d,
+    OPC2_32_RR_CLS_H                             = 0x7e,
+    OPC2_32_RR_CLZ                               = 0x1b,
+    OPC2_32_RR_CLZ_H                             = 0x7c,
+    OPC2_32_RR_NAND                              = 0x09,
+    OPC2_32_RR_NOR                               = 0x0b,
+    OPC2_32_RR_OR                                = 0x0a,
+    OPC2_32_RR_ORN                               = 0x0f,
+    OPC2_32_RR_SH                                = 0x00,
+    OPC2_32_RR_SH_H                              = 0x40,
+    OPC2_32_RR_SHA                               = 0x01,
+    OPC2_32_RR_SHA_H                             = 0x41,
+    OPC2_32_RR_SHAS                              = 0x02,
+    OPC2_32_RR_XNOR                              = 0x0d,
+    OPC2_32_RR_XOR                               = 0x0c,
+};
+/* OPCM_32_RR_ACCUMULATOR                           */
+enum {
+    OPC2_32_RR_ABS                               = 0x1c,
+    OPC2_32_RR_ABS_B                             = 0x5c,
+    OPC2_32_RR_ABS_H                             = 0x7c,
+    OPC2_32_RR_ABSDIF                            = 0x0e,
+    OPC2_32_RR_ABSDIF_B                          = 0x4e,
+    OPC2_32_RR_ABSDIF_H                          = 0x6e,
+    OPC2_32_RR_ABSDIFS                           = 0x0f,
+    OPC2_32_RR_ABSDIFS_H                         = 0x6f,
+    OPC2_32_RR_ABSS                              = 0x1d,
+    OPC2_32_RR_ABSS_H                            = 0x7d,
+    OPC2_32_RR_ADD                               = 0x00,
+    OPC2_32_RR_ADD_B                             = 0x40,
+    OPC2_32_RR_ADD_H                             = 0x60,
+    OPC2_32_RR_ADDC                              = 0x05,
+    OPC2_32_RR_ADDS                              = 0x02,
+    OPC2_32_RR_ADDS_H                            = 0x62,
+    OPC2_32_RR_ADDS_HU                           = 0x63,
+    OPC2_32_RR_ADDS_U                            = 0x03,
+    OPC2_32_RR_ADDX                              = 0x04,
+    OPC2_32_RR_AND_EQ                            = 0x20,
+    OPC2_32_RR_AND_GE                            = 0x24,
+    OPC2_32_RR_AND_GE_U                          = 0x25,
+    OPC2_32_RR_AND_LT                            = 0x22,
+    OPC2_32_RR_AND_LT_U                          = 0x23,
+    OPC2_32_RR_AND_NE                            = 0x21,
+    OPC2_32_RR_EQ                                = 0x10,
+    OPC2_32_RR_EQ_B                              = 0x50,
+    OPC2_32_RR_EQ_H                              = 0x70,
+    OPC2_32_RR_EQ_W                              = 0x90,
+    OPC2_32_RR_EQANY_B                           = 0x56,
+    OPC2_32_RR_EQANY_H                           = 0x76,
+    OPC2_32_RR_GE                                = 0x14,
+    OPC2_32_RR_GE_U                              = 0x15,
+    OPC2_32_RR_LT                                = 0x12,
+    OPC2_32_RR_LT_U                              = 0x13,
+    OPC2_32_RR_LT_B                              = 0x52,
+    OPC2_32_RR_LT_BU                             = 0x53,
+    OPC2_32_RR_LT_H                              = 0x72,
+    OPC2_32_RR_LT_HU                             = 0x73,
+    OPC2_32_RR_LT_W                              = 0x92,
+    OPC2_32_RR_LT_WU                             = 0x93,
+    OPC2_32_RR_MAX                               = 0x1a,
+    OPC2_32_RR_MAX_U                             = 0x1b,
+    OPC2_32_RR_MAX_B                             = 0x5a,
+    OPC2_32_RR_MAX_BU                            = 0x5b,
+    OPC2_32_RR_MAX_H                             = 0x7a,
+    OPC2_32_RR_MAX_HU                            = 0x7b,
+    OPC2_32_RR_MIN                               = 0x19,
+    OPC2_32_RR_MIN_U                             = 0x18,
+    OPC2_32_RR_MIN_B                             = 0x58,
+    OPC2_32_RR_MIN_BU                            = 0x59,
+    OPC2_32_RR_MIN_H                             = 0x78,
+    OPC2_32_RR_MIN_HU                            = 0x79,
+    OPC2_32_RR_MOV                               = 0x1f,
+    OPC2_32_RR_NE                                = 0x11,
+    OPC2_32_RR_OR_EQ                             = 0x27,
+    OPC2_32_RR_OR_GE                             = 0x2b,
+    OPC2_32_RR_OR_GE_U                           = 0x2c,
+    OPC2_32_RR_OR_LT                             = 0x29,
+    OPC2_32_RR_OR_LT_U                           = 0x2a,
+    OPC2_32_RR_OR_NE                             = 0x28,
+    OPC2_32_RR_SAT_B                             = 0x5e,
+    OPC2_32_RR_SAT_BU                            = 0x5f,
+    OPC2_32_RR_SAT_H                             = 0x7e,
+    OPC2_32_RR_SAT_HU                            = 0x7f,
+    OPC2_32_RR_SH_EQ                             = 0x37,
+    OPC2_32_RR_SH_GE                             = 0x3b,
+    OPC2_32_RR_SH_GE_U                           = 0x3c,
+    OPC2_32_RR_SH_LT                             = 0x39,
+    OPC2_32_RR_SH_LT_U                           = 0x3a,
+    OPC2_32_RR_SH_NE                             = 0x38,
+    OPC2_32_RR_SUB                               = 0x08,
+    OPC2_32_RR_SUB_B                             = 0x48,
+    OPC2_32_RR_SUB_H                             = 0x68,
+    OPC2_32_RR_SUBC                              = 0x0d,
+    OPC2_32_RR_SUBS                              = 0x0a,
+    OPC2_32_RR_SUBS_U                            = 0x0b,
+    OPC2_32_RR_SUBS_H                            = 0x6a,
+    OPC2_32_RR_SUBS_HU                           = 0x6b,
+    OPC2_32_RR_SUBX                              = 0x0c,
+    OPC2_32_RR_XOR_EQ                            = 0x2f,
+    OPC2_32_RR_XOR_GE                            = 0x33,
+    OPC2_32_RR_XOR_GE_U                          = 0x34,
+    OPC2_32_RR_XOR_LT                            = 0x31,
+    OPC2_32_RR_XOR_LT_U                          = 0x32,
+    OPC2_32_RR_XOR_NE                            = 0x30,
+};
+/* OPCM_32_RR_ADRESS                                */
+enum {
+    OPC2_32_RR_ADD_A                             = 0x01,
+    OPC2_32_RR_ADDSC_A                           = 0x60,
+    OPC2_32_RR_ADDSC_AT                          = 0x62,
+    OPC2_32_RR_EQ_A                              = 0x40,
+    OPC2_32_RR_EQZ                               = 0x48,
+    OPC2_32_RR_GE_A                              = 0x43,
+    OPC2_32_RR_LT_A                              = 0x42,
+    OPC2_32_RR_MOV_A                             = 0x63,
+    OPC2_32_RR_MOV_AA                            = 0x00,
+    OPC2_32_RR_MOV_D                             = 0x4c,
+    OPC2_32_RR_NE_A                              = 0x41,
+    OPC2_32_RR_NEZ_A                             = 0x49,
+    OPC2_32_RR_SUB_A                             = 0x02,
+};
+/* OPCM_32_RR_FLOAT                                 */
+enum {
+    OPC2_32_RR_BMERGE                            = 0x01,
+    OPC2_32_RR_BSPLIT                            = 0x09,
+    OPC2_32_RR_DVINIT_B                          = 0x5a,
+    OPC2_32_RR_DVINIT_BU                         = 0x4a,
+    OPC2_32_RR_DVINIT_H                          = 0x3a,
+    OPC2_32_RR_DVINIT_HU                         = 0x2a,
+    OPC2_32_RR_DVINIT                            = 0x1a,
+    OPC2_32_RR_DVINIT_U                          = 0x0a,
+    OPC2_32_RR_PARITY                            = 0x02,
+    OPC2_32_RR_UNPACK                            = 0x08,
+};
+/* OPCM_32_RR_IDIRECT                               */
+enum {
+    OPC2_32_RR_JI                                = 0x03,
+    OPC2_32_RR_JLI                               = 0x02,
+    OPC2_32_RR_CALLI                             = 0x00,
+};
+/*
+ * RR1 Format
+ */
+/* OPCM_32_RR1_MUL                                  */
+enum {
+    OPC2_32_RR1_MUL_H_32_LL                      = 0x1a,
+    OPC2_32_RR1_MUL_H_32_LU                      = 0x19,
+    OPC2_32_RR1_MUL_H_32_UL                      = 0x18,
+    OPC2_32_RR1_MUL_H_32_UU                      = 0x1b,
+    OPC2_32_RR1_MULM_H_64_LL                     = 0x1e,
+    OPC2_32_RR1_MULM_H_64_LU                     = 0x1d,
+    OPC2_32_RR1_MULM_H_64_UL                     = 0x1c,
+    OPC2_32_RR1_MULM_H_64_UU                     = 0x1f,
+    OPC2_32_RR1_MULR_H_16_LL                     = 0x0e,
+    OPC2_32_RR1_MULR_H_16_LU                     = 0x0d,
+    OPC2_32_RR1_MULR_H_16_UL                     = 0x0c,
+    OPC2_32_RR1_MULR_H_16_UU                     = 0x0f,
+};
+/* OPCM_32_RR1_MULQ                                 */
+enum {
+    OPC2_32_RR1_MUL_Q_32                         = 0x02,
+    OPC2_32_RR1_MUL_Q_64                         = 0x1b,
+    OPC2_32_RR1_MUL_Q_32_L                       = 0x01,
+    OPC2_32_RR1_MUL_Q_64_L                       = 0x19,
+    OPC2_32_RR1_MUL_Q_32_U                       = 0x00,
+    OPC2_32_RR1_MUL_Q_64_U                       = 0x18,
+    OPC2_32_RR1_MUL_Q_32_LL                      = 0x05,
+    OPC2_32_RR1_MUL_Q_32_UU                      = 0x04,
+    OPC2_32_RR1_MULR_Q_32_L                      = 0x07,
+    OPC2_32_RR1_MULR_Q_32_U                      = 0x06,
+};
+/*
+ * RR2 Format
+ */
+/* OPCM_32_RR2_MUL                                  */
+enum {
+    OPC2_32_RR2_MUL_32                           = 0x0a,
+    OPC2_32_RR2_MUL_64                           = 0x6a,
+    OPC2_32_RR2_MULS_32                          = 0x8a,
+    OPC2_32_RR2_MUL_U_64                         = 0x68,
+    OPC2_32_RR2_MULS_U_32                        = 0x88,
+};
+/*
+ * RRPW Format
+ */
+/* OPCM_32_RRPW_EXTRACT_INSERT                      */
+enum {
+
+    OPC2_32_RRPW_EXTR                            = 0x02,
+    OPC2_32_RRPW_EXTR_U                          = 0x03,
+    OPC2_32_RRPW_IMASK                           = 0x01,
+    OPC2_32_RRPW_INSERT                          = 0x00,
+};
+/*
+ * RRR Format
+ */
+/* OPCM_32_RRR_COND_SELECT                          */
+enum {
+    OPC2_32_RRR_CADD                             = 0x00,
+    OPC2_32_RRR_CADDN                            = 0x01,
+    OPC2_32_RRR_CSUB                             = 0x02,
+    OPC2_32_RRR_CSUBN                            = 0x03,
+    OPC2_32_RRR_SEL                              = 0x04,
+    OPC2_32_RRR_SELN                             = 0x05,
+};
+/* OPCM_32_RRR_FLOAT                                */
+enum {
+    OPC2_32_RRR_DVADJ                            = 0x0d,
+    OPC2_32_RRR_DVSTEP                           = 0x0f,
+    OPC2_32_RRR_DVSTEP_U                         = 0x0e,
+    OPC2_32_RRR_IXMAX                            = 0x0a,
+    OPC2_32_RRR_IXMAX_U                          = 0x0b,
+    OPC2_32_RRR_IXMIN                            = 0x08,
+    OPC2_32_RRR_IXMIN_U                          = 0x09,
+    OPC2_32_RRR_PACK                             = 0x00,
+};
+/*
+ * RRR1 Format
+ */
+/* OPCM_32_RRR1_MADD                                */
+enum {
+    OPC2_32_RRR1_MADD_H_LL                       = 0x1a,
+    OPC2_32_RRR1_MADD_H_LU                       = 0x19,
+    OPC2_32_RRR1_MADD_H_UL                       = 0x18,
+    OPC2_32_RRR1_MADD_H_UU                       = 0x1b,
+    OPC2_32_RRR1_MADDS_H_LL                      = 0x3a,
+    OPC2_32_RRR1_MADDS_H_LU                      = 0x39,
+    OPC2_32_RRR1_MADDS_H_UL                      = 0x38,
+    OPC2_32_RRR1_MADDS_H_UU                      = 0x3b,
+    OPC2_32_RRR1_MADDM_H_LL                      = 0x1e,
+    OPC2_32_RRR1_MADDM_H_LU                      = 0x1d,
+    OPC2_32_RRR1_MADDM_H_UL                      = 0x1c,
+    OPC2_32_RRR1_MADDM_H_UU                      = 0x1f,
+    OPC2_32_RRR1_MADDMS_H_LL                     = 0x3e,
+    OPC2_32_RRR1_MADDMS_H_LU                     = 0x3d,
+    OPC2_32_RRR1_MADDMS_H_UL                     = 0x3c,
+    OPC2_32_RRR1_MADDMS_H_UU                     = 0x3f,
+    OPC2_32_RRR1_MADDR_H_LL                      = 0x0e,
+    OPC2_32_RRR1_MADDR_H_LU                      = 0x0d,
+    OPC2_32_RRR1_MADDR_H_UL                      = 0x0c,
+    OPC2_32_RRR1_MADDR_H_UU                      = 0x0f,
+    OPC2_32_RRR1_MADDRS_H_LL                     = 0x2e,
+    OPC2_32_RRR1_MADDRS_H_LU                     = 0x2d,
+    OPC2_32_RRR1_MADDRS_H_UL                     = 0x2c,
+    OPC2_32_RRR1_MADDRS_H_UU                     = 0x2f,
+};
+/* OPCM_32_RRR1_MADDQ_H                             */
+enum {
+    OPC2_32_RRR1_MADD_Q_32                       = 0x02,
+    OPC2_32_RRR1_MADD_Q_64                       = 0x1b,
+    OPC2_32_RRR1_MADD_Q_32_L                     = 0x01,
+    OPC2_32_RRR1_MADD_Q_64_L                     = 0x19,
+    OPC2_32_RRR1_MADD_Q_32_U                     = 0x00,
+    OPC2_32_RRR1_MADD_Q_64_U                     = 0x18,
+    OPC2_32_RRR1_MADD_Q_32_LL                    = 0x05,
+    OPC2_32_RRR1_MADD_Q_64_LL                    = 0x1d,
+    OPC2_32_RRR1_MADD_Q_32_UU                    = 0x04,
+    OPC2_32_RRR1_MADD_Q_64_UU                    = 0x1c,
+    OPC2_32_RRR1_MADDS_Q_32                      = 0x22,
+    OPC2_32_RRR1_MADDS_Q_64                      = 0x3b,
+    OPC2_32_RRR1_MADDS_Q_32_L                    = 0x21,
+    OPC2_32_RRR1_MADDS_Q_64_L                    = 0x39,
+    OPC2_32_RRR1_MADDS_Q_32_U                    = 0x20,
+    OPC2_32_RRR1_MADDS_Q_64_U                    = 0x38,
+    OPC2_32_RRR1_MADDS_Q_32_LL                   = 0x25,
+    OPC2_32_RRR1_MADDS_Q_64_LL                   = 0x3d,
+    OPC2_32_RRR1_MADDS_Q_32_UU                   = 0x24,
+    OPC2_32_RRR1_MADDS_Q_64_UU                   = 0x3c,
+    OPC2_32_RRR1_MADDR_H_16_UL                   = 0x1e,
+    OPC2_32_RRR1_MADDRS_H_16_UL                  = 0x3e,
+    OPC2_32_RRR1_MADDR_Q_32_L                    = 0x07,
+    OPC2_32_RRR1_MADDR_Q_32_U                    = 0x06,
+    OPC2_32_RRR1_MADDRS_Q_32_LL                  = 0x27,
+    OPC2_32_RRR1_MADDRS_Q_32_UU                  = 0x26,
+};
+/* OPCM_32_RRR1_MADDSU_H                            */
+enum {
+    OPC2_32_RRR1_MADDSU_H_32_LL                  = 0x1a,
+    OPC2_32_RRR1_MADDSU_H_32_LU                  = 0x19,
+    OPC2_32_RRR1_MADDSU_H_32_UL                  = 0x18,
+    OPC2_32_RRR1_MADDSU_H_32_UU                  = 0x1b,
+    OPC2_32_RRR1_MADDSUS_H_32_LL                 = 0x3a,
+    OPC2_32_RRR1_MADDSUS_H_32_LU                 = 0x39,
+    OPC2_32_RRR1_MADDSUS_H_32_UL                 = 0x38,
+    OPC2_32_RRR1_MADDSUS_H_32_UU                 = 0x3b,
+    OPC2_32_RRR1_MADDSUM_H_64_LL                 = 0x1e,
+    OPC2_32_RRR1_MADDSUM_H_64_LU                 = 0x1d,
+    OPC2_32_RRR1_MADDSUM_H_64_UL                 = 0x1c,
+    OPC2_32_RRR1_MADDSUM_H_64_UU                 = 0x1f,
+    OPC2_32_RRR1_MADDSUMS_H_64_LL                = 0x3e,
+    OPC2_32_RRR1_MADDSUMS_H_64_LU                = 0x3d,
+    OPC2_32_RRR1_MADDSUMS_H_64_UL                = 0x3c,
+    OPC2_32_RRR1_MADDSUMS_H_64_UU                = 0x3f,
+    OPC2_32_RRR1_MADDSUR_H_16_LL                 = 0x0e,
+    OPC2_32_RRR1_MADDSUR_H_16_LU                 = 0x0d,
+    OPC2_32_RRR1_MADDSUR_H_16_UL                 = 0x0c,
+    OPC2_32_RRR1_MADDSUR_H_16_UU                 = 0x0f,
+    OPC2_32_RRR1_MADDSURS_H_16_LL                = 0x2e,
+    OPC2_32_RRR1_MADDSURS_H_16_LU                = 0x2d,
+    OPC2_32_RRR1_MADDSURS_H_16_UL                = 0x2c,
+    OPC2_32_RRR1_MADDSURS_H_16_UU                = 0x2f,
+};
+/* OPCM_32_RRR1_MSUB_H                              */
+enum {
+    OPC2_32_RRR1_MSUB_H_32_LL                    = 0x1a,
+    OPC2_32_RRR1_MSUB_H_32_LU                    = 0x19,
+    OPC2_32_RRR1_MSUB_H_32_UL                    = 0x18,
+    OPC2_32_RRR1_MSUB_H_32_UU                    = 0x1b,
+    OPC2_32_RRR1_MSUBS_H_32_LL                   = 0x3a,
+    OPC2_32_RRR1_MSUBS_H_32_LU                   = 0x39,
+    OPC2_32_RRR1_MSUBS_H_32_UL                   = 0x38,
+    OPC2_32_RRR1_MSUBS_H_32_UU                   = 0x3b,
+    OPC2_32_RRR1_MSUBM_H_64_LL                   = 0x1e,
+    OPC2_32_RRR1_MSUBM_H_64_LU                   = 0x1d,
+    OPC2_32_RRR1_MSUBM_H_64_UL                   = 0x1c,
+    OPC2_32_RRR1_MSUBM_H_64_UU                   = 0x1f,
+    OPC2_32_RRR1_MSUBMS_H_64_LL                  = 0x3e,
+    OPC2_32_RRR1_MSUBMS_H_64_LU                  = 0x3d,
+    OPC2_32_RRR1_MSUBMS_H_64_UL                  = 0x3c,
+    OPC2_32_RRR1_MSUBMS_H_64_UU                  = 0x3f,
+    OPC2_32_RRR1_MSUBR_H_16_LL                   = 0x0e,
+    OPC2_32_RRR1_MSUBR_H_16_LU                   = 0x0d,
+    OPC2_32_RRR1_MSUBR_H_16_UL                   = 0x0c,
+    OPC2_32_RRR1_MSUBR_H_16_UU                   = 0x0f,
+    OPC2_32_RRR1_MSUBRS_H_16_LL                  = 0x2e,
+    OPC2_32_RRR1_MSUBRS_H_16_LU                  = 0x2d,
+    OPC2_32_RRR1_MSUBRS_H_16_UL                  = 0x2c,
+    OPC2_32_RRR1_MSUBRS_H_16_UU                  = 0x2f,
+};
+/* OPCM_32_RRR1_MSUB_Q                              */
+enum {
+    OPC2_32_RRR1_MSUB_Q_32                       = 0x02,
+    OPC2_32_RRR1_MSUB_Q_64                       = 0x1b,
+    OPC2_32_RRR1_MSUB_Q_32_L                     = 0x01,
+    OPC2_32_RRR1_MSUB_Q_64_L                     = 0x19,
+    OPC2_32_RRR1_MSUB_Q_32_U                     = 0x00,
+    OPC2_32_RRR1_MSUB_Q_64_U                     = 0x18,
+    OPC2_32_RRR1_MSUB_Q_32_LL                    = 0x05,
+    OPC2_32_RRR1_MSUB_Q_64_LL                    = 0x1d,
+    OPC2_32_RRR1_MSUB_Q_32_UU                    = 0x04,
+    OPC2_32_RRR1_MSUB_Q_64_UU                    = 0x1c,
+    OPC2_32_RRR1_MSUBS_Q_32                      = 0x22,
+    OPC2_32_RRR1_MSUBS_Q_64                      = 0x3b,
+    OPC2_32_RRR1_MSUBS_Q_32_L                    = 0x21,
+    OPC2_32_RRR1_MSUBS_Q_64_L                    = 0x39,
+    OPC2_32_RRR1_MSUBS_Q_32_U                    = 0x20,
+    OPC2_32_RRR1_MSUBS_Q_64_U                    = 0x38,
+    OPC2_32_RRR1_MSUBS_Q_32_LL                   = 0x25,
+    OPC2_32_RRR1_MSUBS_Q_64_LL                   = 0x3d,
+    OPC2_32_RRR1_MSUBS_Q_32_UU                   = 0x24,
+    OPC2_32_RRR1_MSUBS_Q_64_UU                   = 0x3c,
+    OPC2_32_RRR1_MSUBR_H_32_UL                   = 0x1e,
+    OPC2_32_RRR1_MSUBRS_H_32_UL                  = 0x3e,
+    OPC2_32_RRR1_MSUBR_Q_32_LL                   = 0x07,
+    OPC2_32_RRR1_MSUBR_Q_32_UU                   = 0x06,
+    OPC2_32_RRR1_MSUBRS_Q_32_LL                  = 0x27,
+    OPC2_32_RRR1_MSUBRS_Q_32_UU                  = 0x26,
+};
+/* OPCM_32_RRR1_MSUBADS_H                           */
+enum {
+    OPC2_32_RRR1_MSUBAD_H_32_LL                  = 0x1a,
+    OPC2_32_RRR1_MSUBAD_H_32_LU                  = 0x19,
+    OPC2_32_RRR1_MSUBAD_H_32_UL                  = 0x18,
+    OPC2_32_RRR1_MSUBAD_H_32_UU                  = 0x1b,
+    OPC2_32_RRR1_MSUBADS_H_32_LL                 = 0x3a,
+    OPC2_32_RRR1_MSUBADS_H_32_LU                 = 0x39,
+    OPC2_32_RRR1_MSUBADS_H_32_UL                 = 0x38,
+    OPC2_32_RRR1_MSUBADS_H_32_UU                 = 0x3b,
+    OPC2_32_RRR1_MSUBADM_H_64_LL                 = 0x1e,
+    OPC2_32_RRR1_MSUBADM_H_64_LU                 = 0x1d,
+    OPC2_32_RRR1_MSUBADM_H_64_UL                 = 0x1c,
+    OPC2_32_RRR1_MSUBADM_H_64_UU                 = 0x1f,
+    OPC2_32_RRR1_MSUBADMS_H_64_LL                = 0x3e,
+    OPC2_32_RRR1_MSUBADMS_H_64_LU                = 0x3d,
+    OPC2_32_RRR1_MSUBADMS_H_64_UL                = 0x3c,
+    OPC2_32_RRR1_MSUBADMS_H_16_UU                = 0x3f,
+    OPC2_32_RRR1_MSUBADR_H_16_LL                 = 0x0e,
+    OPC2_32_RRR1_MSUBADR_H_16_LU                 = 0x0d,
+    OPC2_32_RRR1_MSUBADR_H_16_UL                 = 0x0c,
+    OPC2_32_RRR1_MSUBADR_H_16_UU                 = 0x0f,
+    OPC2_32_RRR1_MSUBADRS_H_16_LL                = 0x2e,
+    OPC2_32_RRR1_MSUBADRS_H_16_LU                = 0x2d,
+    OPC2_32_RRR1_MSUBADRS_H_16_UL                = 0x2c,
+    OPC2_32_RRR1_MSUBADRS_H_16_UU                = 0x2f,
+};
+/*
+ * RRR2 Format
+ */
+/* OPCM_32_RRR2_MADD                                */
+enum {
+    OPC2_32_RRR2_MADD_32                         = 0x0a,
+    OPC2_32_RRR2_MADD_64                         = 0x6a,
+    OPC2_32_RRR2_MADDS_32                        = 0x8a,
+    OPC2_32_RRR2_MADDS_64                        = 0xea,
+    OPC2_32_RRR2_MADD_U_32                       = 0x68,
+    OPC2_32_RRR2_MADDS_U_32                      = 0x88,
+    OPC2_32_RRR2_MADDS_U_64                      = 0xe8,
+};
+/* OPCM_32_RRR2_MSUB                                */
+enum {
+    OPC2_32_RRR2_MSUB_32                         = 0x0a,
+    OPC2_32_RRR2_MSUB_64                         = 0x6a,
+    OPC2_32_RRR2_MSUBS_32                        = 0x8a,
+    OPC2_32_RRR2_MSUBS_64                        = 0xea,
+    OPC2_32_RRR2_MSUB_U_64                       = 0x68,
+    OPC2_32_RRR2_MSUBS_U_32                      = 0x88,
+    OPC2_32_RRR2_MSUBS_U_64                      = 0xe8,
+};
+/*
+ * RRRR Format
+ */
+/* OPCM_32_RRRR_EXTRACT_INSERT                      */
+enum {
+    OPC2_32_RRRR_DEXTR                           = 0x04,
+    OPC2_32_RRRR_EXTR                            = 0x02,
+    OPC2_32_RRRR_EXTR_U                          = 0x03,
+    OPC2_32_RRRR_INSERT                          = 0x00,
+};
+/*
+ * RRRW Format
+ */
+/* OPCM_32_RRRW_EXTRACT_INSERT                      */
+enum {
+    OPC2_32_RRRW_EXTR                            = 0x02,
+    OPC2_32_RRRW_EXTR_U                          = 0x03,
+    OPC2_32_RRRW_IMASK                           = 0x01,
+    OPC2_32_RRRW_INSERT                          = 0x00,
+};
+/*
+ * SYS Format
+ */
+/* OPCM_32_SYS_INTERRUPTS                           */
+enum {
+    OPC2_32_SYS_DEBUG                            = 0x04,
+    OPC2_32_SYS_DISABLE                          = 0x0d,
+    OPC2_32_SYS_DSYNC                            = 0x12,
+    OPC2_32_SYS_ENABLE                           = 0x0c,
+    OPC2_32_SYS_ISYNC                            = 0x13,
+    OPC2_32_SYS_NOP                              = 0x00,
+    OPC2_32_SYS_RET                              = 0x06,
+    OPC2_32_SYS_RFE                              = 0x07,
+    OPC2_32_SYS_RFM                              = 0x05,
+    OPC2_32_SYS_RSLCX                            = 0x09,
+    OPC2_32_SYS_SVLCX                            = 0x08,
+    OPC2_32_SYS_TRAPSV                           = 0x15,
+    OPC2_32_SYS_TRAPV                            = 0x14,
+};
diff --git a/target-unicore32/cpu-qom.h b/target-unicore32/cpu-qom.h
index f727760d9..ea65b8331 100644
--- a/target-unicore32/cpu-qom.h
+++ b/target-unicore32/cpu-qom.h
@@ -61,6 +61,7 @@ static inline UniCore32CPU *uc32_env_get_cpu(CPUUniCore32State *env)
 #define ENV_OFFSET offsetof(UniCore32CPU, env)
 
 void uc32_cpu_do_interrupt(CPUState *cpu);
+bool uc32_cpu_exec_interrupt(CPUState *cpu, int int_req);
 void uc32_cpu_dump_state(CPUState *cpu, FILE *f,
                          fprintf_function cpu_fprintf, int flags);
 hwaddr uc32_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
diff --git a/target-unicore32/cpu.c b/target-unicore32/cpu.c
index 2d2c429a3..5b3298717 100644
--- a/target-unicore32/cpu.c
+++ b/target-unicore32/cpu.c
@@ -146,6 +146,7 @@ static void uc32_cpu_class_init(ObjectClass *oc, void *data)
     cc->class_by_name = uc32_cpu_class_by_name;
     cc->has_work = uc32_cpu_has_work;
     cc->do_interrupt = uc32_cpu_do_interrupt;
+    cc->cpu_exec_interrupt = uc32_cpu_exec_interrupt;
     cc->dump_state = uc32_cpu_dump_state;
     cc->set_pc = uc32_cpu_set_pc;
 #ifdef CONFIG_USER_ONLY
diff --git a/target-unicore32/helper.c b/target-unicore32/helper.c
index e5ebbf4b1..b4654fa98 100644
--- a/target-unicore32/helper.c
+++ b/target-unicore32/helper.c
@@ -250,3 +250,18 @@ int uc32_cpu_handle_mmu_fault(CPUState *cs, vaddr address,
     return 1;
 }
 #endif
+
+bool uc32_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    if (interrupt_request & CPU_INTERRUPT_HARD) {
+        UniCore32CPU *cpu = UNICORE32_CPU(cs);
+        CPUUniCore32State *env = &cpu->env;
+
+        if (!(env->uncached_asr & ASR_I)) {
+            cs->exception_index = UC32_EXCP_INTR;
+            uc32_cpu_do_interrupt(cs);
+            return true;
+        }
+    }
+    return false;
+}
diff --git a/target-unicore32/translate.c b/target-unicore32/translate.c
index e3643c23c..653c22518 100644
--- a/target-unicore32/translate.c
+++ b/target-unicore32/translate.c
@@ -23,6 +23,9 @@
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
 
+#include "trace-tcg.h"
+
+
 /* internal defines */
 typedef struct DisasContext {
     target_ulong pc;
diff --git a/target-xtensa/core-dc232b.c b/target-xtensa/core-dc232b.c
index c51e11e6d..a3b914bad 100644
--- a/target-xtensa/core-dc232b.c
+++ b/target-xtensa/core-dc232b.c
@@ -33,7 +33,7 @@
 #include "core-dc232b/core-isa.h"
 #include "overlay_tool.h"
 
-static const XtensaConfig dc232b = {
+static const XtensaConfig dc232b __attribute__((unused)) = {
     .name = "dc232b",
     .gdb_regmap = {
         .num_regs = 120,
diff --git a/target-xtensa/core-dc233c.c b/target-xtensa/core-dc233c.c
index 42dd64f03..ac745d106 100644
--- a/target-xtensa/core-dc233c.c
+++ b/target-xtensa/core-dc233c.c
@@ -34,7 +34,7 @@
 #include "core-dc233c/core-isa.h"
 #include "overlay_tool.h"
 
-static const XtensaConfig dc233c = {
+static const XtensaConfig dc233c __attribute__((unused)) = {
     .name = "dc233c",
     .gdb_regmap = {
         .num_regs = 121,
diff --git a/target-xtensa/core-fsf.c b/target-xtensa/core-fsf.c
index 6859bee06..cfcc84025 100644
--- a/target-xtensa/core-fsf.c
+++ b/target-xtensa/core-fsf.c
@@ -33,7 +33,7 @@
 #include "core-fsf/core-isa.h"
 #include "overlay_tool.h"
 
-static const XtensaConfig fsf = {
+static const XtensaConfig fsf __attribute__((unused)) = {
     .name = "fsf",
     /* GDB for this core is not supported currently */
     .clock_freq_khz = 10000,
diff --git a/target-xtensa/cpu-qom.h b/target-xtensa/cpu-qom.h
index f320486a6..9de5c6eb9 100644
--- a/target-xtensa/cpu-qom.h
+++ b/target-xtensa/cpu-qom.h
@@ -84,6 +84,7 @@ static inline XtensaCPU *xtensa_env_get_cpu(const CPUXtensaState *env)
 #define ENV_OFFSET offsetof(XtensaCPU, env)
 
 void xtensa_cpu_do_interrupt(CPUState *cpu);
+bool xtensa_cpu_exec_interrupt(CPUState *cpu, int interrupt_request);
 void xtensa_cpu_dump_state(CPUState *cpu, FILE *f,
                            fprintf_function cpu_fprintf, int flags);
 hwaddr xtensa_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
diff --git a/target-xtensa/cpu.c b/target-xtensa/cpu.c
index 9d8801b70..6a5414f81 100644
--- a/target-xtensa/cpu.c
+++ b/target-xtensa/cpu.c
@@ -119,7 +119,6 @@ static void xtensa_cpu_initfn(Object *obj)
     if (tcg_enabled() && !tcg_inited) {
         tcg_inited = true;
         xtensa_translate_init();
-        cpu_set_debug_excp_handler(xtensa_breakpoint_handler);
     }
 }
 
@@ -143,14 +142,17 @@ static void xtensa_cpu_class_init(ObjectClass *oc, void *data)
     cc->class_by_name = xtensa_cpu_class_by_name;
     cc->has_work = xtensa_cpu_has_work;
     cc->do_interrupt = xtensa_cpu_do_interrupt;
+    cc->cpu_exec_interrupt = xtensa_cpu_exec_interrupt;
     cc->dump_state = xtensa_cpu_dump_state;
     cc->set_pc = xtensa_cpu_set_pc;
     cc->gdb_read_register = xtensa_cpu_gdb_read_register;
     cc->gdb_write_register = xtensa_cpu_gdb_write_register;
+    cc->gdb_stop_before_watchpoint = true;
 #ifndef CONFIG_USER_ONLY
     cc->do_unaligned_access = xtensa_cpu_do_unaligned_access;
     cc->get_phys_page_debug = xtensa_cpu_get_phys_page_debug;
 #endif
+    cc->debug_excp_handler = xtensa_breakpoint_handler;
     dc->vmsd = &vmstate_xtensa_cpu;
 }
 
diff --git a/target-xtensa/cpu.h b/target-xtensa/cpu.h
index d797d2649..ac463f27f 100644
--- a/target-xtensa/cpu.h
+++ b/target-xtensa/cpu.h
@@ -266,6 +266,7 @@ typedef enum {
     INTTYPE_TIMER,
     INTTYPE_DEBUG,
     INTTYPE_WRITE_ERR,
+    INTTYPE_PROFILING,
     INTTYPE_MAX
 } interrupt_type;
 
@@ -390,7 +391,7 @@ static inline CPUXtensaState *cpu_init(const char *cpu_model)
 }
 
 void xtensa_translate_init(void);
-void xtensa_breakpoint_handler(CPUXtensaState *env);
+void xtensa_breakpoint_handler(CPUState *cs);
 int cpu_xtensa_exec(CPUXtensaState *s);
 void xtensa_register_core(XtensaConfigList *node);
 void check_interrupts(CPUXtensaState *s);
@@ -471,6 +472,12 @@ static inline xtensa_tlb_entry *xtensa_tlb_get_entry(CPUXtensaState *env,
         env->itlb[wi] + ei;
 }
 
+static inline uint32_t xtensa_replicate_windowstart(CPUXtensaState *env)
+{
+    return env->sregs[WINDOW_START] |
+        (env->sregs[WINDOW_START] << env->config->nareg / 4);
+}
+
 /* MMU modes definitions */
 #define MMU_MODE0_SUFFIX _ring0
 #define MMU_MODE1_SUFFIX _ring1
diff --git a/target-xtensa/helper.c b/target-xtensa/helper.c
index 94dcd9442..d84d259cf 100644
--- a/target-xtensa/helper.c
+++ b/target-xtensa/helper.c
@@ -79,9 +79,10 @@ static uint32_t check_hw_breakpoints(CPUXtensaState *env)
     return 0;
 }
 
-void xtensa_breakpoint_handler(CPUXtensaState *env)
+void xtensa_breakpoint_handler(CPUState *cs)
 {
-    CPUState *cs = CPU(xtensa_env_get_cpu(env));
+    XtensaCPU *cpu = XTENSA_CPU(cs);
+    CPUXtensaState *env = &cpu->env;
 
     if (cs->watchpoint_hit) {
         if (cs->watchpoint_hit->flags & BP_CPU) {
@@ -255,6 +256,16 @@ void xtensa_cpu_do_interrupt(CPUState *cs)
     check_interrupts(env);
 }
 
+bool xtensa_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    if (interrupt_request & CPU_INTERRUPT_HARD) {
+        cs->exception_index = EXC_IRQ;
+        xtensa_cpu_do_interrupt(cs);
+        return true;
+    }
+    return false;
+}
+
 static void reset_tlb_mmu_all_ways(CPUXtensaState *env,
         const xtensa_tlb *tlb, xtensa_tlb_entry entry[][MAX_TLB_WAY_SIZE])
 {
diff --git a/target-xtensa/import_core.sh b/target-xtensa/import_core.sh
new file mode 100755
index 000000000..73791ec54
--- /dev/null
+++ b/target-xtensa/import_core.sh
@@ -0,0 +1,53 @@
+#! /bin/bash -e
+
+OVERLAY="$1"
+NAME="$2"
+FREQ=40000
+BASE=$(dirname "$0")
+TARGET="$BASE"/core-$NAME
+
+[ $# -ge 2 -a -f "$OVERLAY" ] || { cat <<EOF
+Usage: $0 overlay-archive-to-import core-name [frequency-in-KHz]
+    overlay-archive-to-import:  file name of xtensa-config-overlay.tar.gz
+                                to import configuration from.
+    core-name:                  QEMU name of the imported core. Must be valid
+                                C identifier.
+    frequency-in-KHz:           core frequency (40MHz if not specified).
+EOF
+exit
+}
+
+[ $# -ge 3 ] && FREQ="$3"
+mkdir -p "$TARGET"
+tar -xf "$OVERLAY" -C "$TARGET" --strip-components=1 \
+    --xform='s/core/core-isa/' config/core.h
+tar -xf "$OVERLAY" -O gdb/xtensa-config.c | \
+    sed -n '1,/*\//p;/pc/,/a15/p' > "$TARGET"/gdb-config.c
+NUM_REGS=$(grep XTREG "$TARGET"/gdb-config.c | wc -l)
+
+cat <<EOF > "${TARGET}.c"
+#include "cpu.h"
+#include "exec/exec-all.h"
+#include "exec/gdbstub.h"
+#include "qemu/host-utils.h"
+
+#include "core-$NAME/core-isa.h"
+#include "overlay_tool.h"
+
+static const XtensaConfig $NAME __attribute__((unused)) = {
+    .name = "$NAME",
+    .gdb_regmap = {
+        .num_regs = $NUM_REGS,
+        .reg = {
+#include "core-$NAME/gdb-config.c"
+        }
+    },
+    .clock_freq_khz = $FREQ,
+    DEFAULT_SECTIONS
+};
+
+REGISTER_CORE($NAME)
+EOF
+
+grep -q core-${NAME}.o "$BASE"/Makefile.objs || \
+    echo "obj-y += core-${NAME}.o" >> "$BASE"/Makefile.objs
diff --git a/target-xtensa/op_helper.c b/target-xtensa/op_helper.c
index dae13866e..872e5a823 100644
--- a/target-xtensa/op_helper.c
+++ b/target-xtensa/op_helper.c
@@ -235,6 +235,12 @@ void HELPER(entry)(CPUXtensaState *env, uint32_t pc, uint32_t s, uint32_t imm)
                 pc, env->sregs[PS]);
         HELPER(exception_cause)(env, pc, ILLEGAL_INSTRUCTION_CAUSE);
     } else {
+        uint32_t windowstart = xtensa_replicate_windowstart(env) >>
+            (env->sregs[WINDOW_BASE] + 1);
+
+        if (windowstart & ((1 << callinc) - 1)) {
+            HELPER(window_check)(env, pc, callinc);
+        }
         env->regs[(callinc << 2) | (s & 3)] = env->regs[s] - (imm << 3);
         rotate_window(env, callinc);
         env->sregs[WINDOW_START] |=
diff --git a/target-xtensa/overlay_tool.h b/target-xtensa/overlay_tool.h
index 4c0de7f06..6105d4c8f 100644
--- a/target-xtensa/overlay_tool.h
+++ b/target-xtensa/overlay_tool.h
@@ -108,20 +108,27 @@
 #define XCHAL_WINDOW_UF12_VECOFS        0x00000140
 #endif
 
+#if XCHAL_HAVE_WINDOWED
+#define WINDOW_VECTORS \
+   [EXC_WINDOW_OVERFLOW4] = XCHAL_WINDOW_OF4_VECOFS + \
+       XCHAL_WINDOW_VECTORS_VADDR, \
+   [EXC_WINDOW_UNDERFLOW4] = XCHAL_WINDOW_UF4_VECOFS + \
+       XCHAL_WINDOW_VECTORS_VADDR, \
+   [EXC_WINDOW_OVERFLOW8] = XCHAL_WINDOW_OF8_VECOFS + \
+       XCHAL_WINDOW_VECTORS_VADDR, \
+   [EXC_WINDOW_UNDERFLOW8] = XCHAL_WINDOW_UF8_VECOFS + \
+       XCHAL_WINDOW_VECTORS_VADDR, \
+   [EXC_WINDOW_OVERFLOW12] = XCHAL_WINDOW_OF12_VECOFS + \
+       XCHAL_WINDOW_VECTORS_VADDR, \
+   [EXC_WINDOW_UNDERFLOW12] = XCHAL_WINDOW_UF12_VECOFS + \
+       XCHAL_WINDOW_VECTORS_VADDR,
+#else
+#define WINDOW_VECTORS
+#endif
+
 #define EXCEPTION_VECTORS { \
         [EXC_RESET] = XCHAL_RESET_VECTOR_VADDR, \
-        [EXC_WINDOW_OVERFLOW4] = XCHAL_WINDOW_OF4_VECOFS + \
-            XCHAL_WINDOW_VECTORS_VADDR, \
-        [EXC_WINDOW_UNDERFLOW4] = XCHAL_WINDOW_UF4_VECOFS + \
-            XCHAL_WINDOW_VECTORS_VADDR, \
-        [EXC_WINDOW_OVERFLOW8] = XCHAL_WINDOW_OF8_VECOFS + \
-            XCHAL_WINDOW_VECTORS_VADDR, \
-        [EXC_WINDOW_UNDERFLOW8] = XCHAL_WINDOW_UF8_VECOFS + \
-            XCHAL_WINDOW_VECTORS_VADDR, \
-        [EXC_WINDOW_OVERFLOW12] = XCHAL_WINDOW_OF12_VECOFS + \
-            XCHAL_WINDOW_VECTORS_VADDR, \
-        [EXC_WINDOW_UNDERFLOW12] = XCHAL_WINDOW_UF12_VECOFS + \
-            XCHAL_WINDOW_VECTORS_VADDR, \
+        WINDOW_VECTORS \
         [EXC_KERNEL] = XCHAL_KERNEL_VECTOR_VADDR, \
         [EXC_USER] = XCHAL_USER_VECTOR_VADDR, \
         [EXC_DOUBLE] = XCHAL_DOUBLEEXC_VECTOR_VADDR, \
@@ -163,6 +170,7 @@
 #define XTHAL_INTTYPE_TBD1 INTTYPE_DEBUG
 #define XTHAL_INTTYPE_TBD2 INTTYPE_WRITE_ERR
 #define XTHAL_INTTYPE_WRITE_ERROR INTTYPE_WRITE_ERR
+#define XTHAL_INTTYPE_PROFILING INTTYPE_PROFILING
 
 
 #define INTERRUPT(i) { \
diff --git a/target-xtensa/translate.c b/target-xtensa/translate.c
index 2f22cce84..badca195f 100644
--- a/target-xtensa/translate.c
+++ b/target-xtensa/translate.c
@@ -41,6 +41,9 @@
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
 
+#include "trace-tcg.h"
+
+
 typedef struct DisasContext {
     const XtensaConfig *config;
     TranslationBlock *tb;
diff --git a/tcg/aarch64/tcg-target.c b/tcg/aarch64/tcg-target.c
index 56dae66a3..987c0bd4d 100644
--- a/tcg/aarch64/tcg-target.c
+++ b/tcg/aarch64/tcg-target.c
@@ -1007,7 +1007,7 @@ static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
     tcg_out_adr(s, TCG_REG_X3, lb->raddr);
     tcg_out_call(s, qemu_ld_helpers[opc & ~MO_SIGN]);
     if (opc & MO_SIGN) {
-        tcg_out_sxt(s, TCG_TYPE_I64, size, lb->datalo_reg, TCG_REG_X0);
+        tcg_out_sxt(s, lb->type, size, lb->datalo_reg, TCG_REG_X0);
     } else {
         tcg_out_mov(s, size == MO_64, lb->datalo_reg, TCG_REG_X0);
     }
@@ -1032,7 +1032,7 @@ static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
 }
 
 static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOp opc,
-                                TCGReg data_reg, TCGReg addr_reg,
+                                TCGType ext, TCGReg data_reg, TCGReg addr_reg,
                                 int mem_index, tcg_insn_unit *raddr,
                                 tcg_insn_unit *label_ptr)
 {
@@ -1040,6 +1040,7 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOp opc,
 
     label->is_ld = is_ld;
     label->opc = opc;
+    label->type = ext;
     label->datalo_reg = data_reg;
     label->addrlo_reg = addr_reg;
     label->mem_index = mem_index;
@@ -1108,7 +1109,7 @@ static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, TCGMemOp s_bits,
 
 #endif /* CONFIG_SOFTMMU */
 
-static void tcg_out_qemu_ld_direct(TCGContext *s, TCGMemOp memop,
+static void tcg_out_qemu_ld_direct(TCGContext *s, TCGMemOp memop, TCGType ext,
                                    TCGReg data_r, TCGReg addr_r, TCGReg off_r)
 {
     const TCGMemOp bswap = memop & MO_BSWAP;
@@ -1118,7 +1119,8 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGMemOp memop,
         tcg_out_ldst_r(s, I3312_LDRB, data_r, addr_r, off_r);
         break;
     case MO_SB:
-        tcg_out_ldst_r(s, I3312_LDRSBX, data_r, addr_r, off_r);
+        tcg_out_ldst_r(s, ext ? I3312_LDRSBX : I3312_LDRSBW,
+                       data_r, addr_r, off_r);
         break;
     case MO_UW:
         tcg_out_ldst_r(s, I3312_LDRH, data_r, addr_r, off_r);
@@ -1130,9 +1132,10 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGMemOp memop,
         if (bswap) {
             tcg_out_ldst_r(s, I3312_LDRH, data_r, addr_r, off_r);
             tcg_out_rev16(s, data_r, data_r);
-            tcg_out_sxt(s, TCG_TYPE_I64, MO_16, data_r, data_r);
+            tcg_out_sxt(s, ext, MO_16, data_r, data_r);
         } else {
-            tcg_out_ldst_r(s, I3312_LDRSHX, data_r, addr_r, off_r);
+            tcg_out_ldst_r(s, ext ? I3312_LDRSHX : I3312_LDRSHW,
+                           data_r, addr_r, off_r);
         }
         break;
     case MO_UL:
@@ -1197,18 +1200,18 @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGMemOp memop,
 }
 
 static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
-                            TCGMemOp memop, int mem_index)
+                            TCGMemOp memop, TCGType ext, int mem_index)
 {
 #ifdef CONFIG_SOFTMMU
     TCGMemOp s_bits = memop & MO_SIZE;
     tcg_insn_unit *label_ptr;
 
     tcg_out_tlb_read(s, addr_reg, s_bits, &label_ptr, mem_index, 1);
-    tcg_out_qemu_ld_direct(s, memop, data_reg, addr_reg, TCG_REG_X1);
-    add_qemu_ldst_label(s, true, memop, data_reg, addr_reg,
+    tcg_out_qemu_ld_direct(s, memop, ext, data_reg, addr_reg, TCG_REG_X1);
+    add_qemu_ldst_label(s, true, memop, ext, data_reg, addr_reg,
                         mem_index, s->code_ptr, label_ptr);
 #else /* !CONFIG_SOFTMMU */
-    tcg_out_qemu_ld_direct(s, memop, data_reg, addr_reg,
+    tcg_out_qemu_ld_direct(s, memop, ext, data_reg, addr_reg,
                            GUEST_BASE ? TCG_REG_GUEST_BASE : TCG_REG_XZR);
 #endif /* CONFIG_SOFTMMU */
 }
@@ -1222,7 +1225,7 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
 
     tcg_out_tlb_read(s, addr_reg, s_bits, &label_ptr, mem_index, 0);
     tcg_out_qemu_st_direct(s, memop, data_reg, addr_reg, TCG_REG_X1);
-    add_qemu_ldst_label(s, false, memop, data_reg, addr_reg,
+    add_qemu_ldst_label(s, false, memop, s_bits == MO_64, data_reg, addr_reg,
                         mem_index, s->code_ptr, label_ptr);
 #else /* !CONFIG_SOFTMMU */
     tcg_out_qemu_st_direct(s, memop, data_reg, addr_reg,
@@ -1515,7 +1518,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     case INDEX_op_qemu_ld_i32:
     case INDEX_op_qemu_ld_i64:
-        tcg_out_qemu_ld(s, a0, a1, a2, args[3]);
+        tcg_out_qemu_ld(s, a0, a1, a2, ext, args[3]);
         break;
     case INDEX_op_qemu_st_i32:
     case INDEX_op_qemu_st_i64:
diff --git a/tcg/mips/tcg-target.c b/tcg/mips/tcg-target.c
index 9cce3563a..b7f4d6748 100644
--- a/tcg/mips/tcg-target.c
+++ b/tcg/mips/tcg-target.c
@@ -1302,7 +1302,7 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
        so we can reuse that for the base.  */
     base = (TARGET_LONG_BITS == 32 ? TCG_REG_A1 : TCG_REG_A2);
     tcg_out_tlb_load(s, base, addr_regl, addr_regh, mem_index,
-                     s_bits, label_ptr, 1);
+                     s_bits, label_ptr, 0);
     tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc);
     add_qemu_ldst_label(s, 0, opc, data_regl, data_regh, addr_regl, addr_regh,
                         mem_index, s->code_ptr, label_ptr);
diff --git a/tcg/sparc/tcg-target.c b/tcg/sparc/tcg-target.c
index 40f2ec102..0c4b02858 100644
--- a/tcg/sparc/tcg-target.c
+++ b/tcg/sparc/tcg-target.c
@@ -197,8 +197,8 @@ static const int tcg_target_call_oarg_regs[] = {
 #define ARITH_XOR  (INSN_OP(2) | INSN_OP3(0x03))
 #define ARITH_SUB  (INSN_OP(2) | INSN_OP3(0x04))
 #define ARITH_SUBCC (INSN_OP(2) | INSN_OP3(0x14))
-#define ARITH_ADDX (INSN_OP(2) | INSN_OP3(0x08))
-#define ARITH_SUBX (INSN_OP(2) | INSN_OP3(0x0c))
+#define ARITH_ADDC (INSN_OP(2) | INSN_OP3(0x08))
+#define ARITH_SUBC (INSN_OP(2) | INSN_OP3(0x0c))
 #define ARITH_UMUL (INSN_OP(2) | INSN_OP3(0x0a))
 #define ARITH_SMUL (INSN_OP(2) | INSN_OP3(0x0b))
 #define ARITH_UDIV (INSN_OP(2) | INSN_OP3(0x0e))
@@ -209,6 +209,9 @@ static const int tcg_target_call_oarg_regs[] = {
 #define ARITH_MOVCC (INSN_OP(2) | INSN_OP3(0x2c))
 #define ARITH_MOVR (INSN_OP(2) | INSN_OP3(0x2f))
 
+#define ARITH_ADDXC (INSN_OP(2) | INSN_OP3(0x36) | INSN_OPF(0x11))
+#define ARITH_UMULXHI (INSN_OP(2) | INSN_OP3(0x36) | INSN_OPF(0x16))
+
 #define SHIFT_SLL  (INSN_OP(2) | INSN_OP3(0x25))
 #define SHIFT_SRL  (INSN_OP(2) | INSN_OP3(0x26))
 #define SHIFT_SRA  (INSN_OP(2) | INSN_OP3(0x27))
@@ -262,6 +265,10 @@ static const int tcg_target_call_oarg_regs[] = {
 #define STW_LE     (STWA  | INSN_ASI(ASI_PRIMARY_LITTLE))
 #define STX_LE     (STXA  | INSN_ASI(ASI_PRIMARY_LITTLE))
 
+#ifndef use_vis3_instructions
+bool use_vis3_instructions;
+#endif
+
 static inline int check_fit_i64(int64_t val, unsigned int bits)
 {
     return val == sextract64(val, 0, bits);
@@ -657,7 +664,7 @@ static void tcg_out_movcond_i64(TCGContext *s, TCGCond cond, TCGReg ret,
 static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGReg ret,
                                 TCGReg c1, int32_t c2, int c2const)
 {
-    /* For 32-bit comparisons, we can play games with ADDX/SUBX.  */
+    /* For 32-bit comparisons, we can play games with ADDC/SUBC.  */
     switch (cond) {
     case TCG_COND_LTU:
     case TCG_COND_GEU:
@@ -668,9 +675,12 @@ static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGReg ret,
     case TCG_COND_NE:
         /* For equality, we can transform to inequality vs zero.  */
         if (c2 != 0) {
-            tcg_out_arithc(s, ret, c1, c2, c2const, ARITH_XOR);
+            tcg_out_arithc(s, TCG_REG_T1, c1, c2, c2const, ARITH_XOR);
+            c2 = TCG_REG_T1;
+        } else {
+            c2 = c1;
         }
-        c1 = TCG_REG_G0, c2 = ret, c2const = 0;
+        c1 = TCG_REG_G0, c2const = 0;
         cond = (cond == TCG_COND_EQ ? TCG_COND_GEU : TCG_COND_LTU);
 	break;
 
@@ -698,15 +708,32 @@ static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGReg ret,
 
     tcg_out_cmp(s, c1, c2, c2const);
     if (cond == TCG_COND_LTU) {
-        tcg_out_arithi(s, ret, TCG_REG_G0, 0, ARITH_ADDX);
+        tcg_out_arithi(s, ret, TCG_REG_G0, 0, ARITH_ADDC);
     } else {
-        tcg_out_arithi(s, ret, TCG_REG_G0, -1, ARITH_SUBX);
+        tcg_out_arithi(s, ret, TCG_REG_G0, -1, ARITH_SUBC);
     }
 }
 
 static void tcg_out_setcond_i64(TCGContext *s, TCGCond cond, TCGReg ret,
                                 TCGReg c1, int32_t c2, int c2const)
 {
+    if (use_vis3_instructions) {
+        switch (cond) {
+        case TCG_COND_NE:
+            if (c2 != 0) {
+                break;
+            }
+            c2 = c1, c2const = 0, c1 = TCG_REG_G0;
+            /* FALLTHRU */
+        case TCG_COND_LTU:
+            tcg_out_cmp(s, c1, c2, c2const);
+            tcg_out_arith(s, ret, TCG_REG_G0, TCG_REG_G0, ARITH_ADDXC);
+            return;
+        default:
+            break;
+        }
+    }
+
     /* For 64-bit signed comparisons vs zero, we can avoid the compare
        if the input does not overlap the output.  */
     if (c2 == 0 && !is_unsigned_cond(cond) && c1 != ret) {
@@ -719,9 +746,9 @@ static void tcg_out_setcond_i64(TCGContext *s, TCGCond cond, TCGReg ret,
     }
 }
 
-static void tcg_out_addsub2(TCGContext *s, TCGReg rl, TCGReg rh,
-                            TCGReg al, TCGReg ah, int32_t bl, int blconst,
-                            int32_t bh, int bhconst, int opl, int oph)
+static void tcg_out_addsub2_i32(TCGContext *s, TCGReg rl, TCGReg rh,
+                                TCGReg al, TCGReg ah, int32_t bl, int blconst,
+                                int32_t bh, int bhconst, int opl, int oph)
 {
     TCGReg tmp = TCG_REG_T1;
 
@@ -735,6 +762,54 @@ static void tcg_out_addsub2(TCGContext *s, TCGReg rl, TCGReg rh,
     tcg_out_mov(s, TCG_TYPE_I32, rl, tmp);
 }
 
+static void tcg_out_addsub2_i64(TCGContext *s, TCGReg rl, TCGReg rh,
+                                TCGReg al, TCGReg ah, int32_t bl, int blconst,
+                                int32_t bh, int bhconst, bool is_sub)
+{
+    TCGReg tmp = TCG_REG_T1;
+
+    /* Note that the low parts are fully consumed before tmp is set.  */
+    if (rl != ah && (bhconst || rl != bh)) {
+        tmp = rl;
+    }
+
+    tcg_out_arithc(s, tmp, al, bl, blconst, is_sub ? ARITH_SUBCC : ARITH_ADDCC);
+
+    if (use_vis3_instructions && !is_sub) {
+        /* Note that ADDXC doesn't accept immediates.  */
+        if (bhconst && bh != 0) {
+           tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_T2, bh);
+           bh = TCG_REG_T2;
+        }
+        tcg_out_arith(s, rh, ah, bh, ARITH_ADDXC);
+    } else if (bh == TCG_REG_G0) {
+	/* If we have a zero, we can perform the operation in two insns,
+           with the arithmetic first, and a conditional move into place.  */
+	if (rh == ah) {
+            tcg_out_arithi(s, TCG_REG_T2, ah, 1,
+			   is_sub ? ARITH_SUB : ARITH_ADD);
+            tcg_out_movcc(s, TCG_COND_LTU, MOVCC_XCC, rh, TCG_REG_T2, 0);
+	} else {
+            tcg_out_arithi(s, rh, ah, 1, is_sub ? ARITH_SUB : ARITH_ADD);
+	    tcg_out_movcc(s, TCG_COND_GEU, MOVCC_XCC, rh, ah, 0);
+	}
+    } else {
+        /* Otherwise adjust BH as if there is carry into T2 ... */
+        if (bhconst) {
+            tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_T2, bh + (is_sub ? -1 : 1));
+        } else {
+            tcg_out_arithi(s, TCG_REG_T2, bh, 1,
+                           is_sub ? ARITH_SUB : ARITH_ADD);
+        }
+        /* ... smoosh T2 back to original BH if carry is clear ... */
+        tcg_out_movcc(s, TCG_COND_GEU, MOVCC_XCC, TCG_REG_T2, bh, bhconst);
+	/* ... and finally perform the arithmetic with the new operand.  */
+        tcg_out_arith(s, rh, ah, TCG_REG_T2, is_sub ? ARITH_SUB : ARITH_ADD);
+    }
+
+    tcg_out_mov(s, TCG_TYPE_I64, rl, tmp);
+}
+
 static void tcg_out_call_nodelay(TCGContext *s, tcg_insn_unit *dest)
 {
     ptrdiff_t disp = tcg_pcrel_diff(s, dest);
@@ -1264,12 +1339,14 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         break;
 
     case INDEX_op_add2_i32:
-        tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], const_args[4],
-                        args[5], const_args[5], ARITH_ADDCC, ARITH_ADDX);
+        tcg_out_addsub2_i32(s, args[0], args[1], args[2], args[3],
+                            args[4], const_args[4], args[5], const_args[5],
+                            ARITH_ADDCC, ARITH_ADDC);
         break;
     case INDEX_op_sub2_i32:
-        tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], const_args[4],
-                        args[5], const_args[5], ARITH_SUBCC, ARITH_SUBX);
+        tcg_out_addsub2_i32(s, args[0], args[1], args[2], args[3],
+                            args[4], const_args[4], args[5], const_args[5],
+                            ARITH_SUBCC, ARITH_SUBC);
         break;
     case INDEX_op_mulu2_i32:
         c = ARITH_UMUL;
@@ -1351,6 +1428,17 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_movcond_i64:
         tcg_out_movcond_i64(s, args[5], a0, a1, a2, c2, args[3], const_args[3]);
         break;
+    case INDEX_op_add2_i64:
+        tcg_out_addsub2_i64(s, args[0], args[1], args[2], args[3], args[4],
+                            const_args[4], args[5], const_args[5], false);
+        break;
+    case INDEX_op_sub2_i64:
+        tcg_out_addsub2_i64(s, args[0], args[1], args[2], args[3], args[4],
+                            const_args[4], args[5], const_args[5], true);
+        break;
+    case INDEX_op_muluh_i64:
+        tcg_out_arith(s, args[0], args[1], args[2], ARITH_UMULXHI);
+        break;
 
     gen_arith:
         tcg_out_arithc(s, a0, a1, a2, c2, c);
@@ -1449,6 +1537,10 @@ static const TCGTargetOpDef sparc_op_defs[] = {
     { INDEX_op_setcond_i64, { "R", "RZ", "RJ" } },
     { INDEX_op_movcond_i64, { "R", "RZ", "RJ", "RI", "0" } },
 
+    { INDEX_op_add2_i64, { "R", "R", "RZ", "RZ", "RJ", "RI" } },
+    { INDEX_op_sub2_i64, { "R", "R", "RZ", "RZ", "RJ", "RI" } },
+    { INDEX_op_muluh_i64, { "R", "RZ", "RZ" } },
+
     { INDEX_op_qemu_ld_i32, { "r", "A" } },
     { INDEX_op_qemu_ld_i64, { "R", "A" } },
     { INDEX_op_qemu_st_i32, { "sZ", "A" } },
@@ -1459,6 +1551,15 @@ static const TCGTargetOpDef sparc_op_defs[] = {
 
 static void tcg_target_init(TCGContext *s)
 {
+    /* Only probe for the platform and capabilities if we havn't already
+       determined maximum values at compile time.  */
+#ifndef use_vis3_instructions
+    {
+        unsigned long hwcap = qemu_getauxval(AT_HWCAP);
+        use_vis3_instructions = (hwcap & HWCAP_SPARC_VIS3) != 0;
+    }
+#endif
+
     tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffffffff);
     tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0, ALL_64);
 
diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h
index 089f9761c..0c4c8af0b 100644
--- a/tcg/sparc/tcg-target.h
+++ b/tcg/sparc/tcg-target.h
@@ -85,6 +85,12 @@ typedef enum {
 #define TCG_TARGET_EXTEND_ARGS 1
 #endif
 
+#if defined(__VIS__) && __VIS__ >= 0x300
+#define use_vis3_instructions  1
+#else
+extern bool use_vis3_instructions;
+#endif
+
 /* optional instructions */
 #define TCG_TARGET_HAS_div_i32		1
 #define TCG_TARGET_HAS_rem_i32		0
@@ -133,11 +139,11 @@ typedef enum {
 #define TCG_TARGET_HAS_nor_i64          0
 #define TCG_TARGET_HAS_deposit_i64      0
 #define TCG_TARGET_HAS_movcond_i64      1
-#define TCG_TARGET_HAS_add2_i64         0
-#define TCG_TARGET_HAS_sub2_i64         0
+#define TCG_TARGET_HAS_add2_i64         1
+#define TCG_TARGET_HAS_sub2_i64         1
 #define TCG_TARGET_HAS_mulu2_i64        0
 #define TCG_TARGET_HAS_muls2_i64        0
-#define TCG_TARGET_HAS_muluh_i64        0
+#define TCG_TARGET_HAS_muluh_i64        use_vis3_instructions
 #define TCG_TARGET_HAS_mulsh_i64        0
 
 #define TCG_AREG0 TCG_REG_I0
diff --git a/tcg/tcg-be-ldst.h b/tcg/tcg-be-ldst.h
index 49b3de61e..429cba24d 100644
--- a/tcg/tcg-be-ldst.h
+++ b/tcg/tcg-be-ldst.h
@@ -24,8 +24,9 @@
 #define TCG_MAX_QEMU_LDST       640
 
 typedef struct TCGLabelQemuLdst {
-    bool is_ld:1;           /* qemu_ld: true, qemu_st: false */
-    TCGMemOp opc:4;
+    bool is_ld;             /* qemu_ld: true, qemu_st: false */
+    TCGMemOp opc;
+    TCGType type;           /* result type of a load */
     TCGReg addrlo_reg;      /* reg index for low word of guest virtual addr */
     TCGReg addrhi_reg;      /* reg index for high word of guest virtual addr */
     TCGReg datalo_reg;      /* reg index for low word to be loaded or stored */
diff --git a/tcg/tcg.c b/tcg/tcg.c
index c068990fd..7a84b871f 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -2404,12 +2404,10 @@ static int64_t tcg_table_op_count[NB_OPS];
 static void dump_op_count(void)
 {
     int i;
-    FILE *f;
-    f = fopen("/tmp/op.log", "w");
+
     for(i = INDEX_op_end; i < NB_OPS; i++) {
-        fprintf(f, "%s %" PRId64 "\n", tcg_op_defs[i].name, tcg_table_op_count[i]);
+        qemu_log("%s %" PRId64 "\n", tcg_op_defs[i].name, tcg_table_op_count[i]);
     }
-    fclose(f);
 }
 #endif
 
diff --git a/tcg/tcg.h b/tcg/tcg.h
index 997a70433..7285f71fa 100644
--- a/tcg/tcg.h
+++ b/tcg/tcg.h
@@ -274,75 +274,54 @@ typedef enum TCGMemOp {
 
 typedef tcg_target_ulong TCGArg;
 
-/* Define a type and accessor macros for variables.  Using a struct is
-   nice because it gives some level of type safely.  Ideally the compiler
-   be able to see through all this.  However in practice this is not true,
-   especially on targets with braindamaged ABIs (e.g. i386).
-   We use plain int by default to avoid this runtime overhead.
-   Users of tcg_gen_* don't need to know about any of this, and should
-   treat TCGv as an opaque type.
+/* Define a type and accessor macros for variables.  Using pointer types
+   is nice because it gives some level of type safely.  Converting to and
+   from intptr_t rather than int reduces the number of sign-extension
+   instructions that get implied on 64-bit hosts.  Users of tcg_gen_* don't
+   need to know about any of this, and should treat TCGv as an opaque type.
    In addition we do typechecking for different types of variables.  TCGv_i32
    and TCGv_i64 are 32/64-bit variables respectively.  TCGv and TCGv_ptr
-   are aliases for target_ulong and host pointer sized values respectively.
- */
+   are aliases for target_ulong and host pointer sized values respectively.  */
 
-#ifdef CONFIG_DEBUG_TCG
-#define DEBUG_TCGV 1
-#endif
+typedef struct TCGv_i32_d *TCGv_i32;
+typedef struct TCGv_i64_d *TCGv_i64;
+typedef struct TCGv_ptr_d *TCGv_ptr;
 
-#ifdef DEBUG_TCGV
+static inline TCGv_i32 QEMU_ARTIFICIAL MAKE_TCGV_I32(intptr_t i)
+{
+    return (TCGv_i32)i;
+}
 
-typedef struct
+static inline TCGv_i64 QEMU_ARTIFICIAL MAKE_TCGV_I64(intptr_t i)
 {
-    int i32;
-} TCGv_i32;
+    return (TCGv_i64)i;
+}
 
-typedef struct
+static inline TCGv_ptr QEMU_ARTIFICIAL MAKE_TCGV_PTR(intptr_t i)
 {
-    int i64;
-} TCGv_i64;
-
-typedef struct {
-    int iptr;
-} TCGv_ptr;
-
-#define MAKE_TCGV_I32(i) __extension__                  \
-    ({ TCGv_i32 make_tcgv_tmp = {i}; make_tcgv_tmp;})
-#define MAKE_TCGV_I64(i) __extension__                  \
-    ({ TCGv_i64 make_tcgv_tmp = {i}; make_tcgv_tmp;})
-#define MAKE_TCGV_PTR(i) __extension__                  \
-    ({ TCGv_ptr make_tcgv_tmp = {i}; make_tcgv_tmp; })
-#define GET_TCGV_I32(t) ((t).i32)
-#define GET_TCGV_I64(t) ((t).i64)
-#define GET_TCGV_PTR(t) ((t).iptr)
-#if TCG_TARGET_REG_BITS == 32
-#define TCGV_LOW(t) MAKE_TCGV_I32(GET_TCGV_I64(t))
-#define TCGV_HIGH(t) MAKE_TCGV_I32(GET_TCGV_I64(t) + 1)
-#endif
+    return (TCGv_ptr)i;
+}
+
+static inline intptr_t QEMU_ARTIFICIAL GET_TCGV_I32(TCGv_i32 t)
+{
+    return (intptr_t)t;
+}
 
-#else /* !DEBUG_TCGV */
+static inline intptr_t QEMU_ARTIFICIAL GET_TCGV_I64(TCGv_i64 t)
+{
+    return (intptr_t)t;
+}
 
-typedef int TCGv_i32;
-typedef int TCGv_i64;
-#if TCG_TARGET_REG_BITS == 32
-#define TCGv_ptr TCGv_i32
-#else
-#define TCGv_ptr TCGv_i64
-#endif
-#define MAKE_TCGV_I32(x) (x)
-#define MAKE_TCGV_I64(x) (x)
-#define MAKE_TCGV_PTR(x) (x)
-#define GET_TCGV_I32(t) (t)
-#define GET_TCGV_I64(t) (t)
-#define GET_TCGV_PTR(t) (t)
+static inline intptr_t QEMU_ARTIFICIAL GET_TCGV_PTR(TCGv_ptr t)
+{
+    return (intptr_t)t;
+}
 
 #if TCG_TARGET_REG_BITS == 32
-#define TCGV_LOW(t) (t)
-#define TCGV_HIGH(t) ((t) + 1)
+#define TCGV_LOW(t) MAKE_TCGV_I32(GET_TCGV_I64(t))
+#define TCGV_HIGH(t) MAKE_TCGV_I32(GET_TCGV_I64(t) + 1)
 #endif
 
-#endif /* DEBUG_TCGV */
-
 #define TCGV_EQUAL_I32(a, b) (GET_TCGV_I32(a) == GET_TCGV_I32(b))
 #define TCGV_EQUAL_I64(a, b) (GET_TCGV_I64(a) == GET_TCGV_I64(b))
 #define TCGV_EQUAL_PTR(a, b) (GET_TCGV_PTR(a) == GET_TCGV_PTR(b))
diff --git a/tests/.gitignore b/tests/.gitignore
index c71c11020..e2e495733 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -14,11 +14,14 @@ test-int128
 test-iov
 test-mul64
 test-opts-visitor
+test-qapi-event.[ch]
 test-qapi-types.[ch]
 test-qapi-visit.[ch]
 test-qdev-global-props
+test-qemu-opts
 test-qmp-commands
 test-qmp-commands.h
+test-qmp-event
 test-qmp-input-strict
 test-qmp-input-visitor
 test-qmp-marshal.c
diff --git a/tests/Makefile b/tests/Makefile
index 4b2e1bbea..16f0e4c80 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -58,7 +58,7 @@ check-unit-y += tests/test-int128$(EXESUF)
 # all code tested by test-int128 is inside int128.h
 gcov-files-test-int128-y =
 check-unit-y += tests/test-bitops$(EXESUF)
-check-unit-y += tests/test-qdev-global-props$(EXESUF)
+check-unit-$(CONFIG_HAS_GLIB_SUBPROCESS_TESTS) += tests/test-qdev-global-props$(EXESUF)
 check-unit-y += tests/check-qom-interface$(EXESUF)
 gcov-files-check-qom-interface-y = qom/object.c
 check-unit-$(CONFIG_POSIX) += tests/test-vmstate$(EXESUF)
@@ -132,6 +132,7 @@ check-qtest-i386-y = tests/endianness-test$(EXESUF)
 check-qtest-i386-y += tests/fdc-test$(EXESUF)
 gcov-files-i386-y = hw/block/fdc.c
 check-qtest-i386-y += tests/ide-test$(EXESUF)
+check-qtest-i386-y += tests/ahci-test$(EXESUF)
 check-qtest-i386-y += tests/hd-geo-test$(EXESUF)
 gcov-files-i386-y += hw/block/hd-geometry.c
 check-qtest-i386-y += tests/boot-order-test$(EXESUF)
@@ -139,8 +140,7 @@ check-qtest-i386-y += tests/bios-tables-test$(EXESUF)
 check-qtest-i386-y += tests/rtc-test$(EXESUF)
 check-qtest-i386-y += tests/i440fx-test$(EXESUF)
 check-qtest-i386-y += tests/fw_cfg-test$(EXESUF)
-check-qtest-i386-y += tests/blockdev-test$(EXESUF)
-check-qtest-i386-y += tests/qdev-monitor-test$(EXESUF)
+check-qtest-i386-y += tests/drive_del-test$(EXESUF)
 check-qtest-i386-y += tests/wdt_ib700-test$(EXESUF)
 gcov-files-i386-y += hw/watchdog/watchdog.c hw/watchdog/wdt_ib700.c
 check-qtest-i386-y += $(check-qtest-pci-y)
@@ -155,11 +155,16 @@ check-qtest-i386-y += tests/i82801b11-test$(EXESUF)
 gcov-files-i386-y += hw/pci-bridge/i82801b11.c
 check-qtest-i386-y += tests/ioh3420-test$(EXESUF)
 gcov-files-i386-y += hw/pci-bridge/ioh3420.c
+check-qtest-i386-y += tests/usb-hcd-ohci-test$(EXESUF)
+gcov-files-i386-y += hw/usb/hcd-ohci.c
+check-qtest-i386-y += tests/usb-hcd-uhci-test$(EXESUF)
+gcov-files-i386-y += hw/usb/hcd-uhci.c
 check-qtest-i386-y += tests/usb-hcd-ehci-test$(EXESUF)
 gcov-files-i386-y += hw/usb/hcd-ehci.c
-gcov-files-i386-y += hw/usb/hcd-uhci.c
 gcov-files-i386-y += hw/usb/dev-hid.c
 gcov-files-i386-y += hw/usb/dev-storage.c
+check-qtest-i386-y += tests/usb-hcd-xhci-test$(EXESUF)
+gcov-files-i386-y += hw/usb/hcd-xhci.c
 check-qtest-i386-$(CONFIG_LINUX) += tests/vhost-user-test$(EXESUF)
 check-qtest-x86_64-y = $(check-qtest-i386-y)
 gcov-files-i386-y += i386-softmmu/hw/timer/mc146818rtc.c
@@ -187,26 +192,27 @@ check-qtest-xtensaeb-y = $(check-qtest-xtensa-y)
 
 # qom-test works for all sysemu architectures:
 $(foreach target,$(SYSEMU_TARGET_LIST), \
-    $(eval check-qtest-$(target)-y += tests/qom-test$(EXESUF)))
+	$(if $(findstring tests/qom-test$(EXESUF), $(check-qtest-$(target)-y)),, \
+		$(eval check-qtest-$(target)-y += tests/qom-test$(EXESUF))))
 
 check-qapi-schema-y := $(addprefix tests/qapi-schema/, \
-        comments.json empty.json funny-char.json indented-expr.json \
-        missing-colon.json missing-comma-list.json \
-        missing-comma-object.json non-objects.json \
-        qapi-schema-test.json quoted-structural-chars.json \
-        trailing-comma-list.json trailing-comma-object.json \
-        unclosed-list.json unclosed-object.json unclosed-string.json \
-        duplicate-key.json union-invalid-base.json flat-union-no-base.json \
-        flat-union-invalid-discriminator.json \
-        flat-union-invalid-branch-key.json flat-union-reverse-define.json \
-        flat-union-string-discriminator.json \
-        include-simple.json include-relpath.json include-format-err.json \
-        include-non-file.json include-no-file.json include-before-err.json \
-        include-nested-err.json include-self-cycle.json include-cycle.json \
-        include-repetition.json event-nest-struct.json)
+	comments.json empty.json funny-char.json indented-expr.json \
+	missing-colon.json missing-comma-list.json \
+	missing-comma-object.json non-objects.json \
+	qapi-schema-test.json quoted-structural-chars.json \
+	trailing-comma-list.json trailing-comma-object.json \
+	unclosed-list.json unclosed-object.json unclosed-string.json \
+	duplicate-key.json union-invalid-base.json flat-union-no-base.json \
+	flat-union-invalid-discriminator.json \
+	flat-union-invalid-branch-key.json flat-union-reverse-define.json \
+	flat-union-string-discriminator.json \
+	include-simple.json include-relpath.json include-format-err.json \
+	include-non-file.json include-no-file.json include-before-err.json \
+	include-nested-err.json include-self-cycle.json include-cycle.json \
+	include-repetition.json event-nest-struct.json)
 
 GENERATED_HEADERS += tests/test-qapi-types.h tests/test-qapi-visit.h \
-                     tests/test-qmp-commands.h tests/test-qapi-event.h
+		     tests/test-qmp-commands.h tests/test-qapi-event.h
 
 test-obj-y = tests/check-qint.o tests/check-qstring.o tests/check-qdict.o \
 	tests/check-qlist.o tests/check-qfloat.o tests/check-qjson.o \
@@ -218,7 +224,7 @@ test-obj-y = tests/check-qint.o tests/check-qstring.o tests/check-qdict.o \
 	tests/test-opts-visitor.o tests/test-qmp-event.o
 
 test-qapi-obj-y = tests/test-qapi-visit.o tests/test-qapi-types.o \
-                  tests/test-qapi-event.o
+		  tests/test-qapi-event.o
 
 $(test-obj-y): QEMU_INCLUDES += -Itests
 QEMU_CFLAGS += -I$(SRC_PATH)/tests
@@ -252,8 +258,8 @@ tests/test-qdev-global-props$(EXESUF): tests/test-qdev-global-props.o \
 	$(test-qapi-obj-y) \
 	libqemuutil.a libqemustub.a
 tests/test-vmstate$(EXESUF): tests/test-vmstate.o \
-	vmstate.o qemu-file.o \
-	libqemuutil.a
+	vmstate.o qemu-file.o qemu-file-unix.o \
+	libqemuutil.a libqemustub.a
 
 tests/test-qapi-types.c tests/test-qapi-types.h :\
 $(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-types.py
@@ -282,7 +288,7 @@ tests/test-qmp-event$(EXESUF): tests/test-qmp-event.o $(test-qapi-obj-y) libqemu
 tests/test-qmp-output-visitor$(EXESUF): tests/test-qmp-output-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
 tests/test-qmp-input-visitor$(EXESUF): tests/test-qmp-input-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
 tests/test-qmp-input-strict$(EXESUF): tests/test-qmp-input-strict.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-qmp-commands$(EXESUF): tests/test-qmp-commands.o tests/test-qmp-marshal.o $(test-qapi-obj-y) qapi-types.o qapi-visit.o libqemuutil.a libqemustub.a
+tests/test-qmp-commands$(EXESUF): tests/test-qmp-commands.o tests/test-qmp-marshal.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
 tests/test-visitor-serialization$(EXESUF): tests/test-visitor-serialization.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
 tests/test-opts-visitor$(EXESUF): tests/test-opts-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
 
@@ -294,6 +300,8 @@ libqos-obj-y += tests/libqos/i2c.o
 libqos-pc-obj-y = $(libqos-obj-y) tests/libqos/pci-pc.o
 libqos-pc-obj-y += tests/libqos/malloc-pc.o
 libqos-omap-obj-y = $(libqos-obj-y) tests/libqos/i2c-omap.o
+libqos-virtio-obj-y = $(libqos-obj-y) $(libqos-pc-obj-y) tests/libqos/virtio.o tests/libqos/virtio-pci.o
+libqos-usb-obj-y = $(libqos-pc-obj-y) tests/libqos/usb.o
 
 tests/rtc-test$(EXESUF): tests/rtc-test.o
 tests/m48t59-test$(EXESUF): tests/m48t59-test.o
@@ -301,6 +309,7 @@ tests/endianness-test$(EXESUF): tests/endianness-test.o
 tests/spapr-phb-test$(EXESUF): tests/spapr-phb-test.o $(libqos-obj-y)
 tests/fdc-test$(EXESUF): tests/fdc-test.o
 tests/ide-test$(EXESUF): tests/ide-test.o $(libqos-pc-obj-y)
+tests/ahci-test$(EXESUF): tests/ahci-test.o $(libqos-pc-obj-y)
 tests/hd-geo-test$(EXESUF): tests/hd-geo-test.o
 tests/boot-order-test$(EXESUF): tests/boot-order-test.o $(libqos-obj-y)
 tests/bios-tables-test$(EXESUF): tests/bios-tables-test.o $(libqos-obj-y)
@@ -315,9 +324,9 @@ tests/vmxnet3-test$(EXESUF): tests/vmxnet3-test.o
 tests/ne2000-test$(EXESUF): tests/ne2000-test.o
 tests/wdt_ib700-test$(EXESUF): tests/wdt_ib700-test.o
 tests/virtio-balloon-test$(EXESUF): tests/virtio-balloon-test.o
-tests/virtio-blk-test$(EXESUF): tests/virtio-blk-test.o
-tests/virtio-net-test$(EXESUF): tests/virtio-net-test.o
-tests/virtio-rng-test$(EXESUF): tests/virtio-rng-test.o
+tests/virtio-blk-test$(EXESUF): tests/virtio-blk-test.o $(libqos-virtio-obj-y)
+tests/virtio-net-test$(EXESUF): tests/virtio-net-test.o $(libqos-pc-obj-y)
+tests/virtio-rng-test$(EXESUF): tests/virtio-rng-test.o $(libqos-pc-obj-y)
 tests/virtio-scsi-test$(EXESUF): tests/virtio-scsi-test.o
 tests/virtio-9p-test$(EXESUF): tests/virtio-9p-test.o
 tests/virtio-serial-test$(EXESUF): tests/virtio-serial-test.o
@@ -326,7 +335,7 @@ tests/tpci200-test$(EXESUF): tests/tpci200-test.o
 tests/display-vga-test$(EXESUF): tests/display-vga-test.o
 tests/ipoctal232-test$(EXESUF): tests/ipoctal232-test.o
 tests/qom-test$(EXESUF): tests/qom-test.o
-tests/blockdev-test$(EXESUF): tests/blockdev-test.o $(libqos-pc-obj-y)
+tests/drive_del-test$(EXESUF): tests/drive_del-test.o $(libqos-pc-obj-y)
 tests/qdev-monitor-test$(EXESUF): tests/qdev-monitor-test.o $(libqos-pc-obj-y)
 tests/nvme-test$(EXESUF): tests/nvme-test.o
 tests/pvpanic-test$(EXESUF): tests/pvpanic-test.o
@@ -335,7 +344,10 @@ tests/ac97-test$(EXESUF): tests/ac97-test.o
 tests/es1370-test$(EXESUF): tests/es1370-test.o
 tests/intel-hda-test$(EXESUF): tests/intel-hda-test.o
 tests/ioh3420-test$(EXESUF): tests/ioh3420-test.o
-tests/usb-hcd-ehci-test$(EXESUF): tests/usb-hcd-ehci-test.o $(libqos-pc-obj-y)
+tests/usb-hcd-ohci-test$(EXESUF): tests/usb-hcd-ohci-test.o $(libqos-usb-obj-y)
+tests/usb-hcd-uhci-test$(EXESUF): tests/usb-hcd-uhci-test.o $(libqos-usb-obj-y)
+tests/usb-hcd-ehci-test$(EXESUF): tests/usb-hcd-ehci-test.o $(libqos-usb-obj-y)
+tests/usb-hcd-xhci-test$(EXESUF): tests/usb-hcd-xhci-test.o $(libqos-usb-obj-y)
 tests/vhost-user-test$(EXESUF): tests/vhost-user-test.o qemu-char.o qemu-timer.o $(qtest-obj-y)
 tests/qemu-iotests/socket_scm_helper$(EXESUF): tests/qemu-iotests/socket_scm_helper.o
 tests/test-qemu-opts$(EXESUF): tests/test-qemu-opts.o libqemuutil.a libqemustub.a
diff --git a/tests/acpi-test-data/pc/DSDT b/tests/acpi-test-data/pc/DSDT
index d37ec3445..ee9cc6781 100644
--- a/tests/acpi-test-data/pc/DSDT
+++ b/tests/acpi-test-data/pc/DSDT
diff --git a/tests/acpi-test-data/pc/SSDT b/tests/acpi-test-data/pc/SSDT
index eb2d8b698..558e4c85b 100644
--- a/tests/acpi-test-data/pc/SSDT
+++ b/tests/acpi-test-data/pc/SSDT
diff --git a/tests/acpi-test-data/q35/DSDT b/tests/acpi-test-data/q35/DSDT
index 2d2bc4ada..ef0c75f42 100644
--- a/tests/acpi-test-data/q35/DSDT
+++ b/tests/acpi-test-data/q35/DSDT
diff --git a/tests/acpi-test-data/q35/SSDT b/tests/acpi-test-data/q35/SSDT
index 778b79bf4..4e4551018 100644
--- a/tests/acpi-test-data/q35/SSDT
+++ b/tests/acpi-test-data/q35/SSDT
diff --git a/tests/acpi-test-data/rebuild-expected-aml.sh b/tests/acpi-test-data/rebuild-expected-aml.sh
index ab9849888..11bf74391 100755
--- a/tests/acpi-test-data/rebuild-expected-aml.sh
+++ b/tests/acpi-test-data/rebuild-expected-aml.sh
@@ -23,13 +23,13 @@ else
     exit 1;
 fi
 
-if [ ! -e "tests/acpi-test" ]; then
-    echo "Test: acpi-test is required! Run make check before this script."
+if [ ! -e "tests/bios-tables-test" ]; then
+    echo "Test: bios-tables-test is required! Run make check before this script."
     echo "Run this script from the build directory."
     exit 1;
 fi
 
-TEST_ACPI_REBUILD_AML=y QTEST_QEMU_BINARY=$qemu tests/acpi-test
+TEST_ACPI_REBUILD_AML=y QTEST_QEMU_BINARY=$qemu tests/bios-tables-test
 
 echo "The files were rebuilt and can be added to git."
 echo "However, if new files were created, please copy them manually" \
diff --git a/tests/ahci-test.c b/tests/ahci-test.c
new file mode 100644
index 000000000..4c77ebec7
--- /dev/null
+++ b/tests/ahci-test.c
@@ -0,0 +1,1561 @@
+/*
+ * AHCI test cases
+ *
+ * Copyright (c) 2014 John Snow <jsnow@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <stdio.h>
+#include <getopt.h>
+#include <glib.h>
+
+#include "libqtest.h"
+#include "libqos/pci-pc.h"
+#include "libqos/malloc-pc.h"
+
+#include "qemu-common.h"
+#include "qemu/host-utils.h"
+
+#include "hw/pci/pci_ids.h"
+#include "hw/pci/pci_regs.h"
+
+/* Test-specific defines. */
+#define TEST_IMAGE_SIZE    (64 * 1024 * 1024)
+
+/*** Supplementary PCI Config Space IDs & Masks ***/
+#define PCI_DEVICE_ID_INTEL_Q35_AHCI   (0x2922)
+#define PCI_MSI_FLAGS_RESERVED         (0xFF00)
+#define PCI_PM_CTRL_RESERVED             (0xFC)
+#define PCI_BCC(REG32)          ((REG32) >> 24)
+#define PCI_PI(REG32)   (((REG32) >> 8) & 0xFF)
+#define PCI_SCC(REG32) (((REG32) >> 16) & 0xFF)
+
+/*** Recognized AHCI Device Types ***/
+#define AHCI_INTEL_ICH9 (PCI_DEVICE_ID_INTEL_Q35_AHCI << 16 | \
+                         PCI_VENDOR_ID_INTEL)
+
+/*** AHCI/HBA Register Offsets and Bitmasks ***/
+#define AHCI_CAP                          (0)
+#define AHCI_CAP_NP                    (0x1F)
+#define AHCI_CAP_SXS                   (0x20)
+#define AHCI_CAP_EMS                   (0x40)
+#define AHCI_CAP_CCCS                  (0x80)
+#define AHCI_CAP_NCS                 (0x1F00)
+#define AHCI_CAP_PSC                 (0x2000)
+#define AHCI_CAP_SSC                 (0x4000)
+#define AHCI_CAP_PMD                 (0x8000)
+#define AHCI_CAP_FBSS               (0x10000)
+#define AHCI_CAP_SPM                (0x20000)
+#define AHCI_CAP_SAM                (0x40000)
+#define AHCI_CAP_RESERVED           (0x80000)
+#define AHCI_CAP_ISS               (0xF00000)
+#define AHCI_CAP_SCLO             (0x1000000)
+#define AHCI_CAP_SAL              (0x2000000)
+#define AHCI_CAP_SALP             (0x4000000)
+#define AHCI_CAP_SSS              (0x8000000)
+#define AHCI_CAP_SMPS            (0x10000000)
+#define AHCI_CAP_SSNTF           (0x20000000)
+#define AHCI_CAP_SNCQ            (0x40000000)
+#define AHCI_CAP_S64A            (0x80000000)
+
+#define AHCI_GHC                          (1)
+#define AHCI_GHC_HR                    (0x01)
+#define AHCI_GHC_IE                    (0x02)
+#define AHCI_GHC_MRSM                  (0x04)
+#define AHCI_GHC_RESERVED        (0x7FFFFFF8)
+#define AHCI_GHC_AE              (0x80000000)
+
+#define AHCI_IS                           (2)
+#define AHCI_PI                           (3)
+#define AHCI_VS                           (4)
+
+#define AHCI_CCCCTL                       (5)
+#define AHCI_CCCCTL_EN                 (0x01)
+#define AHCI_CCCCTL_RESERVED           (0x06)
+#define AHCI_CCCCTL_CC               (0xFF00)
+#define AHCI_CCCCTL_TV           (0xFFFF0000)
+
+#define AHCI_CCCPORTS                     (6)
+#define AHCI_EMLOC                        (7)
+
+#define AHCI_EMCTL                        (8)
+#define AHCI_EMCTL_STSMR               (0x01)
+#define AHCI_EMCTL_CTLTM              (0x100)
+#define AHCI_EMCTL_CTLRST             (0x200)
+#define AHCI_EMCTL_RESERVED      (0xF0F0FCFE)
+
+#define AHCI_CAP2                         (9)
+#define AHCI_CAP2_BOH                  (0x01)
+#define AHCI_CAP2_NVMP                 (0x02)
+#define AHCI_CAP2_APST                 (0x04)
+#define AHCI_CAP2_RESERVED       (0xFFFFFFF8)
+
+#define AHCI_BOHC                        (10)
+#define AHCI_RESERVED                    (11)
+#define AHCI_NVMHCI                      (24)
+#define AHCI_VENDOR                      (40)
+#define AHCI_PORTS                       (64)
+
+/*** Port Memory Offsets & Bitmasks ***/
+#define AHCI_PX_CLB                       (0)
+#define AHCI_PX_CLB_RESERVED          (0x1FF)
+
+#define AHCI_PX_CLBU                      (1)
+
+#define AHCI_PX_FB                        (2)
+#define AHCI_PX_FB_RESERVED            (0xFF)
+
+#define AHCI_PX_FBU                       (3)
+
+#define AHCI_PX_IS                        (4)
+#define AHCI_PX_IS_DHRS                 (0x1)
+#define AHCI_PX_IS_PSS                  (0x2)
+#define AHCI_PX_IS_DSS                  (0x4)
+#define AHCI_PX_IS_SDBS                 (0x8)
+#define AHCI_PX_IS_UFS                 (0x10)
+#define AHCI_PX_IS_DPS                 (0x20)
+#define AHCI_PX_IS_PCS                 (0x40)
+#define AHCI_PX_IS_DMPS                (0x80)
+#define AHCI_PX_IS_RESERVED       (0x23FFF00)
+#define AHCI_PX_IS_PRCS            (0x400000)
+#define AHCI_PX_IS_IPMS            (0x800000)
+#define AHCI_PX_IS_OFS            (0x1000000)
+#define AHCI_PX_IS_INFS           (0x4000000)
+#define AHCI_PX_IS_IFS            (0x8000000)
+#define AHCI_PX_IS_HBDS          (0x10000000)
+#define AHCI_PX_IS_HBFS          (0x20000000)
+#define AHCI_PX_IS_TFES          (0x40000000)
+#define AHCI_PX_IS_CPDS          (0x80000000)
+
+#define AHCI_PX_IE                        (5)
+#define AHCI_PX_IE_DHRE                 (0x1)
+#define AHCI_PX_IE_PSE                  (0x2)
+#define AHCI_PX_IE_DSE                  (0x4)
+#define AHCI_PX_IE_SDBE                 (0x8)
+#define AHCI_PX_IE_UFE                 (0x10)
+#define AHCI_PX_IE_DPE                 (0x20)
+#define AHCI_PX_IE_PCE                 (0x40)
+#define AHCI_PX_IE_DMPE                (0x80)
+#define AHCI_PX_IE_RESERVED       (0x23FFF00)
+#define AHCI_PX_IE_PRCE            (0x400000)
+#define AHCI_PX_IE_IPME            (0x800000)
+#define AHCI_PX_IE_OFE            (0x1000000)
+#define AHCI_PX_IE_INFE           (0x4000000)
+#define AHCI_PX_IE_IFE            (0x8000000)
+#define AHCI_PX_IE_HBDE          (0x10000000)
+#define AHCI_PX_IE_HBFE          (0x20000000)
+#define AHCI_PX_IE_TFEE          (0x40000000)
+#define AHCI_PX_IE_CPDE          (0x80000000)
+
+#define AHCI_PX_CMD                       (6)
+#define AHCI_PX_CMD_ST                  (0x1)
+#define AHCI_PX_CMD_SUD                 (0x2)
+#define AHCI_PX_CMD_POD                 (0x4)
+#define AHCI_PX_CMD_CLO                 (0x8)
+#define AHCI_PX_CMD_FRE                (0x10)
+#define AHCI_PX_CMD_RESERVED           (0xE0)
+#define AHCI_PX_CMD_CCS              (0x1F00)
+#define AHCI_PX_CMD_MPSS             (0x2000)
+#define AHCI_PX_CMD_FR               (0x4000)
+#define AHCI_PX_CMD_CR               (0x8000)
+#define AHCI_PX_CMD_CPS             (0x10000)
+#define AHCI_PX_CMD_PMA             (0x20000)
+#define AHCI_PX_CMD_HPCP            (0x40000)
+#define AHCI_PX_CMD_MPSP            (0x80000)
+#define AHCI_PX_CMD_CPD            (0x100000)
+#define AHCI_PX_CMD_ESP            (0x200000)
+#define AHCI_PX_CMD_FBSCP          (0x400000)
+#define AHCI_PX_CMD_APSTE          (0x800000)
+#define AHCI_PX_CMD_ATAPI         (0x1000000)
+#define AHCI_PX_CMD_DLAE          (0x2000000)
+#define AHCI_PX_CMD_ALPE          (0x4000000)
+#define AHCI_PX_CMD_ASP           (0x8000000)
+#define AHCI_PX_CMD_ICC          (0xF0000000)
+
+#define AHCI_PX_RES1                      (7)
+
+#define AHCI_PX_TFD                       (8)
+#define AHCI_PX_TFD_STS                (0xFF)
+#define AHCI_PX_TFD_STS_ERR            (0x01)
+#define AHCI_PX_TFD_STS_CS1            (0x06)
+#define AHCI_PX_TFD_STS_DRQ            (0x08)
+#define AHCI_PX_TFD_STS_CS2            (0x70)
+#define AHCI_PX_TFD_STS_BSY            (0x80)
+#define AHCI_PX_TFD_ERR              (0xFF00)
+#define AHCI_PX_TFD_RESERVED     (0xFFFF0000)
+
+#define AHCI_PX_SIG                       (9)
+#define AHCI_PX_SIG_SECTOR_COUNT       (0xFF)
+#define AHCI_PX_SIG_LBA_LOW          (0xFF00)
+#define AHCI_PX_SIG_LBA_MID        (0xFF0000)
+#define AHCI_PX_SIG_LBA_HIGH     (0xFF000000)
+
+#define AHCI_PX_SSTS                     (10)
+#define AHCI_PX_SSTS_DET               (0x0F)
+#define AHCI_PX_SSTS_SPD               (0xF0)
+#define AHCI_PX_SSTS_IPM              (0xF00)
+#define AHCI_PX_SSTS_RESERVED    (0xFFFFF000)
+#define SSTS_DET_NO_DEVICE             (0x00)
+#define SSTS_DET_PRESENT               (0x01)
+#define SSTS_DET_ESTABLISHED           (0x03)
+#define SSTS_DET_OFFLINE               (0x04)
+
+#define AHCI_PX_SCTL                     (11)
+
+#define AHCI_PX_SERR                     (12)
+#define AHCI_PX_SERR_ERR             (0xFFFF)
+#define AHCI_PX_SERR_DIAG        (0xFFFF0000)
+#define AHCI_PX_SERR_DIAG_X      (0x04000000)
+
+#define AHCI_PX_SACT                     (13)
+#define AHCI_PX_CI                       (14)
+#define AHCI_PX_SNTF                     (15)
+
+#define AHCI_PX_FBS                      (16)
+#define AHCI_PX_FBS_EN                  (0x1)
+#define AHCI_PX_FBS_DEC                 (0x2)
+#define AHCI_PX_FBS_SDE                 (0x4)
+#define AHCI_PX_FBS_DEV               (0xF00)
+#define AHCI_PX_FBS_ADO              (0xF000)
+#define AHCI_PX_FBS_DWE             (0xF0000)
+#define AHCI_PX_FBS_RESERVED     (0xFFF000F8)
+
+#define AHCI_PX_RES2                     (17)
+#define AHCI_PX_VS                       (28)
+
+#define HBA_DATA_REGION_SIZE            (256)
+#define HBA_PORT_DATA_SIZE              (128)
+#define HBA_PORT_NUM_REG (HBA_PORT_DATA_SIZE/4)
+
+#define AHCI_VERSION_0_95        (0x00000905)
+#define AHCI_VERSION_1_0         (0x00010000)
+#define AHCI_VERSION_1_1         (0x00010100)
+#define AHCI_VERSION_1_2         (0x00010200)
+#define AHCI_VERSION_1_3         (0x00010300)
+
+/*** Structures ***/
+
+/**
+ * Generic FIS structure.
+ */
+typedef struct FIS {
+    uint8_t fis_type;
+    uint8_t flags;
+    char data[0];
+} __attribute__((__packed__)) FIS;
+
+/**
+ * Register device-to-host FIS structure.
+ */
+typedef struct RegD2HFIS {
+    /* DW0 */
+    uint8_t fis_type;
+    uint8_t flags;
+    uint8_t status;
+    uint8_t error;
+    /* DW1 */
+    uint8_t lba_low;
+    uint8_t lba_mid;
+    uint8_t lba_high;
+    uint8_t device;
+    /* DW2 */
+    uint8_t lba3;
+    uint8_t lba4;
+    uint8_t lba5;
+    uint8_t res1;
+    /* DW3 */
+    uint16_t count;
+    uint8_t res2;
+    uint8_t res3;
+    /* DW4 */
+    uint16_t res4;
+    uint16_t res5;
+} __attribute__((__packed__)) RegD2HFIS;
+
+/**
+ * Register host-to-device FIS structure.
+ */
+typedef struct RegH2DFIS {
+    /* DW0 */
+    uint8_t fis_type;
+    uint8_t flags;
+    uint8_t command;
+    uint8_t feature_low;
+    /* DW1 */
+    uint8_t lba_low;
+    uint8_t lba_mid;
+    uint8_t lba_high;
+    uint8_t device;
+    /* DW2 */
+    uint8_t lba3;
+    uint8_t lba4;
+    uint8_t lba5;
+    uint8_t feature_high;
+    /* DW3 */
+    uint16_t count;
+    uint8_t icc;
+    uint8_t control;
+    /* DW4 */
+    uint32_t aux;
+} __attribute__((__packed__)) RegH2DFIS;
+
+/**
+ * Command List entry structure.
+ * The command list contains between 1-32 of these structures.
+ */
+typedef struct AHCICommand {
+    uint8_t b1;
+    uint8_t b2;
+    uint16_t prdtl; /* Phys Region Desc. Table Length */
+    uint32_t prdbc; /* Phys Region Desc. Byte Count */
+    uint32_t ctba;  /* Command Table Descriptor Base Address */
+    uint32_t ctbau; /*                                    '' Upper */
+    uint32_t res[4];
+} __attribute__((__packed__)) AHCICommand;
+
+/**
+ * Physical Region Descriptor; pointed to by the Command List Header,
+ * struct ahci_command.
+ */
+typedef struct PRD {
+    uint32_t dba;  /* Data Base Address */
+    uint32_t dbau; /* Data Base Address Upper */
+    uint32_t res;  /* Reserved */
+    uint32_t dbc;  /* Data Byte Count (0-indexed) & Interrupt Flag (bit 2^31) */
+} PRD;
+
+typedef struct HBACap {
+    uint32_t cap;
+    uint32_t cap2;
+} HBACap;
+
+/*** Globals ***/
+static QGuestAllocator *guest_malloc;
+static QPCIBus *pcibus;
+static uint64_t barsize;
+static char tmp_path[] = "/tmp/qtest.XXXXXX";
+static bool ahci_pedantic;
+static uint32_t ahci_fingerprint;
+
+/*** Macro Utilities ***/
+#define BITANY(data, mask) (((data) & (mask)) != 0)
+#define BITSET(data, mask) (((data) & (mask)) == (mask))
+#define BITCLR(data, mask) (((data) & (mask)) == 0)
+#define ASSERT_BIT_SET(data, mask) g_assert_cmphex((data) & (mask), ==, (mask))
+#define ASSERT_BIT_CLEAR(data, mask) g_assert_cmphex((data) & (mask), ==, 0)
+
+/*** IO macros for the AHCI memory registers. ***/
+#define AHCI_READ(OFST) qpci_io_readl(ahci, hba_base + (OFST))
+#define AHCI_WRITE(OFST, VAL) qpci_io_writel(ahci, hba_base + (OFST), (VAL))
+#define AHCI_RREG(regno)      AHCI_READ(4 * (regno))
+#define AHCI_WREG(regno, val) AHCI_WRITE(4 * (regno), (val))
+#define AHCI_SET(regno, mask) AHCI_WREG((regno), AHCI_RREG(regno) | (mask))
+#define AHCI_CLR(regno, mask) AHCI_WREG((regno), AHCI_RREG(regno) & ~(mask))
+
+/*** IO macros for port-specific offsets inside of AHCI memory. ***/
+#define PX_OFST(port, regno) (HBA_PORT_NUM_REG * (port) + AHCI_PORTS + (regno))
+#define PX_RREG(port, regno)      AHCI_RREG(PX_OFST((port), (regno)))
+#define PX_WREG(port, regno, val) AHCI_WREG(PX_OFST((port), (regno)), (val))
+#define PX_SET(port, reg, mask)   PX_WREG((port), (reg),                \
+                                          PX_RREG((port), (reg)) | (mask));
+#define PX_CLR(port, reg, mask)   PX_WREG((port), (reg),                \
+                                          PX_RREG((port), (reg)) & ~(mask));
+
+/* For calculating how big the PRD table needs to be: */
+#define CMD_TBL_SIZ(n) ((0x80 + ((n) * sizeof(PRD)) + 0x7F) & ~0x7F)
+
+
+/*** Function Declarations ***/
+static QPCIDevice *get_ahci_device(void);
+static QPCIDevice *start_ahci_device(QPCIDevice *dev, void **hba_base);
+static void free_ahci_device(QPCIDevice *dev);
+static void ahci_test_port_spec(QPCIDevice *ahci, void *hba_base,
+                                HBACap *hcap, uint8_t port);
+static void ahci_test_pci_spec(QPCIDevice *ahci);
+static void ahci_test_pci_caps(QPCIDevice *ahci, uint16_t header,
+                               uint8_t offset);
+static void ahci_test_satacap(QPCIDevice *ahci, uint8_t offset);
+static void ahci_test_msicap(QPCIDevice *ahci, uint8_t offset);
+static void ahci_test_pmcap(QPCIDevice *ahci, uint8_t offset);
+
+/*** Utilities ***/
+
+static void string_bswap16(uint16_t *s, size_t bytes)
+{
+    g_assert_cmphex((bytes & 1), ==, 0);
+    bytes /= 2;
+
+    while (bytes--) {
+        *s = bswap16(*s);
+        s++;
+    }
+}
+
+/**
+ * Locate, verify, and return a handle to the AHCI device.
+ */
+static QPCIDevice *get_ahci_device(void)
+{
+    QPCIDevice *ahci;
+
+    pcibus = qpci_init_pc();
+
+    /* Find the AHCI PCI device and verify it's the right one. */
+    ahci = qpci_device_find(pcibus, QPCI_DEVFN(0x1F, 0x02));
+    g_assert(ahci != NULL);
+
+    ahci_fingerprint = qpci_config_readl(ahci, PCI_VENDOR_ID);
+
+    switch (ahci_fingerprint) {
+    case AHCI_INTEL_ICH9:
+        break;
+    default:
+        /* Unknown device. */
+        g_assert_not_reached();
+    }
+
+    return ahci;
+}
+
+static void free_ahci_device(QPCIDevice *ahci)
+{
+    /* libqos doesn't have a function for this, so free it manually */
+    g_free(ahci);
+
+    if (pcibus) {
+        qpci_free_pc(pcibus);
+        pcibus = NULL;
+    }
+
+    /* Clear our cached barsize information. */
+    barsize = 0;
+}
+
+/*** Test Setup & Teardown ***/
+
+/**
+ * Launch QEMU with the given command line,
+ * and then set up interrupts and our guest malloc interface.
+ */
+static void qtest_boot(const char *cmdline_fmt, ...)
+{
+    va_list ap;
+    char *cmdline;
+
+    va_start(ap, cmdline_fmt);
+    cmdline = g_strdup_vprintf(cmdline_fmt, ap);
+    va_end(ap);
+
+    qtest_start(cmdline);
+    qtest_irq_intercept_in(global_qtest, "ioapic");
+    guest_malloc = pc_alloc_init();
+
+    g_free(cmdline);
+}
+
+/**
+ * Tear down the QEMU instance.
+ */
+static void qtest_shutdown(void)
+{
+    g_free(guest_malloc);
+    guest_malloc = NULL;
+    qtest_end();
+}
+
+/**
+ * Start a Q35 machine and bookmark a handle to the AHCI device.
+ */
+static QPCIDevice *ahci_boot(void)
+{
+    qtest_boot("-drive if=none,id=drive0,file=%s,cache=writeback,serial=%s"
+               " -M q35 "
+               "-device ide-hd,drive=drive0 "
+               "-global ide-hd.ver=%s",
+               tmp_path, "testdisk", "version");
+
+    /* Verify that we have an AHCI device present. */
+    return get_ahci_device();
+}
+
+/**
+ * Clean up the PCI device, then terminate the QEMU instance.
+ */
+static void ahci_shutdown(QPCIDevice *ahci)
+{
+    free_ahci_device(ahci);
+    qtest_shutdown();
+}
+
+/*** Logical Device Initialization ***/
+
+/**
+ * Start the PCI device and sanity-check default operation.
+ */
+static void ahci_pci_enable(QPCIDevice *ahci, void **hba_base)
+{
+    uint8_t reg;
+
+    start_ahci_device(ahci, hba_base);
+
+    switch (ahci_fingerprint) {
+    case AHCI_INTEL_ICH9:
+        /* ICH9 has a register at PCI 0x92 that
+         * acts as a master port enabler mask. */
+        reg = qpci_config_readb(ahci, 0x92);
+        reg |= 0x3F;
+        qpci_config_writeb(ahci, 0x92, reg);
+        /* 0...0111111b -- bit significant, ports 0-5 enabled. */
+        ASSERT_BIT_SET(qpci_config_readb(ahci, 0x92), 0x3F);
+        break;
+    }
+
+}
+
+/**
+ * Map BAR5/ABAR, and engage the PCI device.
+ */
+static QPCIDevice *start_ahci_device(QPCIDevice *ahci, void **hba_base)
+{
+    /* Map AHCI's ABAR (BAR5) */
+    *hba_base = qpci_iomap(ahci, 5, &barsize);
+
+    /* turns on pci.cmd.iose, pci.cmd.mse and pci.cmd.bme */
+    qpci_device_enable(ahci);
+
+    return ahci;
+}
+
+/**
+ * Test and initialize the AHCI's HBA memory areas.
+ * Initialize and start any ports with devices attached.
+ * Bring the HBA into the idle state.
+ */
+static void ahci_hba_enable(QPCIDevice *ahci, void *hba_base)
+{
+    /* Bits of interest in this section:
+     * GHC.AE     Global Host Control / AHCI Enable
+     * PxCMD.ST   Port Command: Start
+     * PxCMD.SUD  "Spin Up Device"
+     * PxCMD.POD  "Power On Device"
+     * PxCMD.FRE  "FIS Receive Enable"
+     * PxCMD.FR   "FIS Receive Running"
+     * PxCMD.CR   "Command List Running"
+     */
+
+    g_assert(ahci != NULL);
+    g_assert(hba_base != NULL);
+
+    uint32_t reg, ports_impl, clb, fb;
+    uint16_t i;
+    uint8_t num_cmd_slots;
+
+    g_assert(hba_base != 0);
+
+    /* Set GHC.AE to 1 */
+    AHCI_SET(AHCI_GHC, AHCI_GHC_AE);
+    reg = AHCI_RREG(AHCI_GHC);
+    ASSERT_BIT_SET(reg, AHCI_GHC_AE);
+
+    /* Read CAP.NCS, how many command slots do we have? */
+    reg = AHCI_RREG(AHCI_CAP);
+    num_cmd_slots = ((reg & AHCI_CAP_NCS) >> ctzl(AHCI_CAP_NCS)) + 1;
+    g_test_message("Number of Command Slots: %u", num_cmd_slots);
+
+    /* Determine which ports are implemented. */
+    ports_impl = AHCI_RREG(AHCI_PI);
+
+    for (i = 0; ports_impl; ports_impl >>= 1, ++i) {
+        if (!(ports_impl & 0x01)) {
+            continue;
+        }
+
+        g_test_message("Initializing port %u", i);
+
+        reg = PX_RREG(i, AHCI_PX_CMD);
+        if (BITCLR(reg, AHCI_PX_CMD_ST | AHCI_PX_CMD_CR |
+                   AHCI_PX_CMD_FRE | AHCI_PX_CMD_FR)) {
+            g_test_message("port is idle");
+        } else {
+            g_test_message("port needs to be idled");
+            PX_CLR(i, AHCI_PX_CMD, (AHCI_PX_CMD_ST | AHCI_PX_CMD_FRE));
+            /* The port has 500ms to disengage. */
+            usleep(500000);
+            reg = PX_RREG(i, AHCI_PX_CMD);
+            ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_CR);
+            ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_FR);
+            g_test_message("port is now idle");
+            /* The spec does allow for possibly needing a PORT RESET
+             * or HBA reset if we fail to idle the port. */
+        }
+
+        /* Allocate Memory for the Command List Buffer & FIS Buffer */
+        /* PxCLB space ... 0x20 per command, as in 4.2.2 p 36 */
+        clb = guest_alloc(guest_malloc, num_cmd_slots * 0x20);
+        g_test_message("CLB: 0x%08x", clb);
+        PX_WREG(i, AHCI_PX_CLB, clb);
+        g_assert_cmphex(clb, ==, PX_RREG(i, AHCI_PX_CLB));
+
+        /* PxFB space ... 0x100, as in 4.2.1 p 35 */
+        fb = guest_alloc(guest_malloc, 0x100);
+        g_test_message("FB: 0x%08x", fb);
+        PX_WREG(i, AHCI_PX_FB, fb);
+        g_assert_cmphex(fb, ==, PX_RREG(i, AHCI_PX_FB));
+
+        /* Clear PxSERR, PxIS, then IS.IPS[x] by writing '1's. */
+        PX_WREG(i, AHCI_PX_SERR, 0xFFFFFFFF);
+        PX_WREG(i, AHCI_PX_IS, 0xFFFFFFFF);
+        AHCI_WREG(AHCI_IS, (1 << i));
+
+        /* Verify Interrupts Cleared */
+        reg = PX_RREG(i, AHCI_PX_SERR);
+        g_assert_cmphex(reg, ==, 0);
+
+        reg = PX_RREG(i, AHCI_PX_IS);
+        g_assert_cmphex(reg, ==, 0);
+
+        reg = AHCI_RREG(AHCI_IS);
+        ASSERT_BIT_CLEAR(reg, (1 << i));
+
+        /* Enable All Interrupts: */
+        PX_WREG(i, AHCI_PX_IE, 0xFFFFFFFF);
+        reg = PX_RREG(i, AHCI_PX_IE);
+        g_assert_cmphex(reg, ==, ~((uint32_t)AHCI_PX_IE_RESERVED));
+
+        /* Enable the FIS Receive Engine. */
+        PX_SET(i, AHCI_PX_CMD, AHCI_PX_CMD_FRE);
+        reg = PX_RREG(i, AHCI_PX_CMD);
+        ASSERT_BIT_SET(reg, AHCI_PX_CMD_FR);
+
+        /* AHCI 1.3 spec: if !STS.BSY, !STS.DRQ and PxSSTS.DET indicates
+         * physical presence, a device is present and may be started. However,
+         * PxSERR.DIAG.X /may/ need to be cleared a priori. */
+        reg = PX_RREG(i, AHCI_PX_SERR);
+        if (BITSET(reg, AHCI_PX_SERR_DIAG_X)) {
+            PX_SET(i, AHCI_PX_SERR, AHCI_PX_SERR_DIAG_X);
+        }
+
+        reg = PX_RREG(i, AHCI_PX_TFD);
+        if (BITCLR(reg, AHCI_PX_TFD_STS_BSY | AHCI_PX_TFD_STS_DRQ)) {
+            reg = PX_RREG(i, AHCI_PX_SSTS);
+            if ((reg & AHCI_PX_SSTS_DET) == SSTS_DET_ESTABLISHED) {
+                /* Device Found: set PxCMD.ST := 1 */
+                PX_SET(i, AHCI_PX_CMD, AHCI_PX_CMD_ST);
+                ASSERT_BIT_SET(PX_RREG(i, AHCI_PX_CMD), AHCI_PX_CMD_CR);
+                g_test_message("Started Device %u", i);
+            } else if ((reg & AHCI_PX_SSTS_DET)) {
+                /* Device present, but in some unknown state. */
+                g_assert_not_reached();
+            }
+        }
+    }
+
+    /* Enable GHC.IE */
+    AHCI_SET(AHCI_GHC, AHCI_GHC_IE);
+    reg = AHCI_RREG(AHCI_GHC);
+    ASSERT_BIT_SET(reg, AHCI_GHC_IE);
+
+    /* TODO: The device should now be idling and waiting for commands.
+     * In the future, a small test-case to inspect the Register D2H FIS
+     * and clear the initial interrupts might be good. */
+}
+
+/*** Specification Adherence Tests ***/
+
+/**
+ * Implementation for test_pci_spec. Ensures PCI configuration space is sane.
+ */
+static void ahci_test_pci_spec(QPCIDevice *ahci)
+{
+    uint8_t datab;
+    uint16_t data;
+    uint32_t datal;
+
+    /* Most of these bits should start cleared until we turn them on. */
+    data = qpci_config_readw(ahci, PCI_COMMAND);
+    ASSERT_BIT_CLEAR(data, PCI_COMMAND_MEMORY);
+    ASSERT_BIT_CLEAR(data, PCI_COMMAND_MASTER);
+    ASSERT_BIT_CLEAR(data, PCI_COMMAND_SPECIAL);     /* Reserved */
+    ASSERT_BIT_CLEAR(data, PCI_COMMAND_VGA_PALETTE); /* Reserved */
+    ASSERT_BIT_CLEAR(data, PCI_COMMAND_PARITY);
+    ASSERT_BIT_CLEAR(data, PCI_COMMAND_WAIT);        /* Reserved */
+    ASSERT_BIT_CLEAR(data, PCI_COMMAND_SERR);
+    ASSERT_BIT_CLEAR(data, PCI_COMMAND_FAST_BACK);
+    ASSERT_BIT_CLEAR(data, PCI_COMMAND_INTX_DISABLE);
+    ASSERT_BIT_CLEAR(data, 0xF800);                  /* Reserved */
+
+    data = qpci_config_readw(ahci, PCI_STATUS);
+    ASSERT_BIT_CLEAR(data, 0x01 | 0x02 | 0x04);     /* Reserved */
+    ASSERT_BIT_CLEAR(data, PCI_STATUS_INTERRUPT);
+    ASSERT_BIT_SET(data, PCI_STATUS_CAP_LIST);      /* must be set */
+    ASSERT_BIT_CLEAR(data, PCI_STATUS_UDF);         /* Reserved */
+    ASSERT_BIT_CLEAR(data, PCI_STATUS_PARITY);
+    ASSERT_BIT_CLEAR(data, PCI_STATUS_SIG_TARGET_ABORT);
+    ASSERT_BIT_CLEAR(data, PCI_STATUS_REC_TARGET_ABORT);
+    ASSERT_BIT_CLEAR(data, PCI_STATUS_REC_MASTER_ABORT);
+    ASSERT_BIT_CLEAR(data, PCI_STATUS_SIG_SYSTEM_ERROR);
+    ASSERT_BIT_CLEAR(data, PCI_STATUS_DETECTED_PARITY);
+
+    /* RID occupies the low byte, CCs occupy the high three. */
+    datal = qpci_config_readl(ahci, PCI_CLASS_REVISION);
+    if (ahci_pedantic) {
+        /* AHCI 1.3 specifies that at-boot, the RID should reset to 0x00,
+         * Though in practice this is likely seldom true. */
+        ASSERT_BIT_CLEAR(datal, 0xFF);
+    }
+
+    /* BCC *must* equal 0x01. */
+    g_assert_cmphex(PCI_BCC(datal), ==, 0x01);
+    if (PCI_SCC(datal) == 0x01) {
+        /* IDE */
+        ASSERT_BIT_SET(0x80000000, datal);
+        ASSERT_BIT_CLEAR(0x60000000, datal);
+    } else if (PCI_SCC(datal) == 0x04) {
+        /* RAID */
+        g_assert_cmphex(PCI_PI(datal), ==, 0);
+    } else if (PCI_SCC(datal) == 0x06) {
+        /* AHCI */
+        g_assert_cmphex(PCI_PI(datal), ==, 0x01);
+    } else {
+        g_assert_not_reached();
+    }
+
+    datab = qpci_config_readb(ahci, PCI_CACHE_LINE_SIZE);
+    g_assert_cmphex(datab, ==, 0);
+
+    datab = qpci_config_readb(ahci, PCI_LATENCY_TIMER);
+    g_assert_cmphex(datab, ==, 0);
+
+    /* Only the bottom 7 bits must be off. */
+    datab = qpci_config_readb(ahci, PCI_HEADER_TYPE);
+    ASSERT_BIT_CLEAR(datab, 0x7F);
+
+    /* BIST is optional, but the low 7 bits must always start off regardless. */
+    datab = qpci_config_readb(ahci, PCI_BIST);
+    ASSERT_BIT_CLEAR(datab, 0x7F);
+
+    /* BARS 0-4 do not have a boot spec, but ABAR/BAR5 must be clean. */
+    datal = qpci_config_readl(ahci, PCI_BASE_ADDRESS_5);
+    g_assert_cmphex(datal, ==, 0);
+
+    qpci_config_writel(ahci, PCI_BASE_ADDRESS_5, 0xFFFFFFFF);
+    datal = qpci_config_readl(ahci, PCI_BASE_ADDRESS_5);
+    /* ABAR must be 32-bit, memory mapped, non-prefetchable and
+     * must be >= 512 bytes. To that end, bits 0-8 must be off. */
+    ASSERT_BIT_CLEAR(datal, 0xFF);
+
+    /* Capability list MUST be present, */
+    datal = qpci_config_readl(ahci, PCI_CAPABILITY_LIST);
+    /* But these bits are reserved. */
+    ASSERT_BIT_CLEAR(datal, ~0xFF);
+    g_assert_cmphex(datal, !=, 0);
+
+    /* Check specification adherence for capability extenstions. */
+    data = qpci_config_readw(ahci, datal);
+
+    switch (ahci_fingerprint) {
+    case AHCI_INTEL_ICH9:
+        /* Intel ICH9 Family Datasheet 14.1.19 p.550 */
+        g_assert_cmphex((data & 0xFF), ==, PCI_CAP_ID_MSI);
+        break;
+    default:
+        /* AHCI 1.3, Section 2.1.14 -- CAP must point to PMCAP. */
+        g_assert_cmphex((data & 0xFF), ==, PCI_CAP_ID_PM);
+    }
+
+    ahci_test_pci_caps(ahci, data, (uint8_t)datal);
+
+    /* Reserved. */
+    datal = qpci_config_readl(ahci, PCI_CAPABILITY_LIST + 4);
+    g_assert_cmphex(datal, ==, 0);
+
+    /* IPIN might vary, but ILINE must be off. */
+    datab = qpci_config_readb(ahci, PCI_INTERRUPT_LINE);
+    g_assert_cmphex(datab, ==, 0);
+}
+
+/**
+ * Test PCI capabilities for AHCI specification adherence.
+ */
+static void ahci_test_pci_caps(QPCIDevice *ahci, uint16_t header,
+                               uint8_t offset)
+{
+    uint8_t cid = header & 0xFF;
+    uint8_t next = header >> 8;
+
+    g_test_message("CID: %02x; next: %02x", cid, next);
+
+    switch (cid) {
+    case PCI_CAP_ID_PM:
+        ahci_test_pmcap(ahci, offset);
+        break;
+    case PCI_CAP_ID_MSI:
+        ahci_test_msicap(ahci, offset);
+        break;
+    case PCI_CAP_ID_SATA:
+        ahci_test_satacap(ahci, offset);
+        break;
+
+    default:
+        g_test_message("Unknown CAP 0x%02x", cid);
+    }
+
+    if (next) {
+        ahci_test_pci_caps(ahci, qpci_config_readw(ahci, next), next);
+    }
+}
+
+/**
+ * Test SATA PCI capabilitity for AHCI specification adherence.
+ */
+static void ahci_test_satacap(QPCIDevice *ahci, uint8_t offset)
+{
+    uint16_t dataw;
+    uint32_t datal;
+
+    g_test_message("Verifying SATACAP");
+
+    /* Assert that the SATACAP version is 1.0, And reserved bits are empty. */
+    dataw = qpci_config_readw(ahci, offset + 2);
+    g_assert_cmphex(dataw, ==, 0x10);
+
+    /* Grab the SATACR1 register. */
+    datal = qpci_config_readw(ahci, offset + 4);
+
+    switch (datal & 0x0F) {
+    case 0x04: /* BAR0 */
+    case 0x05: /* BAR1 */
+    case 0x06:
+    case 0x07:
+    case 0x08:
+    case 0x09: /* BAR5 */
+    case 0x0F: /* Immediately following SATACR1 in PCI config space. */
+        break;
+    default:
+        /* Invalid BARLOC for the Index Data Pair. */
+        g_assert_not_reached();
+    }
+
+    /* Reserved. */
+    g_assert_cmphex((datal >> 24), ==, 0x00);
+}
+
+/**
+ * Test MSI PCI capability for AHCI specification adherence.
+ */
+static void ahci_test_msicap(QPCIDevice *ahci, uint8_t offset)
+{
+    uint16_t dataw;
+    uint32_t datal;
+
+    g_test_message("Verifying MSICAP");
+
+    dataw = qpci_config_readw(ahci, offset + PCI_MSI_FLAGS);
+    ASSERT_BIT_CLEAR(dataw, PCI_MSI_FLAGS_ENABLE);
+    ASSERT_BIT_CLEAR(dataw, PCI_MSI_FLAGS_QSIZE);
+    ASSERT_BIT_CLEAR(dataw, PCI_MSI_FLAGS_RESERVED);
+
+    datal = qpci_config_readl(ahci, offset + PCI_MSI_ADDRESS_LO);
+    g_assert_cmphex(datal, ==, 0);
+
+    if (dataw & PCI_MSI_FLAGS_64BIT) {
+        g_test_message("MSICAP is 64bit");
+        datal = qpci_config_readl(ahci, offset + PCI_MSI_ADDRESS_HI);
+        g_assert_cmphex(datal, ==, 0);
+        dataw = qpci_config_readw(ahci, offset + PCI_MSI_DATA_64);
+        g_assert_cmphex(dataw, ==, 0);
+    } else {
+        g_test_message("MSICAP is 32bit");
+        dataw = qpci_config_readw(ahci, offset + PCI_MSI_DATA_32);
+        g_assert_cmphex(dataw, ==, 0);
+    }
+}
+
+/**
+ * Test Power Management PCI capability for AHCI specification adherence.
+ */
+static void ahci_test_pmcap(QPCIDevice *ahci, uint8_t offset)
+{
+    uint16_t dataw;
+
+    g_test_message("Verifying PMCAP");
+
+    dataw = qpci_config_readw(ahci, offset + PCI_PM_PMC);
+    ASSERT_BIT_CLEAR(dataw, PCI_PM_CAP_PME_CLOCK);
+    ASSERT_BIT_CLEAR(dataw, PCI_PM_CAP_RESERVED);
+    ASSERT_BIT_CLEAR(dataw, PCI_PM_CAP_D1);
+    ASSERT_BIT_CLEAR(dataw, PCI_PM_CAP_D2);
+
+    dataw = qpci_config_readw(ahci, offset + PCI_PM_CTRL);
+    ASSERT_BIT_CLEAR(dataw, PCI_PM_CTRL_STATE_MASK);
+    ASSERT_BIT_CLEAR(dataw, PCI_PM_CTRL_RESERVED);
+    ASSERT_BIT_CLEAR(dataw, PCI_PM_CTRL_DATA_SEL_MASK);
+    ASSERT_BIT_CLEAR(dataw, PCI_PM_CTRL_DATA_SCALE_MASK);
+}
+
+static void ahci_test_hba_spec(QPCIDevice *ahci, void *hba_base)
+{
+    HBACap hcap;
+    unsigned i;
+    uint32_t cap, cap2, reg;
+    uint32_t ports;
+    uint8_t nports_impl;
+    uint8_t maxports;
+
+    g_assert(ahci != 0);
+    g_assert(hba_base != 0);
+
+    /*
+     * Note that the AHCI spec does expect the BIOS to set up a few things:
+     * CAP.SSS    - Support for staggered spin-up            (t/f)
+     * CAP.SMPS   - Support for mechanical presence switches (t/f)
+     * PI         - Ports Implemented                        (1-32)
+     * PxCMD.HPCP - Hot Plug Capable Port
+     * PxCMD.MPSP - Mechanical Presence Switch Present
+     * PxCMD.CPD  - Cold Presence Detection support
+     *
+     * Additional items are touched if CAP.SSS is on, see AHCI 10.1.1 p.97:
+     * Foreach Port Implemented:
+     * -PxCMD.ST, PxCMD.CR, PxCMD.FRE, PxCMD.FR, PxSCTL.DET are 0
+     * -PxCLB/U and PxFB/U are set to valid regions in memory
+     * -PxSUD is set to 1.
+     * -PxSSTS.DET is polled for presence; if detected, we continue:
+     * -PxSERR is cleared with 1's.
+     * -If PxTFD.STS.BSY, PxTFD.STS.DRQ, and PxTFD.STS.ERR are all zero,
+     *  the device is ready.
+     */
+
+    /* 1 CAP - Capabilities Register */
+    cap = AHCI_RREG(AHCI_CAP);
+    ASSERT_BIT_CLEAR(cap, AHCI_CAP_RESERVED);
+
+    /* 2 GHC - Global Host Control */
+    reg = AHCI_RREG(AHCI_GHC);
+    ASSERT_BIT_CLEAR(reg, AHCI_GHC_HR);
+    ASSERT_BIT_CLEAR(reg, AHCI_GHC_IE);
+    ASSERT_BIT_CLEAR(reg, AHCI_GHC_MRSM);
+    if (BITSET(cap, AHCI_CAP_SAM)) {
+        g_test_message("Supports AHCI-Only Mode: GHC_AE is Read-Only.");
+        ASSERT_BIT_SET(reg, AHCI_GHC_AE);
+    } else {
+        g_test_message("Supports AHCI/Legacy mix.");
+        ASSERT_BIT_CLEAR(reg, AHCI_GHC_AE);
+    }
+
+    /* 3 IS - Interrupt Status */
+    reg = AHCI_RREG(AHCI_IS);
+    g_assert_cmphex(reg, ==, 0);
+
+    /* 4 PI - Ports Implemented */
+    ports = AHCI_RREG(AHCI_PI);
+    /* Ports Implemented must be non-zero. */
+    g_assert_cmphex(ports, !=, 0);
+    /* Ports Implemented must be <= Number of Ports. */
+    nports_impl = ctpopl(ports);
+    g_assert_cmpuint(((AHCI_CAP_NP & cap) + 1), >=, nports_impl);
+
+    g_assert_cmphex(barsize, >, 0);
+    /* Ports must be within the proper range. Given a mapping of SIZE,
+     * 256 bytes are used for global HBA control, and the rest is used
+     * for ports data, at 0x80 bytes each. */
+    maxports = (barsize - HBA_DATA_REGION_SIZE) / HBA_PORT_DATA_SIZE;
+    /* e.g, 30 ports for 4K of memory. (4096 - 256) / 128 = 30 */
+    g_assert_cmphex((reg >> maxports), ==, 0);
+
+    /* 5 AHCI Version */
+    reg = AHCI_RREG(AHCI_VS);
+    switch (reg) {
+    case AHCI_VERSION_0_95:
+    case AHCI_VERSION_1_0:
+    case AHCI_VERSION_1_1:
+    case AHCI_VERSION_1_2:
+    case AHCI_VERSION_1_3:
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    /* 6 Command Completion Coalescing Control: depends on CAP.CCCS. */
+    reg = AHCI_RREG(AHCI_CCCCTL);
+    if (BITSET(cap, AHCI_CAP_CCCS)) {
+        ASSERT_BIT_CLEAR(reg, AHCI_CCCCTL_EN);
+        ASSERT_BIT_CLEAR(reg, AHCI_CCCCTL_RESERVED);
+        ASSERT_BIT_SET(reg, AHCI_CCCCTL_CC);
+        ASSERT_BIT_SET(reg, AHCI_CCCCTL_TV);
+    } else {
+        g_assert_cmphex(reg, ==, 0);
+    }
+
+    /* 7 CCC_PORTS */
+    reg = AHCI_RREG(AHCI_CCCPORTS);
+    /* Must be zeroes initially regardless of CAP.CCCS */
+    g_assert_cmphex(reg, ==, 0);
+
+    /* 8 EM_LOC */
+    reg = AHCI_RREG(AHCI_EMLOC);
+    if (BITCLR(cap, AHCI_CAP_EMS)) {
+        g_assert_cmphex(reg, ==, 0);
+    }
+
+    /* 9 EM_CTL */
+    reg = AHCI_RREG(AHCI_EMCTL);
+    if (BITSET(cap, AHCI_CAP_EMS)) {
+        ASSERT_BIT_CLEAR(reg, AHCI_EMCTL_STSMR);
+        ASSERT_BIT_CLEAR(reg, AHCI_EMCTL_CTLTM);
+        ASSERT_BIT_CLEAR(reg, AHCI_EMCTL_CTLRST);
+        ASSERT_BIT_CLEAR(reg, AHCI_EMCTL_RESERVED);
+    } else {
+        g_assert_cmphex(reg, ==, 0);
+    }
+
+    /* 10 CAP2 -- Capabilities Extended */
+    cap2 = AHCI_RREG(AHCI_CAP2);
+    ASSERT_BIT_CLEAR(cap2, AHCI_CAP2_RESERVED);
+
+    /* 11 BOHC -- Bios/OS Handoff Control */
+    reg = AHCI_RREG(AHCI_BOHC);
+    g_assert_cmphex(reg, ==, 0);
+
+    /* 12 -- 23: Reserved */
+    g_test_message("Verifying HBA reserved area is empty.");
+    for (i = AHCI_RESERVED; i < AHCI_NVMHCI; ++i) {
+        reg = AHCI_RREG(i);
+        g_assert_cmphex(reg, ==, 0);
+    }
+
+    /* 24 -- 39: NVMHCI */
+    if (BITCLR(cap2, AHCI_CAP2_NVMP)) {
+        g_test_message("Verifying HBA/NVMHCI area is empty.");
+        for (i = AHCI_NVMHCI; i < AHCI_VENDOR; ++i) {
+            reg = AHCI_RREG(i);
+            g_assert_cmphex(reg, ==, 0);
+        }
+    }
+
+    /* 40 -- 63: Vendor */
+    g_test_message("Verifying HBA/Vendor area is empty.");
+    for (i = AHCI_VENDOR; i < AHCI_PORTS; ++i) {
+        reg = AHCI_RREG(i);
+        g_assert_cmphex(reg, ==, 0);
+    }
+
+    /* 64 -- XX: Port Space */
+    hcap.cap = cap;
+    hcap.cap2 = cap2;
+    for (i = 0; ports || (i < maxports); ports >>= 1, ++i) {
+        if (BITSET(ports, 0x1)) {
+            g_test_message("Testing port %u for spec", i);
+            ahci_test_port_spec(ahci, hba_base, &hcap, i);
+        } else {
+            uint16_t j;
+            uint16_t low = AHCI_PORTS + (32 * i);
+            uint16_t high = AHCI_PORTS + (32 * (i + 1));
+            g_test_message("Asserting unimplemented port %u "
+                           "(reg [%u-%u]) is empty.",
+                           i, low, high - 1);
+            for (j = low; j < high; ++j) {
+                reg = AHCI_RREG(j);
+                g_assert_cmphex(reg, ==, 0);
+            }
+        }
+    }
+}
+
+/**
+ * Test the memory space for one port for specification adherence.
+ */
+static void ahci_test_port_spec(QPCIDevice *ahci, void *hba_base,
+                                HBACap *hcap, uint8_t port)
+{
+    uint32_t reg;
+    unsigned i;
+
+    /* (0) CLB */
+    reg = PX_RREG(port, AHCI_PX_CLB);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_CLB_RESERVED);
+
+    /* (1) CLBU */
+    if (BITCLR(hcap->cap, AHCI_CAP_S64A)) {
+        reg = PX_RREG(port, AHCI_PX_CLBU);
+        g_assert_cmphex(reg, ==, 0);
+    }
+
+    /* (2) FB */
+    reg = PX_RREG(port, AHCI_PX_FB);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_FB_RESERVED);
+
+    /* (3) FBU */
+    if (BITCLR(hcap->cap, AHCI_CAP_S64A)) {
+        reg = PX_RREG(port, AHCI_PX_FBU);
+        g_assert_cmphex(reg, ==, 0);
+    }
+
+    /* (4) IS */
+    reg = PX_RREG(port, AHCI_PX_IS);
+    g_assert_cmphex(reg, ==, 0);
+
+    /* (5) IE */
+    reg = PX_RREG(port, AHCI_PX_IE);
+    g_assert_cmphex(reg, ==, 0);
+
+    /* (6) CMD */
+    reg = PX_RREG(port, AHCI_PX_CMD);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_FRE);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_RESERVED);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_CCS);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_FR);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_CR);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_PMA); /* And RW only if CAP.SPM */
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_APSTE); /* RW only if CAP2.APST */
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_ATAPI);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_DLAE);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_ALPE);  /* RW only if CAP.SALP */
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_ASP);   /* RW only if CAP.SALP */
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_ICC);
+    /* If CPDetect support does not exist, CPState must be off. */
+    if (BITCLR(reg, AHCI_PX_CMD_CPD)) {
+        ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_CPS);
+    }
+    /* If MPSPresence is not set, MPSState must be off. */
+    if (BITCLR(reg, AHCI_PX_CMD_MPSP)) {
+        ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_MPSS);
+    }
+    /* If we do not support MPS, MPSS and MPSP must be off. */
+    if (BITCLR(hcap->cap, AHCI_CAP_SMPS)) {
+        ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_MPSS);
+        ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_MPSP);
+    }
+    /* If, via CPD or MPSP we detect a drive, HPCP must be on. */
+    if (BITANY(reg, AHCI_PX_CMD_CPD || AHCI_PX_CMD_MPSP)) {
+        ASSERT_BIT_SET(reg, AHCI_PX_CMD_HPCP);
+    }
+    /* HPCP and ESP cannot both be active. */
+    g_assert(!BITSET(reg, AHCI_PX_CMD_HPCP | AHCI_PX_CMD_ESP));
+    /* If CAP.FBSS is not set, FBSCP must not be set. */
+    if (BITCLR(hcap->cap, AHCI_CAP_FBSS)) {
+        ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_FBSCP);
+    }
+
+    /* (7) RESERVED */
+    reg = PX_RREG(port, AHCI_PX_RES1);
+    g_assert_cmphex(reg, ==, 0);
+
+    /* (8) TFD */
+    reg = PX_RREG(port, AHCI_PX_TFD);
+    /* At boot, prior to an FIS being received, the TFD register should be 0x7F,
+     * which breaks down as follows, as seen in AHCI 1.3 sec 3.3.8, p. 27. */
+    ASSERT_BIT_SET(reg, AHCI_PX_TFD_STS_ERR);
+    ASSERT_BIT_SET(reg, AHCI_PX_TFD_STS_CS1);
+    ASSERT_BIT_SET(reg, AHCI_PX_TFD_STS_DRQ);
+    ASSERT_BIT_SET(reg, AHCI_PX_TFD_STS_CS2);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_TFD_STS_BSY);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_TFD_ERR);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_TFD_RESERVED);
+
+    /* (9) SIG */
+    /* Though AHCI specifies the boot value should be 0xFFFFFFFF,
+     * Even when GHC.ST is zero, the AHCI HBA may receive the initial
+     * D2H register FIS and update the signature asynchronously,
+     * so we cannot expect a value here. AHCI 1.3, sec 3.3.9, pp 27-28 */
+
+    /* (10) SSTS / SCR0: SStatus */
+    reg = PX_RREG(port, AHCI_PX_SSTS);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_SSTS_RESERVED);
+    /* Even though the register should be 0 at boot, it is asynchronous and
+     * prone to change, so we cannot test any well known value. */
+
+    /* (11) SCTL / SCR2: SControl */
+    reg = PX_RREG(port, AHCI_PX_SCTL);
+    g_assert_cmphex(reg, ==, 0);
+
+    /* (12) SERR / SCR1: SError */
+    reg = PX_RREG(port, AHCI_PX_SERR);
+    g_assert_cmphex(reg, ==, 0);
+
+    /* (13) SACT / SCR3: SActive */
+    reg = PX_RREG(port, AHCI_PX_SACT);
+    g_assert_cmphex(reg, ==, 0);
+
+    /* (14) CI */
+    reg = PX_RREG(port, AHCI_PX_CI);
+    g_assert_cmphex(reg, ==, 0);
+
+    /* (15) SNTF */
+    reg = PX_RREG(port, AHCI_PX_SNTF);
+    g_assert_cmphex(reg, ==, 0);
+
+    /* (16) FBS */
+    reg = PX_RREG(port, AHCI_PX_FBS);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_FBS_EN);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_FBS_DEC);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_FBS_SDE);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_FBS_DEV);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_FBS_DWE);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_FBS_RESERVED);
+    if (BITSET(hcap->cap, AHCI_CAP_FBSS)) {
+        /* if Port-Multiplier FIS-based switching avail, ADO must >= 2 */
+        g_assert((reg & AHCI_PX_FBS_ADO) >> ctzl(AHCI_PX_FBS_ADO) >= 2);
+    }
+
+    /* [17 -- 27] RESERVED */
+    for (i = AHCI_PX_RES2; i < AHCI_PX_VS; ++i) {
+        reg = PX_RREG(port, i);
+        g_assert_cmphex(reg, ==, 0);
+    }
+
+    /* [28 -- 31] Vendor-Specific */
+    for (i = AHCI_PX_VS; i < 32; ++i) {
+        reg = PX_RREG(port, i);
+        if (reg) {
+            g_test_message("INFO: Vendor register %u non-empty", i);
+        }
+    }
+}
+
+/**
+ * Utilizing an initialized AHCI HBA, issue an IDENTIFY command to the first
+ * device we see, then read and check the response.
+ */
+static void ahci_test_identify(QPCIDevice *ahci, void *hba_base)
+{
+    RegD2HFIS *d2h = g_malloc0(0x20);
+    RegD2HFIS *pio = g_malloc0(0x20);
+    RegH2DFIS fis;
+    AHCICommand cmd;
+    PRD prd;
+    uint32_t ports, reg, clb, table, fb, data_ptr;
+    uint16_t buff[256];
+    unsigned i;
+    int rc;
+
+    g_assert(ahci != NULL);
+    g_assert(hba_base != NULL);
+
+    /* We need to:
+     * (1) Create a Command Table Buffer and update the Command List Slot #0
+     *     to point to this buffer.
+     * (2) Construct an FIS host-to-device command structure, and write it to
+     *     the top of the command table buffer.
+     * (3) Create a data buffer for the IDENTIFY response to be sent to
+     * (4) Create a Physical Region Descriptor that points to the data buffer,
+     *     and write it to the bottom (offset 0x80) of the command table.
+     * (5) Now, PxCLB points to the command list, command 0 points to
+     *     our table, and our table contains an FIS instruction and a
+     *     PRD that points to our rx buffer.
+     * (6) We inform the HBA via PxCI that there is a command ready in slot #0.
+     */
+
+    /* Pick the first implemented and running port */
+    ports = AHCI_RREG(AHCI_PI);
+    for (i = 0; i < 32; ports >>= 1, ++i) {
+        if (ports == 0) {
+            i = 32;
+        }
+
+        if (!(ports & 0x01)) {
+            continue;
+        }
+
+        reg = PX_RREG(i, AHCI_PX_CMD);
+        if (BITSET(reg, AHCI_PX_CMD_ST)) {
+            break;
+        }
+    }
+    g_assert_cmphex(i, <, 32);
+    g_test_message("Selected port %u for test", i);
+
+    /* Clear out this port's interrupts (ignore the init register d2h fis) */
+    reg = PX_RREG(i, AHCI_PX_IS);
+    PX_WREG(i, AHCI_PX_IS, reg);
+    g_assert_cmphex(PX_RREG(i, AHCI_PX_IS), ==, 0);
+
+    /* Wipe the FIS-Recieve Buffer */
+    fb = PX_RREG(i, AHCI_PX_FB);
+    g_assert_cmphex(fb, !=, 0);
+    qmemset(fb, 0x00, 0x100);
+
+    /* Create a Command Table buffer. 0x80 is the smallest with a PRDTL of 0. */
+    /* We need at least one PRD, so round up to the nearest 0x80 multiple.    */
+    table = guest_alloc(guest_malloc, CMD_TBL_SIZ(1));
+    g_assert(table);
+    ASSERT_BIT_CLEAR(table, 0x7F);
+
+    /* Create a data buffer ... where we will dump the IDENTIFY data to. */
+    data_ptr = guest_alloc(guest_malloc, 512);
+    g_assert(data_ptr);
+
+    /* Grab the Command List Buffer pointer */
+    clb = PX_RREG(i, AHCI_PX_CLB);
+    g_assert(clb);
+
+    /* Copy the existing Command #0 structure from the CLB into local memory,
+     * and build a new command #0. */
+    memread(clb, &cmd, sizeof(cmd));
+    cmd.b1 = 5;    /* reg_h2d_fis is 5 double-words long */
+    cmd.b2 = 0x04; /* clear PxTFD.STS.BSY when done */
+    cmd.prdtl = cpu_to_le16(1); /* One PRD table entry. */
+    cmd.prdbc = 0;
+    cmd.ctba = cpu_to_le32(table);
+    cmd.ctbau = 0;
+
+    /* Construct our PRD, noting that DBC is 0-indexed. */
+    prd.dba = cpu_to_le32(data_ptr);
+    prd.dbau = 0;
+    prd.res = 0;
+    /* 511+1 bytes, request DPS interrupt */
+    prd.dbc = cpu_to_le32(511 | 0x80000000);
+
+    /* Construct our Command FIS, Based on http://wiki.osdev.org/AHCI */
+    memset(&fis, 0x00, sizeof(fis));
+    fis.fis_type = 0x27; /* Register Host-to-Device FIS */
+    fis.command = 0xEC;  /* IDENTIFY */
+    fis.device = 0;
+    fis.flags = 0x80;    /* Indicate this is a command FIS */
+
+    /* We've committed nothing yet, no interrupts should be posted yet. */
+    g_assert_cmphex(PX_RREG(i, AHCI_PX_IS), ==, 0);
+
+    /* Commit the Command FIS to the Command Table */
+    memwrite(table, &fis, sizeof(fis));
+
+    /* Commit the PRD entry to the Command Table */
+    memwrite(table + 0x80, &prd, sizeof(prd));
+
+    /* Commit Command #0, pointing to the Table, to the Command List Buffer. */
+    memwrite(clb, &cmd, sizeof(cmd));
+
+    /* Everything is in place, but we haven't given the go-ahead yet. */
+    g_assert_cmphex(PX_RREG(i, AHCI_PX_IS), ==, 0);
+
+    /* Issue Command #0 via PxCI */
+    PX_WREG(i, AHCI_PX_CI, (1 << 0));
+    while (BITSET(PX_RREG(i, AHCI_PX_TFD), AHCI_PX_TFD_STS_BSY)) {
+        usleep(50);
+    }
+
+    /* Check for expected interrupts */
+    reg = PX_RREG(i, AHCI_PX_IS);
+    ASSERT_BIT_SET(reg, AHCI_PX_IS_DHRS);
+    ASSERT_BIT_SET(reg, AHCI_PX_IS_PSS);
+    /* BUG: we expect AHCI_PX_IS_DPS to be set. */
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_IS_DPS);
+
+    /* Clear expected interrupts and assert all interrupts now cleared. */
+    PX_WREG(i, AHCI_PX_IS, AHCI_PX_IS_DHRS | AHCI_PX_IS_PSS | AHCI_PX_IS_DPS);
+    g_assert_cmphex(PX_RREG(i, AHCI_PX_IS), ==, 0);
+
+    /* Check for errors. */
+    reg = PX_RREG(i, AHCI_PX_SERR);
+    g_assert_cmphex(reg, ==, 0);
+    reg = PX_RREG(i, AHCI_PX_TFD);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_TFD_STS_ERR);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_TFD_ERR);
+
+    /* Investigate CMD #0, assert that we read 512 bytes */
+    memread(clb, &cmd, sizeof(cmd));
+    g_assert_cmphex(512, ==, le32_to_cpu(cmd.prdbc));
+
+    /* Investigate FIS responses */
+    memread(fb + 0x20, pio, 0x20);
+    memread(fb + 0x40, d2h, 0x20);
+    g_assert_cmphex(pio->fis_type, ==, 0x5f);
+    g_assert_cmphex(d2h->fis_type, ==, 0x34);
+    g_assert_cmphex(pio->flags, ==, d2h->flags);
+    g_assert_cmphex(pio->status, ==, d2h->status);
+    g_assert_cmphex(pio->error, ==, d2h->error);
+
+    reg = PX_RREG(i, AHCI_PX_TFD);
+    g_assert_cmphex((reg & AHCI_PX_TFD_ERR), ==, pio->error);
+    g_assert_cmphex((reg & AHCI_PX_TFD_STS), ==, pio->status);
+    /* The PIO Setup FIS contains a "bytes read" field, which is a
+     * 16-bit value. The Physical Region Descriptor Byte Count is
+     * 32-bit, but for small transfers using one PRD, it should match. */
+    g_assert_cmphex(le16_to_cpu(pio->res4), ==, le32_to_cpu(cmd.prdbc));
+
+    /* Last, but not least: Investigate the IDENTIFY response data. */
+    memread(data_ptr, &buff, 512);
+
+    /* Check serial number/version in the buffer */
+    /* NB: IDENTIFY strings are packed in 16bit little endian chunks.
+     * Since we copy byte-for-byte in ahci-test, on both LE and BE, we need to
+     * unchunk this data. By contrast, ide-test copies 2 bytes at a time, and
+     * as a consequence, only needs to unchunk the data on LE machines. */
+    string_bswap16(&buff[10], 20);
+    rc = memcmp(&buff[10], "testdisk            ", 20);
+    g_assert_cmphex(rc, ==, 0);
+
+    string_bswap16(&buff[23], 8);
+    rc = memcmp(&buff[23], "version ", 8);
+    g_assert_cmphex(rc, ==, 0);
+
+    g_free(d2h);
+    g_free(pio);
+}
+
+/******************************************************************************/
+/* Test Interfaces                                                            */
+/******************************************************************************/
+
+/**
+ * Basic sanity test to boot a machine, find an AHCI device, and shutdown.
+ */
+static void test_sanity(void)
+{
+    QPCIDevice *ahci;
+    ahci = ahci_boot();
+    ahci_shutdown(ahci);
+}
+
+/**
+ * Ensure that the PCI configuration space for the AHCI device is in-line with
+ * the AHCI 1.3 specification for initial values.
+ */
+static void test_pci_spec(void)
+{
+    QPCIDevice *ahci;
+    ahci = ahci_boot();
+    ahci_test_pci_spec(ahci);
+    ahci_shutdown(ahci);
+}
+
+/**
+ * Engage the PCI AHCI device and sanity check the response.
+ * Perform additional PCI config space bringup for the HBA.
+ */
+static void test_pci_enable(void)
+{
+    QPCIDevice *ahci;
+    void *hba_base;
+    ahci = ahci_boot();
+    ahci_pci_enable(ahci, &hba_base);
+    ahci_shutdown(ahci);
+}
+
+/**
+ * Investigate the memory mapped regions of the HBA,
+ * and test them for AHCI specification adherence.
+ */
+static void test_hba_spec(void)
+{
+    QPCIDevice *ahci;
+    void *hba_base;
+
+    ahci = ahci_boot();
+    ahci_pci_enable(ahci, &hba_base);
+    ahci_test_hba_spec(ahci, hba_base);
+    ahci_shutdown(ahci);
+}
+
+/**
+ * Engage the HBA functionality of the AHCI PCI device,
+ * and bring it into a functional idle state.
+ */
+static void test_hba_enable(void)
+{
+    QPCIDevice *ahci;
+    void *hba_base;
+
+    ahci = ahci_boot();
+    ahci_pci_enable(ahci, &hba_base);
+    ahci_hba_enable(ahci, hba_base);
+    ahci_shutdown(ahci);
+}
+
+/**
+ * Bring up the device and issue an IDENTIFY command.
+ * Inspect the state of the HBA device and the data returned.
+ */
+static void test_identify(void)
+{
+    QPCIDevice *ahci;
+    void *hba_base;
+
+    ahci = ahci_boot();
+    ahci_pci_enable(ahci, &hba_base);
+    ahci_hba_enable(ahci, hba_base);
+    ahci_test_identify(ahci, hba_base);
+    ahci_shutdown(ahci);
+}
+
+/******************************************************************************/
+
+int main(int argc, char **argv)
+{
+    const char *arch;
+    int fd;
+    int ret;
+    int c;
+
+    static struct option long_options[] = {
+        {"pedantic", no_argument, 0, 'p' },
+        {0, 0, 0, 0},
+    };
+
+    /* Should be first to utilize g_test functionality, So we can see errors. */
+    g_test_init(&argc, &argv, NULL);
+
+    while (1) {
+        c = getopt_long(argc, argv, "", long_options, NULL);
+        if (c == -1) {
+            break;
+        }
+        switch (c) {
+        case -1:
+            break;
+        case 'p':
+            ahci_pedantic = 1;
+            break;
+        default:
+            fprintf(stderr, "Unrecognized ahci_test option.\n");
+            g_assert_not_reached();
+        }
+    }
+
+    /* Check architecture */
+    arch = qtest_get_arch();
+    if (strcmp(arch, "i386") && strcmp(arch, "x86_64")) {
+        g_test_message("Skipping test for non-x86");
+        return 0;
+    }
+
+    /* Create a temporary raw image */
+    fd = mkstemp(tmp_path);
+    g_assert(fd >= 0);
+    ret = ftruncate(fd, TEST_IMAGE_SIZE);
+    g_assert(ret == 0);
+    close(fd);
+
+    /* Run the tests */
+    qtest_add_func("/ahci/sanity",     test_sanity);
+    qtest_add_func("/ahci/pci_spec",   test_pci_spec);
+    qtest_add_func("/ahci/pci_enable", test_pci_enable);
+    qtest_add_func("/ahci/hba_spec",   test_hba_spec);
+    qtest_add_func("/ahci/hba_enable", test_hba_enable);
+    qtest_add_func("/ahci/identify",   test_identify);
+
+    ret = g_test_run();
+
+    /* Cleanup */
+    unlink(tmp_path);
+
+    return ret;
+}
diff --git a/tests/bios-tables-test.c b/tests/bios-tables-test.c
index 045eb2757..9e4d20592 100644
--- a/tests/bios-tables-test.c
+++ b/tests/bios-tables-test.c
@@ -714,14 +714,12 @@ static void test_acpi_one(const char *params, test_data *data)
     uint8_t signature_high;
     uint16_t signature;
     int i;
-    const char *device = "";
 
-    if (!g_strcmp0(data->machine, MACHINE_Q35)) {
-        device = ",id=hd -device ide-hd,drive=hd";
-    }
+    args = g_strdup_printf("-net none -display none %s "
+                           "-drive id=hd0,if=none,file=%s "
+                           "-device ide-hd,drive=hd0 ",
+                           params ? params : "", disk);
 
-    args = g_strdup_printf("-net none -display none %s -drive file=%s%s,",
-                           params ? params : "", disk, device);
     qtest_start(args);
 
    /* Wait at most 1 minute */
@@ -790,6 +788,11 @@ int main(int argc, char *argv[])
     const char *arch = qtest_get_arch();
     FILE *f = fopen(disk, "w");
     int ret;
+
+    if (!f) {
+        fprintf(stderr, "Couldn't open \"%s\": %s", disk, strerror(errno));
+        return 1;
+    }
     fwrite(boot_sector, 1, sizeof boot_sector, f);
     fclose(f);
 
diff --git a/tests/blockdev-test.c b/tests/blockdev-test.c
deleted file mode 100644
index c940e0069..000000000
--- a/tests/blockdev-test.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * blockdev.c test cases
- *
- * Copyright (C) 2013 Red Hat Inc.
- *
- * Authors:
- *  Stefan Hajnoczi <stefanha@redhat.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
-
-static void test_drive_add_empty(void)
-{
-    QDict *response;
-    const char *response_return;
-
-    /* Start with an empty drive */
-    qtest_start("-drive if=none,id=drive0");
-
-    /* Delete the drive */
-    response = qmp("{\"execute\": \"human-monitor-command\","
-                   " \"arguments\": {"
-                   "   \"command-line\": \"drive_del drive0\""
-                   "}}");
-    g_assert(response);
-    response_return = qdict_get_try_str(response, "return");
-    g_assert(response_return);
-    g_assert(strcmp(response_return, "") == 0);
-    QDECREF(response);
-
-    /* Ensure re-adding the drive works - there should be no duplicate ID error
-     * because the old drive must be gone.
-     */
-    response = qmp("{\"execute\": \"human-monitor-command\","
-                   " \"arguments\": {"
-                   "   \"command-line\": \"drive_add 0 if=none,id=drive0\""
-                   "}}");
-    g_assert(response);
-    response_return = qdict_get_try_str(response, "return");
-    g_assert(response_return);
-    g_assert(strcmp(response_return, "OK\r\n") == 0);
-    QDECREF(response);
-
-    qtest_end();
-}
-
-int main(int argc, char **argv)
-{
-    g_test_init(&argc, &argv, NULL);
-
-    qtest_add_func("/qmp/drive_add_empty", test_drive_add_empty);
-
-    return g_test_run();
-}
diff --git a/tests/drive_del-test.c b/tests/drive_del-test.c
new file mode 100644
index 000000000..53fa96926
--- /dev/null
+++ b/tests/drive_del-test.c
@@ -0,0 +1,137 @@
+/*
+ * blockdev.c test cases
+ *
+ * Copyright (C) 2013-2014 Red Hat Inc.
+ *
+ * Authors:
+ *  Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#include <glib.h>
+#include <string.h>
+#include "libqtest.h"
+
+static void drive_add(void)
+{
+    QDict *response;
+
+    response = qmp("{'execute': 'human-monitor-command',"
+                   " 'arguments': {"
+                   "   'command-line': 'drive_add 0 if=none,id=drive0'"
+                   "}}");
+    g_assert(response);
+    g_assert_cmpstr(qdict_get_try_str(response, "return"), ==, "OK\r\n");
+    QDECREF(response);
+}
+
+static void drive_del(void)
+{
+    QDict *response;
+
+    response = qmp("{'execute': 'human-monitor-command',"
+                   " 'arguments': {"
+                   "   'command-line': 'drive_del drive0'"
+                   "}}");
+    g_assert(response);
+    g_assert_cmpstr(qdict_get_try_str(response, "return"), ==, "");
+    QDECREF(response);
+}
+
+static void device_del(void)
+{
+    QDict *response;
+
+    /* Complication: ignore DEVICE_DELETED event */
+    qmp_discard_response("{'execute': 'device_del',"
+                         " 'arguments': { 'id': 'dev0' } }");
+    response = qmp_receive();
+    g_assert(response);
+    g_assert(qdict_haskey(response, "return"));
+    QDECREF(response);
+}
+
+static void test_drive_without_dev(void)
+{
+    /* Start with an empty drive */
+    qtest_start("-drive if=none,id=drive0");
+
+    /* Delete the drive */
+    drive_del();
+
+    /* Ensure re-adding the drive works - there should be no duplicate ID error
+     * because the old drive must be gone.
+     */
+    drive_add();
+
+    qtest_end();
+}
+
+static void test_after_failed_device_add(void)
+{
+    QDict *response;
+    QDict *error;
+
+    qtest_start("-drive if=none,id=drive0");
+
+    /* Make device_add fail.  If this leaks the virtio-blk-pci device then a
+     * reference to drive0 will also be held (via qdev properties).
+     */
+    response = qmp("{'execute': 'device_add',"
+                   " 'arguments': {"
+                   "   'driver': 'virtio-blk-pci',"
+                   "   'drive': 'drive0'"
+                   "}}");
+    g_assert(response);
+    error = qdict_get_qdict(response, "error");
+    g_assert_cmpstr(qdict_get_try_str(error, "class"), ==, "GenericError");
+    QDECREF(response);
+
+    /* Delete the drive */
+    drive_del();
+
+    /* Try to re-add the drive.  This fails with duplicate IDs if a leaked
+     * virtio-blk-pci exists that holds a reference to the old drive0.
+     */
+    drive_add();
+
+    qtest_end();
+}
+
+static void test_drive_del_device_del(void)
+{
+    /* Start with a drive used by a device that unplugs instantaneously */
+    qtest_start("-drive if=none,id=drive0,file=/dev/null"
+                " -device virtio-scsi-pci"
+                " -device scsi-hd,drive=drive0,id=dev0");
+
+    /*
+     * Delete the drive, and then the device
+     * Doing it in this order takes notoriously tricky special paths
+     */
+    drive_del();
+    device_del();
+
+    qtest_end();
+}
+
+int main(int argc, char **argv)
+{
+    const char *arch = qtest_get_arch();
+
+    g_test_init(&argc, &argv, NULL);
+
+    qtest_add_func("/drive_del/without-dev", test_drive_without_dev);
+
+    /* TODO I guess any arch with PCI would do */
+    if (!strcmp(arch, "i386") || !strcmp(arch, "x86_64")) {
+        qtest_add_func("/drive_del/after_failed_device_add",
+                       test_after_failed_device_add);
+        qtest_add_func("/blockdev/drive_del_device_del",
+                       test_drive_del_device_del);
+    }
+
+    return g_test_run();
+}
diff --git a/tests/ide-test.c b/tests/ide-test.c
index 4a0d97f19..b7a97e936 100644
--- a/tests/ide-test.c
+++ b/tests/ide-test.c
@@ -106,6 +106,7 @@ static QPCIBus *pcibus = NULL;
 static QGuestAllocator *guest_malloc;
 
 static char tmp_path[] = "/tmp/qtest.XXXXXX";
+static char debug_path[] = "/tmp/qtest-blkdebug.XXXXXX";
 
 static void ide_test_start(const char *cmdline_fmt, ...)
 {
@@ -119,10 +120,14 @@ static void ide_test_start(const char *cmdline_fmt, ...)
     qtest_start(cmdline);
     qtest_irq_intercept_in(global_qtest, "ioapic");
     guest_malloc = pc_alloc_init();
+
+    g_free(cmdline);
 }
 
 static void ide_test_quit(void)
 {
+    pc_alloc_uninit(guest_malloc);
+    guest_malloc = NULL;
     qtest_end();
 }
 
@@ -145,7 +150,7 @@ static QPCIDevice *get_pci_device(uint16_t *bmdma_base)
     g_assert(device_id == PCI_DEVICE_ID_INTEL_82371SB_1);
 
     /* Map bmdma BAR */
-    *bmdma_base = (uint16_t)(uintptr_t) qpci_iomap(dev, 4);
+    *bmdma_base = (uint16_t)(uintptr_t) qpci_iomap(dev, 4, NULL);
 
     qpci_device_enable(dev);
 
@@ -489,6 +494,91 @@ static void test_flush(void)
     ide_test_quit();
 }
 
+static void prepare_blkdebug_script(const char *debug_fn, const char *event)
+{
+    FILE *debug_file = fopen(debug_fn, "w");
+    int ret;
+
+    fprintf(debug_file, "[inject-error]\n");
+    fprintf(debug_file, "event = \"%s\"\n", event);
+    fprintf(debug_file, "errno = \"5\"\n");
+    fprintf(debug_file, "state = \"1\"\n");
+    fprintf(debug_file, "immediately = \"off\"\n");
+    fprintf(debug_file, "once = \"on\"\n");
+
+    fprintf(debug_file, "[set-state]\n");
+    fprintf(debug_file, "event = \"%s\"\n", event);
+    fprintf(debug_file, "new_state = \"2\"\n");
+    fflush(debug_file);
+    g_assert(!ferror(debug_file));
+
+    ret = fclose(debug_file);
+    g_assert(ret == 0);
+}
+
+static void test_retry_flush(void)
+{
+    uint8_t data;
+    const char *s;
+    QDict *response;
+
+    prepare_blkdebug_script(debug_path, "flush_to_disk");
+
+    ide_test_start(
+        "-vnc none "
+        "-drive file=blkdebug:%s:%s,if=ide,cache=writeback,rerror=stop,werror=stop",
+        debug_path, tmp_path);
+
+    /* FLUSH CACHE command on device 0*/
+    outb(IDE_BASE + reg_device, 0);
+    outb(IDE_BASE + reg_command, CMD_FLUSH_CACHE);
+
+    /* Check status while request is in flight*/
+    data = inb(IDE_BASE + reg_status);
+    assert_bit_set(data, BSY | DRDY);
+    assert_bit_clear(data, DF | ERR | DRQ);
+
+    for (;; response = NULL) {
+        response = qmp_receive();
+        if ((qdict_haskey(response, "event")) &&
+            (strcmp(qdict_get_str(response, "event"), "STOP") == 0)) {
+            QDECREF(response);
+            break;
+        }
+        QDECREF(response);
+    }
+
+    /* Complete the command */
+    s = "{'execute':'cont' }";
+    qmp_discard_response(s);
+
+    /* Check registers */
+    data = inb(IDE_BASE + reg_device);
+    g_assert_cmpint(data & DEV, ==, 0);
+
+    do {
+        data = inb(IDE_BASE + reg_status);
+    } while (data & BSY);
+
+    assert_bit_set(data, DRDY);
+    assert_bit_clear(data, BSY | DF | ERR | DRQ);
+
+    ide_test_quit();
+}
+
+static void test_flush_nodev(void)
+{
+    ide_test_start("");
+
+    /* FLUSH CACHE command on device 0*/
+    outb(IDE_BASE + reg_device, 0);
+    outb(IDE_BASE + reg_command, CMD_FLUSH_CACHE);
+
+    /* Just testing that qemu doesn't crash... */
+
+    ide_test_quit();
+}
+
 int main(int argc, char **argv)
 {
     const char *arch = qtest_get_arch();
@@ -501,6 +591,11 @@ int main(int argc, char **argv)
         return 0;
     }
 
+    /* Create temporary blkdebug instructions */
+    fd = mkstemp(debug_path);
+    g_assert(fd >= 0);
+    close(fd);
+
     /* Create a temporary raw image */
     fd = mkstemp(tmp_path);
     g_assert(fd >= 0);
@@ -521,11 +616,15 @@ int main(int argc, char **argv)
     qtest_add_func("/ide/bmdma/teardown", test_bmdma_teardown);
 
     qtest_add_func("/ide/flush", test_flush);
+    qtest_add_func("/ide/flush_nodev", test_flush_nodev);
+
+    qtest_add_func("/ide/retry/flush", test_retry_flush);
 
     ret = g_test_run();
 
     /* Cleanup */
     unlink(tmp_path);
+    unlink(debug_path);
 
     return ret;
 }
diff --git a/tests/image-fuzzer/qcow2/__init__.py b/tests/image-fuzzer/qcow2/__init__.py
new file mode 100644
index 000000000..e2ebe1931
--- /dev/null
+++ b/tests/image-fuzzer/qcow2/__init__.py
@@ -0,0 +1 @@
+from layout import create_image
diff --git a/tests/image-fuzzer/qcow2/fuzz.py b/tests/image-fuzzer/qcow2/fuzz.py
new file mode 100644
index 000000000..20eba6bc1
--- /dev/null
+++ b/tests/image-fuzzer/qcow2/fuzz.py
@@ -0,0 +1,367 @@
+# Fuzzing functions for qcow2 fields
+#
+# Copyright (C) 2014 Maria Kustova <maria.k@catit.be>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import random
+
+UINT8 = 0xff
+UINT16 = 0xffff
+UINT32 = 0xffffffff
+UINT64 = 0xffffffffffffffff
+# Most significant bit orders
+UINT32_M = 31
+UINT64_M = 63
+# Fuzz vectors
+UINT8_V = [0, 0x10, UINT8/4, UINT8/2 - 1, UINT8/2, UINT8/2 + 1, UINT8 - 1,
+           UINT8]
+UINT16_V = [0, 0x100, 0x1000, UINT16/4, UINT16/2 - 1, UINT16/2, UINT16/2 + 1,
+            UINT16 - 1, UINT16]
+UINT32_V = [0, 0x100, 0x1000, 0x10000, 0x100000, UINT32/4, UINT32/2 - 1,
+            UINT32/2, UINT32/2 + 1, UINT32 - 1, UINT32]
+UINT64_V = UINT32_V + [0x1000000, 0x10000000, 0x100000000, UINT64/4,
+                       UINT64/2 - 1, UINT64/2, UINT64/2 + 1, UINT64 - 1,
+                       UINT64]
+STRING_V = ['%s%p%x%d', '.1024d', '%.2049d', '%p%p%p%p', '%x%x%x%x',
+            '%d%d%d%d', '%s%s%s%s', '%99999999999s', '%08x', '%%20d', '%%20n',
+            '%%20x', '%%20s', '%s%s%s%s%s%s%s%s%s%s', '%p%p%p%p%p%p%p%p%p%p',
+            '%#0123456x%08x%x%s%p%d%n%o%u%c%h%l%q%j%z%Z%t%i%e%g%f%a%C%S%08x%%',
+            '%s x 129', '%x x 257']
+
+
+def random_from_intervals(intervals):
+    """Select a random integer number from the list of specified intervals.
+
+    Each interval is a tuple of lower and upper limits of the interval. The
+    limits are included. Intervals in a list should not overlap.
+    """
+    total = reduce(lambda x, y: x + y[1] - y[0] + 1, intervals, 0)
+    r = random.randint(0, total - 1) + intervals[0][0]
+    for x in zip(intervals, intervals[1:]):
+        r = r + (r > x[0][1]) * (x[1][0] - x[0][1] - 1)
+    return r
+
+
+def random_bits(bit_ranges):
+    """Generate random binary mask with ones in the specified bit ranges.
+
+    Each bit_ranges is a list of tuples of lower and upper limits of bit
+    positions will be fuzzed. The limits are included. Random amount of bits
+    in range limits will be set to ones. The mask is returned in decimal
+    integer format.
+    """
+    bit_numbers = []
+    # Select random amount of random positions in bit_ranges
+    for rng in bit_ranges:
+        bit_numbers += random.sample(range(rng[0], rng[1] + 1),
+                                     random.randint(0, rng[1] - rng[0] + 1))
+    val = 0
+    # Set bits on selected positions to ones
+    for bit in bit_numbers:
+        val |= 1 << bit
+    return val
+
+
+def truncate_string(strings, length):
+    """Return strings truncated to specified length."""
+    if type(strings) == list:
+        return [s[:length] for s in strings]
+    else:
+        return strings[:length]
+
+
+def validator(current, pick, choices):
+    """Return a value not equal to the current selected by the pick
+    function from choices.
+    """
+    while True:
+        val = pick(choices)
+        if not val == current:
+            return val
+
+
+def int_validator(current, intervals):
+    """Return a random value from intervals not equal to the current.
+
+    This function is useful for selection from valid values except current one.
+    """
+    return validator(current, random_from_intervals, intervals)
+
+
+def bit_validator(current, bit_ranges):
+    """Return a random bit mask not equal to the current.
+
+    This function is useful for selection from valid values except current one.
+    """
+    return validator(current, random_bits, bit_ranges)
+
+
+def string_validator(current, strings):
+    """Return a random string value from the list not equal to the current.
+
+    This function is useful for selection from valid values except current one.
+    """
+    return validator(current, random.choice, strings)
+
+
+def selector(current, constraints, validate=int_validator):
+    """Select one value from all defined by constraints.
+
+    Each constraint produces one random value satisfying to it. The function
+    randomly selects one value satisfying at least one constraint (depending on
+    constraints overlaps).
+    """
+    def iter_validate(c):
+        """Apply validate() only to constraints represented as lists.
+
+        This auxiliary function replaces short circuit conditions not supported
+        in Python 2.4
+        """
+        if type(c) == list:
+            return validate(current, c)
+        else:
+            return c
+
+    fuzz_values = [iter_validate(c) for c in constraints]
+    # Remove current for cases it's implicitly specified in constraints
+    # Duplicate validator functionality to prevent decreasing of probability
+    # to get one of allowable values
+    # TODO: remove validators after implementation of intelligent selection
+    # of fields will be fuzzed
+    try:
+        fuzz_values.remove(current)
+    except ValueError:
+        pass
+    return random.choice(fuzz_values)
+
+
+def magic(current):
+    """Fuzz magic header field.
+
+    The function just returns the current magic value and provides uniformity
+    of calls for all fuzzing functions.
+    """
+    return current
+
+
+def version(current):
+    """Fuzz version header field."""
+    constraints = UINT32_V + [
+        [(2, 3)],  # correct values
+        [(0, 1), (4, UINT32)]
+    ]
+    return selector(current, constraints)
+
+
+def backing_file_offset(current):
+    """Fuzz backing file offset header field."""
+    constraints = UINT64_V
+    return selector(current, constraints)
+
+
+def backing_file_size(current):
+    """Fuzz backing file size header field."""
+    constraints = UINT32_V
+    return selector(current, constraints)
+
+
+def cluster_bits(current):
+    """Fuzz cluster bits header field."""
+    constraints = UINT32_V + [
+        [(9, 20)],  # correct values
+        [(0, 9), (20, UINT32)]
+    ]
+    return selector(current, constraints)
+
+
+def size(current):
+    """Fuzz image size header field."""
+    constraints = UINT64_V
+    return selector(current, constraints)
+
+
+def crypt_method(current):
+    """Fuzz crypt method header field."""
+    constraints = UINT32_V + [
+        1,
+        [(2, UINT32)]
+    ]
+    return selector(current, constraints)
+
+
+def l1_size(current):
+    """Fuzz L1 table size header field."""
+    constraints = UINT32_V
+    return selector(current, constraints)
+
+
+def l1_table_offset(current):
+    """Fuzz L1 table offset header field."""
+    constraints = UINT64_V
+    return selector(current, constraints)
+
+
+def refcount_table_offset(current):
+    """Fuzz refcount table offset header field."""
+    constraints = UINT64_V
+    return selector(current, constraints)
+
+
+def refcount_table_clusters(current):
+    """Fuzz refcount table clusters header field."""
+    constraints = UINT32_V
+    return selector(current, constraints)
+
+
+def nb_snapshots(current):
+    """Fuzz number of snapshots header field."""
+    constraints = UINT32_V
+    return selector(current, constraints)
+
+
+def snapshots_offset(current):
+    """Fuzz snapshots offset header field."""
+    constraints = UINT64_V
+    return selector(current, constraints)
+
+
+def incompatible_features(current):
+    """Fuzz incompatible features header field."""
+    constraints = [
+        [(0, 1)],  # allowable values
+        [(0, UINT64_M)]
+    ]
+    return selector(current, constraints, bit_validator)
+
+
+def compatible_features(current):
+    """Fuzz compatible features header field."""
+    constraints = [
+        [(0, UINT64_M)]
+    ]
+    return selector(current, constraints, bit_validator)
+
+
+def autoclear_features(current):
+    """Fuzz autoclear features header field."""
+    constraints = [
+        [(0, UINT64_M)]
+    ]
+    return selector(current, constraints, bit_validator)
+
+
+def refcount_order(current):
+    """Fuzz number of refcount order header field."""
+    constraints = UINT32_V
+    return selector(current, constraints)
+
+
+def header_length(current):
+    """Fuzz number of refcount order header field."""
+    constraints = UINT32_V + [
+        72,
+        104,
+        [(0, UINT32)]
+    ]
+    return selector(current, constraints)
+
+
+def bf_name(current):
+    """Fuzz the backing file name."""
+    constraints = [
+        truncate_string(STRING_V, len(current))
+    ]
+    return selector(current, constraints, string_validator)
+
+
+def ext_magic(current):
+    """Fuzz magic field of a header extension."""
+    constraints = UINT32_V
+    return selector(current, constraints)
+
+
+def ext_length(current):
+    """Fuzz length field of a header extension."""
+    constraints = UINT32_V
+    return selector(current, constraints)
+
+
+def bf_format(current):
+    """Fuzz backing file format in the corresponding header extension."""
+    constraints = [
+        truncate_string(STRING_V, len(current)),
+        truncate_string(STRING_V, (len(current) + 7) & ~7)  # Fuzz padding
+    ]
+    return selector(current, constraints, string_validator)
+
+
+def feature_type(current):
+    """Fuzz feature type field of a feature name table header extension."""
+    constraints = UINT8_V
+    return selector(current, constraints)
+
+
+def feature_bit_number(current):
+    """Fuzz bit number field of a feature name table header extension."""
+    constraints = UINT8_V
+    return selector(current, constraints)
+
+
+def feature_name(current):
+    """Fuzz feature name field of a feature name table header extension."""
+    constraints = [
+        truncate_string(STRING_V, len(current)),
+        truncate_string(STRING_V, 46)  # Fuzz padding (field length = 46)
+    ]
+    return selector(current, constraints, string_validator)
+
+
+def l1_entry(current):
+    """Fuzz an entry of the L1 table."""
+    constraints = UINT64_V
+    # Reserved bits are ignored
+    # Added a possibility when only flags are fuzzed
+    offset = 0x7fffffffffffffff & \
+             random.choice([selector(current, constraints), current])
+    is_cow = random.randint(0, 1)
+    return offset + (is_cow << UINT64_M)
+
+
+def l2_entry(current):
+    """Fuzz an entry of an L2 table."""
+    constraints = UINT64_V
+    # Reserved bits are ignored
+    # Add a possibility when only flags are fuzzed
+    offset = 0x3ffffffffffffffe & \
+             random.choice([selector(current, constraints), current])
+    is_compressed = random.randint(0, 1)
+    is_cow = random.randint(0, 1)
+    is_zero = random.randint(0, 1)
+    value = offset + (is_cow << UINT64_M) + \
+            (is_compressed << UINT64_M - 1) + is_zero
+    return value
+
+
+def refcount_table_entry(current):
+    """Fuzz an entry of the refcount table."""
+    constraints = UINT64_V
+    return selector(current, constraints)
+
+
+def refcount_block_entry(current):
+    """Fuzz an entry of a refcount block."""
+    constraints = UINT16_V
+    return selector(current, constraints)
diff --git a/tests/image-fuzzer/qcow2/layout.py b/tests/image-fuzzer/qcow2/layout.py
new file mode 100644
index 000000000..63e801f4e
--- /dev/null
+++ b/tests/image-fuzzer/qcow2/layout.py
@@ -0,0 +1,612 @@
+# Generator of fuzzed qcow2 images
+#
+# Copyright (C) 2014 Maria Kustova <maria.k@catit.be>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import random
+import struct
+import fuzz
+from math import ceil
+from os import urandom
+from itertools import chain
+
+MAX_IMAGE_SIZE = 10 * (1 << 20)
+# Standard sizes
+UINT32_S = 4
+UINT64_S = 8
+
+
+class Field(object):
+
+    """Atomic image element (field).
+
+    The class represents an image field as quadruple of a data format
+    of value necessary for its packing to binary form, an offset from
+    the beginning of the image, a value and a name.
+
+    The field can be iterated as a list [format, offset, value, name].
+    """
+
+    __slots__ = ('fmt', 'offset', 'value', 'name')
+
+    def __init__(self, fmt, offset, val, name):
+        self.fmt = fmt
+        self.offset = offset
+        self.value = val
+        self.name = name
+
+    def __iter__(self):
+        return iter([self.fmt, self.offset, self.value, self.name])
+
+    def __repr__(self):
+        return "Field(fmt='%s', offset=%d, value=%s, name=%s)" % \
+            (self.fmt, self.offset, str(self.value), self.name)
+
+
+class FieldsList(object):
+
+    """List of fields.
+
+    The class allows access to a field in the list by its name.
+    """
+
+    def __init__(self, meta_data=None):
+        if meta_data is None:
+            self.data = []
+        else:
+            self.data = [Field(*f)
+                         for f in meta_data]
+
+    def __getitem__(self, name):
+        return [x for x in self.data if x.name == name]
+
+    def __iter__(self):
+        return iter(self.data)
+
+    def __len__(self):
+        return len(self.data)
+
+
+class Image(object):
+
+    """ Qcow2 image object.
+
+    This class allows to create qcow2 images with random valid structures and
+    values, fuzz them via external qcow2.fuzz module and write the result to
+    a file.
+    """
+
+    def __init__(self, backing_file_name=None):
+        """Create a random valid qcow2 image with the correct header and stored
+        backing file name.
+        """
+        cluster_bits, self.image_size = self._size_params()
+        self.cluster_size = 1 << cluster_bits
+        self.header = FieldsList()
+        self.backing_file_name = FieldsList()
+        self.backing_file_format = FieldsList()
+        self.feature_name_table = FieldsList()
+        self.end_of_extension_area = FieldsList()
+        self.l2_tables = FieldsList()
+        self.l1_table = FieldsList()
+        self.refcount_table = FieldsList()
+        self.refcount_blocks = FieldsList()
+        self.ext_offset = 0
+        self.create_header(cluster_bits, backing_file_name)
+        self.set_backing_file_name(backing_file_name)
+        self.data_clusters = self._alloc_data(self.image_size,
+                                              self.cluster_size)
+        # Percentage of fields will be fuzzed
+        self.bias = random.uniform(0.2, 0.5)
+
+    def __iter__(self):
+        return chain(self.header, self.backing_file_format,
+                     self.feature_name_table, self.end_of_extension_area,
+                     self.backing_file_name, self.l1_table, self.l2_tables,
+                     self.refcount_table, self.refcount_blocks)
+
+    def create_header(self, cluster_bits, backing_file_name=None):
+        """Generate a random valid header."""
+        meta_header = [
+            ['>4s', 0, "QFI\xfb", 'magic'],
+            ['>I', 4, random.randint(2, 3), 'version'],
+            ['>Q', 8, 0, 'backing_file_offset'],
+            ['>I', 16, 0, 'backing_file_size'],
+            ['>I', 20, cluster_bits, 'cluster_bits'],
+            ['>Q', 24, self.image_size, 'size'],
+            ['>I', 32, 0, 'crypt_method'],
+            ['>I', 36, 0, 'l1_size'],
+            ['>Q', 40, 0, 'l1_table_offset'],
+            ['>Q', 48, 0, 'refcount_table_offset'],
+            ['>I', 56, 0, 'refcount_table_clusters'],
+            ['>I', 60, 0, 'nb_snapshots'],
+            ['>Q', 64, 0, 'snapshots_offset'],
+            ['>Q', 72, 0, 'incompatible_features'],
+            ['>Q', 80, 0, 'compatible_features'],
+            ['>Q', 88, 0, 'autoclear_features'],
+            # Only refcount_order = 4 is supported by current (07.2014)
+            # implementation of QEMU
+            ['>I', 96, 4, 'refcount_order'],
+            ['>I', 100, 0, 'header_length']
+        ]
+        self.header = FieldsList(meta_header)
+
+        if self.header['version'][0].value == 2:
+            self.header['header_length'][0].value = 72
+        else:
+            self.header['incompatible_features'][0].value = \
+                                                        random.getrandbits(2)
+            self.header['compatible_features'][0].value = random.getrandbits(1)
+            self.header['header_length'][0].value = 104
+        # Extensions start at the header last field offset and the field size
+        self.ext_offset = struct.calcsize(
+            self.header['header_length'][0].fmt) + \
+            self.header['header_length'][0].offset
+        end_of_extension_area_len = 2 * UINT32_S
+        free_space = self.cluster_size - self.ext_offset - \
+                     end_of_extension_area_len
+        # If the backing file name specified and there is enough space for it
+        # in the first cluster, then it's placed in the very end of the first
+        # cluster.
+        if (backing_file_name is not None) and \
+           (free_space >= len(backing_file_name)):
+            self.header['backing_file_size'][0].value = len(backing_file_name)
+            self.header['backing_file_offset'][0].value = \
+                                    self.cluster_size - len(backing_file_name)
+
+    def set_backing_file_name(self, backing_file_name=None):
+        """Add the name of the backing file at the offset specified
+        in the header.
+        """
+        if (backing_file_name is not None) and \
+           (not self.header['backing_file_offset'][0].value == 0):
+            data_len = len(backing_file_name)
+            data_fmt = '>' + str(data_len) + 's'
+            self.backing_file_name = FieldsList([
+                [data_fmt, self.header['backing_file_offset'][0].value,
+                 backing_file_name, 'bf_name']
+            ])
+
+    def set_backing_file_format(self, backing_file_fmt=None):
+        """Generate the header extension for the backing file format."""
+        if backing_file_fmt is not None:
+            # Calculation of the free space available in the first cluster
+            end_of_extension_area_len = 2 * UINT32_S
+            high_border = (self.header['backing_file_offset'][0].value or
+                           (self.cluster_size - 1)) - \
+                end_of_extension_area_len
+            free_space = high_border - self.ext_offset
+            ext_size = 2 * UINT32_S + ((len(backing_file_fmt) + 7) & ~7)
+
+            if free_space >= ext_size:
+                ext_data_len = len(backing_file_fmt)
+                ext_data_fmt = '>' + str(ext_data_len) + 's'
+                ext_padding_len = 7 - (ext_data_len - 1) % 8
+                self.backing_file_format = FieldsList([
+                    ['>I', self.ext_offset, 0xE2792ACA, 'ext_magic'],
+                    ['>I', self.ext_offset + UINT32_S, ext_data_len,
+                     'ext_length'],
+                    [ext_data_fmt, self.ext_offset + UINT32_S * 2,
+                     backing_file_fmt, 'bf_format']
+                ])
+                self.ext_offset = \
+                        struct.calcsize(
+                            self.backing_file_format['bf_format'][0].fmt) + \
+                        ext_padding_len + \
+                        self.backing_file_format['bf_format'][0].offset
+
+    def create_feature_name_table(self):
+        """Generate a random header extension for names of features used in
+        the image.
+        """
+        def gen_feat_ids():
+            """Return random feature type and feature bit."""
+            return (random.randint(0, 2), random.randint(0, 63))
+
+        end_of_extension_area_len = 2 * UINT32_S
+        high_border = (self.header['backing_file_offset'][0].value or
+                       (self.cluster_size - 1)) - \
+            end_of_extension_area_len
+        free_space = high_border - self.ext_offset
+        # Sum of sizes of 'magic' and 'length' header extension fields
+        ext_header_len = 2 * UINT32_S
+        fnt_entry_size = 6 * UINT64_S
+        num_fnt_entries = min(10, (free_space - ext_header_len) /
+                              fnt_entry_size)
+        if not num_fnt_entries == 0:
+            feature_tables = []
+            feature_ids = []
+            inner_offset = self.ext_offset + ext_header_len
+            feat_name = 'some cool feature'
+            while len(feature_tables) < num_fnt_entries * 3:
+                feat_type, feat_bit = gen_feat_ids()
+                # Remove duplicates
+                while (feat_type, feat_bit) in feature_ids:
+                    feat_type, feat_bit = gen_feat_ids()
+                feature_ids.append((feat_type, feat_bit))
+                feat_fmt = '>' + str(len(feat_name)) + 's'
+                feature_tables += [['B', inner_offset,
+                                    feat_type, 'feature_type'],
+                                   ['B', inner_offset + 1, feat_bit,
+                                    'feature_bit_number'],
+                                   [feat_fmt, inner_offset + 2,
+                                    feat_name, 'feature_name']
+                ]
+                inner_offset += fnt_entry_size
+            # No padding for the extension is necessary, because
+            # the extension length is multiple of 8
+            self.feature_name_table = FieldsList([
+                ['>I', self.ext_offset, 0x6803f857, 'ext_magic'],
+                # One feature table contains 3 fields and takes 48 bytes
+                ['>I', self.ext_offset + UINT32_S,
+                 len(feature_tables) / 3 * 48, 'ext_length']
+            ] + feature_tables)
+            self.ext_offset = inner_offset
+
+    def set_end_of_extension_area(self):
+        """Generate a mandatory header extension marking end of header
+        extensions.
+        """
+        self.end_of_extension_area = FieldsList([
+            ['>I', self.ext_offset, 0, 'ext_magic'],
+            ['>I', self.ext_offset + UINT32_S, 0, 'ext_length']
+        ])
+
+    def create_l_structures(self):
+        """Generate random valid L1 and L2 tables."""
+        def create_l2_entry(host, guest, l2_cluster):
+            """Generate one L2 entry."""
+            offset = l2_cluster * self.cluster_size
+            l2_size = self.cluster_size / UINT64_S
+            entry_offset = offset + UINT64_S * (guest % l2_size)
+            cluster_descriptor = host * self.cluster_size
+            if not self.header['version'][0].value == 2:
+                cluster_descriptor += random.randint(0, 1)
+            # While snapshots are not supported, bit #63 = 1
+            # Compressed clusters are not supported => bit #62 = 0
+            entry_val = (1 << 63) + cluster_descriptor
+            return ['>Q', entry_offset, entry_val, 'l2_entry']
+
+        def create_l1_entry(l2_cluster, l1_offset, guest):
+            """Generate one L1 entry."""
+            l2_size = self.cluster_size / UINT64_S
+            entry_offset = l1_offset + UINT64_S * (guest / l2_size)
+            # While snapshots are not supported bit #63 = 1
+            entry_val = (1 << 63) + l2_cluster * self.cluster_size
+            return ['>Q', entry_offset, entry_val, 'l1_entry']
+
+        if len(self.data_clusters) == 0:
+            # All metadata for an empty guest image needs 4 clusters:
+            # header, rfc table, rfc block, L1 table.
+            # Header takes cluster #0, other clusters ##1-3 can be used
+            l1_offset = random.randint(1, 3) * self.cluster_size
+            l1 = [['>Q', l1_offset, 0, 'l1_entry']]
+            l2 = []
+        else:
+            meta_data = self._get_metadata()
+            guest_clusters = random.sample(range(self.image_size /
+                                                 self.cluster_size),
+                                           len(self.data_clusters))
+            # Number of entries in a L1/L2 table
+            l_size = self.cluster_size / UINT64_S
+            # Number of clusters necessary for L1 table
+            l1_size = int(ceil((max(guest_clusters) + 1) / float(l_size**2)))
+            l1_start = self._get_adjacent_clusters(self.data_clusters |
+                                                   meta_data, l1_size)
+            meta_data |= set(range(l1_start, l1_start + l1_size))
+            l1_offset = l1_start * self.cluster_size
+            # Indices of L2 tables
+            l2_ids = []
+            # Host clusters allocated for L2 tables
+            l2_clusters = []
+            # L1 entries
+            l1 = []
+            # L2 entries
+            l2 = []
+            for host, guest in zip(self.data_clusters, guest_clusters):
+                l2_id = guest / l_size
+                if l2_id not in l2_ids:
+                    l2_ids.append(l2_id)
+                    l2_clusters.append(self._get_adjacent_clusters(
+                        self.data_clusters | meta_data | set(l2_clusters),
+                        1))
+                    l1.append(create_l1_entry(l2_clusters[-1], l1_offset,
+                                              guest))
+                l2.append(create_l2_entry(host, guest,
+                                          l2_clusters[l2_ids.index(l2_id)]))
+        self.l2_tables = FieldsList(l2)
+        self.l1_table = FieldsList(l1)
+        self.header['l1_size'][0].value = int(ceil(UINT64_S * self.image_size /
+                                                float(self.cluster_size**2)))
+        self.header['l1_table_offset'][0].value = l1_offset
+
+    def create_refcount_structures(self):
+        """Generate random refcount blocks and refcount table."""
+        def allocate_rfc_blocks(data, size):
+            """Return indices of clusters allocated for refcount blocks."""
+            cluster_ids = set()
+            diff = block_ids = set([x / size for x in data])
+            while len(diff) != 0:
+                # Allocate all yet not allocated clusters
+                new = self._get_available_clusters(data | cluster_ids,
+                                                   len(diff))
+                # Indices of new refcount blocks necessary to cover clusters
+                # in 'new'
+                diff = set([x / size for x in new]) - block_ids
+                cluster_ids |= new
+                block_ids |= diff
+            return cluster_ids, block_ids
+
+        def allocate_rfc_table(data, init_blocks, block_size):
+            """Return indices of clusters allocated for the refcount table
+            and updated indices of clusters allocated for blocks and indices
+            of blocks.
+            """
+            blocks = set(init_blocks)
+            clusters = set()
+            # Number of entries in one cluster of the refcount table
+            size = self.cluster_size / UINT64_S
+            # Number of clusters necessary for the refcount table based on
+            # the current number of refcount blocks
+            table_size = int(ceil((max(blocks) + 1) / float(size)))
+            # Index of the first cluster of the refcount table
+            table_start = self._get_adjacent_clusters(data, table_size + 1)
+            # Clusters allocated for the current length of the refcount table
+            table_clusters = set(range(table_start, table_start + table_size))
+            # Clusters allocated for the refcount table including
+            # last optional one for potential l1 growth
+            table_clusters_allocated = set(range(table_start, table_start +
+                                                 table_size + 1))
+            # New refcount blocks necessary for clusters occupied by the
+            # refcount table
+            diff = set([c / block_size for c in table_clusters]) - blocks
+            blocks |= diff
+            while len(diff) != 0:
+                # Allocate clusters for new refcount blocks
+                new = self._get_available_clusters((data | clusters) |
+                                                   table_clusters_allocated,
+                                                   len(diff))
+                # Indices of new refcount blocks necessary to cover
+                # clusters in 'new'
+                diff = set([x / block_size for x in new]) - blocks
+                clusters |= new
+                blocks |= diff
+                # Check if the refcount table needs one more cluster
+                if int(ceil((max(blocks) + 1) / float(size))) > table_size:
+                    new_block_id = (table_start + table_size) / block_size
+                    # Check if the additional table cluster needs
+                    # one more refcount block
+                    if new_block_id not in blocks:
+                        diff.add(new_block_id)
+                    table_clusters.add(table_start + table_size)
+                    table_size += 1
+            return table_clusters, blocks, clusters
+
+        def create_table_entry(table_offset, block_cluster, block_size,
+                               cluster):
+            """Generate a refcount table entry."""
+            offset = table_offset + UINT64_S * (cluster / block_size)
+            return ['>Q', offset, block_cluster * self.cluster_size,
+                    'refcount_table_entry']
+
+        def create_block_entry(block_cluster, block_size, cluster):
+            """Generate a list of entries for the current block."""
+            entry_size = self.cluster_size / block_size
+            offset = block_cluster * self.cluster_size
+            entry_offset = offset + entry_size * (cluster % block_size)
+            # While snapshots are not supported all refcounts are set to 1
+            return ['>H', entry_offset, 1, 'refcount_block_entry']
+        # Size of a block entry in bits
+        refcount_bits = 1 << self.header['refcount_order'][0].value
+        # Number of refcount entries per refcount block
+        # Convert self.cluster_size from bytes to bits to have the same
+        # base for the numerator and denominator
+        block_size = self.cluster_size * 8 / refcount_bits
+        meta_data = self._get_metadata()
+        if len(self.data_clusters) == 0:
+            # All metadata for an empty guest image needs 4 clusters:
+            # header, rfc table, rfc block, L1 table.
+            # Header takes cluster #0, other clusters ##1-3 can be used
+            block_clusters = set([random.choice(list(set(range(1, 4)) -
+                                                     meta_data))])
+            block_ids = set([0])
+            table_clusters = set([random.choice(list(set(range(1, 4)) -
+                                                     meta_data -
+                                                     block_clusters))])
+        else:
+            block_clusters, block_ids = \
+                                allocate_rfc_blocks(self.data_clusters |
+                                                    meta_data, block_size)
+            table_clusters, block_ids, new_clusters = \
+                                    allocate_rfc_table(self.data_clusters |
+                                                       meta_data |
+                                                       block_clusters,
+                                                       block_ids,
+                                                       block_size)
+            block_clusters |= new_clusters
+
+        meta_data |= block_clusters | table_clusters
+        table_offset = min(table_clusters) * self.cluster_size
+        block_id = None
+        # Clusters allocated for refcount blocks
+        block_clusters = list(block_clusters)
+        # Indices of refcount blocks
+        block_ids = list(block_ids)
+        # Refcount table entries
+        rfc_table = []
+        # Refcount entries
+        rfc_blocks = []
+
+        for cluster in sorted(self.data_clusters | meta_data):
+            if cluster / block_size != block_id:
+                block_id = cluster / block_size
+                block_cluster = block_clusters[block_ids.index(block_id)]
+                rfc_table.append(create_table_entry(table_offset,
+                                                    block_cluster,
+                                                    block_size, cluster))
+            rfc_blocks.append(create_block_entry(block_cluster, block_size,
+                                                 cluster))
+        self.refcount_table = FieldsList(rfc_table)
+        self.refcount_blocks = FieldsList(rfc_blocks)
+
+        self.header['refcount_table_offset'][0].value = table_offset
+        self.header['refcount_table_clusters'][0].value = len(table_clusters)
+
+    def fuzz(self, fields_to_fuzz=None):
+        """Fuzz an image by corrupting values of a random subset of its fields.
+
+        Without parameters the method fuzzes an entire image.
+
+        If 'fields_to_fuzz' is specified then only fields in this list will be
+        fuzzed. 'fields_to_fuzz' can contain both individual fields and more
+        general image elements as a header or tables.
+
+        In the first case the field will be fuzzed always.
+        In the second a random subset of fields will be selected and fuzzed.
+        """
+        def coin():
+            """Return boolean value proportional to a portion of fields to be
+            fuzzed.
+            """
+            return random.random() < self.bias
+
+        if fields_to_fuzz is None:
+            for field in self:
+                if coin():
+                    field.value = getattr(fuzz, field.name)(field.value)
+        else:
+            for item in fields_to_fuzz:
+                if len(item) == 1:
+                    for field in getattr(self, item[0]):
+                        if coin():
+                            field.value = getattr(fuzz,
+                                                  field.name)(field.value)
+                else:
+                    # If fields with the requested name were not generated
+                    # getattr(self, item[0])[item[1]] returns an empty list
+                    for field in getattr(self, item[0])[item[1]]:
+                        field.value = getattr(fuzz, field.name)(field.value)
+
+    def write(self, filename):
+        """Write an entire image to the file."""
+        image_file = open(filename, 'w')
+        for field in self:
+            image_file.seek(field.offset)
+            image_file.write(struct.pack(field.fmt, field.value))
+
+        for cluster in sorted(self.data_clusters):
+            image_file.seek(cluster * self.cluster_size)
+            image_file.write(urandom(self.cluster_size))
+
+        # Align the real image size to the cluster size
+        image_file.seek(0, 2)
+        size = image_file.tell()
+        rounded = (size + self.cluster_size - 1) & ~(self.cluster_size - 1)
+        if rounded > size:
+            image_file.seek(rounded - 1)
+            image_file.write("\0")
+        image_file.close()
+
+    @staticmethod
+    def _size_params():
+        """Generate a random image size aligned to a random correct
+        cluster size.
+        """
+        cluster_bits = random.randrange(9, 21)
+        cluster_size = 1 << cluster_bits
+        img_size = random.randrange(0, MAX_IMAGE_SIZE + 1, cluster_size)
+        return (cluster_bits, img_size)
+
+    @staticmethod
+    def _get_available_clusters(used, number):
+        """Return a set of indices of not allocated clusters.
+
+        'used' contains indices of currently allocated clusters.
+        All clusters that cannot be allocated between 'used' clusters will have
+        indices appended to the end of 'used'.
+        """
+        append_id = max(used) + 1
+        free = set(range(1, append_id)) - used
+        if len(free) >= number:
+            return set(random.sample(free, number))
+        else:
+            return free | set(range(append_id, append_id + number - len(free)))
+
+    @staticmethod
+    def _get_adjacent_clusters(used, size):
+        """Return an index of the first cluster in the sequence of free ones.
+
+        'used' contains indices of currently allocated clusters. 'size' is the
+        length of the sequence of free clusters.
+        If the sequence of 'size' is not available between 'used' clusters, its
+        first index will be append to the end of 'used'.
+        """
+        def get_cluster_id(lst, length):
+            """Return the first index of the sequence of the specified length
+            or None if the sequence cannot be inserted in the list.
+            """
+            if len(lst) != 0:
+                pairs = []
+                pair = (lst[0], 1)
+                for i in range(1, len(lst)):
+                    if lst[i] == lst[i-1] + 1:
+                        pair = (lst[i], pair[1] + 1)
+                    else:
+                        pairs.append(pair)
+                        pair = (lst[i], 1)
+                pairs.append(pair)
+                random.shuffle(pairs)
+                for x, s in pairs:
+                    if s >= length:
+                        return x - length + 1
+            return None
+
+        append_id = max(used) + 1
+        free = list(set(range(1, append_id)) - used)
+        idx = get_cluster_id(free, size)
+        if idx is None:
+            return append_id
+        else:
+            return idx
+
+    @staticmethod
+    def _alloc_data(img_size, cluster_size):
+        """Return a set of random indices of clusters allocated for guest data.
+        """
+        num_of_cls = img_size/cluster_size
+        return set(random.sample(range(1, num_of_cls + 1),
+                                 random.randint(0, num_of_cls)))
+
+    def _get_metadata(self):
+        """Return indices of clusters allocated for image metadata."""
+        ids = set()
+        for x in self:
+            ids.add(x.offset/self.cluster_size)
+        return ids
+
+
+def create_image(test_img_path, backing_file_name=None, backing_file_fmt=None,
+                 fields_to_fuzz=None):
+    """Create a fuzzed image and write it to the specified file."""
+    image = Image(backing_file_name)
+    image.set_backing_file_format(backing_file_fmt)
+    image.create_feature_name_table()
+    image.set_end_of_extension_area()
+    image.create_l_structures()
+    image.create_refcount_structures()
+    image.fuzz(fields_to_fuzz)
+    image.write(test_img_path)
+    return image.image_size
diff --git a/tests/image-fuzzer/runner.py b/tests/image-fuzzer/runner.py
new file mode 100755
index 000000000..0a8743ef4
--- /dev/null
+++ b/tests/image-fuzzer/runner.py
@@ -0,0 +1,437 @@
+#!/usr/bin/env python
+
+# Tool for running fuzz tests
+#
+# Copyright (C) 2014 Maria Kustova <maria.k@catit.be>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import sys
+import os
+import signal
+import subprocess
+import random
+import shutil
+from itertools import count
+import time
+import getopt
+import StringIO
+import resource
+
+try:
+    import json
+except ImportError:
+    try:
+        import simplejson as json
+    except ImportError:
+        print >>sys.stderr, \
+            "Warning: Module for JSON processing is not found.\n" \
+            "'--config' and '--command' options are not supported."
+
+# Backing file sizes in MB
+MAX_BACKING_FILE_SIZE = 10
+MIN_BACKING_FILE_SIZE = 1
+
+
+def multilog(msg, *output):
+    """ Write an object to all of specified file descriptors."""
+    for fd in output:
+        fd.write(msg)
+        fd.flush()
+
+
+def str_signal(sig):
+    """ Convert a numeric value of a system signal to the string one
+    defined by the current operational system.
+    """
+    for k, v in signal.__dict__.items():
+        if v == sig:
+            return k
+
+
+def run_app(fd, q_args):
+    """Start an application with specified arguments and return its exit code
+    or kill signal depending on the result of execution.
+    """
+
+    class Alarm(Exception):
+        """Exception for signal.alarm events."""
+        pass
+
+    def handler(*args):
+        """Notify that an alarm event occurred."""
+        raise Alarm
+
+    signal.signal(signal.SIGALRM, handler)
+    signal.alarm(600)
+    term_signal = signal.SIGKILL
+    devnull = open('/dev/null', 'r+')
+    process = subprocess.Popen(q_args, stdin=devnull,
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    try:
+        out, err = process.communicate()
+        signal.alarm(0)
+        fd.write(out)
+        fd.write(err)
+        fd.flush()
+        return process.returncode
+
+    except Alarm:
+        os.kill(process.pid, term_signal)
+        fd.write('The command was terminated by timeout.\n')
+        fd.flush()
+        return -term_signal
+
+
+class TestException(Exception):
+    """Exception for errors risen by TestEnv objects."""
+    pass
+
+
+class TestEnv(object):
+
+    """Test object.
+
+    The class sets up test environment, generates backing and test images
+    and executes application under tests with specified arguments and a test
+    image provided.
+
+    All logs are collected.
+
+    The summary log will contain short descriptions and statuses of tests in
+    a run.
+
+    The test log will include application (e.g. 'qemu-img') logs besides info
+    sent to the summary log.
+    """
+
+    def __init__(self, test_id, seed, work_dir, run_log,
+                 cleanup=True, log_all=False):
+        """Set test environment in a specified work directory.
+
+        Path to qemu-img and qemu-io will be retrieved from 'QEMU_IMG' and
+        'QEMU_IO' environment variables.
+        """
+        if seed is not None:
+            self.seed = seed
+        else:
+            self.seed = str(random.randint(0, sys.maxint))
+        random.seed(self.seed)
+
+        self.init_path = os.getcwd()
+        self.work_dir = work_dir
+        self.current_dir = os.path.join(work_dir, 'test-' + test_id)
+        self.qemu_img = \
+            os.environ.get('QEMU_IMG', 'qemu-img').strip().split(' ')
+        self.qemu_io = os.environ.get('QEMU_IO', 'qemu-io').strip().split(' ')
+        self.commands = [['qemu-img', 'check', '-f', 'qcow2', '$test_img'],
+                         ['qemu-img', 'info', '-f', 'qcow2', '$test_img'],
+                         ['qemu-io', '$test_img', '-c', 'read $off $len'],
+                         ['qemu-io', '$test_img', '-c', 'write $off $len'],
+                         ['qemu-io', '$test_img', '-c',
+                          'aio_read $off $len'],
+                         ['qemu-io', '$test_img', '-c',
+                          'aio_write $off $len'],
+                         ['qemu-io', '$test_img', '-c', 'flush'],
+                         ['qemu-io', '$test_img', '-c',
+                          'discard $off $len'],
+                         ['qemu-io', '$test_img', '-c',
+                          'truncate $off']]
+        for fmt in ['raw', 'vmdk', 'vdi', 'qcow2', 'file', 'qed', 'vpc']:
+            self.commands.append(
+                ['qemu-img', 'convert', '-f', 'qcow2', '-O', fmt,
+                 '$test_img', 'converted_image.' + fmt])
+
+        try:
+            os.makedirs(self.current_dir)
+        except OSError, e:
+            print >>sys.stderr, \
+                "Error: The working directory '%s' cannot be used. Reason: %s"\
+                % (self.work_dir, e[1])
+            raise TestException
+        self.log = open(os.path.join(self.current_dir, "test.log"), "w")
+        self.parent_log = open(run_log, "a")
+        self.failed = False
+        self.cleanup = cleanup
+        self.log_all = log_all
+
+    def _create_backing_file(self):
+        """Create a backing file in the current directory.
+
+        Return a tuple of a backing file name and format.
+
+        Format of a backing file is randomly chosen from all formats supported
+        by 'qemu-img create'.
+        """
+        # All formats supported by the 'qemu-img create' command.
+        backing_file_fmt = random.choice(['raw', 'vmdk', 'vdi', 'qcow2',
+                                          'file', 'qed', 'vpc'])
+        backing_file_name = 'backing_img.' + backing_file_fmt
+        backing_file_size = random.randint(MIN_BACKING_FILE_SIZE,
+                                           MAX_BACKING_FILE_SIZE) * (1 << 20)
+        cmd = self.qemu_img + ['create', '-f', backing_file_fmt,
+                               backing_file_name, str(backing_file_size)]
+        temp_log = StringIO.StringIO()
+        retcode = run_app(temp_log, cmd)
+        if retcode == 0:
+            temp_log.close()
+            return (backing_file_name, backing_file_fmt)
+        else:
+            multilog("Warning: The %s backing file was not created.\n\n"
+                     % backing_file_fmt, sys.stderr, self.log, self.parent_log)
+            self.log.write("Log for the failure:\n" + temp_log.getvalue() +
+                           '\n\n')
+            temp_log.close()
+            return (None, None)
+
+    def execute(self, input_commands=None, fuzz_config=None):
+        """ Execute a test.
+
+        The method creates backing and test images, runs test app and analyzes
+        its exit status. If the application was killed by a signal, the test
+        is marked as failed.
+        """
+        if input_commands is None:
+            commands = self.commands
+        else:
+            commands = input_commands
+
+        os.chdir(self.current_dir)
+        backing_file_name, backing_file_fmt = self._create_backing_file()
+        img_size = image_generator.create_image(
+            'test.img', backing_file_name, backing_file_fmt, fuzz_config)
+        for item in commands:
+            shutil.copy('test.img', 'copy.img')
+            # 'off' and 'len' are multiple of the sector size
+            sector_size = 512
+            start = random.randrange(0, img_size + 1, sector_size)
+            end = random.randrange(start, img_size + 1, sector_size)
+
+            if item[0] == 'qemu-img':
+                current_cmd = list(self.qemu_img)
+            elif item[0] == 'qemu-io':
+                current_cmd = list(self.qemu_io)
+            else:
+                multilog("Warning: test command '%s' is not defined.\n"
+                         % item[0], sys.stderr, self.log, self.parent_log)
+                continue
+            # Replace all placeholders with their real values
+            for v in item[1:]:
+                c = (v
+                     .replace('$test_img', 'copy.img')
+                     .replace('$off', str(start))
+                     .replace('$len', str(end - start)))
+                current_cmd.append(c)
+
+            # Log string with the test header
+            test_summary = "Seed: %s\nCommand: %s\nTest directory: %s\n" \
+                           "Backing file: %s\n" \
+                           % (self.seed, " ".join(current_cmd),
+                              self.current_dir, backing_file_name)
+            temp_log = StringIO.StringIO()
+            try:
+                retcode = run_app(temp_log, current_cmd)
+            except OSError, e:
+                multilog("%sError: Start of '%s' failed. Reason: %s\n\n"
+                         % (test_summary, os.path.basename(current_cmd[0]),
+                            e[1]),
+                         sys.stderr, self.log, self.parent_log)
+                raise TestException
+
+            if retcode < 0:
+                self.log.write(temp_log.getvalue())
+                multilog("%sFAIL: Test terminated by signal %s\n\n"
+                         % (test_summary, str_signal(-retcode)),
+                         sys.stderr, self.log, self.parent_log)
+                self.failed = True
+            else:
+                if self.log_all:
+                    self.log.write(temp_log.getvalue())
+                    multilog("%sPASS: Application exited with the code " \
+                             "'%d'\n\n" % (test_summary, retcode),
+                             sys.stdout, self.log, self.parent_log)
+            temp_log.close()
+            os.remove('copy.img')
+
+    def finish(self):
+        """Restore the test environment after a test execution."""
+        self.log.close()
+        self.parent_log.close()
+        os.chdir(self.init_path)
+        if self.cleanup and not self.failed:
+            shutil.rmtree(self.current_dir)
+
+if __name__ == '__main__':
+
+    def usage():
+        print """
+        Usage: runner.py [OPTION...] TEST_DIR IMG_GENERATOR
+
+        Set up test environment in TEST_DIR and run a test in it. A module for
+        test image generation should be specified via IMG_GENERATOR.
+
+        Example:
+          runner.py -c '[["qemu-img", "info", "$test_img"]]' /tmp/test qcow2
+
+        Optional arguments:
+          -h, --help                    display this help and exit
+          -d, --duration=NUMBER         finish tests after NUMBER of seconds
+          -c, --command=JSON            run tests for all commands specified in
+                                        the JSON array
+          -s, --seed=STRING             seed for a test image generation,
+                                        by default will be generated randomly
+          --config=JSON                 take fuzzer configuration from the JSON
+                                        array
+          -k, --keep_passed             don't remove folders of passed tests
+          -v, --verbose                 log information about passed tests
+
+        JSON:
+
+        '--command' accepts a JSON array of commands. Each command presents
+        an application under test with all its paramaters as a list of strings,
+        e.g. ["qemu-io", "$test_img", "-c", "write $off $len"].
+
+        Supported application aliases: 'qemu-img' and 'qemu-io'.
+
+        Supported argument aliases: $test_img for the fuzzed image, $off
+        for an offset, $len for length.
+
+        Values for $off and $len will be generated based on the virtual disk
+        size of the fuzzed image.
+
+        Paths to 'qemu-img' and 'qemu-io' are retrevied from 'QEMU_IMG' and
+        'QEMU_IO' environment variables.
+
+        '--config' accepts a JSON array of fields to be fuzzed, e.g.
+        '[["header"], ["header", "version"]]'.
+
+        Each of the list elements can consist of a complex image element only
+        as ["header"] or ["feature_name_table"] or an exact field as
+        ["header", "version"]. In the first case random portion of the element
+        fields will be fuzzed, in the second one the specified field will be
+        fuzzed always.
+
+        If '--config' argument is specified, fields not listed in
+        the configuration array will not be fuzzed.
+        """
+
+    def run_test(test_id, seed, work_dir, run_log, cleanup, log_all,
+                 command, fuzz_config):
+        """Setup environment for one test and execute this test."""
+        try:
+            test = TestEnv(test_id, seed, work_dir, run_log, cleanup,
+                           log_all)
+        except TestException:
+            sys.exit(1)
+
+        # Python 2.4 doesn't support 'finally' and 'except' in the same 'try'
+        # block
+        try:
+            try:
+                test.execute(command, fuzz_config)
+            except TestException:
+                sys.exit(1)
+        finally:
+            test.finish()
+
+    def should_continue(duration, start_time):
+        """Return True if a new test can be started and False otherwise."""
+        current_time = int(time.time())
+        return (duration is None) or (current_time - start_time < duration)
+
+    try:
+        opts, args = getopt.gnu_getopt(sys.argv[1:], 'c:hs:kvd:',
+                                       ['command=', 'help', 'seed=', 'config=',
+                                        'keep_passed', 'verbose', 'duration='])
+    except getopt.error, e:
+        print >>sys.stderr, \
+            "Error: %s\n\nTry 'runner.py --help' for more information" % e
+        sys.exit(1)
+
+    command = None
+    cleanup = True
+    log_all = False
+    seed = None
+    config = None
+    duration = None
+    for opt, arg in opts:
+        if opt in ('-h', '--help'):
+            usage()
+            sys.exit()
+        elif opt in ('-c', '--command'):
+            try:
+                command = json.loads(arg)
+            except (TypeError, ValueError, NameError), e:
+                print >>sys.stderr, \
+                    "Error: JSON array of test commands cannot be loaded.\n" \
+                    "Reason: %s" % e
+                sys.exit(1)
+        elif opt in ('-k', '--keep_passed'):
+            cleanup = False
+        elif opt in ('-v', '--verbose'):
+            log_all = True
+        elif opt in ('-s', '--seed'):
+            seed = arg
+        elif opt in ('-d', '--duration'):
+            duration = int(arg)
+        elif opt == '--config':
+            try:
+                config = json.loads(arg)
+            except (TypeError, ValueError, NameError), e:
+                print >>sys.stderr, \
+                    "Error: JSON array with the fuzzer configuration cannot" \
+                    " be loaded\nReason: %s" % e
+                sys.exit(1)
+
+    if not len(args) == 2:
+        print >>sys.stderr, \
+            "Expected two parameters\nTry 'runner.py --help'" \
+            " for more information."
+        sys.exit(1)
+
+    work_dir = os.path.realpath(args[0])
+    # run_log is created in 'main', because multiple tests are expected to
+    # log in it
+    run_log = os.path.join(work_dir, 'run.log')
+
+    # Add the path to the image generator module to sys.path
+    sys.path.append(os.path.realpath(os.path.dirname(args[1])))
+    # Remove a script extension from image generator module if any
+    generator_name = os.path.splitext(os.path.basename(args[1]))[0]
+
+    try:
+        image_generator = __import__(generator_name)
+    except ImportError, e:
+        print >>sys.stderr, \
+            "Error: The image generator '%s' cannot be imported.\n" \
+            "Reason: %s" % (generator_name, e)
+        sys.exit(1)
+
+    # Enable core dumps
+    resource.setrlimit(resource.RLIMIT_CORE, (-1, -1))
+    # If a seed is specified, only one test will be executed.
+    # Otherwise runner will terminate after a keyboard interruption
+    start_time = int(time.time())
+    test_id = count(1)
+    while should_continue(duration, start_time):
+        try:
+            run_test(str(test_id.next()), seed, work_dir, run_log, cleanup,
+                     log_all, command, config)
+        except (KeyboardInterrupt, SystemExit):
+            sys.exit(1)
+
+        if seed is not None:
+            break
diff --git a/tests/libqos/malloc-pc.c b/tests/libqos/malloc-pc.c
index db1496c66..f4218c645 100644
--- a/tests/libqos/malloc-pc.c
+++ b/tests/libqos/malloc-pc.c
@@ -17,45 +17,294 @@
 #include "hw/nvram/fw_cfg.h"
 
 #include "qemu-common.h"
+#include "qemu/queue.h"
 #include <glib.h>
 
 #define PAGE_SIZE (4096)
 
+#define MLIST_ENTNAME entries
+typedef QTAILQ_HEAD(MemList, MemBlock) MemList;
+typedef struct MemBlock {
+    QTAILQ_ENTRY(MemBlock) MLIST_ENTNAME;
+    uint64_t size;
+    uint64_t addr;
+} MemBlock;
+
 typedef struct PCAlloc
 {
     QGuestAllocator alloc;
-
+    PCAllocOpts opts;
     uint64_t start;
     uint64_t end;
+
+    MemList used;
+    MemList free;
 } PCAlloc;
 
-static uint64_t pc_alloc(QGuestAllocator *allocator, size_t size)
+static MemBlock *mlist_new(uint64_t addr, uint64_t size)
 {
-    PCAlloc *s = container_of(allocator, PCAlloc, alloc);
-    uint64_t addr;
+    MemBlock *block;
+
+    if (!size) {
+        return NULL;
+    }
+    block = g_malloc0(sizeof(MemBlock));
 
+    block->addr = addr;
+    block->size = size;
 
-    size += (PAGE_SIZE - 1);
-    size &= PAGE_SIZE;
+    return block;
+}
+
+static void mlist_delete(MemList *list, MemBlock *node)
+{
+    g_assert(list && node);
+    QTAILQ_REMOVE(list, node, MLIST_ENTNAME);
+    g_free(node);
+}
+
+static MemBlock *mlist_find_key(MemList *head, uint64_t addr)
+{
+    MemBlock *node;
+    QTAILQ_FOREACH(node, head, MLIST_ENTNAME) {
+        if (node->addr == addr) {
+            return node;
+        }
+    }
+    return NULL;
+}
+
+static MemBlock *mlist_find_space(MemList *head, uint64_t size)
+{
+    MemBlock *node;
+
+    QTAILQ_FOREACH(node, head, MLIST_ENTNAME) {
+        if (node->size >= size) {
+            return node;
+        }
+    }
+    return NULL;
+}
+
+static MemBlock *mlist_sort_insert(MemList *head, MemBlock *insr)
+{
+    MemBlock *node;
+    g_assert(head && insr);
+
+    QTAILQ_FOREACH(node, head, MLIST_ENTNAME) {
+        if (insr->addr < node->addr) {
+            QTAILQ_INSERT_BEFORE(node, insr, MLIST_ENTNAME);
+            return insr;
+        }
+    }
+
+    QTAILQ_INSERT_TAIL(head, insr, MLIST_ENTNAME);
+    return insr;
+}
+
+static inline uint64_t mlist_boundary(MemBlock *node)
+{
+    return node->size + node->addr;
+}
+
+static MemBlock *mlist_join(MemList *head, MemBlock *left, MemBlock *right)
+{
+    g_assert(head && left && right);
+
+    left->size += right->size;
+    mlist_delete(head, right);
+    return left;
+}
+
+static void mlist_coalesce(MemList *head, MemBlock *node)
+{
+    g_assert(node);
+    MemBlock *left;
+    MemBlock *right;
+    char merge;
+
+    do {
+        merge = 0;
+        left = QTAILQ_PREV(node, MemList, MLIST_ENTNAME);
+        right = QTAILQ_NEXT(node, MLIST_ENTNAME);
+
+        /* clowns to the left of me */
+        if (left && mlist_boundary(left) == node->addr) {
+            node = mlist_join(head, left, node);
+            merge = 1;
+        }
+
+        /* jokers to the right */
+        if (right && mlist_boundary(node) == right->addr) {
+            node = mlist_join(head, node, right);
+            merge = 1;
+        }
+
+    } while (merge);
+}
+
+static uint64_t pc_mlist_fulfill(PCAlloc *s, MemBlock *freenode, uint64_t size)
+{
+    uint64_t addr;
+    MemBlock *usednode;
 
-    g_assert_cmpint((s->start + size), <=, s->end);
+    g_assert(freenode);
+    g_assert_cmpint(freenode->size, >=, size);
 
-    addr = s->start;
-    s->start += size;
+    addr = freenode->addr;
+    if (freenode->size == size) {
+        /* re-use this freenode as our used node */
+        QTAILQ_REMOVE(&s->free, freenode, MLIST_ENTNAME);
+        usednode = freenode;
+    } else {
+        /* adjust the free node and create a new used node */
+        freenode->addr += size;
+        freenode->size -= size;
+        usednode = mlist_new(addr, size);
+    }
 
+    mlist_sort_insert(&s->used, usednode);
     return addr;
 }
 
+/* To assert the correctness of the list.
+ * Used only if PC_ALLOC_PARANOID is set. */
+static void pc_mlist_check(PCAlloc *s)
+{
+    MemBlock *node;
+    uint64_t addr = s->start > 0 ? s->start - 1 : 0;
+    uint64_t next = s->start;
+
+    QTAILQ_FOREACH(node, &s->free, MLIST_ENTNAME) {
+        g_assert_cmpint(node->addr, >, addr);
+        g_assert_cmpint(node->addr, >=, next);
+        addr = node->addr;
+        next = node->addr + node->size;
+    }
+
+    addr = s->start > 0 ? s->start - 1 : 0;
+    next = s->start;
+    QTAILQ_FOREACH(node, &s->used, MLIST_ENTNAME) {
+        g_assert_cmpint(node->addr, >, addr);
+        g_assert_cmpint(node->addr, >=, next);
+        addr = node->addr;
+        next = node->addr + node->size;
+    }
+}
+
+static uint64_t pc_mlist_alloc(PCAlloc *s, uint64_t size)
+{
+    MemBlock *node;
+
+    node = mlist_find_space(&s->free, size);
+    if (!node) {
+        fprintf(stderr, "Out of guest memory.\n");
+        g_assert_not_reached();
+    }
+    return pc_mlist_fulfill(s, node, size);
+}
+
+static void pc_mlist_free(PCAlloc *s, uint64_t addr)
+{
+    MemBlock *node;
+
+    if (addr == 0) {
+        return;
+    }
+
+    node = mlist_find_key(&s->used, addr);
+    if (!node) {
+        fprintf(stderr, "Error: no record found for an allocation at "
+                "0x%016" PRIx64 ".\n",
+                addr);
+        g_assert_not_reached();
+    }
+
+    /* Rip it out of the used list and re-insert back into the free list. */
+    QTAILQ_REMOVE(&s->used, node, MLIST_ENTNAME);
+    mlist_sort_insert(&s->free, node);
+    mlist_coalesce(&s->free, node);
+}
+
+static uint64_t pc_alloc(QGuestAllocator *allocator, size_t size)
+{
+    PCAlloc *s = container_of(allocator, PCAlloc, alloc);
+    uint64_t rsize = size;
+    uint64_t naddr;
+
+    rsize += (PAGE_SIZE - 1);
+    rsize &= -PAGE_SIZE;
+    g_assert_cmpint((s->start + rsize), <=, s->end);
+    g_assert_cmpint(rsize, >=, size);
+
+    naddr = pc_mlist_alloc(s, rsize);
+    if (s->opts & PC_ALLOC_PARANOID) {
+        pc_mlist_check(s);
+    }
+
+    return naddr;
+}
+
 static void pc_free(QGuestAllocator *allocator, uint64_t addr)
 {
+    PCAlloc *s = container_of(allocator, PCAlloc, alloc);
+
+    pc_mlist_free(s, addr);
+    if (s->opts & PC_ALLOC_PARANOID) {
+        pc_mlist_check(s);
+    }
+}
+
+/*
+ * Mostly for valgrind happiness, but it does offer
+ * a chokepoint for debugging guest memory leaks, too.
+ */
+void pc_alloc_uninit(QGuestAllocator *allocator)
+{
+    PCAlloc *s = container_of(allocator, PCAlloc, alloc);
+    MemBlock *node;
+    MemBlock *tmp;
+    PCAllocOpts mask;
+
+    /* Check for guest leaks, and destroy the list. */
+    QTAILQ_FOREACH_SAFE(node, &s->used, MLIST_ENTNAME, tmp) {
+        if (s->opts & (PC_ALLOC_LEAK_WARN | PC_ALLOC_LEAK_ASSERT)) {
+            fprintf(stderr, "guest malloc leak @ 0x%016" PRIx64 "; "
+                    "size 0x%016" PRIx64 ".\n",
+                    node->addr, node->size);
+        }
+        if (s->opts & (PC_ALLOC_LEAK_ASSERT)) {
+            g_assert_not_reached();
+        }
+        g_free(node);
+    }
+
+    /* If we have previously asserted that there are no leaks, then there
+     * should be only one node here with a specific address and size. */
+    mask = PC_ALLOC_LEAK_ASSERT | PC_ALLOC_PARANOID;
+    QTAILQ_FOREACH_SAFE(node, &s->free, MLIST_ENTNAME, tmp) {
+        if ((s->opts & mask) == mask) {
+            if ((node->addr != s->start) ||
+                (node->size != s->end - s->start)) {
+                fprintf(stderr, "Free list is corrupted.\n");
+                g_assert_not_reached();
+            }
+        }
+
+        g_free(node);
+    }
+
+    g_free(s);
 }
 
-QGuestAllocator *pc_alloc_init(void)
+QGuestAllocator *pc_alloc_init_flags(PCAllocOpts flags)
 {
     PCAlloc *s = g_malloc0(sizeof(*s));
     uint64_t ram_size;
     QFWCFG *fw_cfg = pc_fw_cfg_init();
+    MemBlock *node;
 
+    s->opts = flags;
     s->alloc.alloc = pc_alloc;
     s->alloc.free = pc_free;
 
@@ -67,5 +316,19 @@ QGuestAllocator *pc_alloc_init(void)
     /* Respect PCI hole */
     s->end = MIN(ram_size, 0xE0000000);
 
+    /* clean-up */
+    g_free(fw_cfg);
+
+    QTAILQ_INIT(&s->used);
+    QTAILQ_INIT(&s->free);
+
+    node = mlist_new(s->start, s->end - s->start);
+    QTAILQ_INSERT_HEAD(&s->free, node, MLIST_ENTNAME);
+
     return &s->alloc;
 }
+
+inline QGuestAllocator *pc_alloc_init(void)
+{
+    return pc_alloc_init_flags(PC_ALLOC_NO_FLAGS);
+}
diff --git a/tests/libqos/malloc-pc.h b/tests/libqos/malloc-pc.h
index ff964abe5..9f525e3b9 100644
--- a/tests/libqos/malloc-pc.h
+++ b/tests/libqos/malloc-pc.h
@@ -15,6 +15,15 @@
 
 #include "libqos/malloc.h"
 
+typedef enum {
+    PC_ALLOC_NO_FLAGS    = 0x00,
+    PC_ALLOC_LEAK_WARN   = 0x01,
+    PC_ALLOC_LEAK_ASSERT = 0x02,
+    PC_ALLOC_PARANOID    = 0x04
+} PCAllocOpts;
+
 QGuestAllocator *pc_alloc_init(void);
+QGuestAllocator *pc_alloc_init_flags(PCAllocOpts flags);
+void             pc_alloc_uninit(QGuestAllocator *allocator);
 
 #endif
diff --git a/tests/libqos/malloc.h b/tests/libqos/malloc.h
index 46f600076..556538121 100644
--- a/tests/libqos/malloc.h
+++ b/tests/libqos/malloc.h
@@ -32,7 +32,7 @@ static inline uint64_t guest_alloc(QGuestAllocator *allocator, size_t size)
 
 static inline void guest_free(QGuestAllocator *allocator, uint64_t addr)
 {
-    allocator->alloc(allocator, addr);
+    allocator->free(allocator, addr);
 }
 
 #endif
diff --git a/tests/libqos/pci-pc.c b/tests/libqos/pci-pc.c
index 4adf4006a..6dba0db00 100644
--- a/tests/libqos/pci-pc.c
+++ b/tests/libqos/pci-pc.c
@@ -20,6 +20,9 @@
 
 #include <glib.h>
 
+#define ACPI_PCIHP_ADDR         0xae00
+#define PCI_EJ_BASE             0x0008
+
 typedef struct QPCIBusPC
 {
     QPCIBus bus;
@@ -144,7 +147,7 @@ static void qpci_pc_config_writel(QPCIBus *bus, int devfn, uint8_t offset, uint3
     outl(0xcfc, value);
 }
 
-static void *qpci_pc_iomap(QPCIBus *bus, QPCIDevice *dev, int barno)
+static void *qpci_pc_iomap(QPCIBus *bus, QPCIDevice *dev, int barno, uint64_t *sizeptr)
 {
     QPCIBusPC *s = container_of(bus, QPCIBusPC, bus);
     static const int bar_reg_map[] = {
@@ -173,6 +176,9 @@ static void *qpci_pc_iomap(QPCIBus *bus, QPCIDevice *dev, int barno)
     if (size == 0) {
         return NULL;
     }
+    if (sizeptr) {
+        *sizeptr = size;
+    }
 
     if (io_type == PCI_BASE_ADDRESS_SPACE_IO) {
         uint16_t loc;
@@ -237,3 +243,56 @@ QPCIBus *qpci_init_pc(void)
 
     return &ret->bus;
 }
+
+void qpci_free_pc(QPCIBus *bus)
+{
+    QPCIBusPC *s = container_of(bus, QPCIBusPC, bus);
+
+    g_free(s);
+}
+
+void qpci_plug_device_test(const char *driver, const char *id,
+                           uint8_t slot, const char *opts)
+{
+    QDict *response;
+    char *cmd;
+
+    cmd = g_strdup_printf("{'execute': 'device_add',"
+                          " 'arguments': {"
+                          "   'driver': '%s',"
+                          "   'addr': '%d',"
+                          "   %s%s"
+                          "   'id': '%s'"
+                          "}}", driver, slot,
+                          opts ? opts : "", opts ? "," : "",
+                          id);
+    response = qmp(cmd);
+    g_free(cmd);
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+}
+
+void qpci_unplug_acpi_device_test(const char *id, uint8_t slot)
+{
+    QDict *response;
+    char *cmd;
+
+    cmd = g_strdup_printf("{'execute': 'device_del',"
+                          " 'arguments': {"
+                          "   'id': '%s'"
+                          "}}", id);
+    response = qmp(cmd);
+    g_free(cmd);
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    outb(ACPI_PCIHP_ADDR + PCI_EJ_BASE, 1 << slot);
+
+    response = qmp("");
+    g_assert(response);
+    g_assert(qdict_haskey(response, "event"));
+    g_assert(!strcmp(qdict_get_str(response, "event"), "DEVICE_DELETED"));
+    QDECREF(response);
+}
diff --git a/tests/libqos/pci-pc.h b/tests/libqos/pci-pc.h
index 4f7475f6f..26211790c 100644
--- a/tests/libqos/pci-pc.h
+++ b/tests/libqos/pci-pc.h
@@ -16,5 +16,6 @@
 #include "libqos/pci.h"
 
 QPCIBus *qpci_init_pc(void);
+void     qpci_free_pc(QPCIBus *bus);
 
 #endif
diff --git a/tests/libqos/pci.c b/tests/libqos/pci.c
index c9a0b9134..4e630c250 100644
--- a/tests/libqos/pci.c
+++ b/tests/libqos/pci.c
@@ -15,8 +15,6 @@
 #include "hw/pci/pci_regs.h"
 #include <glib.h>
 
-#include <stdio.h>
-
 void qpci_device_foreach(QPCIBus *bus, int vendor_id, int device_id,
                          void (*func)(QPCIDevice *dev, int devfn, void *data),
                          void *data)
@@ -73,6 +71,121 @@ void qpci_device_enable(QPCIDevice *dev)
     cmd = qpci_config_readw(dev, PCI_COMMAND);
     cmd |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER;
     qpci_config_writew(dev, PCI_COMMAND, cmd);
+
+    /* Verify the bits are now set. */
+    cmd = qpci_config_readw(dev, PCI_COMMAND);
+    g_assert_cmphex(cmd & PCI_COMMAND_IO, ==, PCI_COMMAND_IO);
+    g_assert_cmphex(cmd & PCI_COMMAND_MEMORY, ==, PCI_COMMAND_MEMORY);
+    g_assert_cmphex(cmd & PCI_COMMAND_MASTER, ==, PCI_COMMAND_MASTER);
+}
+
+uint8_t qpci_find_capability(QPCIDevice *dev, uint8_t id)
+{
+    uint8_t cap;
+    uint8_t addr = qpci_config_readb(dev, PCI_CAPABILITY_LIST);
+
+    do {
+        cap = qpci_config_readb(dev, addr);
+        if (cap != id) {
+            addr = qpci_config_readb(dev, addr + PCI_CAP_LIST_NEXT);
+        }
+    } while (cap != id && addr != 0);
+
+    return addr;
+}
+
+void qpci_msix_enable(QPCIDevice *dev)
+{
+    uint8_t addr;
+    uint16_t val;
+    uint32_t table;
+    uint8_t bir_table;
+    uint8_t bir_pba;
+    void *offset;
+
+    addr = qpci_find_capability(dev, PCI_CAP_ID_MSIX);
+    g_assert_cmphex(addr, !=, 0);
+
+    val = qpci_config_readw(dev, addr + PCI_MSIX_FLAGS);
+    qpci_config_writew(dev, addr + PCI_MSIX_FLAGS, val | PCI_MSIX_FLAGS_ENABLE);
+
+    table = qpci_config_readl(dev, addr + PCI_MSIX_TABLE);
+    bir_table = table & PCI_MSIX_FLAGS_BIRMASK;
+    offset = qpci_iomap(dev, bir_table, NULL);
+    dev->msix_table = offset + (table & ~PCI_MSIX_FLAGS_BIRMASK);
+
+    table = qpci_config_readl(dev, addr + PCI_MSIX_PBA);
+    bir_pba = table & PCI_MSIX_FLAGS_BIRMASK;
+    if (bir_pba != bir_table) {
+        offset = qpci_iomap(dev, bir_pba, NULL);
+    }
+    dev->msix_pba = offset + (table & ~PCI_MSIX_FLAGS_BIRMASK);
+
+    g_assert(dev->msix_table != NULL);
+    g_assert(dev->msix_pba != NULL);
+    dev->msix_enabled = true;
+}
+
+void qpci_msix_disable(QPCIDevice *dev)
+{
+    uint8_t addr;
+    uint16_t val;
+
+    g_assert(dev->msix_enabled);
+    addr = qpci_find_capability(dev, PCI_CAP_ID_MSIX);
+    g_assert_cmphex(addr, !=, 0);
+    val = qpci_config_readw(dev, addr + PCI_MSIX_FLAGS);
+    qpci_config_writew(dev, addr + PCI_MSIX_FLAGS,
+                                                val & ~PCI_MSIX_FLAGS_ENABLE);
+
+    qpci_iounmap(dev, dev->msix_table);
+    qpci_iounmap(dev, dev->msix_pba);
+    dev->msix_enabled = 0;
+    dev->msix_table = NULL;
+    dev->msix_pba = NULL;
+}
+
+bool qpci_msix_pending(QPCIDevice *dev, uint16_t entry)
+{
+    uint32_t pba_entry;
+    uint8_t bit_n = entry % 32;
+    void *addr = dev->msix_pba + (entry / 32) * PCI_MSIX_ENTRY_SIZE / 4;
+
+    g_assert(dev->msix_enabled);
+    pba_entry = qpci_io_readl(dev, addr);
+    qpci_io_writel(dev, addr, pba_entry & ~(1 << bit_n));
+    return (pba_entry & (1 << bit_n)) != 0;
+}
+
+bool qpci_msix_masked(QPCIDevice *dev, uint16_t entry)
+{
+    uint8_t addr;
+    uint16_t val;
+    void *vector_addr = dev->msix_table + (entry * PCI_MSIX_ENTRY_SIZE);
+
+    g_assert(dev->msix_enabled);
+    addr = qpci_find_capability(dev, PCI_CAP_ID_MSIX);
+    g_assert_cmphex(addr, !=, 0);
+    val = qpci_config_readw(dev, addr + PCI_MSIX_FLAGS);
+
+    if (val & PCI_MSIX_FLAGS_MASKALL) {
+        return true;
+    } else {
+        return (qpci_io_readl(dev, vector_addr + PCI_MSIX_ENTRY_VECTOR_CTRL)
+                                            & PCI_MSIX_ENTRY_CTRL_MASKBIT) != 0;
+    }
+}
+
+uint16_t qpci_msix_table_size(QPCIDevice *dev)
+{
+    uint8_t addr;
+    uint16_t control;
+
+    addr = qpci_find_capability(dev, PCI_CAP_ID_MSIX);
+    g_assert_cmphex(addr, !=, 0);
+
+    control = qpci_config_readw(dev, addr + PCI_MSIX_FLAGS);
+    return (control & PCI_MSIX_FLAGS_QSIZE) + 1;
 }
 
 uint8_t qpci_config_readb(QPCIDevice *dev, uint8_t offset)
@@ -138,9 +251,9 @@ void qpci_io_writel(QPCIDevice *dev, void *data, uint32_t value)
     dev->bus->io_writel(dev->bus, data, value);
 }
 
-void *qpci_iomap(QPCIDevice *dev, int barno)
+void *qpci_iomap(QPCIDevice *dev, int barno, uint64_t *sizeptr)
 {
-    return dev->bus->iomap(dev->bus, dev, barno);
+    return dev->bus->iomap(dev->bus, dev, barno, sizeptr);
 }
 
 void qpci_iounmap(QPCIDevice *dev, void *data)
diff --git a/tests/libqos/pci.h b/tests/libqos/pci.h
index 343943154..dfaee9ec3 100644
--- a/tests/libqos/pci.h
+++ b/tests/libqos/pci.h
@@ -14,6 +14,7 @@
 #define LIBQOS_PCI_H
 
 #include <stdint.h>
+#include "libqtest.h"
 
 #define QPCI_DEVFN(dev, fn) (((dev) << 3) | (fn))
 
@@ -41,7 +42,7 @@ struct QPCIBus
     void (*config_writel)(QPCIBus *bus, int devfn,
                           uint8_t offset, uint32_t value);
 
-    void *(*iomap)(QPCIBus *bus, QPCIDevice *dev, int barno);
+    void *(*iomap)(QPCIBus *bus, QPCIDevice *dev, int barno, uint64_t *sizeptr);
     void (*iounmap)(QPCIBus *bus, void *data);
 };
 
@@ -49,6 +50,9 @@ struct QPCIDevice
 {
     QPCIBus *bus;
     int devfn;
+    bool msix_enabled;
+    void *msix_table;
+    void *msix_pba;
 };
 
 void qpci_device_foreach(QPCIBus *bus, int vendor_id, int device_id,
@@ -57,6 +61,12 @@ void qpci_device_foreach(QPCIBus *bus, int vendor_id, int device_id,
 QPCIDevice *qpci_device_find(QPCIBus *bus, int devfn);
 
 void qpci_device_enable(QPCIDevice *dev);
+uint8_t qpci_find_capability(QPCIDevice *dev, uint8_t id);
+void qpci_msix_enable(QPCIDevice *dev);
+void qpci_msix_disable(QPCIDevice *dev);
+bool qpci_msix_pending(QPCIDevice *dev, uint16_t entry);
+bool qpci_msix_masked(QPCIDevice *dev, uint16_t entry);
+uint16_t qpci_msix_table_size(QPCIDevice *dev);
 
 uint8_t qpci_config_readb(QPCIDevice *dev, uint8_t offset);
 uint16_t qpci_config_readw(QPCIDevice *dev, uint8_t offset);
@@ -74,7 +84,10 @@ void qpci_io_writeb(QPCIDevice *dev, void *data, uint8_t value);
 void qpci_io_writew(QPCIDevice *dev, void *data, uint16_t value);
 void qpci_io_writel(QPCIDevice *dev, void *data, uint32_t value);
 
-void *qpci_iomap(QPCIDevice *dev, int barno);
+void *qpci_iomap(QPCIDevice *dev, int barno, uint64_t *sizeptr);
 void qpci_iounmap(QPCIDevice *dev, void *data);
 
+void qpci_plug_device_test(const char *driver, const char *id,
+                           uint8_t slot, const char *opts);
+void qpci_unplug_acpi_device_test(const char *id, uint8_t slot);
 #endif
diff --git a/tests/libqos/usb.c b/tests/libqos/usb.c
new file mode 100644
index 000000000..41d89b848
--- /dev/null
+++ b/tests/libqos/usb.c
@@ -0,0 +1,71 @@
+/*
+ * common code shared by usb tests
+ *
+ * Copyright (c) 2014 Red Hat, Inc
+ *
+ * Authors:
+ *     Gerd Hoffmann <kraxel@redhat.com>
+ *     John Snow <jsnow@redhat.com>
+ *     Igor Mammedov <imammedo@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#include <glib.h>
+#include <string.h>
+#include "libqtest.h"
+#include "qemu/osdep.h"
+#include "hw/usb/uhci-regs.h"
+#include "libqos/usb.h"
+
+void qusb_pci_init_one(QPCIBus *pcibus, struct qhc *hc, uint32_t devfn, int bar)
+{
+    hc->dev = qpci_device_find(pcibus, devfn);
+    g_assert(hc->dev != NULL);
+    qpci_device_enable(hc->dev);
+    hc->base = qpci_iomap(hc->dev, bar, NULL);
+    g_assert(hc->base != NULL);
+}
+
+void uhci_port_test(struct qhc *hc, int port, uint16_t expect)
+{
+    void *addr = hc->base + 0x10 + 2 * port;
+    uint16_t value = qpci_io_readw(hc->dev, addr);
+    uint16_t mask = ~(UHCI_PORT_WRITE_CLEAR | UHCI_PORT_RSVD1);
+
+    g_assert((value & mask) == (expect & mask));
+}
+
+void usb_test_hotplug(const char *hcd_id, const int port,
+                      void (*port_check)(void))
+{
+    QDict *response;
+    char  *cmd;
+
+    cmd = g_strdup_printf("{'execute': 'device_add',"
+                          " 'arguments': {"
+                          "   'driver': 'usb-tablet',"
+                          "   'port': '%d',"
+                          "   'bus': '%s.0',"
+                          "   'id': 'usbdev%d'"
+                          "}}", port, hcd_id, port);
+    response = qmp(cmd);
+    g_free(cmd);
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    if (port_check) {
+        port_check();
+    }
+
+    cmd = g_strdup_printf("{'execute': 'device_del',"
+                           " 'arguments': {"
+                           "   'id': 'usbdev%d'"
+                           "}}", port);
+    response = qmp(cmd);
+    g_free(cmd);
+    g_assert(response);
+    g_assert(qdict_haskey(response, "event"));
+    g_assert(!strcmp(qdict_get_str(response, "event"), "DEVICE_DELETED"));
+}
diff --git a/tests/libqos/usb.h b/tests/libqos/usb.h
new file mode 100644
index 000000000..8fe56872b
--- /dev/null
+++ b/tests/libqos/usb.h
@@ -0,0 +1,17 @@
+#ifndef LIBQOS_USB_H
+#define LIBQOS_USB_H
+
+#include "libqos/pci-pc.h"
+
+struct qhc {
+    QPCIDevice *dev;
+    void *base;
+};
+
+void qusb_pci_init_one(QPCIBus *pcibus, struct qhc *hc,
+                       uint32_t devfn, int bar);
+void uhci_port_test(struct qhc *hc, int port, uint16_t expect);
+
+void usb_test_hotplug(const char *bus_name, const int port,
+                      void (*port_check)(void));
+#endif
diff --git a/tests/libqos/virtio-pci.c b/tests/libqos/virtio-pci.c
new file mode 100644
index 000000000..788ebaff4
--- /dev/null
+++ b/tests/libqos/virtio-pci.c
@@ -0,0 +1,343 @@
+/*
+ * libqos virtio PCI driver
+ *
+ * Copyright (c) 2014 Marc Marí
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include <glib.h>
+#include <stdio.h>
+#include "libqtest.h"
+#include "libqos/virtio.h"
+#include "libqos/virtio-pci.h"
+#include "libqos/pci.h"
+#include "libqos/pci-pc.h"
+#include "libqos/malloc.h"
+#include "libqos/malloc-pc.h"
+
+#include "hw/pci/pci_regs.h"
+
+typedef struct QVirtioPCIForeachData {
+    void (*func)(QVirtioDevice *d, void *data);
+    uint16_t device_type;
+    void *user_data;
+} QVirtioPCIForeachData;
+
+static QVirtioPCIDevice *qpcidevice_to_qvirtiodevice(QPCIDevice *pdev)
+{
+    QVirtioPCIDevice *vpcidev;
+    vpcidev = g_malloc0(sizeof(*vpcidev));
+
+    if (pdev) {
+        vpcidev->pdev = pdev;
+        vpcidev->vdev.device_type =
+                            qpci_config_readw(vpcidev->pdev, PCI_SUBSYSTEM_ID);
+    }
+
+    vpcidev->config_msix_entry = -1;
+
+    return vpcidev;
+}
+
+static void qvirtio_pci_foreach_callback(
+                        QPCIDevice *dev, int devfn, void *data)
+{
+    QVirtioPCIForeachData *d = data;
+    QVirtioPCIDevice *vpcidev = qpcidevice_to_qvirtiodevice(dev);
+
+    if (vpcidev->vdev.device_type == d->device_type) {
+        d->func(&vpcidev->vdev, d->user_data);
+    } else {
+        g_free(vpcidev);
+    }
+}
+
+static void qvirtio_pci_assign_device(QVirtioDevice *d, void *data)
+{
+    QVirtioPCIDevice **vpcidev = data;
+    *vpcidev = (QVirtioPCIDevice *)d;
+}
+
+static uint8_t qvirtio_pci_config_readb(QVirtioDevice *d, void *addr)
+{
+    QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
+    return qpci_io_readb(dev->pdev, addr);
+}
+
+static uint16_t qvirtio_pci_config_readw(QVirtioDevice *d, void *addr)
+{
+    QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
+    return qpci_io_readw(dev->pdev, addr);
+}
+
+static uint32_t qvirtio_pci_config_readl(QVirtioDevice *d, void *addr)
+{
+    QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
+    return qpci_io_readl(dev->pdev, addr);
+}
+
+static uint64_t qvirtio_pci_config_readq(QVirtioDevice *d, void *addr)
+{
+    QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
+    int i;
+    uint64_t u64 = 0;
+
+    if (qtest_big_endian()) {
+        for (i = 0; i < 8; ++i) {
+            u64 |= (uint64_t)qpci_io_readb(dev->pdev, addr + i) << (7 - i) * 8;
+        }
+    } else {
+        for (i = 0; i < 8; ++i) {
+            u64 |= (uint64_t)qpci_io_readb(dev->pdev, addr + i) << i * 8;
+        }
+    }
+
+    return u64;
+}
+
+static uint32_t qvirtio_pci_get_features(QVirtioDevice *d)
+{
+    QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
+    return qpci_io_readl(dev->pdev, dev->addr + QVIRTIO_DEVICE_FEATURES);
+}
+
+static void qvirtio_pci_set_features(QVirtioDevice *d, uint32_t features)
+{
+    QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
+    qpci_io_writel(dev->pdev, dev->addr + QVIRTIO_GUEST_FEATURES, features);
+}
+
+static uint32_t qvirtio_pci_get_guest_features(QVirtioDevice *d)
+{
+    QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
+    return qpci_io_readl(dev->pdev, dev->addr + QVIRTIO_GUEST_FEATURES);
+}
+
+static uint8_t qvirtio_pci_get_status(QVirtioDevice *d)
+{
+    QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
+    return qpci_io_readb(dev->pdev, dev->addr + QVIRTIO_DEVICE_STATUS);
+}
+
+static void qvirtio_pci_set_status(QVirtioDevice *d, uint8_t status)
+{
+    QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
+    qpci_io_writeb(dev->pdev, dev->addr + QVIRTIO_DEVICE_STATUS, status);
+}
+
+static bool qvirtio_pci_get_queue_isr_status(QVirtioDevice *d, QVirtQueue *vq)
+{
+    QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
+    QVirtQueuePCI *vqpci = (QVirtQueuePCI *)vq;
+    uint32_t data;
+
+    if (dev->pdev->msix_enabled) {
+        g_assert_cmpint(vqpci->msix_entry, !=, -1);
+        if (qpci_msix_masked(dev->pdev, vqpci->msix_entry)) {
+            /* No ISR checking should be done if masked, but read anyway */
+            return qpci_msix_pending(dev->pdev, vqpci->msix_entry);
+        } else {
+            data = readl(vqpci->msix_addr);
+            writel(vqpci->msix_addr, 0);
+            return data == vqpci->msix_data;
+        }
+    } else {
+        return qpci_io_readb(dev->pdev, dev->addr + QVIRTIO_ISR_STATUS) & 1;
+    }
+}
+
+static bool qvirtio_pci_get_config_isr_status(QVirtioDevice *d)
+{
+    QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
+    uint32_t data;
+
+    if (dev->pdev->msix_enabled) {
+        g_assert_cmpint(dev->config_msix_entry, !=, -1);
+        if (qpci_msix_masked(dev->pdev, dev->config_msix_entry)) {
+            /* No ISR checking should be done if masked, but read anyway */
+            return qpci_msix_pending(dev->pdev, dev->config_msix_entry);
+        } else {
+            data = readl(dev->config_msix_addr);
+            writel(dev->config_msix_addr, 0);
+            return data == dev->config_msix_data;
+        }
+    } else {
+        return qpci_io_readb(dev->pdev, dev->addr + QVIRTIO_ISR_STATUS) & 2;
+    }
+}
+
+static void qvirtio_pci_queue_select(QVirtioDevice *d, uint16_t index)
+{
+    QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
+    qpci_io_writeb(dev->pdev, dev->addr + QVIRTIO_QUEUE_SELECT, index);
+}
+
+static uint16_t qvirtio_pci_get_queue_size(QVirtioDevice *d)
+{
+    QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
+    return qpci_io_readw(dev->pdev, dev->addr + QVIRTIO_QUEUE_SIZE);
+}
+
+static void qvirtio_pci_set_queue_address(QVirtioDevice *d, uint32_t pfn)
+{
+    QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
+    qpci_io_writel(dev->pdev, dev->addr + QVIRTIO_QUEUE_ADDRESS, pfn);
+}
+
+static QVirtQueue *qvirtio_pci_virtqueue_setup(QVirtioDevice *d,
+                                        QGuestAllocator *alloc, uint16_t index)
+{
+    uint32_t feat;
+    uint64_t addr;
+    QVirtQueuePCI *vqpci;
+
+    vqpci = g_malloc0(sizeof(*vqpci));
+    feat = qvirtio_pci_get_guest_features(d);
+
+    qvirtio_pci_queue_select(d, index);
+    vqpci->vq.index = index;
+    vqpci->vq.size = qvirtio_pci_get_queue_size(d);
+    vqpci->vq.free_head = 0;
+    vqpci->vq.num_free = vqpci->vq.size;
+    vqpci->vq.align = QVIRTIO_PCI_ALIGN;
+    vqpci->vq.indirect = (feat & QVIRTIO_F_RING_INDIRECT_DESC) != 0;
+    vqpci->vq.event = (feat & QVIRTIO_F_RING_EVENT_IDX) != 0;
+
+    vqpci->msix_entry = -1;
+    vqpci->msix_addr = 0;
+    vqpci->msix_data = 0x12345678;
+
+    /* Check different than 0 */
+    g_assert_cmpint(vqpci->vq.size, !=, 0);
+
+    /* Check power of 2 */
+    g_assert_cmpint(vqpci->vq.size & (vqpci->vq.size - 1), ==, 0);
+
+    addr = guest_alloc(alloc, qvring_size(vqpci->vq.size, QVIRTIO_PCI_ALIGN));
+    qvring_init(alloc, &vqpci->vq, addr);
+    qvirtio_pci_set_queue_address(d, vqpci->vq.desc / QVIRTIO_PCI_ALIGN);
+
+    return &vqpci->vq;
+}
+
+static void qvirtio_pci_virtqueue_kick(QVirtioDevice *d, QVirtQueue *vq)
+{
+    QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
+    qpci_io_writew(dev->pdev, dev->addr + QVIRTIO_QUEUE_NOTIFY, vq->index);
+}
+
+const QVirtioBus qvirtio_pci = {
+    .config_readb = qvirtio_pci_config_readb,
+    .config_readw = qvirtio_pci_config_readw,
+    .config_readl = qvirtio_pci_config_readl,
+    .config_readq = qvirtio_pci_config_readq,
+    .get_features = qvirtio_pci_get_features,
+    .set_features = qvirtio_pci_set_features,
+    .get_guest_features = qvirtio_pci_get_guest_features,
+    .get_status = qvirtio_pci_get_status,
+    .set_status = qvirtio_pci_set_status,
+    .get_queue_isr_status = qvirtio_pci_get_queue_isr_status,
+    .get_config_isr_status = qvirtio_pci_get_config_isr_status,
+    .queue_select = qvirtio_pci_queue_select,
+    .get_queue_size = qvirtio_pci_get_queue_size,
+    .set_queue_address = qvirtio_pci_set_queue_address,
+    .virtqueue_setup = qvirtio_pci_virtqueue_setup,
+    .virtqueue_kick = qvirtio_pci_virtqueue_kick,
+};
+
+void qvirtio_pci_foreach(QPCIBus *bus, uint16_t device_type,
+                void (*func)(QVirtioDevice *d, void *data), void *data)
+{
+    QVirtioPCIForeachData d = { .func = func,
+                                .device_type = device_type,
+                                .user_data = data };
+
+    qpci_device_foreach(bus, QVIRTIO_VENDOR_ID, -1,
+                                qvirtio_pci_foreach_callback, &d);
+}
+
+QVirtioPCIDevice *qvirtio_pci_device_find(QPCIBus *bus, uint16_t device_type)
+{
+    QVirtioPCIDevice *dev = NULL;
+    qvirtio_pci_foreach(bus, device_type, qvirtio_pci_assign_device, &dev);
+
+    return dev;
+}
+
+void qvirtio_pci_device_enable(QVirtioPCIDevice *d)
+{
+    qpci_device_enable(d->pdev);
+    d->addr = qpci_iomap(d->pdev, 0, NULL);
+    g_assert(d->addr != NULL);
+}
+
+void qvirtio_pci_device_disable(QVirtioPCIDevice *d)
+{
+    qpci_iounmap(d->pdev, d->addr);
+    d->addr = NULL;
+}
+
+void qvirtqueue_pci_msix_setup(QVirtioPCIDevice *d, QVirtQueuePCI *vqpci,
+                                        QGuestAllocator *alloc, uint16_t entry)
+{
+    uint16_t vector;
+    uint32_t control;
+    void *addr;
+
+    g_assert(d->pdev->msix_enabled);
+    addr = d->pdev->msix_table + (entry * 16);
+
+    g_assert_cmpint(entry, >=, 0);
+    g_assert_cmpint(entry, <, qpci_msix_table_size(d->pdev));
+    vqpci->msix_entry = entry;
+
+    vqpci->msix_addr = guest_alloc(alloc, 4);
+    qpci_io_writel(d->pdev, addr + PCI_MSIX_ENTRY_LOWER_ADDR,
+                                                    vqpci->msix_addr & ~0UL);
+    qpci_io_writel(d->pdev, addr + PCI_MSIX_ENTRY_UPPER_ADDR,
+                                            (vqpci->msix_addr >> 32) & ~0UL);
+    qpci_io_writel(d->pdev, addr + PCI_MSIX_ENTRY_DATA, vqpci->msix_data);
+
+    control = qpci_io_readl(d->pdev, addr + PCI_MSIX_ENTRY_VECTOR_CTRL);
+    qpci_io_writel(d->pdev, addr + PCI_MSIX_ENTRY_VECTOR_CTRL,
+                                        control & ~PCI_MSIX_ENTRY_CTRL_MASKBIT);
+
+    qvirtio_pci_queue_select(&d->vdev, vqpci->vq.index);
+    qpci_io_writew(d->pdev, d->addr + QVIRTIO_MSIX_QUEUE_VECTOR, entry);
+    vector = qpci_io_readw(d->pdev, d->addr + QVIRTIO_MSIX_QUEUE_VECTOR);
+    g_assert_cmphex(vector, !=, QVIRTIO_MSI_NO_VECTOR);
+}
+
+void qvirtio_pci_set_msix_configuration_vector(QVirtioPCIDevice *d,
+                                        QGuestAllocator *alloc, uint16_t entry)
+{
+    uint16_t vector;
+    uint32_t control;
+    void *addr;
+
+    g_assert(d->pdev->msix_enabled);
+    addr = d->pdev->msix_table + (entry * 16);
+
+    g_assert_cmpint(entry, >=, 0);
+    g_assert_cmpint(entry, <, qpci_msix_table_size(d->pdev));
+    d->config_msix_entry = entry;
+
+    d->config_msix_data = 0x12345678;
+    d->config_msix_addr = guest_alloc(alloc, 4);
+
+    qpci_io_writel(d->pdev, addr + PCI_MSIX_ENTRY_LOWER_ADDR,
+                                                    d->config_msix_addr & ~0UL);
+    qpci_io_writel(d->pdev, addr + PCI_MSIX_ENTRY_UPPER_ADDR,
+                                            (d->config_msix_addr >> 32) & ~0UL);
+    qpci_io_writel(d->pdev, addr + PCI_MSIX_ENTRY_DATA, d->config_msix_data);
+
+    control = qpci_io_readl(d->pdev, addr + PCI_MSIX_ENTRY_VECTOR_CTRL);
+    qpci_io_writel(d->pdev, addr + PCI_MSIX_ENTRY_VECTOR_CTRL,
+                                        control & ~PCI_MSIX_ENTRY_CTRL_MASKBIT);
+
+    qpci_io_writew(d->pdev, d->addr + QVIRTIO_MSIX_CONF_VECTOR, entry);
+    vector = qpci_io_readw(d->pdev, d->addr + QVIRTIO_MSIX_CONF_VECTOR);
+    g_assert_cmphex(vector, !=, QVIRTIO_MSI_NO_VECTOR);
+}
diff --git a/tests/libqos/virtio-pci.h b/tests/libqos/virtio-pci.h
new file mode 100644
index 000000000..883f7ff26
--- /dev/null
+++ b/tests/libqos/virtio-pci.h
@@ -0,0 +1,61 @@
+/*
+ * libqos virtio PCI definitions
+ *
+ * Copyright (c) 2014 Marc Marí
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef LIBQOS_VIRTIO_PCI_H
+#define LIBQOS_VIRTIO_PCI_H
+
+#include "libqos/virtio.h"
+#include "libqos/pci.h"
+
+#define QVIRTIO_DEVICE_FEATURES         0x00
+#define QVIRTIO_GUEST_FEATURES          0x04
+#define QVIRTIO_QUEUE_ADDRESS           0x08
+#define QVIRTIO_QUEUE_SIZE              0x0C
+#define QVIRTIO_QUEUE_SELECT            0x0E
+#define QVIRTIO_QUEUE_NOTIFY            0x10
+#define QVIRTIO_DEVICE_STATUS           0x12
+#define QVIRTIO_ISR_STATUS              0x13
+#define QVIRTIO_MSIX_CONF_VECTOR        0x14
+#define QVIRTIO_MSIX_QUEUE_VECTOR       0x16
+#define QVIRTIO_DEVICE_SPECIFIC_MSIX    0x18
+#define QVIRTIO_DEVICE_SPECIFIC_NO_MSIX 0x14
+
+#define QVIRTIO_PCI_ALIGN   4096
+
+#define QVIRTIO_MSI_NO_VECTOR   0xFFFF
+
+typedef struct QVirtioPCIDevice {
+    QVirtioDevice vdev;
+    QPCIDevice *pdev;
+    void *addr;
+    uint16_t config_msix_entry;
+    uint64_t config_msix_addr;
+    uint32_t config_msix_data;
+} QVirtioPCIDevice;
+
+typedef struct QVirtQueuePCI {
+    QVirtQueue vq;
+    uint16_t msix_entry;
+    uint64_t msix_addr;
+    uint32_t msix_data;
+} QVirtQueuePCI;
+
+extern const QVirtioBus qvirtio_pci;
+
+void qvirtio_pci_foreach(QPCIBus *bus, uint16_t device_type,
+                void (*func)(QVirtioDevice *d, void *data), void *data);
+QVirtioPCIDevice *qvirtio_pci_device_find(QPCIBus *bus, uint16_t device_type);
+void qvirtio_pci_device_enable(QVirtioPCIDevice *d);
+void qvirtio_pci_device_disable(QVirtioPCIDevice *d);
+
+void qvirtio_pci_set_msix_configuration_vector(QVirtioPCIDevice *d,
+                                        QGuestAllocator *alloc, uint16_t entry);
+void qvirtqueue_pci_msix_setup(QVirtioPCIDevice *d, QVirtQueuePCI *vqpci,
+                                        QGuestAllocator *alloc, uint16_t entry);
+#endif
diff --git a/tests/libqos/virtio.c b/tests/libqos/virtio.c
new file mode 100644
index 000000000..a06128924
--- /dev/null
+++ b/tests/libqos/virtio.c
@@ -0,0 +1,281 @@
+/*
+ * libqos virtio driver
+ *
+ * Copyright (c) 2014 Marc Marí
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include <glib.h>
+#include "libqtest.h"
+#include "libqos/virtio.h"
+
+uint8_t qvirtio_config_readb(const QVirtioBus *bus, QVirtioDevice *d,
+                                                                void *addr)
+{
+    return bus->config_readb(d, addr);
+}
+
+uint16_t qvirtio_config_readw(const QVirtioBus *bus, QVirtioDevice *d,
+                                                                void *addr)
+{
+    return bus->config_readw(d, addr);
+}
+
+uint32_t qvirtio_config_readl(const QVirtioBus *bus, QVirtioDevice *d,
+                                                                void *addr)
+{
+    return bus->config_readl(d, addr);
+}
+
+uint64_t qvirtio_config_readq(const QVirtioBus *bus, QVirtioDevice *d,
+                                                                void *addr)
+{
+    return bus->config_readq(d, addr);
+}
+
+uint32_t qvirtio_get_features(const QVirtioBus *bus, QVirtioDevice *d)
+{
+    return bus->get_features(d);
+}
+
+void qvirtio_set_features(const QVirtioBus *bus, QVirtioDevice *d,
+                                                            uint32_t features)
+{
+    bus->set_features(d, features);
+}
+
+QVirtQueue *qvirtqueue_setup(const QVirtioBus *bus, QVirtioDevice *d,
+                                        QGuestAllocator *alloc, uint16_t index)
+{
+    return bus->virtqueue_setup(d, alloc, index);
+}
+
+void qvirtio_reset(const QVirtioBus *bus, QVirtioDevice *d)
+{
+    bus->set_status(d, QVIRTIO_RESET);
+    g_assert_cmphex(bus->get_status(d), ==, QVIRTIO_RESET);
+}
+
+void qvirtio_set_acknowledge(const QVirtioBus *bus, QVirtioDevice *d)
+{
+    bus->set_status(d, bus->get_status(d) | QVIRTIO_ACKNOWLEDGE);
+    g_assert_cmphex(bus->get_status(d), ==, QVIRTIO_ACKNOWLEDGE);
+}
+
+void qvirtio_set_driver(const QVirtioBus *bus, QVirtioDevice *d)
+{
+    bus->set_status(d, bus->get_status(d) | QVIRTIO_DRIVER);
+    g_assert_cmphex(bus->get_status(d), ==,
+                                    QVIRTIO_DRIVER | QVIRTIO_ACKNOWLEDGE);
+}
+
+void qvirtio_set_driver_ok(const QVirtioBus *bus, QVirtioDevice *d)
+{
+    bus->set_status(d, bus->get_status(d) | QVIRTIO_DRIVER_OK);
+    g_assert_cmphex(bus->get_status(d), ==,
+                QVIRTIO_DRIVER_OK | QVIRTIO_DRIVER | QVIRTIO_ACKNOWLEDGE);
+}
+
+void qvirtio_wait_queue_isr(const QVirtioBus *bus, QVirtioDevice *d,
+                            QVirtQueue *vq, gint64 timeout_us)
+{
+    gint64 start_time = g_get_monotonic_time();
+
+    for (;;) {
+        clock_step(100);
+        if (bus->get_queue_isr_status(d, vq)) {
+            return;
+        }
+        g_assert(g_get_monotonic_time() - start_time <= timeout_us);
+    }
+}
+
+/* Wait for the status byte at given guest memory address to be set
+ *
+ * The virtqueue interrupt must not be raised, making this useful for testing
+ * event_index functionality.
+ */
+uint8_t qvirtio_wait_status_byte_no_isr(const QVirtioBus *bus,
+                                        QVirtioDevice *d,
+                                        QVirtQueue *vq,
+                                        uint64_t addr,
+                                        gint64 timeout_us)
+{
+    gint64 start_time = g_get_monotonic_time();
+    uint8_t val;
+
+    while ((val = readb(addr)) == 0xff) {
+        clock_step(100);
+        g_assert(!bus->get_queue_isr_status(d, vq));
+        g_assert(g_get_monotonic_time() - start_time <= timeout_us);
+    }
+    return val;
+}
+
+void qvirtio_wait_config_isr(const QVirtioBus *bus, QVirtioDevice *d,
+                             gint64 timeout_us)
+{
+    gint64 start_time = g_get_monotonic_time();
+
+    for (;;) {
+        clock_step(100);
+        if (bus->get_config_isr_status(d)) {
+            return;
+        }
+        g_assert(g_get_monotonic_time() - start_time <= timeout_us);
+    }
+}
+
+void qvring_init(const QGuestAllocator *alloc, QVirtQueue *vq, uint64_t addr)
+{
+    int i;
+
+    vq->desc = addr;
+    vq->avail = vq->desc + vq->size*sizeof(QVRingDesc);
+    vq->used = (uint64_t)((vq->avail + sizeof(uint16_t) * (3 + vq->size)
+        + vq->align - 1) & ~(vq->align - 1));
+
+    for (i = 0; i < vq->size - 1; i++) {
+        /* vq->desc[i].addr */
+        writew(vq->desc + (16 * i), 0);
+        /* vq->desc[i].next */
+        writew(vq->desc + (16 * i) + 14, i + 1);
+    }
+
+    /* vq->avail->flags */
+    writew(vq->avail, 0);
+    /* vq->avail->idx */
+    writew(vq->avail + 2, 0);
+    /* vq->avail->used_event */
+    writew(vq->avail + 4 + (2 * vq->size), 0);
+
+    /* vq->used->flags */
+    writew(vq->used, 0);
+    /* vq->used->avail_event */
+    writew(vq->used+2+(sizeof(struct QVRingUsedElem)*vq->size), 0);
+}
+
+QVRingIndirectDesc *qvring_indirect_desc_setup(QVirtioDevice *d,
+                                        QGuestAllocator *alloc, uint16_t elem)
+{
+    int i;
+    QVRingIndirectDesc *indirect = g_malloc(sizeof(*indirect));
+
+    indirect->index = 0;
+    indirect->elem = elem;
+    indirect->desc = guest_alloc(alloc, sizeof(QVRingDesc)*elem);
+
+    for (i = 0; i < elem - 1; ++i) {
+        /* indirect->desc[i].addr */
+        writeq(indirect->desc + (16 * i), 0);
+        /* indirect->desc[i].flags */
+        writew(indirect->desc + (16 * i) + 12, QVRING_DESC_F_NEXT);
+        /* indirect->desc[i].next */
+        writew(indirect->desc + (16 * i) + 14, i + 1);
+    }
+
+    return indirect;
+}
+
+void qvring_indirect_desc_add(QVRingIndirectDesc *indirect, uint64_t data,
+                                                    uint32_t len, bool write)
+{
+    uint16_t flags;
+
+    g_assert_cmpint(indirect->index, <, indirect->elem);
+
+    flags = readw(indirect->desc + (16 * indirect->index) + 12);
+
+    if (write) {
+        flags |= QVRING_DESC_F_WRITE;
+    }
+
+    /* indirect->desc[indirect->index].addr */
+    writeq(indirect->desc + (16 * indirect->index), data);
+    /* indirect->desc[indirect->index].len */
+    writel(indirect->desc + (16 * indirect->index) + 8, len);
+    /* indirect->desc[indirect->index].flags */
+    writew(indirect->desc + (16 * indirect->index) + 12, flags);
+
+    indirect->index++;
+}
+
+uint32_t qvirtqueue_add(QVirtQueue *vq, uint64_t data, uint32_t len, bool write,
+                                                                    bool next)
+{
+    uint16_t flags = 0;
+    vq->num_free--;
+
+    if (write) {
+        flags |= QVRING_DESC_F_WRITE;
+    }
+
+    if (next) {
+        flags |= QVRING_DESC_F_NEXT;
+    }
+
+    /* vq->desc[vq->free_head].addr */
+    writeq(vq->desc + (16 * vq->free_head), data);
+    /* vq->desc[vq->free_head].len */
+    writel(vq->desc + (16 * vq->free_head) + 8, len);
+    /* vq->desc[vq->free_head].flags */
+    writew(vq->desc + (16 * vq->free_head) + 12, flags);
+
+    return vq->free_head++; /* Return and increase, in this order */
+}
+
+uint32_t qvirtqueue_add_indirect(QVirtQueue *vq, QVRingIndirectDesc *indirect)
+{
+    g_assert(vq->indirect);
+    g_assert_cmpint(vq->size, >=, indirect->elem);
+    g_assert_cmpint(indirect->index, ==, indirect->elem);
+
+    vq->num_free--;
+
+    /* vq->desc[vq->free_head].addr */
+    writeq(vq->desc + (16 * vq->free_head), indirect->desc);
+    /* vq->desc[vq->free_head].len */
+    writel(vq->desc + (16 * vq->free_head) + 8,
+                                        sizeof(QVRingDesc) * indirect->elem);
+    /* vq->desc[vq->free_head].flags */
+    writew(vq->desc + (16 * vq->free_head) + 12, QVRING_DESC_F_INDIRECT);
+
+    return vq->free_head++; /* Return and increase, in this order */
+}
+
+void qvirtqueue_kick(const QVirtioBus *bus, QVirtioDevice *d, QVirtQueue *vq,
+                                                            uint32_t free_head)
+{
+    /* vq->avail->idx */
+    uint16_t idx = readl(vq->avail + 2);
+    /* vq->used->flags */
+    uint16_t flags;
+    /* vq->used->avail_event */
+    uint16_t avail_event;
+
+    /* vq->avail->ring[idx % vq->size] */
+    writel(vq->avail + 4 + (2 * (idx % vq->size)), free_head);
+    /* vq->avail->idx */
+    writel(vq->avail + 2, idx + 1);
+
+    /* Must read after idx is updated */
+    flags = readw(vq->avail);
+    avail_event = readw(vq->used + 4 +
+                                (sizeof(struct QVRingUsedElem) * vq->size));
+
+    /* < 1 because we add elements to avail queue one by one */
+    if ((flags & QVRING_USED_F_NO_NOTIFY) == 0 &&
+                            (!vq->event || (uint16_t)(idx-avail_event) < 1)) {
+        bus->virtqueue_kick(d, vq);
+    }
+}
+
+void qvirtqueue_set_used_event(QVirtQueue *vq, uint16_t idx)
+{
+    g_assert(vq->event);
+
+    /* vq->avail->used_event */
+    writew(vq->avail + 4 + (2 * vq->size), idx);
+}
diff --git a/tests/libqos/virtio.h b/tests/libqos/virtio.h
new file mode 100644
index 000000000..29fbacbc9
--- /dev/null
+++ b/tests/libqos/virtio.h
@@ -0,0 +1,187 @@
+/*
+ * libqos virtio definitions
+ *
+ * Copyright (c) 2014 Marc Marí
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef LIBQOS_VIRTIO_H
+#define LIBQOS_VIRTIO_H
+
+#include "libqos/malloc.h"
+
+#define QVIRTIO_VENDOR_ID       0x1AF4
+
+#define QVIRTIO_RESET           0x0
+#define QVIRTIO_ACKNOWLEDGE     0x1
+#define QVIRTIO_DRIVER          0x2
+#define QVIRTIO_DRIVER_OK       0x4
+
+#define QVIRTIO_NET_DEVICE_ID   0x1
+#define QVIRTIO_BLK_DEVICE_ID   0x2
+
+#define QVIRTIO_F_NOTIFY_ON_EMPTY       0x01000000
+#define QVIRTIO_F_ANY_LAYOUT            0x08000000
+#define QVIRTIO_F_RING_INDIRECT_DESC    0x10000000
+#define QVIRTIO_F_RING_EVENT_IDX        0x20000000
+#define QVIRTIO_F_BAD_FEATURE           0x40000000
+
+#define QVRING_DESC_F_NEXT      0x1
+#define QVRING_DESC_F_WRITE     0x2
+#define QVRING_DESC_F_INDIRECT  0x4
+
+#define QVIRTIO_F_NOTIFY_ON_EMPTY       0x01000000
+#define QVIRTIO_F_ANY_LAYOUT            0x08000000
+#define QVIRTIO_F_RING_INDIRECT_DESC    0x10000000
+#define QVIRTIO_F_RING_EVENT_IDX        0x20000000
+#define QVIRTIO_F_BAD_FEATURE           0x40000000
+
+#define QVRING_AVAIL_F_NO_INTERRUPT     1
+
+#define QVRING_USED_F_NO_NOTIFY     1
+
+typedef struct QVirtioDevice {
+    /* Device type */
+    uint16_t device_type;
+} QVirtioDevice;
+
+typedef struct QVRingDesc {
+    uint64_t addr;
+    uint32_t len;
+    uint16_t flags;
+    uint16_t next;
+} QVRingDesc;
+
+typedef struct QVRingAvail {
+    uint16_t flags;
+    uint16_t idx;
+    uint16_t ring[0]; /* This is an array of uint16_t */
+    uint16_t used_event;
+} QVRingAvail;
+
+typedef struct QVRingUsedElem {
+    uint32_t id;
+    uint32_t len;
+} QVRingUsedElem;
+
+typedef struct QVRingUsed {
+    uint16_t flags;
+    uint16_t idx;
+    QVRingUsedElem ring[0]; /* This is an array of QVRingUsedElem structs */
+    uint16_t avail_event;
+} QVRingUsed;
+
+typedef struct QVirtQueue {
+    uint64_t desc; /* This points to an array of QVRingDesc */
+    uint64_t avail; /* This points to a QVRingAvail */
+    uint64_t used; /* This points to a QVRingDesc */
+    uint16_t index;
+    uint32_t size;
+    uint32_t free_head;
+    uint32_t num_free;
+    uint32_t align;
+    bool indirect;
+    bool event;
+} QVirtQueue;
+
+typedef struct QVRingIndirectDesc {
+    uint64_t desc; /* This points to an array fo QVRingDesc */
+    uint16_t index;
+    uint16_t elem;
+} QVRingIndirectDesc;
+
+typedef struct QVirtioBus {
+    uint8_t (*config_readb)(QVirtioDevice *d, void *addr);
+    uint16_t (*config_readw)(QVirtioDevice *d, void *addr);
+    uint32_t (*config_readl)(QVirtioDevice *d, void *addr);
+    uint64_t (*config_readq)(QVirtioDevice *d, void *addr);
+
+    /* Get features of the device */
+    uint32_t (*get_features)(QVirtioDevice *d);
+
+    /* Set features of the device */
+    void (*set_features)(QVirtioDevice *d, uint32_t features);
+
+    /* Get features of the guest */
+    uint32_t (*get_guest_features)(QVirtioDevice *d);
+
+    /* Get status of the device */
+    uint8_t (*get_status)(QVirtioDevice *d);
+
+    /* Set status of the device  */
+    void (*set_status)(QVirtioDevice *d, uint8_t status);
+
+    /* Get the queue ISR status of the device */
+    bool (*get_queue_isr_status)(QVirtioDevice *d, QVirtQueue *vq);
+
+    /* Get the configuration ISR status of the device */
+    bool (*get_config_isr_status)(QVirtioDevice *d);
+
+    /* Select a queue to work on */
+    void (*queue_select)(QVirtioDevice *d, uint16_t index);
+
+    /* Get the size of the selected queue */
+    uint16_t (*get_queue_size)(QVirtioDevice *d);
+
+    /* Set the address of the selected queue */
+    void (*set_queue_address)(QVirtioDevice *d, uint32_t pfn);
+
+    /* Setup the virtqueue specified by index */
+    QVirtQueue *(*virtqueue_setup)(QVirtioDevice *d, QGuestAllocator *alloc,
+                                                                uint16_t index);
+
+    /* Notify changes in virtqueue */
+    void (*virtqueue_kick)(QVirtioDevice *d, QVirtQueue *vq);
+} QVirtioBus;
+
+static inline uint32_t qvring_size(uint32_t num, uint32_t align)
+{
+    return ((sizeof(struct QVRingDesc) * num + sizeof(uint16_t) * (3 + num)
+        + align - 1) & ~(align - 1))
+        + sizeof(uint16_t) * 3 + sizeof(struct QVRingUsedElem) * num;
+}
+
+uint8_t qvirtio_config_readb(const QVirtioBus *bus, QVirtioDevice *d,
+                                                                void *addr);
+uint16_t qvirtio_config_readw(const QVirtioBus *bus, QVirtioDevice *d,
+                                                                void *addr);
+uint32_t qvirtio_config_readl(const QVirtioBus *bus, QVirtioDevice *d,
+                                                                void *addr);
+uint64_t qvirtio_config_readq(const QVirtioBus *bus, QVirtioDevice *d,
+                                                                void *addr);
+uint32_t qvirtio_get_features(const QVirtioBus *bus, QVirtioDevice *d);
+void qvirtio_set_features(const QVirtioBus *bus, QVirtioDevice *d,
+                                                            uint32_t features);
+
+void qvirtio_reset(const QVirtioBus *bus, QVirtioDevice *d);
+void qvirtio_set_acknowledge(const QVirtioBus *bus, QVirtioDevice *d);
+void qvirtio_set_driver(const QVirtioBus *bus, QVirtioDevice *d);
+void qvirtio_set_driver_ok(const QVirtioBus *bus, QVirtioDevice *d);
+
+void qvirtio_wait_queue_isr(const QVirtioBus *bus, QVirtioDevice *d,
+                            QVirtQueue *vq, gint64 timeout_us);
+uint8_t qvirtio_wait_status_byte_no_isr(const QVirtioBus *bus,
+                                        QVirtioDevice *d,
+                                        QVirtQueue *vq,
+                                        uint64_t addr,
+                                        gint64 timeout_us);
+void qvirtio_wait_config_isr(const QVirtioBus *bus, QVirtioDevice *d,
+                             gint64 timeout_us);
+QVirtQueue *qvirtqueue_setup(const QVirtioBus *bus, QVirtioDevice *d,
+                                        QGuestAllocator *alloc, uint16_t index);
+
+void qvring_init(const QGuestAllocator *alloc, QVirtQueue *vq, uint64_t addr);
+QVRingIndirectDesc *qvring_indirect_desc_setup(QVirtioDevice *d,
+                                        QGuestAllocator *alloc, uint16_t elem);
+void qvring_indirect_desc_add(QVRingIndirectDesc *indirect, uint64_t data,
+                                                    uint32_t len, bool write);
+uint32_t qvirtqueue_add(QVirtQueue *vq, uint64_t data, uint32_t len, bool write,
+                                                                    bool next);
+uint32_t qvirtqueue_add_indirect(QVirtQueue *vq, QVRingIndirectDesc *indirect);
+void qvirtqueue_kick(const QVirtioBus *bus, QVirtioDevice *d, QVirtQueue *vq,
+                                                            uint32_t free_head);
+
+void qvirtqueue_set_used_event(QVirtQueue *vq, uint16_t idx);
+#endif
diff --git a/tests/libqtest.c b/tests/libqtest.c
index 98e8f4b64..9a92aa70e 100644
--- a/tests/libqtest.c
+++ b/tests/libqtest.c
@@ -165,13 +165,15 @@ QTestState *qtest_init(const char *extra_args)
 
     s->qemu_pid = fork();
     if (s->qemu_pid == 0) {
+        setenv("QEMU_AUDIO_DRV", "none", true);
         command = g_strdup_printf("exec %s "
                                   "-qtest unix:%s,nowait "
-                                  "-qtest-log /dev/null "
+                                  "-qtest-log %s "
                                   "-qmp unix:%s,nowait "
                                   "-machine accel=qtest "
                                   "-display none "
                                   "%s", qemu_binary, socket_path,
+                                  getenv("QTEST_LOG") ? "/dev/fd/2" : "/dev/null",
                                   qmp_socket_path,
                                   extra_args ?: "");
         execlp("/bin/sh", "sh", "-c", command, NULL);
@@ -358,6 +360,7 @@ static void qmp_response(JSONMessageParser *parser, QList *tokens)
 QDict *qtest_qmp_receive(QTestState *s)
 {
     QMPResponseParser qmp;
+    bool log = getenv("QTEST_LOG") != NULL;
 
     qmp.response = NULL;
     json_message_parser_init(&qmp.parser, qmp_response);
@@ -375,6 +378,9 @@ QDict *qtest_qmp_receive(QTestState *s)
             exit(1);
         }
 
+        if (log) {
+            len = write(2, &c, 1);
+        }
         json_message_parser_feed(&qmp.parser, &c, 1);
     }
     json_message_parser_destroy(&qmp.parser);
@@ -397,10 +403,14 @@ QDict *qtest_qmpv(QTestState *s, const char *fmt, va_list ap)
 
     /* No need to send anything for an empty QObject.  */
     if (qobj) {
+        int log = getenv("QTEST_LOG") != NULL;
         QString *qstr = qobject_to_json(qobj);
         const char *str = qstring_get_str(qstr);
         size_t size = qstring_get_length(qstr);
 
+        if (log) {
+            fprintf(stderr, "%s", str);
+        }
         /* Send QMP request */
         socket_send(s->qmp_fd, str, size);
 
@@ -639,6 +649,7 @@ void qtest_add_func(const char *str, void (*fn))
 {
     gchar *path = g_strdup_printf("/%s/%s", qtest_get_arch(), str);
     g_test_add_func(path, fn);
+    g_free(path);
 }
 
 void qtest_memwrite(QTestState *s, uint64_t addr, const void *data, size_t size)
@@ -654,6 +665,18 @@ void qtest_memwrite(QTestState *s, uint64_t addr, const void *data, size_t size)
     qtest_rsp(s, 0);
 }
 
+void qtest_memset(QTestState *s, uint64_t addr, uint8_t pattern, size_t size)
+{
+    size_t i;
+
+    qtest_sendf(s, "write 0x%" PRIx64 " 0x%zx 0x", addr, size);
+    for (i = 0; i < size; i++) {
+        qtest_sendf(s, "%02x", pattern);
+    }
+    qtest_sendf(s, "\n");
+    qtest_rsp(s, 0);
+}
+
 QDict *qmp(const char *fmt, ...)
 {
     va_list ap;
@@ -673,3 +696,51 @@ void qmp_discard_response(const char *fmt, ...)
     qtest_qmpv_discard_response(global_qtest, fmt, ap);
     va_end(ap);
 }
+
+bool qtest_big_endian(void)
+{
+    const char *arch = qtest_get_arch();
+    int i;
+
+    static const struct {
+        const char *arch;
+        bool big_endian;
+    } endianness[] = {
+        { "aarch64", false },
+        { "alpha", false },
+        { "arm", false },
+        { "cris", false },
+        { "i386", false },
+        { "lm32", true },
+        { "m68k", true },
+        { "microblaze", true },
+        { "microblazeel", false },
+        { "mips", true },
+        { "mips64", true },
+        { "mips64el", false },
+        { "mipsel", false },
+        { "moxie", true },
+        { "or32", true },
+        { "ppc", true },
+        { "ppc64", true },
+        { "ppcemb", true },
+        { "s390x", true },
+        { "sh4", false },
+        { "sh4eb", true },
+        { "sparc", true },
+        { "sparc64", true },
+        { "unicore32", false },
+        { "x86_64", false },
+        { "xtensa", false },
+        { "xtensaeb", true },
+        {},
+    };
+
+    for (i = 0; endianness[i].arch; i++) {
+        if (strcmp(endianness[i].arch, arch) == 0) {
+            return endianness[i].big_endian;
+        }
+    }
+
+    return false;
+}
diff --git a/tests/libqtest.h b/tests/libqtest.h
index 8f323c703..e7413d52d 100644
--- a/tests/libqtest.h
+++ b/tests/libqtest.h
@@ -23,6 +23,7 @@
 #include <stdarg.h>
 #include <sys/types.h>
 #include "qapi/qmp/qdict.h"
+#include "glib-compat.h"
 
 typedef struct QTestState QTestState;
 
@@ -283,6 +284,17 @@ void qtest_memread(QTestState *s, uint64_t addr, void *data, size_t size);
 void qtest_memwrite(QTestState *s, uint64_t addr, const void *data, size_t size);
 
 /**
+ * qtest_memset:
+ * @s: #QTestState instance to operate on.
+ * @addr: Guest address to write to.
+ * @patt: Byte pattern to fill the guest memory region with.
+ * @size: Number of bytes to write.
+ *
+ * Write a pattern to guest memory.
+ */
+void qtest_memset(QTestState *s, uint64_t addr, uint8_t patt, size_t size);
+
+/**
  * qtest_clock_step_next:
  * @s: #QTestState instance to operate on.
  *
@@ -621,6 +633,19 @@ static inline void memwrite(uint64_t addr, const void *data, size_t size)
 }
 
 /**
+ * qmemset:
+ * @addr: Guest address to write to.
+ * @patt: Byte pattern to fill the guest memory region with.
+ * @size: Number of bytes to write.
+ *
+ * Write a pattern to guest memory.
+ */
+static inline void qmemset(uint64_t addr, uint8_t patt, size_t size)
+{
+    qtest_memset(global_qtest, addr, patt, size);
+}
+
+/**
  * clock_step_next:
  *
  * Advance the QEMU_CLOCK_VIRTUAL to the next deadline.
@@ -658,4 +683,11 @@ static inline int64_t clock_set(int64_t val)
     return qtest_clock_set(global_qtest, val);
 }
 
+/**
+ * qtest_big_endian:
+ *
+ * Returns: True if the architecture under test has a big endian configuration.
+ */
+bool qtest_big_endian(void);
+
 #endif
diff --git a/tests/qapi-schema/qapi-schema-test.json b/tests/qapi-schema/qapi-schema-test.json
index ab4d3d96b..d43b5fd2e 100644
--- a/tests/qapi-schema/qapi-schema-test.json
+++ b/tests/qapi-schema/qapi-schema-test.json
@@ -33,6 +33,9 @@
 { 'type': 'UserDefB',
   'data': { 'integer': 'int' } }
 
+{ 'type': 'UserDefC',
+  'data': { 'string1': 'str', 'string2': 'str' } }
+
 { 'union': 'UserDefUnion',
   'base': 'UserDefZero',
   'data': { 'a' : 'UserDefA', 'b' : 'UserDefB' } }
@@ -47,6 +50,13 @@
 # FIXME generated struct UserDefFlatUnion has members for direct base
 # UserDefOne, but lacks members for indirect base UserDefZero
 
+# this variant of UserDefFlatUnion defaults to a union that uses fields with
+# allocated types to test corner cases in the cleanup/dealloc visitor
+{ 'union': 'UserDefFlatUnion2',
+  'base': 'UserDefUnionBase',
+  'discriminator': 'enum1',
+  'data': { 'value1' : 'UserDefC', 'value2' : 'UserDefB', 'value3' : 'UserDefA' } }
+
 { 'union': 'UserDefAnonUnion',
   'discriminator': {},
   'data': { 'uda': 'UserDefA', 's': 'str', 'i': 'int' } }
diff --git a/tests/qapi-schema/qapi-schema-test.out b/tests/qapi-schema/qapi-schema-test.out
index 95e989925..08d7304df 100644
--- a/tests/qapi-schema/qapi-schema-test.out
+++ b/tests/qapi-schema/qapi-schema-test.out
@@ -6,9 +6,11 @@
  OrderedDict([('type', 'UserDefNested'), ('data', OrderedDict([('string0', 'str'), ('dict1', OrderedDict([('string1', 'str'), ('dict2', OrderedDict([('userdef1', 'UserDefOne'), ('string2', 'str')])), ('*dict3', OrderedDict([('userdef2', 'UserDefOne'), ('string3', 'str')]))]))]))]),
  OrderedDict([('type', 'UserDefA'), ('data', OrderedDict([('boolean', 'bool')]))]),
  OrderedDict([('type', 'UserDefB'), ('data', OrderedDict([('integer', 'int')]))]),
+ OrderedDict([('type', 'UserDefC'), ('data', OrderedDict([('string1', 'str'), ('string2', 'str')]))]),
  OrderedDict([('union', 'UserDefUnion'), ('base', 'UserDefZero'), ('data', OrderedDict([('a', 'UserDefA'), ('b', 'UserDefB')]))]),
  OrderedDict([('type', 'UserDefUnionBase'), ('data', OrderedDict([('string', 'str'), ('enum1', 'EnumOne')]))]),
  OrderedDict([('union', 'UserDefFlatUnion'), ('base', 'UserDefUnionBase'), ('discriminator', 'enum1'), ('data', OrderedDict([('value1', 'UserDefA'), ('value2', 'UserDefB'), ('value3', 'UserDefB')]))]),
+ OrderedDict([('union', 'UserDefFlatUnion2'), ('base', 'UserDefUnionBase'), ('discriminator', 'enum1'), ('data', OrderedDict([('value1', 'UserDefC'), ('value2', 'UserDefB'), ('value3', 'UserDefA')]))]),
  OrderedDict([('union', 'UserDefAnonUnion'), ('discriminator', OrderedDict()), ('data', OrderedDict([('uda', 'UserDefA'), ('s', 'str'), ('i', 'int')]))]),
  OrderedDict([('union', 'UserDefNativeListUnion'), ('data', OrderedDict([('integer', ['int']), ('s8', ['int8']), ('s16', ['int16']), ('s32', ['int32']), ('s64', ['int64']), ('u8', ['uint8']), ('u16', ['uint16']), ('u32', ['uint32']), ('u64', ['uint64']), ('number', ['number']), ('boolean', ['bool']), ('string', ['str'])]))]),
  OrderedDict([('command', 'user_def_cmd'), ('data', OrderedDict())]),
@@ -32,6 +34,7 @@
  OrderedDict([('type', 'UserDefNested'), ('data', OrderedDict([('string0', 'str'), ('dict1', OrderedDict([('string1', 'str'), ('dict2', OrderedDict([('userdef1', 'UserDefOne'), ('string2', 'str')])), ('*dict3', OrderedDict([('userdef2', 'UserDefOne'), ('string3', 'str')]))]))]))]),
  OrderedDict([('type', 'UserDefA'), ('data', OrderedDict([('boolean', 'bool')]))]),
  OrderedDict([('type', 'UserDefB'), ('data', OrderedDict([('integer', 'int')]))]),
+ OrderedDict([('type', 'UserDefC'), ('data', OrderedDict([('string1', 'str'), ('string2', 'str')]))]),
  OrderedDict([('type', 'UserDefUnionBase'), ('data', OrderedDict([('string', 'str'), ('enum1', 'EnumOne')]))]),
  OrderedDict([('type', 'UserDefOptions'), ('data', OrderedDict([('*i64', ['int']), ('*u64', ['uint64']), ('*u16', ['uint16']), ('*i64x', 'int'), ('*u64x', 'uint64')]))]),
  OrderedDict([('type', 'EventStructOne'), ('data', OrderedDict([('struct1', 'UserDefOne'), ('string', 'str'), ('*enum2', 'EnumOne')]))])]
diff --git a/tests/qdev-monitor-test.c b/tests/qdev-monitor-test.c
deleted file mode 100644
index e20ffd67a..000000000
--- a/tests/qdev-monitor-test.c
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * qdev-monitor.c test cases
- *
- * Copyright (C) 2013 Red Hat Inc.
- *
- * Authors:
- *  Stefan Hajnoczi <stefanha@redhat.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include <string.h>
-#include <glib.h>
-#include "libqtest.h"
-#include "qapi/qmp/qjson.h"
-
-static void test_device_add(void)
-{
-    QDict *response;
-    QDict *error;
-
-    qtest_start("-drive if=none,id=drive0");
-
-    /* Make device_add fail.  If this leaks the virtio-blk-pci device then a
-     * reference to drive0 will also be held (via qdev properties).
-     */
-    response = qmp("{\"execute\": \"device_add\","
-                   " \"arguments\": {"
-                   "   \"driver\": \"virtio-blk-pci\","
-                   "   \"drive\": \"drive0\""
-                   "}}");
-    g_assert(response);
-    error = qdict_get_qdict(response, "error");
-    g_assert_cmpstr(qdict_get_try_str(error, "class"), ==, "GenericError");
-    QDECREF(response);
-
-    /* Delete the drive */
-    response = qmp("{\"execute\": \"human-monitor-command\","
-                   " \"arguments\": {"
-                   "   \"command-line\": \"drive_del drive0\""
-                   "}}");
-    g_assert(response);
-    g_assert_cmpstr(qdict_get_try_str(response, "return"), ==, "");
-    QDECREF(response);
-
-    /* Try to re-add the drive.  This fails with duplicate IDs if a leaked
-     * virtio-blk-pci exists that holds a reference to the old drive0.
-     */
-    response = qmp("{\"execute\": \"human-monitor-command\","
-                   " \"arguments\": {"
-                   "   \"command-line\": \"drive_add pci-addr=auto if=none,id=drive0\""
-                   "}}");
-    g_assert(response);
-    g_assert_cmpstr(qdict_get_try_str(response, "return"), ==, "OK\r\n");
-    QDECREF(response);
-
-    qtest_end();
-}
-
-int main(int argc, char **argv)
-{
-    const char *arch = qtest_get_arch();
-
-    /* Check architecture */
-    if (strcmp(arch, "i386") && strcmp(arch, "x86_64")) {
-        g_test_message("Skipping test for non-x86\n");
-        return 0;
-    }
-
-    /* Run the tests */
-    g_test_init(&argc, &argv, NULL);
-
-    qtest_add_func("/qmp/device_add", test_device_add);
-
-    return g_test_run();
-}
diff --git a/tests/qemu-iotests-quick.sh b/tests/qemu-iotests-quick.sh
index 8a9a4c68e..12af731c6 100755
--- a/tests/qemu-iotests-quick.sh
+++ b/tests/qemu-iotests-quick.sh
@@ -3,6 +3,6 @@
 cd tests/qemu-iotests
 
 ret=0
-./check -T -nocache -qcow2 -g quick || ret=1
+./check -T -qcow2 -g quick || ret=1
 
 exit $ret
diff --git a/tests/qemu-iotests/025 b/tests/qemu-iotests/025
index a5f45b454..467a4b709 100755
--- a/tests/qemu-iotests/025
+++ b/tests/qemu-iotests/025
@@ -40,7 +40,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.pattern
 
 _supported_fmt raw qcow2 qed
-_supported_proto file sheepdog rbd nfs
+_supported_proto file sheepdog rbd nfs archipelago
 _supported_os Linux
 
 echo "=== Creating image"
diff --git a/tests/qemu-iotests/028 b/tests/qemu-iotests/028
index 9e701e1c2..a1f4423d4 100755
--- a/tests/qemu-iotests/028
+++ b/tests/qemu-iotests/028
@@ -113,6 +113,7 @@ QEMU_COMM_TIMEOUT=1
 # Silence output since it contains the disk image path and QEMU's readline
 # character echoing makes it very hard to filter the output
 _send_qemu_cmd $h "drive_backup disk ${TEST_IMG}.copy" "(qemu)" >/dev/null
+_send_qemu_cmd $h "" "Formatting" | _filter_img_create
 qemu_cmd_repeat=20 _send_qemu_cmd $h "info block-jobs" "No active jobs"
 _send_qemu_cmd $h 'quit' ""
 
diff --git a/tests/qemu-iotests/028.out b/tests/qemu-iotests/028.out
index 0e1a5ae65..e8d02459b 100644
--- a/tests/qemu-iotests/028.out
+++ b/tests/qemu-iotests/028.out
@@ -468,7 +468,8 @@ No errors were found on the image.
 
 block-backup
 
-Formatting 'TEST_DIR/t.qcow2.copy', fmt=qcow2 size=4294968832 backing_file='TEST_DIR/t.qcow2.base' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
+Formatting 'TEST_DIR/t.IMGFMT.copy', fmt=IMGFMT size=4294968832 backing_file='TEST_DIR/t.IMGFMT.base' backing_fmt='IMGFMT'
+(qemu) 
 (qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block-info block-jinfo block-joinfo block-jobinfo block-jobs
 Type backup, device disk: Completed 0 of 4294968832 bytes, speed limit 0 bytes/s
 iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block-info block-jinfo block-joinfo block-jobinfo block-jobs
diff --git a/tests/qemu-iotests/039.out b/tests/qemu-iotests/039.out
index 67e774430..0adf1535e 100644
--- a/tests/qemu-iotests/039.out
+++ b/tests/qemu-iotests/039.out
@@ -25,7 +25,10 @@ read 512/512 bytes at offset 0
 incompatible_features     0x1
 
 == Repairing the image file must succeed ==
-Repairing cluster 5 refcount=0 reference=1
+ERROR cluster 5 refcount=0 reference=1
+Rebuilding refcount structure
+Repairing cluster 1 refcount=1 reference=0
+Repairing cluster 2 refcount=1 reference=0
 The following inconsistencies were found and repaired:
 
     0 leaked clusters
@@ -45,7 +48,10 @@ wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 ./039: Aborted                 ( ulimit -c 0; exec "$@" )
 incompatible_features     0x1
-Repairing cluster 5 refcount=0 reference=1
+ERROR cluster 5 refcount=0 reference=1
+Rebuilding refcount structure
+Repairing cluster 1 refcount=1 reference=0
+Repairing cluster 2 refcount=1 reference=0
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 incompatible_features     0x0
diff --git a/tests/qemu-iotests/040 b/tests/qemu-iotests/040
index f1e16c11c..2b432ad7a 100755
--- a/tests/qemu-iotests/040
+++ b/tests/qemu-iotests/040
@@ -43,8 +43,7 @@ class ImageCommitTestCase(iotests.QMPTestCase):
                 if event['event'] == 'BLOCK_JOB_COMPLETED':
                     self.assert_qmp(event, 'data/type', 'commit')
                     self.assert_qmp(event, 'data/device', 'drive0')
-                    self.assert_qmp(event, 'data/offset', self.image_len)
-                    self.assert_qmp(event, 'data/len', self.image_len)
+                    self.assert_qmp(event, 'data/offset', event['data']['len'])
                     if need_ready:
                         self.assertTrue(ready, "Expecting BLOCK_JOB_COMPLETED event")
                     completed = True
@@ -52,7 +51,6 @@ class ImageCommitTestCase(iotests.QMPTestCase):
                     ready = True
                     self.assert_qmp(event, 'data/type', 'commit')
                     self.assert_qmp(event, 'data/device', 'drive0')
-                    self.assert_qmp(event, 'data/len', self.image_len)
                     self.vm.qmp('block-job-complete', device='drive0')
 
         self.assert_no_active_block_jobs()
diff --git a/tests/qemu-iotests/041 b/tests/qemu-iotests/041
index 5dbd4ee91..59a8f733f 100755
--- a/tests/qemu-iotests/041
+++ b/tests/qemu-iotests/041
@@ -52,8 +52,7 @@ class ImageMirroringTestCase(iotests.QMPTestCase):
         event = self.cancel_and_wait(drive=drive)
         self.assertEquals(event['event'], 'BLOCK_JOB_COMPLETED')
         self.assert_qmp(event, 'data/type', 'mirror')
-        self.assert_qmp(event, 'data/offset', self.image_len)
-        self.assert_qmp(event, 'data/len', self.image_len)
+        self.assert_qmp(event, 'data/offset', event['data']['len'])
 
     def complete_and_wait(self, drive='drive0', wait_ready=True):
         '''Complete a block job and wait for it to finish'''
@@ -417,7 +416,6 @@ new_state = "1"
                     self.assert_qmp(event, 'data/type', 'mirror')
                     self.assert_qmp(event, 'data/device', 'drive0')
                     self.assert_qmp(event, 'data/error', 'Input/output error')
-                    self.assert_qmp(event, 'data/len', self.image_len)
                     completed = True
 
         self.assert_no_active_block_jobs()
@@ -568,7 +566,6 @@ new_state = "1"
                     self.assert_qmp(event, 'data/type', 'mirror')
                     self.assert_qmp(event, 'data/device', 'drive0')
                     self.assert_qmp(event, 'data/error', 'Input/output error')
-                    self.assert_qmp(event, 'data/len', self.image_len)
                     completed = True
 
         self.assert_no_active_block_jobs()
diff --git a/tests/qemu-iotests/049.out b/tests/qemu-iotests/049.out
index 71ca44d76..09ca0aed4 100644
--- a/tests/qemu-iotests/049.out
+++ b/tests/qemu-iotests/049.out
@@ -179,7 +179,7 @@ qemu-img create -f qcow2 -o preallocation=metadata TEST_DIR/t.qcow2 64M
 Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation='metadata' lazy_refcounts=off 
 
 qemu-img create -f qcow2 -o preallocation=1234 TEST_DIR/t.qcow2 64M
-qemu-img: TEST_DIR/t.qcow2: Invalid preallocation mode: '1234'
+qemu-img: TEST_DIR/t.qcow2: invalid parameter value: 1234
 Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation='1234' lazy_refcounts=off 
 
 == Check encryption option ==
diff --git a/tests/qemu-iotests/051 b/tests/qemu-iotests/051
index a41334e02..11c858f27 100755
--- a/tests/qemu-iotests/051
+++ b/tests/qemu-iotests/051
@@ -199,6 +199,29 @@ run_qemu -drive file.driver=raw
 run_qemu -drive foo=bar
 
 echo
+echo === Specifying both an option and its legacy alias ===
+echo
+
+run_qemu -drive file="$TEST_IMG",iops=1234,throttling.iops-total=5678
+run_qemu -drive file="$TEST_IMG",iops_rd=1234,throttling.iops-read=5678
+run_qemu -drive file="$TEST_IMG",iops_wr=1234,throttling.iops-write=5678
+
+run_qemu -drive file="$TEST_IMG",bps=1234,throttling.bps-total=5678
+run_qemu -drive file="$TEST_IMG",bps_rd=1234,throttling.bps-read=5678
+run_qemu -drive file="$TEST_IMG",bps_wr=1234,throttling.bps-write=5678
+
+run_qemu -drive file="$TEST_IMG",iops_max=1234,throttling.iops-total-max=5678
+run_qemu -drive file="$TEST_IMG",iops_rd_max=1234,throttling.iops-read-max=5678
+run_qemu -drive file="$TEST_IMG",iops_wr_max=1234,throttling.iops-write-max=5678
+
+run_qemu -drive file="$TEST_IMG",bps_max=1234,throttling.bps-total-max=5678
+run_qemu -drive file="$TEST_IMG",bps_rd_max=1234,throttling.bps-read-max=5678
+run_qemu -drive file="$TEST_IMG",bps_wr_max=1234,throttling.bps-write-max=5678
+
+run_qemu -drive file="$TEST_IMG",iops_size=1234,throttling.iops-size=5678
+run_qemu -drive file="$TEST_IMG",readonly=on,read-only=off
+
+echo
 echo === Parsing protocol from file name ===
 echo
 
diff --git a/tests/qemu-iotests/051.out b/tests/qemu-iotests/051.out
index d7b0f503a..2c7e80876 100644
--- a/tests/qemu-iotests/051.out
+++ b/tests/qemu-iotests/051.out
@@ -149,13 +149,11 @@ QEMU_PROG: -device ide-hd,drive=disk: Device 'ide-hd' could not be initialized
 Testing: -drive if=none,id=disk -device lsi53c895a -device scsi-disk,drive=disk
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) QEMU_PROG: -device scsi-disk,drive=disk: Device needs media, but drive is empty
-QEMU_PROG: -device scsi-disk,drive=disk: Device initialization failed.
 QEMU_PROG: -device scsi-disk,drive=disk: Device 'scsi-disk' could not be initialized
 
 Testing: -drive if=none,id=disk -device lsi53c895a -device scsi-hd,drive=disk
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) QEMU_PROG: -device scsi-hd,drive=disk: Device needs media, but drive is empty
-QEMU_PROG: -device scsi-hd,drive=disk: Device initialization failed.
 QEMU_PROG: -device scsi-hd,drive=disk: Device 'scsi-hd' could not be initialized
 
 
@@ -276,6 +274,51 @@ Testing: -drive foo=bar
 QEMU_PROG: -drive foo=bar: could not open disk image ide0-hd0: Must specify either driver or file
 
 
+=== Specifying both an option and its legacy alias ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops=1234,throttling.iops-total=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops=1234,throttling.iops-total=5678: 'throttling.iops-total' and its alias 'iops' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops_rd=1234,throttling.iops-read=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops_rd=1234,throttling.iops-read=5678: 'throttling.iops-read' and its alias 'iops_rd' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops_wr=1234,throttling.iops-write=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops_wr=1234,throttling.iops-write=5678: 'throttling.iops-write' and its alias 'iops_wr' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=1234,throttling.bps-total=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps=1234,throttling.bps-total=5678: 'throttling.bps-total' and its alias 'bps' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_rd=1234,throttling.bps-read=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_rd=1234,throttling.bps-read=5678: 'throttling.bps-read' and its alias 'bps_rd' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_wr=1234,throttling.bps-write=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_wr=1234,throttling.bps-write=5678: 'throttling.bps-write' and its alias 'bps_wr' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops_max=1234,throttling.iops-total-max=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops_max=1234,throttling.iops-total-max=5678: 'throttling.iops-total-max' and its alias 'iops_max' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops_rd_max=1234,throttling.iops-read-max=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops_rd_max=1234,throttling.iops-read-max=5678: 'throttling.iops-read-max' and its alias 'iops_rd_max' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops_wr_max=1234,throttling.iops-write-max=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops_wr_max=1234,throttling.iops-write-max=5678: 'throttling.iops-write-max' and its alias 'iops_wr_max' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_max=1234,throttling.bps-total-max=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_max=1234,throttling.bps-total-max=5678: 'throttling.bps-total-max' and its alias 'bps_max' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_rd_max=1234,throttling.bps-read-max=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_rd_max=1234,throttling.bps-read-max=5678: 'throttling.bps-read-max' and its alias 'bps_rd_max' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_wr_max=1234,throttling.bps-write-max=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_wr_max=1234,throttling.bps-write-max=5678: 'throttling.bps-write-max' and its alias 'bps_wr_max' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops_size=1234,throttling.iops-size=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops_size=1234,throttling.iops-size=5678: 'throttling.iops-size' and its alias 'iops_size' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,readonly=on,read-only=off
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,readonly=on,read-only=off: 'read-only' and its alias 'readonly' can't be used at the same time
+
+
 === Parsing protocol from file name ===
 
 Testing: -hda foo:bar
diff --git a/tests/qemu-iotests/052 b/tests/qemu-iotests/052
index 6bdae9278..61959e286 100755
--- a/tests/qemu-iotests/052
+++ b/tests/qemu-iotests/052
@@ -41,8 +41,9 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt generic
 _supported_proto file
 _supported_os Linux
-_default_cache_mode "writethrough"
-_supported_cache_modes "writethrough"
+
+# Don't do O_DIRECT on tmpfs
+_supported_cache_modes "writeback" "writethrough" "unsafe"
 
 size=128M
 _make_test_img $size
diff --git a/tests/qemu-iotests/059 b/tests/qemu-iotests/059
index 26a2fd3e0..3c053c29b 100755
--- a/tests/qemu-iotests/059
+++ b/tests/qemu-iotests/059
@@ -114,6 +114,10 @@ echo
 echo "=== Testing version 3 ==="
 _use_sample_img iotest-version3.vmdk.bz2
 _img_info
+for i in {0..99}; do
+    $QEMU_IO -r -c "read -P $(( i % 10 + 0x30 )) $(( i * 64 * 1024 * 10 + i * 512 )) 512" $TEST_IMG \
+        | _filter_qemu_io
+done
 
 echo
 echo "=== Testing 4TB monolithicFlat creation and IO ==="
diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out
index eba0dedda..0dadba658 100644
--- a/tests/qemu-iotests/059.out
+++ b/tests/qemu-iotests/059.out
@@ -2056,8 +2056,208 @@ wrote 512/512 bytes at offset 10240
 === Testing version 3 ===
 image: TEST_DIR/iotest-version3.IMGFMT
 file format: IMGFMT
-virtual size: 1.0G (1073741824 bytes)
+virtual size: 16G (17179869184 bytes)
 cluster_size: 65536
+read 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 655872
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 1311744
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 1967616
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 2623488
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 3279360
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 3935232
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4591104
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 5246976
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 5902848
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 6558720
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 7214592
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 7870464
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 8526336
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 9182208
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 9838080
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 10493952
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 11149824
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 11805696
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 12461568
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 13117440
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 13773312
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 14429184
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 15085056
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 15740928
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 16396800
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 17052672
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 17708544
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 18364416
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 19020288
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 19676160
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 20332032
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 20987904
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 21643776
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 22299648
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 22955520
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 23611392
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 24267264
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 24923136
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 25579008
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 26234880
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 26890752
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 27546624
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 28202496
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 28858368
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 29514240
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 30170112
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 30825984
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 31481856
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 32137728
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 32793600
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 33449472
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 34105344
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 34761216
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 35417088
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 36072960
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 36728832
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 37384704
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 38040576
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 38696448
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 39352320
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 40008192
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 40664064
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 41319936
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 41975808
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 42631680
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 43287552
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 43943424
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 44599296
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 45255168
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 45911040
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 46566912
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 47222784
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 47878656
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 48534528
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 49190400
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 49846272
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 50502144
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 51158016
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 51813888
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 52469760
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 53125632
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 53781504
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 54437376
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 55093248
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 55749120
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 56404992
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 57060864
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 57716736
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 58372608
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 59028480
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 59684352
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 60340224
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 60996096
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 61651968
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 62307840
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 62963712
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 63619584
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 64275456
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 64931328
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Testing 4TB monolithicFlat creation and IO ===
 Formatting 'TEST_DIR/iotest-version3.IMGFMT', fmt=IMGFMT size=4398046511104
diff --git a/tests/qemu-iotests/060 b/tests/qemu-iotests/060
index 3cffc12fe..9772d365a 100755
--- a/tests/qemu-iotests/060
+++ b/tests/qemu-iotests/060
@@ -76,6 +76,9 @@ $QEMU_IO -c "$OPEN_RW" -c "write -P 0x2a 0 512" | _filter_qemu_io
 # The corrupt bit must now be set
 $PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
 
+# This information should be available through qemu-img info
+$QEMU_IMG info "$TEST_IMG" | _filter_testdir
+
 # Try to open the image R/W (which should fail)
 $QEMU_IO -c "$OPEN_RW" -c "read 0 512" 2>&1 | _filter_qemu_io \
                                             | _filter_testdir \
@@ -164,6 +167,67 @@ wait_break 0
 write 64k 64k
 resume 0" | $QEMU_IO | _filter_qemu_io
 
+echo
+echo "=== Testing unallocated image header ==="
+echo
+_make_test_img 64M
+# Create L1/L2
+$QEMU_IO -c "write 0 64k" "$TEST_IMG" | _filter_qemu_io
+poke_file "$TEST_IMG" "$rb_offset" "\x00\x00"
+$QEMU_IO -c "write 64k 64k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "=== Testing unaligned L1 entry ==="
+echo
+_make_test_img 64M
+$QEMU_IO -c "write 0 64k" "$TEST_IMG" | _filter_qemu_io
+# This will be masked with ~(512 - 1) = ~0x1ff, so whether the lower 9 bits are
+# aligned or not does not matter
+poke_file "$TEST_IMG" "$l1_offset" "\x80\x00\x00\x00\x00\x04\x2a\x00"
+$QEMU_IO -c "read 0 64k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "=== Testing unaligned L2 entry ==="
+echo
+_make_test_img 64M
+$QEMU_IO -c "write 0 64k" "$TEST_IMG" | _filter_qemu_io
+poke_file "$TEST_IMG" "$l2_offset" "\x80\x00\x00\x00\x00\x05\x2a\x00"
+$QEMU_IO -c "read 0 64k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "=== Testing unaligned reftable entry ==="
+echo
+_make_test_img 64M
+poke_file "$TEST_IMG" "$rt_offset" "\x00\x00\x00\x00\x00\x02\x2a\x00"
+$QEMU_IO -c "write 0 64k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "=== Testing non-fatal corruption on freeing ==="
+echo
+_make_test_img 64M
+$QEMU_IO -c "write 0 64k" "$TEST_IMG" | _filter_qemu_io
+poke_file "$TEST_IMG" "$l2_offset" "\x80\x00\x00\x00\x00\x05\x2a\x00"
+$QEMU_IO -c "discard 0 64k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "=== Testing read-only corruption report ==="
+echo
+_make_test_img 64M
+$QEMU_IO -c "write 0 64k" "$TEST_IMG" | _filter_qemu_io
+poke_file "$TEST_IMG" "$l2_offset" "\x80\x00\x00\x00\x00\x05\x2a\x00"
+# Should only emit a single error message
+$QEMU_IO -c "$OPEN_RO" -c "read 0 64k" -c "read 0 64k" | _filter_qemu_io
+
+echo
+echo "=== Testing non-fatal and then fatal corruption report ==="
+echo
+_make_test_img 64M
+$QEMU_IO -c "write 0 128k" "$TEST_IMG" | _filter_qemu_io
+poke_file "$TEST_IMG" "$l2_offset"        "\x80\x00\x00\x00\x00\x05\x2a\x00"
+poke_file "$TEST_IMG" "$(($l2_offset+8))" "\x80\x00\x00\x00\x00\x06\x2a\x00"
+# Should emit two error messages
+$QEMU_IO -c "discard 0 64k" -c "read 64k 64k" "$TEST_IMG" | _filter_qemu_io
+
 # success, all done
 echo "*** done"
 rm -f $seq.full
diff --git a/tests/qemu-iotests/060.out b/tests/qemu-iotests/060.out
index a51794803..9419da1b4 100644
--- a/tests/qemu-iotests/060.out
+++ b/tests/qemu-iotests/060.out
@@ -8,9 +8,18 @@ ERROR cluster 3 refcount=1 reference=3
 1 errors were found on the image.
 Data may be corrupted, or further writes to the image may corrupt it.
 incompatible_features     0x0
-qcow2: Preventing invalid write on metadata (overlaps with active L1 table); image marked as corrupt.
+qcow2: Marking image as corrupt: Preventing invalid write on metadata (overlaps with active L1 table); further corruption events will be suppressed
 write failed: Input/output error
 incompatible_features     0x2
+image: TEST_DIR/t.qcow2
+file format: qcow2
+virtual size: 64M (67108864 bytes)
+disk size: 196K
+cluster_size: 65536
+Format specific information:
+    compat: 1.1
+    lazy refcounts: false
+    corrupt: true
 qemu-io: can't open device TEST_DIR/t.IMGFMT: IMGFMT: Image is corrupt; cannot be opened read/write
 read 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -24,14 +33,18 @@ ERROR cluster 2 refcount=1 reference=2
 2 errors were found on the image.
 Data may be corrupted, or further writes to the image may corrupt it.
 incompatible_features     0x0
-qcow2: Preventing invalid write on metadata (overlaps with refcount block); image marked as corrupt.
+qcow2: Marking image as corrupt: Preventing invalid write on metadata (overlaps with refcount block); further corruption events will be suppressed
 write failed: Input/output error
 incompatible_features     0x2
-Repairing refcount block 0 refcount=2
+ERROR refcount block 0 refcount=2
+ERROR cluster 2 refcount=1 reference=2
+Rebuilding refcount structure
+Repairing cluster 1 refcount=1 reference=0
+Repairing cluster 2 refcount=2 reference=1
 The following inconsistencies were found and repaired:
 
     0 leaked clusters
-    1 corruptions
+    2 corruptions
 
 Double checking the fixed image now...
 No errors were found on the image.
@@ -56,9 +69,11 @@ Data may be corrupted, or further writes to the image may corrupt it.
 1 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 incompatible_features     0x0
-qcow2: Preventing invalid write on metadata (overlaps with inactive L2 table); image marked as corrupt.
+qcow2: Marking image as corrupt: Preventing invalid write on metadata (overlaps with inactive L2 table); further corruption events will be suppressed
 write failed: Input/output error
 incompatible_features     0x2
+ERROR cluster 4 refcount=1 reference=2
+Leaked cluster 9 refcount=1 reference=0
 Repairing cluster 4 refcount=1 reference=2
 Repairing cluster 9 refcount=1 reference=0
 Repairing OFLAG_COPIED data cluster: l2_entry=8000000000040000 refcount=2
@@ -88,9 +103,68 @@ wrote 65536/65536 bytes at offset 536870912
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 discard 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-qcow2: Preventing invalid write on metadata (overlaps with active L2 table); image marked as corrupt.
+qcow2: Marking image as corrupt: Preventing invalid write on metadata (overlaps with active L2 table); further corruption events will be suppressed
 blkdebug: Suspended request '0'
 write failed: Input/output error
 blkdebug: Resuming request '0'
 aio_write failed: No medium found
+
+=== Testing unallocated image header ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qcow2: Marking image as corrupt: Preventing invalid write on metadata (overlaps with qcow2_header); further corruption events will be suppressed
+write failed: Input/output error
+
+=== Testing unaligned L1 entry ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qcow2: Marking image as corrupt: L2 table offset 0x42a00 unaligned (L1 index: 0); further corruption events will be suppressed
+read failed: Input/output error
+
+=== Testing unaligned L2 entry ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qcow2: Marking image as corrupt: Data cluster offset 0x52a00 unaligned (L2 offset: 0x40000, L2 index: 0); further corruption events will be suppressed
+read failed: Input/output error
+
+=== Testing unaligned reftable entry ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+qcow2: Marking image as corrupt: Refblock offset 0x22a00 unaligned (reftable index: 0); further corruption events will be suppressed
+write failed: Input/output error
+
+=== Testing non-fatal corruption on freeing ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qcow2: Image is corrupt: Cannot free unaligned cluster 0x52a00; further non-fatal corruption events will be suppressed
+discard 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Testing read-only corruption report ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qcow2: Image is corrupt: Data cluster offset 0x52a00 unaligned (L2 offset: 0x40000, L2 index: 0); further non-fatal corruption events will be suppressed
+read failed: Input/output error
+read failed: Input/output error
+
+=== Testing non-fatal and then fatal corruption report ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+wrote 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qcow2: Image is corrupt: Cannot free unaligned cluster 0x52a00; further non-fatal corruption events will be suppressed
+qcow2: Marking image as corrupt: Data cluster offset 0x62a00 unaligned (L2 offset: 0x40000, L2 index: 0x1); further corruption events will be suppressed
+discard 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read failed: Input/output error
 *** done
diff --git a/tests/qemu-iotests/061 b/tests/qemu-iotests/061
index ab98def6d..8d37f8a65 100755
--- a/tests/qemu-iotests/061
+++ b/tests/qemu-iotests/061
@@ -209,6 +209,31 @@ $QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
 _check_test_img
 $QEMU_IO -c "read -P 0 0 64M" "$TEST_IMG" | _filter_qemu_io
 
+echo
+echo "=== Testing progress report without snapshot ==="
+echo
+IMGOPTS="compat=1.1" TEST_IMG="$TEST_IMG.base" _make_test_img 4G
+IMGOPTS="compat=1.1" _make_test_img -b "$TEST_IMG.base" 4G
+$QEMU_IO -c "write -z 0  64k" \
+         -c "write -z 1G 64k" \
+         -c "write -z 2G 64k" \
+         -c "write -z 3G 64k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG amend -p -o "compat=0.10" "$TEST_IMG"
+_check_test_img
+
+echo
+echo "=== Testing progress report with snapshot ==="
+echo
+IMGOPTS="compat=1.1" TEST_IMG="$TEST_IMG.base" _make_test_img 4G
+IMGOPTS="compat=1.1" _make_test_img -b "$TEST_IMG.base" 4G
+$QEMU_IO -c "write -z 0  64k" \
+         -c "write -z 1G 64k" \
+         -c "write -z 2G 64k" \
+         -c "write -z 3G 64k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG snapshot -c foo "$TEST_IMG"
+$QEMU_IMG amend -p -o "compat=0.10" "$TEST_IMG"
+_check_test_img
+
 # success, all done
 echo "*** done"
 rm -f $seq.full
diff --git a/tests/qemu-iotests/061.out b/tests/qemu-iotests/061.out
index e3724700b..9045544df 100644
--- a/tests/qemu-iotests/061.out
+++ b/tests/qemu-iotests/061.out
@@ -76,8 +76,11 @@ autoclear_features        0x0
 refcount_order            4
 header_length             104
 
-Repairing cluster 5 refcount=0 reference=1
-Repairing cluster 6 refcount=0 reference=1
+ERROR cluster 5 refcount=0 reference=1
+ERROR cluster 6 refcount=0 reference=1
+Rebuilding refcount structure
+Repairing cluster 1 refcount=1 reference=0
+Repairing cluster 2 refcount=1 reference=0
 magic                     0x514649fb
 version                   2
 backing_file_offset       0x0
@@ -87,7 +90,7 @@ size                      67108864
 crypt_method              0
 l1_size                   1
 l1_table_offset           0x30000
-refcount_table_offset     0x10000
+refcount_table_offset     0x80000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
@@ -230,8 +233,11 @@ autoclear_features        0x0
 refcount_order            4
 header_length             104
 
-Repairing cluster 5 refcount=0 reference=1
-Repairing cluster 6 refcount=0 reference=1
+ERROR cluster 5 refcount=0 reference=1
+ERROR cluster 6 refcount=0 reference=1
+Rebuilding refcount structure
+Repairing cluster 1 refcount=1 reference=0
+Repairing cluster 2 refcount=1 reference=0
 magic                     0x514649fb
 version                   3
 backing_file_offset       0x0
@@ -241,7 +247,7 @@ size                      67108864
 crypt_method              0
 l1_size                   1
 l1_table_offset           0x30000
-refcount_table_offset     0x10000
+refcount_table_offset     0x80000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
@@ -384,4 +390,34 @@ wrote 67108864/67108864 bytes at offset 0
 No errors were found on the image.
 read 67108864/67108864 bytes at offset 0
 64 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Testing progress report without snapshot ===
+
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=4294967296 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 backing_file='TEST_DIR/t.IMGFMT.base' 
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 1073741824
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 2147483648
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 3221225472
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+    (0.00/100%)
    (12.50/100%)
    (25.00/100%)
    (37.50/100%)
    (50.00/100%)
    (62.50/100%)
    (75.00/100%)
    (87.50/100%)
    (100.00/100%)
    (100.00/100%)
+No errors were found on the image.
+
+=== Testing progress report with snapshot ===
+
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=4294967296 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 backing_file='TEST_DIR/t.IMGFMT.base' 
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 1073741824
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 2147483648
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 3221225472
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+    (0.00/100%)
    (6.25/100%)
    (12.50/100%)
    (18.75/100%)
    (25.00/100%)
    (31.25/100%)
    (37.50/100%)
    (43.75/100%)
    (50.00/100%)
    (56.25/100%)
    (62.50/100%)
    (68.75/100%)
    (75.00/100%)
    (81.25/100%)
    (87.50/100%)
    (93.75/100%)
    (100.00/100%)
    (100.00/100%)
+No errors were found on the image.
 *** done
diff --git a/tests/qemu-iotests/065 b/tests/qemu-iotests/065
index e89b61d70..8d3a9c9af 100755
--- a/tests/qemu-iotests/065
+++ b/tests/qemu-iotests/065
@@ -94,28 +94,28 @@ class TestQCow2(TestQemuImgInfo):
 class TestQCow3NotLazy(TestQemuImgInfo):
     '''Testing a qcow2 version 3 image with lazy refcounts disabled'''
     img_options = 'compat=1.1,lazy_refcounts=off'
-    json_compare = { 'compat': '1.1', 'lazy-refcounts': False }
-    human_compare = [ 'compat: 1.1', 'lazy refcounts: false' ]
+    json_compare = { 'compat': '1.1', 'lazy-refcounts': False, 'corrupt': False }
+    human_compare = [ 'compat: 1.1', 'lazy refcounts: false', 'corrupt: false' ]
 
 class TestQCow3Lazy(TestQemuImgInfo):
     '''Testing a qcow2 version 3 image with lazy refcounts enabled'''
     img_options = 'compat=1.1,lazy_refcounts=on'
-    json_compare = { 'compat': '1.1', 'lazy-refcounts': True }
-    human_compare = [ 'compat: 1.1', 'lazy refcounts: true' ]
+    json_compare = { 'compat': '1.1', 'lazy-refcounts': True, 'corrupt': False }
+    human_compare = [ 'compat: 1.1', 'lazy refcounts: true', 'corrupt: false' ]
 
 class TestQCow3NotLazyQMP(TestQMP):
     '''Testing a qcow2 version 3 image with lazy refcounts disabled, opening
        with lazy refcounts enabled'''
     img_options = 'compat=1.1,lazy_refcounts=off'
     qemu_options = 'lazy-refcounts=on'
-    compare = { 'compat': '1.1', 'lazy-refcounts': False }
+    compare = { 'compat': '1.1', 'lazy-refcounts': False, 'corrupt': False }
 
 class TestQCow3LazyQMP(TestQMP):
     '''Testing a qcow2 version 3 image with lazy refcounts enabled, opening
        with lazy refcounts disabled'''
     img_options = 'compat=1.1,lazy_refcounts=on'
     qemu_options = 'lazy-refcounts=off'
-    compare = { 'compat': '1.1', 'lazy-refcounts': True }
+    compare = { 'compat': '1.1', 'lazy-refcounts': True, 'corrupt': False }
 
 TestImageInfoSpecific = None
 TestQemuImgInfo = None
diff --git a/tests/qemu-iotests/067.out b/tests/qemu-iotests/067.out
index 7e090b95a..0f72dcf63 100644
--- a/tests/qemu-iotests/067.out
+++ b/tests/qemu-iotests/067.out
@@ -6,7 +6,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk -device virtio-blk-pci,drive=disk,id=virtio0
 QMP_VERSION
 {"return": {}}
-{"return": [{"io-status": "ok", "device": "disk", "locked": false, "removable": false, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "type": "unknown"}, {"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}]}
+{"return": [{"io-status": "ok", "device": "disk", "locked": false, "removable": false, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false, "corrupt": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "type": "unknown"}, {"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}]}
 {"return": {}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"path": "/machine/peripheral/virtio0/virtio-backend"}}
@@ -24,7 +24,7 @@ QMP_VERSION
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk
 QMP_VERSION
 {"return": {}}
-{"return": [{"device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}, {"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}]}
+{"return": [{"device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false, "corrupt": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}, {"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}]}
 {"return": {}}
 {"return": {}}
 {"return": {}}
@@ -44,7 +44,7 @@ Testing:
 QMP_VERSION
 {"return": {}}
 {"return": "OK\r\n"}
-{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}]}
+{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false, "corrupt": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}]}
 {"return": {}}
 {"return": {}}
 {"return": {}}
@@ -64,14 +64,14 @@ Testing:
 QMP_VERSION
 {"return": {}}
 {"return": {}}
-{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}]}
+{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false, "corrupt": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}]}
 {"return": {}}
 {"return": {}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"path": "/machine/peripheral/virtio0/virtio-backend"}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"device": "virtio0", "path": "/machine/peripheral/virtio0"}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "RESET"}
-{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"io-status": "ok", "device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}]}
+{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"io-status": "ok", "device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false, "corrupt": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}]}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
diff --git a/tests/qemu-iotests/069 b/tests/qemu-iotests/069
index e661598c4..ce9e0541b 100755
--- a/tests/qemu-iotests/069
+++ b/tests/qemu-iotests/069
@@ -38,7 +38,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.rc
 . ./common.filter
 
-_supported_fmt cow qed qcow qcow2 vmdk
+_supported_fmt qed qcow qcow2 vmdk
 _supported_proto file
 _supported_os Linux
 _unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat"
diff --git a/tests/qemu-iotests/070 b/tests/qemu-iotests/070
index ea0dae7e9..d649ddf9b 100755
--- a/tests/qemu-iotests/070
+++ b/tests/qemu-iotests/070
@@ -77,7 +77,7 @@ _use_sample_img test-disk2vhd.vhdx.bz2
 
 echo
 echo "=== Verify image created by Disk2VHD can be opened ==="
-$QEMU_IMG info "$TEST_IMG" 2>&1 | _filter_testdir | _filter_qemu
+_img_info
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/070.out b/tests/qemu-iotests/070.out
index 15f1fc147..ca743831c 100644
--- a/tests/qemu-iotests/070.out
+++ b/tests/qemu-iotests/070.out
@@ -20,9 +20,8 @@ read 18874368/18874368 bytes at offset 0
 18 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Verify image created by Disk2VHD can be opened ===
-image: TEST_DIR/test-disk2vhd.vhdx
-file format: vhdx
+image: TEST_DIR/test-disk2vhd.IMGFMT
+file format: IMGFMT
 virtual size: 256M (268435456 bytes)
-disk size: 260M
 cluster_size: 2097152
 *** done
diff --git a/tests/qemu-iotests/072 b/tests/qemu-iotests/072
index 58faa8b5a..e4a723d73 100755
--- a/tests/qemu-iotests/072
+++ b/tests/qemu-iotests/072
@@ -38,7 +38,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.rc
 . ./common.filter
 
-_supported_fmt vpc vmdk vhdx vdi qed qcow2 qcow cow
+_supported_fmt vpc vmdk vhdx vdi qed qcow2 qcow
 _supported_proto file
 _supported_os Linux
 
diff --git a/tests/qemu-iotests/075 b/tests/qemu-iotests/075
index 40032c563..6117660c5 100755
--- a/tests/qemu-iotests/075
+++ b/tests/qemu-iotests/075
@@ -39,7 +39,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.filter
 
 _supported_fmt cloop
-_supported_proto generic
+_supported_proto file
 _supported_os Linux
 
 block_size_offset=128
diff --git a/tests/qemu-iotests/076 b/tests/qemu-iotests/076
index b614a7dd6..ed2be3581 100755
--- a/tests/qemu-iotests/076
+++ b/tests/qemu-iotests/076
@@ -39,7 +39,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.filter
 
 _supported_fmt parallels
-_supported_proto generic
+_supported_proto file
 _supported_os Linux
 
 tracks_offset=$((0x1c))
@@ -47,29 +47,34 @@ catalog_entries_offset=$((0x20))
 nb_sectors_offset=$((0x24))
 
 echo
-echo "== Read from a valid (enough) image =="
-_use_sample_img fake.parallels.bz2
+echo "== Read from a valid v1 image =="
+_use_sample_img parallels-v1.bz2
 { $QEMU_IO -c "read -P 0x11 0 64k" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
 
 echo
 echo "== Negative catalog size =="
-_use_sample_img fake.parallels.bz2
+_use_sample_img parallels-v1.bz2
 poke_file "$TEST_IMG" "$catalog_entries_offset" "\xff\xff\xff\xff"
 { $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
 
 echo
 echo "== Overflow in catalog allocation =="
-_use_sample_img fake.parallels.bz2
+_use_sample_img parallels-v1.bz2
 poke_file "$TEST_IMG" "$nb_sectors_offset" "\xff\xff\xff\xff"
 poke_file "$TEST_IMG" "$catalog_entries_offset" "\x01\x00\x00\x40"
 { $QEMU_IO -c "read 64M 64M" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
 
 echo
 echo "== Zero sectors per track =="
-_use_sample_img fake.parallels.bz2
+_use_sample_img parallels-v1.bz2
 poke_file "$TEST_IMG" "$tracks_offset" "\x00\x00\x00\x00"
 { $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
 
+echo
+echo "== Read from a valid v2 image =="
+_use_sample_img parallels-v2.bz2
+{ $QEMU_IO -c "read -P 0x11 0 64k" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+
 # success, all done
 echo "*** done"
 rm -f $seq.full
diff --git a/tests/qemu-iotests/076.out b/tests/qemu-iotests/076.out
index f7745d8b0..32ade0856 100644
--- a/tests/qemu-iotests/076.out
+++ b/tests/qemu-iotests/076.out
@@ -1,18 +1,22 @@
 QA output created by 076
 
-== Read from a valid (enough) image ==
+== Read from a valid v1 image ==
 read 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 == Negative catalog size ==
-qemu-io: can't open device TEST_DIR/fake.parallels: Catalog too large
+qemu-io: can't open device TEST_DIR/parallels-v1: Catalog too large
 no file open, try 'help open'
 
 == Overflow in catalog allocation ==
-qemu-io: can't open device TEST_DIR/fake.parallels: Catalog too large
+qemu-io: can't open device TEST_DIR/parallels-v1: Catalog too large
 no file open, try 'help open'
 
 == Zero sectors per track ==
-qemu-io: can't open device TEST_DIR/fake.parallels: Invalid image: Zero sectors per track
+qemu-io: can't open device TEST_DIR/parallels-v1: Invalid image: Zero sectors per track
 no file open, try 'help open'
+
+== Read from a valid v2 image ==
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 *** done
diff --git a/tests/qemu-iotests/078 b/tests/qemu-iotests/078
index d4d6da7b0..7be2c3f69 100755
--- a/tests/qemu-iotests/078
+++ b/tests/qemu-iotests/078
@@ -39,7 +39,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.filter
 
 _supported_fmt bochs
-_supported_proto generic
+_supported_proto file
 _supported_os Linux
 
 catalog_size_offset=$((0x48))
diff --git a/tests/qemu-iotests/079 b/tests/qemu-iotests/079
index 2142bbb37..6613cfb18 100755
--- a/tests/qemu-iotests/079
+++ b/tests/qemu-iotests/079
@@ -39,7 +39,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.filter
 
 _supported_fmt qcow2
-_supported_proto file
+_supported_proto file nfs
 _supported_os Linux
 
 function test_qemu_img()
diff --git a/tests/qemu-iotests/080 b/tests/qemu-iotests/080
index 6b3a3e77a..9de337c40 100755
--- a/tests/qemu-iotests/080
+++ b/tests/qemu-iotests/080
@@ -40,7 +40,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.filter
 
 _supported_fmt qcow2
-_supported_proto generic
+_supported_proto file
 _supported_os Linux
 
 header_size=104
diff --git a/tests/qemu-iotests/081 b/tests/qemu-iotests/081
index 7ae4be205..ed3c29e13 100755
--- a/tests/qemu-iotests/081
+++ b/tests/qemu-iotests/081
@@ -41,7 +41,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.filter
 
 _supported_fmt raw
-_supported_proto generic
+_supported_proto file
 _supported_os Linux
 
 function do_run_qemu()
diff --git a/tests/qemu-iotests/082 b/tests/qemu-iotests/082
index f6eb75f62..e64de2773 100755
--- a/tests/qemu-iotests/082
+++ b/tests/qemu-iotests/082
@@ -39,7 +39,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.filter
 
 _supported_fmt qcow2
-_supported_proto file
+_supported_proto file nfs
 _supported_os Linux
 
 function run_qemu_img()
@@ -56,7 +56,7 @@ echo === create: Options specified more than once ===
 
 # Last -f should win
 run_qemu_img create -f foo -f $IMGFMT "$TEST_IMG" $size
-run_qemu_img info "$TEST_IMG"
+_img_info
 
 # Multiple -o should be merged
 run_qemu_img create -f $IMGFMT -o cluster_size=4k -o lazy_refcounts=on "$TEST_IMG" $size
@@ -66,7 +66,7 @@ run_qemu_img info "$TEST_IMG"
 run_qemu_img create -f $IMGFMT -o cluster_size=4k -o lazy_refcounts=on -o cluster_size=8k "$TEST_IMG" $size
 run_qemu_img info "$TEST_IMG"
 run_qemu_img create -f $IMGFMT -o cluster_size=4k,cluster_size=8k "$TEST_IMG" $size
-run_qemu_img info "$TEST_IMG"
+_img_info
 
 echo
 echo === create: help for -o ===
@@ -106,11 +106,11 @@ run_qemu_img create -f $IMGFMT "$TEST_IMG" $size
 
 # Last -f should win
 run_qemu_img convert -f foo -f $IMGFMT "$TEST_IMG" "$TEST_IMG".base
-run_qemu_img info "$TEST_IMG".base
+TEST_IMG="${TEST_IMG}.base" _img_info
 
 # Last -O should win
 run_qemu_img convert -O foo -O $IMGFMT "$TEST_IMG" "$TEST_IMG".base
-run_qemu_img info "$TEST_IMG".base
+TEST_IMG="${TEST_IMG}.base" _img_info
 
 # Multiple -o should be merged
 run_qemu_img convert -O $IMGFMT -o cluster_size=4k -o lazy_refcounts=on "$TEST_IMG" "$TEST_IMG".base
@@ -120,7 +120,7 @@ run_qemu_img info "$TEST_IMG".base
 run_qemu_img convert -O $IMGFMT -o cluster_size=4k -o lazy_refcounts=on -o cluster_size=8k "$TEST_IMG" "$TEST_IMG".base
 run_qemu_img info "$TEST_IMG".base
 run_qemu_img convert -O $IMGFMT -o cluster_size=4k,cluster_size=8k "$TEST_IMG" "$TEST_IMG".base
-run_qemu_img info "$TEST_IMG".base
+TEST_IMG="${TEST_IMG}.base" _img_info
 
 echo
 echo === convert: help for -o ===
@@ -167,7 +167,7 @@ run_qemu_img info "$TEST_IMG"
 run_qemu_img amend -f $IMGFMT -o size=8M -o lazy_refcounts=on -o size=132M "$TEST_IMG"
 run_qemu_img info "$TEST_IMG"
 run_qemu_img amend -f $IMGFMT -o size=4M,size=148M "$TEST_IMG"
-run_qemu_img info "$TEST_IMG"
+_img_info
 
 echo
 echo === amend: help for -o ===
diff --git a/tests/qemu-iotests/082.out b/tests/qemu-iotests/082.out
index 413e7ef39..0a3ab5ac9 100644
--- a/tests/qemu-iotests/082.out
+++ b/tests/qemu-iotests/082.out
@@ -4,16 +4,10 @@ QA output created by 082
 
 Testing: create -f foo -f qcow2 TEST_DIR/t.qcow2 128M
 Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 encryption=off cluster_size=65536 lazy_refcounts=off 
-
-Testing: info TEST_DIR/t.qcow2
-image: TEST_DIR/t.qcow2
-file format: qcow2
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
 virtual size: 128M (134217728 bytes)
-disk size: 196K
 cluster_size: 65536
-Format specific information:
-    compat: 1.1
-    lazy refcounts: false
 
 Testing: create -f qcow2 -o cluster_size=4k -o lazy_refcounts=on TEST_DIR/t.qcow2 128M
 Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 encryption=off cluster_size=4096 lazy_refcounts=on 
@@ -27,6 +21,7 @@ cluster_size: 4096
 Format specific information:
     compat: 1.1
     lazy refcounts: true
+    corrupt: false
 
 Testing: create -f qcow2 -o cluster_size=4k -o lazy_refcounts=on -o cluster_size=8k TEST_DIR/t.qcow2 128M
 Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 encryption=off cluster_size=8192 lazy_refcounts=on 
@@ -40,19 +35,14 @@ cluster_size: 8192
 Format specific information:
     compat: 1.1
     lazy refcounts: true
+    corrupt: false
 
 Testing: create -f qcow2 -o cluster_size=4k,cluster_size=8k TEST_DIR/t.qcow2 128M
 Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 encryption=off cluster_size=8192 lazy_refcounts=off 
-
-Testing: info TEST_DIR/t.qcow2
-image: TEST_DIR/t.qcow2
-file format: qcow2
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
 virtual size: 128M (134217728 bytes)
-disk size: 28K
 cluster_size: 8192
-Format specific information:
-    compat: 1.1
-    lazy refcounts: false
 
 === create: help for -o ===
 
@@ -64,7 +54,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -76,7 +66,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -88,7 +78,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -100,7 +90,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -112,7 +102,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -124,7 +114,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -136,7 +126,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -148,7 +138,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -175,7 +165,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 
 Testing: create -o help
@@ -188,24 +178,15 @@ Testing: create -f qcow2 TEST_DIR/t.qcow2 128M
 Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 encryption=off cluster_size=65536 lazy_refcounts=off 
 
 Testing: convert -f foo -f qcow2 TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.base
-
-Testing: info TEST_DIR/t.qcow2.base
-image: TEST_DIR/t.qcow2.base
+image: TEST_DIR/t.IMGFMT.base
 file format: raw
 virtual size: 128M (134217728 bytes)
-disk size: 0
 
 Testing: convert -O foo -O qcow2 TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.base
-
-Testing: info TEST_DIR/t.qcow2.base
-image: TEST_DIR/t.qcow2.base
-file format: qcow2
+image: TEST_DIR/t.IMGFMT.base
+file format: IMGFMT
 virtual size: 128M (134217728 bytes)
-disk size: 196K
 cluster_size: 65536
-Format specific information:
-    compat: 1.1
-    lazy refcounts: false
 
 Testing: convert -O qcow2 -o cluster_size=4k -o lazy_refcounts=on TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.base
 
@@ -218,6 +199,7 @@ cluster_size: 4096
 Format specific information:
     compat: 1.1
     lazy refcounts: true
+    corrupt: false
 
 Testing: convert -O qcow2 -o cluster_size=4k -o lazy_refcounts=on -o cluster_size=8k TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.base
 
@@ -230,18 +212,13 @@ cluster_size: 8192
 Format specific information:
     compat: 1.1
     lazy refcounts: true
+    corrupt: false
 
 Testing: convert -O qcow2 -o cluster_size=4k,cluster_size=8k TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.base
-
-Testing: info TEST_DIR/t.qcow2.base
-image: TEST_DIR/t.qcow2.base
-file format: qcow2
+image: TEST_DIR/t.IMGFMT.base
+file format: IMGFMT
 virtual size: 128M (134217728 bytes)
-disk size: 28K
 cluster_size: 8192
-Format specific information:
-    compat: 1.1
-    lazy refcounts: false
 
 === convert: help for -o ===
 
@@ -253,7 +230,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -265,7 +242,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -277,7 +254,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -289,7 +266,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -301,7 +278,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -313,7 +290,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -325,7 +302,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -337,7 +314,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -364,7 +341,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 
 Testing: convert -o help
@@ -384,6 +361,7 @@ cluster_size: 65536
 Format specific information:
     compat: 1.1
     lazy refcounts: true
+    corrupt: false
 
 Testing: amend -f qcow2 -o size=130M -o lazy_refcounts=off TEST_DIR/t.qcow2
 
@@ -396,6 +374,7 @@ cluster_size: 65536
 Format specific information:
     compat: 1.1
     lazy refcounts: false
+    corrupt: false
 
 Testing: amend -f qcow2 -o size=8M -o lazy_refcounts=on -o size=132M TEST_DIR/t.qcow2
 
@@ -408,18 +387,13 @@ cluster_size: 65536
 Format specific information:
     compat: 1.1
     lazy refcounts: true
+    corrupt: false
 
 Testing: amend -f qcow2 -o size=4M,size=148M TEST_DIR/t.qcow2
-
-Testing: info TEST_DIR/t.qcow2
-image: TEST_DIR/t.qcow2
-file format: qcow2
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
 virtual size: 148M (155189248 bytes)
-disk size: 196K
 cluster_size: 65536
-Format specific information:
-    compat: 1.1
-    lazy refcounts: true
 
 === amend: help for -o ===
 
@@ -431,7 +405,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -443,7 +417,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -455,7 +429,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -467,7 +441,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -479,7 +453,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -491,7 +465,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -503,7 +477,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -515,7 +489,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 nocow            Turn off copy-on-write (valid only on btrfs)
 
@@ -544,7 +518,7 @@ backing_file     File name of a base image
 backing_fmt      Image format of the base image
 encryption       Encrypt the image
 cluster_size     qcow2 cluster size
-preallocation    Preallocation mode (allowed values: off, metadata)
+preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
 
 Testing: convert -o help
diff --git a/tests/qemu-iotests/084 b/tests/qemu-iotests/084
index cb4d7b729..733018d4a 100755
--- a/tests/qemu-iotests/084
+++ b/tests/qemu-iotests/084
@@ -1,6 +1,7 @@
 #!/bin/bash
 #
-# Test case for VDI header corruption; image too large, and too many blocks
+# Test case for VDI header corruption; image too large, and too many blocks.
+# Also simple test for creating dynamic and static VDI images.
 #
 # Copyright (C) 2013 Red Hat, Inc.
 #
@@ -40,29 +41,40 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 
 # This tests vdi-specific header fields
 _supported_fmt vdi
-_supported_proto generic
+_supported_proto file
 _supported_os Linux
 
+size=64M
 ds_offset=368  # disk image size field offset
 bs_offset=376  # block size field offset
 bii_offset=384 # block in image field offset
 
 echo
+echo "=== Statically allocated image creation ==="
+echo
+_make_test_img $size -o static
+_img_info
+stat -c"disk image file size in bytes: %s" "${TEST_IMG}"
+_cleanup_test_img
+
+echo
 echo "=== Testing image size bounds ==="
 echo
-_make_test_img 64M
+_make_test_img $size
+_img_info
+stat -c"disk image file size in bytes: %s" "${TEST_IMG}"
 
 # check for image size too large
 # poke max image size, and appropriate blocks_in_image value
-echo "Test 1: Maximum size (1024 TB):"
-poke_file "$TEST_IMG" "$ds_offset" "\x00\x00\xf0\xff\xff\xff\x03\x00"
-poke_file "$TEST_IMG" "$bii_offset" "\xff\xff\xff\x3f"
+echo "Test 1: Maximum size (512 TB - 128 MB):"
+poke_file "$TEST_IMG" "$ds_offset" "\x00\x00\x00\xf8\xff\xff\x01\x00"
+poke_file "$TEST_IMG" "$bii_offset" "\x80\xff\xff\x1f"
 _img_info
 
 echo
-echo "Test 2: Size too large (1024TB + 1)"
+echo "Test 2: Size too large (512 TB - 128 MB + 64 kB)"
 # This should be too large (-EINVAL):
-poke_file "$TEST_IMG" "$ds_offset" "\x00\x00\xf1\xff\xff\xff\x03\x00"
+poke_file "$TEST_IMG" "$ds_offset" "\x00\x00\x01\xf8\xff\xff\x01\x00"
 _img_info
 
 echo
@@ -77,9 +89,9 @@ _img_info
 
 echo
 echo "Test 4: Size valid (64M), but Blocks In Image exceeds max allowed"
-# Now check the bounds of blocks_in_image - 0x3fffffff should be the max
+# Now check the bounds of blocks_in_image - 0x1fffff80 should be the max
 # value here, and we should get -ENOTSUP
-poke_file "$TEST_IMG" "$bii_offset" "\x00\x00\x00\x40"
+poke_file "$TEST_IMG" "$bii_offset" "\x81\xff\xff\x1f"
 _img_info
 
 # Finally, 1MB is the only block size supported.  Verify that
diff --git a/tests/qemu-iotests/084.out b/tests/qemu-iotests/084.out
index c7120d9b0..5ece8299c 100644
--- a/tests/qemu-iotests/084.out
+++ b/tests/qemu-iotests/084.out
@@ -1,19 +1,36 @@
 QA output created by 084
 
+=== Statically allocated image creation ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
+virtual size: 64M (67108864 bytes)
+cluster_size: 1048576
+disk image file size in bytes: 67109888
+
 === Testing image size bounds ===
 
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-Test 1: Maximum size (1024 TB):
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Could not open 'TEST_DIR/t.IMGFMT': Invalid argument
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
+virtual size: 64M (67108864 bytes)
+cluster_size: 1048576
+disk image file size in bytes: 1024
+Test 1: Maximum size (512 TB - 128 MB):
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
+virtual size: 512T (562949819203584 bytes)
+cluster_size: 1048576
 
-Test 2: Size too large (1024TB + 1)
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported VDI image size (size is 0x3fffffff10000, max supported is 0x3fffffff00000)
+Test 2: Size too large (512 TB - 128 MB + 64 kB)
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported VDI image size (size is 0x1fffff8010000, max supported is 0x1fffff8000000)
 
 Test 3: Size valid (64M), but Blocks In Image too small (63)
 qemu-img: Could not open 'TEST_DIR/t.IMGFMT': unsupported VDI image (disk size 67108864, image bitmap has room for 66060288)
 
 Test 4: Size valid (64M), but Blocks In Image exceeds max allowed
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': unsupported VDI image (too many blocks 1073741824, max is 1073741823)
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': unsupported VDI image (too many blocks 536870785, max is 536870784)
 
 Test 5: Valid Image: 64MB, Blocks In Image 64, Block Size 1MB
 image: TEST_DIR/t.IMGFMT
diff --git a/tests/qemu-iotests/086 b/tests/qemu-iotests/086
index d9a80cf86..234eb9a91 100755
--- a/tests/qemu-iotests/086
+++ b/tests/qemu-iotests/086
@@ -39,7 +39,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.filter
 
 _supported_fmt qcow2
-_supported_proto file
+_supported_proto file nfs
 _supported_os Linux
 
 function run_qemu_img()
diff --git a/tests/qemu-iotests/087 b/tests/qemu-iotests/087
index 82c56b139..d7454d13d 100755
--- a/tests/qemu-iotests/087
+++ b/tests/qemu-iotests/087
@@ -218,6 +218,23 @@ run_qemu <<EOF
 { "execute": "quit" }
 EOF
 
+echo
+echo === Missing driver ===
+echo
+
+_make_test_img -o encryption=on $size
+run_qemu -S <<EOF
+{ "execute": "qmp_capabilities" }
+{ "execute": "blockdev-add",
+  "arguments": {
+      "options": {
+        "id": "disk"
+      }
+    }
+  }
+{ "execute": "quit" }
+EOF
+
 # success, all done
 echo "*** done"
 rm -f $seq.full
diff --git a/tests/qemu-iotests/087.out b/tests/qemu-iotests/087.out
index 7fbee3ff5..e8795b3a1 100644
--- a/tests/qemu-iotests/087.out
+++ b/tests/qemu-iotests/087.out
@@ -20,7 +20,7 @@ QMP_VERSION
 {"return": {}}
 {"return": {}}
 {"error": {"class": "GenericError", "desc": "Device with id 'disk' already exists"}}
-{"error": {"class": "GenericError", "desc": "Device with node-name 'test-node' already exists"}}
+{"error": {"class": "GenericError", "desc": "Device name 'test-node' conflicts with an existing node name"}}
 main-loop: WARNING: I/O thread spun for 1000 iterations
 {"error": {"class": "GenericError", "desc": "could not open disk image disk2: node-name=disk is conflicting with a device id"}}
 {"error": {"class": "GenericError", "desc": "could not open disk image disk2: Duplicate node name"}}
@@ -64,4 +64,17 @@ QMP_VERSION
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
+
+=== Missing driver ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 encryption=on 
+Testing: -S
+QMP_VERSION
+{"return": {}}
+{"error": {"class": "GenericError", "desc": "Invalid parameter type for 'driver', expected: string"}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
+
 *** done
diff --git a/tests/qemu-iotests/088 b/tests/qemu-iotests/088
index c09adf802..f9c312918 100755
--- a/tests/qemu-iotests/088
+++ b/tests/qemu-iotests/088
@@ -40,7 +40,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.filter
 
 _supported_fmt vpc
-_supported_proto generic
+_supported_proto file
 _supported_os Linux
 
 offset_block_size=$((512 + 32))
diff --git a/tests/qemu-iotests/089.out b/tests/qemu-iotests/089.out
index 4ca2f88e6..b2b039081 100644
--- a/tests/qemu-iotests/089.out
+++ b/tests/qemu-iotests/089.out
@@ -41,10 +41,12 @@ vm state offset: 512 MiB
 Format specific information:
     compat: 1.1
     lazy refcounts: false
+    corrupt: false
 format name: IMGFMT
 cluster size: 64 KiB
 vm state offset: 512 MiB
 Format specific information:
     compat: 1.1
     lazy refcounts: false
+    corrupt: false
 *** done
diff --git a/tests/qemu-iotests/090 b/tests/qemu-iotests/090
index 8d032f811..70b5a6fd7 100755
--- a/tests/qemu-iotests/090
+++ b/tests/qemu-iotests/090
@@ -39,7 +39,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.filter
 
 _supported_fmt qcow2
-_supported_proto file
+_supported_proto file nfs
 _supported_os Linux
 
 IMG_SIZE=128K
diff --git a/tests/qemu-iotests/092 b/tests/qemu-iotests/092
index a8c0c9ca2..52c529bae 100755
--- a/tests/qemu-iotests/092
+++ b/tests/qemu-iotests/092
@@ -40,7 +40,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.filter
 
 _supported_fmt qcow
-_supported_proto generic
+_supported_proto file
 _supported_os Linux
 
 offset_backing_file_offset=8
diff --git a/tests/qemu-iotests/095 b/tests/qemu-iotests/095
index acc7dbf18..6630181a7 100755
--- a/tests/qemu-iotests/095
+++ b/tests/qemu-iotests/095
@@ -60,7 +60,7 @@ _make_test_img -b "${TEST_IMG}.snp1" $size_larger
 
 echo
 echo "=== Base image info before commit and resize ==="
-$QEMU_IMG info "${TEST_IMG}.base" | _filter_testdir
+TEST_IMG="${TEST_IMG}.base" _img_info
 
 echo
 echo === Running QEMU Live Commit Test ===
@@ -78,7 +78,7 @@ _send_qemu_cmd $h "{ 'execute': 'block-commit',
 
 echo
 echo "=== Base image info after commit and resize ==="
-$QEMU_IMG info "${TEST_IMG}.base" | _filter_testdir
+TEST_IMG="${TEST_IMG}.base" _img_info
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/095.out b/tests/qemu-iotests/095.out
index 5864ddac2..cc86efacc 100644
--- a/tests/qemu-iotests/095.out
+++ b/tests/qemu-iotests/095.out
@@ -4,14 +4,10 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=104857600 backing_file='TEST_DIR
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=104857600 backing_file='TEST_DIR/t.IMGFMT.snp1' 
 
 === Base image info before commit and resize ===
-image: TEST_DIR/t.qcow2.base
-file format: qcow2
+image: TEST_DIR/t.IMGFMT.base
+file format: IMGFMT
 virtual size: 5.0M (5242880 bytes)
-disk size: 196K
 cluster_size: 65536
-Format specific information:
-    compat: 1.1
-    lazy refcounts: false
 
 === Running QEMU Live Commit Test ===
 
@@ -20,12 +16,8 @@ Format specific information:
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "test", "len": 104857600, "offset": 104857600, "speed": 0, "type": "commit"}}
 
 === Base image info after commit and resize ===
-image: TEST_DIR/t.qcow2.base
-file format: qcow2
+image: TEST_DIR/t.IMGFMT.base
+file format: IMGFMT
 virtual size: 100M (104857600 bytes)
-disk size: 196K
 cluster_size: 65536
-Format specific information:
-    compat: 1.1
-    lazy refcounts: false
 *** done
diff --git a/tests/qemu-iotests/097 b/tests/qemu-iotests/097
new file mode 100755
index 000000000..c7a613b7e
--- /dev/null
+++ b/tests/qemu-iotests/097
@@ -0,0 +1,122 @@
+#!/bin/bash
+#
+# Commit changes into backing chains and empty the top image if the
+# backing image is not explicitly specified
+#
+# Copyright (C) 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+    _rm_test_img "$TEST_IMG.itmd"
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.pattern
+
+# Any format supporting backing files and bdrv_make_empty
+_supported_fmt qcow qcow2
+_supported_proto file
+_supported_os Linux
+
+
+# Four passes:
+#  0: Two-layer backing chain, commit to upper backing file (implicitly)
+#     (in this case, the top image will be emptied)
+#  1: Two-layer backing chain, commit to upper backing file (explicitly)
+#     (in this case, the top image will implicitly stay unchanged)
+#  2: Two-layer backing chain, commit to upper backing file (implicitly with -d)
+#     (in this case, the top image will explicitly stay unchanged)
+#  3: Two-layer backing chain, commit to lower backing file
+#     (in this case, the top image will implicitly stay unchanged)
+#
+# 020 already tests committing, so this only tests whether image chains are
+# working properly and that all images above the base are emptied; therefore,
+# no complicated patterns are necessary
+for i in 0 1 2 3; do
+
+echo
+echo "=== Test pass $i ==="
+echo
+
+TEST_IMG="$TEST_IMG.base" _make_test_img 64M
+TEST_IMG="$TEST_IMG.itmd" _make_test_img -b "$TEST_IMG.base" 64M
+_make_test_img -b "$TEST_IMG.itmd" 64M
+
+$QEMU_IO -c 'write -P 1 0 192k' "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c 'write -P 2 64k 128k' "$TEST_IMG.itmd" | _filter_qemu_io
+$QEMU_IO -c 'write -P 3 128k 64k' "$TEST_IMG" | _filter_qemu_io
+
+if [ $i -lt 3 ]; then
+    if [ $i == 0 ]; then
+        # -b "$TEST_IMG.itmd" should be the default (that is, committing to the
+        # first backing file in the chain)
+        $QEMU_IMG commit "$TEST_IMG"
+    elif [ $i == 1 ]; then
+        # explicitly specify the commit target (this should imply -d)
+        $QEMU_IMG commit -b "$TEST_IMG.itmd" "$TEST_IMG"
+    else
+        # do not explicitly specify the commit target, but use -d to leave the
+        # top image unchanged
+        $QEMU_IMG commit -d "$TEST_IMG"
+    fi
+
+    # Bottom should be unchanged
+    $QEMU_IO -c 'read -P 1 0 192k' "$TEST_IMG.base" | _filter_qemu_io
+
+    # Intermediate should contain changes from top
+    $QEMU_IO -c 'read -P 1 0 64k' "$TEST_IMG.itmd" | _filter_qemu_io
+    $QEMU_IO -c 'read -P 2 64k 64k' "$TEST_IMG.itmd" | _filter_qemu_io
+    $QEMU_IO -c 'read -P 3 128k 64k' "$TEST_IMG.itmd" | _filter_qemu_io
+
+    # And in pass 0, the top image should be empty, whereas in both other passes
+    # it should be unchanged (which is both checked by qemu-img map)
+else
+    $QEMU_IMG commit -b "$TEST_IMG.base" "$TEST_IMG"
+
+    # Bottom should contain all changes
+    $QEMU_IO -c 'read -P 1 0 64k' "$TEST_IMG.base" | _filter_qemu_io
+    $QEMU_IO -c 'read -P 2 64k 64k' "$TEST_IMG.base" | _filter_qemu_io
+    $QEMU_IO -c 'read -P 3 128k 64k' "$TEST_IMG.base" | _filter_qemu_io
+
+    # Both top and intermediate should be unchanged
+fi
+
+$QEMU_IMG map "$TEST_IMG.base" | _filter_qemu_img_map
+$QEMU_IMG map "$TEST_IMG.itmd" | _filter_qemu_img_map
+$QEMU_IMG map "$TEST_IMG" | _filter_qemu_img_map
+
+done
+
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/097.out b/tests/qemu-iotests/097.out
new file mode 100644
index 000000000..1cb7764fe
--- /dev/null
+++ b/tests/qemu-iotests/097.out
@@ -0,0 +1,119 @@
+QA output created by 097
+
+=== Test pass 0 ===
+
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.itmd' 
+wrote 196608/196608 bytes at offset 0
+192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 131072/131072 bytes at offset 65536
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 131072
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Image committed.
+read 196608/196608 bytes at offset 0
+192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 131072
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Offset          Length          File
+0               0x30000         TEST_DIR/t.IMGFMT.base
+Offset          Length          File
+0               0x10000         TEST_DIR/t.IMGFMT.base
+0x10000         0x20000         TEST_DIR/t.IMGFMT.itmd
+Offset          Length          File
+0               0x10000         TEST_DIR/t.IMGFMT.base
+0x10000         0x20000         TEST_DIR/t.IMGFMT.itmd
+
+=== Test pass 1 ===
+
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.itmd' 
+wrote 196608/196608 bytes at offset 0
+192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 131072/131072 bytes at offset 65536
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 131072
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Image committed.
+read 196608/196608 bytes at offset 0
+192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 131072
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Offset          Length          File
+0               0x30000         TEST_DIR/t.IMGFMT.base
+Offset          Length          File
+0               0x10000         TEST_DIR/t.IMGFMT.base
+0x10000         0x20000         TEST_DIR/t.IMGFMT.itmd
+Offset          Length          File
+0               0x10000         TEST_DIR/t.IMGFMT.base
+0x10000         0x10000         TEST_DIR/t.IMGFMT.itmd
+0x20000         0x10000         TEST_DIR/t.IMGFMT
+
+=== Test pass 2 ===
+
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.itmd' 
+wrote 196608/196608 bytes at offset 0
+192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 131072/131072 bytes at offset 65536
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 131072
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Image committed.
+read 196608/196608 bytes at offset 0
+192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 131072
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Offset          Length          File
+0               0x30000         TEST_DIR/t.IMGFMT.base
+Offset          Length          File
+0               0x10000         TEST_DIR/t.IMGFMT.base
+0x10000         0x20000         TEST_DIR/t.IMGFMT.itmd
+Offset          Length          File
+0               0x10000         TEST_DIR/t.IMGFMT.base
+0x10000         0x10000         TEST_DIR/t.IMGFMT.itmd
+0x20000         0x10000         TEST_DIR/t.IMGFMT
+
+=== Test pass 3 ===
+
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.itmd' 
+wrote 196608/196608 bytes at offset 0
+192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 131072/131072 bytes at offset 65536
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 131072
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Image committed.
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 131072
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Offset          Length          File
+0               0x30000         TEST_DIR/t.IMGFMT.base
+Offset          Length          File
+0               0x10000         TEST_DIR/t.IMGFMT.base
+0x10000         0x20000         TEST_DIR/t.IMGFMT.itmd
+Offset          Length          File
+0               0x10000         TEST_DIR/t.IMGFMT.base
+0x10000         0x10000         TEST_DIR/t.IMGFMT.itmd
+0x20000         0x10000         TEST_DIR/t.IMGFMT
+*** done
diff --git a/tests/qemu-iotests/098 b/tests/qemu-iotests/098
new file mode 100755
index 000000000..e2230ad60
--- /dev/null
+++ b/tests/qemu-iotests/098
@@ -0,0 +1,82 @@
+#!/bin/bash
+#
+# Test qcow2's bdrv_make_empty for images without internal snapshots
+#
+# Copyright (C) 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+    rm -f "$TEST_DIR/blkdebug.conf"
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.pattern
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+IMGOPTS="compat=1.1"
+
+for event in l1_update empty_image_prepare reftable_update refblock_alloc; do
+
+echo
+echo "=== $event ==="
+echo
+
+TEST_IMG="$TEST_IMG.base" _make_test_img 64M
+_make_test_img -b "$TEST_IMG.base" 64M
+
+# Some data that can be leaked when emptying the top image
+$QEMU_IO -c 'write 0 64k' "$TEST_IMG" | _filter_qemu_io
+
+cat > "$TEST_DIR/blkdebug.conf" <<EOF
+[inject-error]
+event = "$event"
+EOF
+
+$QEMU_IMG commit "blkdebug:$TEST_DIR/blkdebug.conf:$TEST_IMG" 2>&1 \
+    | _filter_testdir | _filter_imgfmt
+
+# There may be errors, but they should be fixed by opening the image
+$QEMU_IO -c close "$TEST_IMG"
+
+_check_test_img
+
+done
+
+
+rm -f "$TEST_DIR/blkdebug.conf"
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/098.out b/tests/qemu-iotests/098.out
new file mode 100644
index 000000000..586dfc809
--- /dev/null
+++ b/tests/qemu-iotests/098.out
@@ -0,0 +1,52 @@
+QA output created by 098
+
+=== l1_update ===
+
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-img: Could not empty blkdebug:TEST_DIR/blkdebug.conf:TEST_DIR/t.IMGFMT: Input/output error
+No errors were found on the image.
+
+=== empty_image_prepare ===
+
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-img: Could not empty blkdebug:TEST_DIR/blkdebug.conf:TEST_DIR/t.IMGFMT: Input/output error
+Leaked cluster 4 refcount=1 reference=0
+Leaked cluster 5 refcount=1 reference=0
+Repairing cluster 4 refcount=1 reference=0
+Repairing cluster 5 refcount=1 reference=0
+No errors were found on the image.
+
+=== reftable_update ===
+
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-img: Could not empty blkdebug:TEST_DIR/blkdebug.conf:TEST_DIR/t.IMGFMT: Input/output error
+ERROR cluster 0 refcount=0 reference=1
+ERROR cluster 1 refcount=0 reference=1
+ERROR cluster 3 refcount=0 reference=1
+Rebuilding refcount structure
+Repairing cluster 1 refcount=1 reference=0
+No errors were found on the image.
+
+=== refblock_alloc ===
+
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-img: Could not empty blkdebug:TEST_DIR/blkdebug.conf:TEST_DIR/t.IMGFMT: Input/output error
+ERROR cluster 0 refcount=0 reference=1
+ERROR cluster 1 refcount=0 reference=1
+ERROR cluster 3 refcount=0 reference=1
+Rebuilding refcount structure
+Repairing cluster 1 refcount=1 reference=0
+No errors were found on the image.
+*** done
diff --git a/tests/qemu-iotests/099 b/tests/qemu-iotests/099
new file mode 100755
index 000000000..ffc7ea795
--- /dev/null
+++ b/tests/qemu-iotests/099
@@ -0,0 +1,116 @@
+#!/bin/bash
+#
+# Test valid filenames for blkdebug and blkverify representatively for
+# other protocols (such as NBD) when queried
+#
+# Copyright (C) 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+# Basically all formats, but "raw" has issues with _filter_imgfmt regarding the
+# raw comparison image for blkverify; also, all images have to support creation
+_supported_fmt qcow qcow2 qed vdi vhdx vmdk vpc
+_supported_proto file
+_supported_os Linux
+
+
+function do_run_qemu()
+{
+    $QEMU -nographic -qmp stdio -serial none "$@"
+}
+
+function run_qemu()
+{
+    # Get the "file": "foo" entry ($foo may only contain escaped double quotes,
+    # which is how we can extract it)
+    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_imgfmt | _filter_qmp \
+        | grep "drv0" \
+        | sed -e 's/^.*"file": "\(\(\\"\|[^"]\)*\)".*$/\1/' -e 's/\\"/"/g'
+}
+
+function test_qemu()
+{
+    run_qemu -drive "if=none,id=drv0,$1" <<EOF
+        { 'execute': 'qmp_capabilities' }
+        { 'execute': 'query-block' }
+        { 'execute': 'quit' }
+EOF
+}
+
+
+
+IMG_SIZE=128K
+
+_make_test_img $IMG_SIZE
+$QEMU_IMG create -f raw "$TEST_IMG.compare" $IMG_SIZE \
+    | _filter_testdir | _filter_imgfmt
+
+echo
+echo '=== Testing simple filename for blkverify ==='
+echo
+
+# This should return simply the filename itself
+test_qemu "file=blkverify:$TEST_IMG.compare:$TEST_IMG"
+
+echo
+echo '=== Testing filename reconstruction for blkverify ==='
+echo
+
+# This should return the same filename as above
+test_qemu "file.driver=blkverify,file.raw.filename=$TEST_IMG.compare,file.test.file.filename=$TEST_IMG"
+
+echo
+echo '=== Testing JSON filename for blkdebug ==='
+echo
+
+# blkdebug cannot create a configuration file, therefore it is unable to
+# generate a plain filename here; thus this should return a JSON filename
+test_qemu "file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=l1_update"
+
+echo
+echo '=== Testing indirectly enforced JSON filename ==='
+echo
+
+# Because blkdebug cannot return a plain filename, blkverify is forced to
+# generate a JSON object here as well
+test_qemu "file.driver=blkverify,file.raw.filename=$TEST_IMG.compare,file.test.file.driver=blkdebug,file.test.file.image.filename=$TEST_IMG,file.test.file.inject-error.0.event=l1_update"
+
+
+rm -f "$TEST_IMG.compare"
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/099.out b/tests/qemu-iotests/099.out
new file mode 100644
index 000000000..55be4d4c5
--- /dev/null
+++ b/tests/qemu-iotests/099.out
@@ -0,0 +1,20 @@
+QA output created by 099
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=131072 
+Formatting 'TEST_DIR/t.IMGFMT.compare', fmt=raw size=131072 
+
+=== Testing simple filename for blkverify ===
+
+blkverify:TEST_DIR/t.IMGFMT.compare:TEST_DIR/t.IMGFMT
+
+=== Testing filename reconstruction for blkverify ===
+
+blkverify:TEST_DIR/t.IMGFMT.compare:TEST_DIR/t.IMGFMT
+
+=== Testing JSON filename for blkdebug ===
+
+json:{"driver": "IMGFMT", "file": {"inject-error": [{"immediately": false, "once": false, "state": 0, "sector": -1, "event": "l1_update", "errno": 5}], "image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug"}}
+
+=== Testing indirectly enforced JSON filename ===
+
+json:{"driver": "raw", "file": {"test": {"driver": "IMGFMT", "file": {"inject-error": [{"immediately": false, "once": false, "state": 0, "sector": -1, "event": "l1_update", "errno": 5}], "image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug"}}, "driver": "blkverify", "raw": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT.compare"}}}
+*** done
diff --git a/tests/qemu-iotests/100 b/tests/qemu-iotests/100
new file mode 100755
index 000000000..9124aba76
--- /dev/null
+++ b/tests/qemu-iotests/100
@@ -0,0 +1,134 @@
+#!/bin/bash
+#
+# Test simple read/write using plain bdrv_read/bdrv_write
+#
+# Copyright (C) 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=stefanha@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt generic
+_supported_proto generic
+_supported_os Linux
+
+
+size=128M
+
+echo
+echo "== Single request =="
+_make_test_img $size
+$QEMU_IO -c "multiwrite 0 4k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "== verify pattern =="
+$QEMU_IO -c "read -P 0xcd 0 4k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 4k 4k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "== Sequential requests =="
+_make_test_img $size
+$QEMU_IO -c "multiwrite 0 4k ; 4k 4k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "== verify pattern =="
+$QEMU_IO -c "read -P 0xcd 0 4k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0xce 4k 4k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 8k 4k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "== Superset overlapping requests =="
+_make_test_img $size
+$QEMU_IO -c "multiwrite 0 4k ; 1k 2k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "== verify pattern =="
+# Order of overlapping in-flight requests is not guaranteed so we cannot verify
+# [1k, 3k) since it could have either pattern 0xcd or 0xce.
+$QEMU_IO -c "read -P 0xcd 0 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0xcd 3k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 4k 4k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "== Subset overlapping requests =="
+_make_test_img $size
+$QEMU_IO -c "multiwrite 1k 2k ; 0k 4k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "== verify pattern =="
+# Order of overlapping in-flight requests is not guaranteed so we cannot verify
+# [1k, 3k) since it could have either pattern 0xcd or 0xce.
+$QEMU_IO -c "read -P 0xce 0 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0xce 3k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 4k 4k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "== Head overlapping requests =="
+_make_test_img $size
+$QEMU_IO -c "multiwrite 0k 2k ; 0k 4k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "== verify pattern =="
+# Order of overlapping in-flight requests is not guaranteed so we cannot verify
+# [0k, 2k) since it could have either pattern 0xcd or 0xce.
+$QEMU_IO -c "read -P 0xce 2k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 4k 4k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "== Tail overlapping requests =="
+_make_test_img $size
+$QEMU_IO -c "multiwrite 2k 2k ; 0k 4k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "== verify pattern =="
+# Order of overlapping in-flight requests is not guaranteed so we cannot verify
+# [2k, 4k) since it could have either pattern 0xcd or 0xce.
+$QEMU_IO -c "read -P 0xce 0k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 4k 4k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "== Disjoint requests =="
+_make_test_img $size
+$QEMU_IO -c "multiwrite 0 4k ; 64k 4k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "== verify pattern =="
+$QEMU_IO -c "read -P 0xcd 0 4k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 4k 60k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0xce 64k 4k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 68k 4k" "$TEST_IMG" | _filter_qemu_io
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/100.out b/tests/qemu-iotests/100.out
new file mode 100644
index 000000000..2d6e9f0a7
--- /dev/null
+++ b/tests/qemu-iotests/100.out
@@ -0,0 +1,89 @@
+QA output created by 100
+
+== Single request ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verify pattern ==
+read 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 4096/4096 bytes at offset 4096
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== Sequential requests ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+wrote 8192/8192 bytes at offset 0
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verify pattern ==
+read 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 4096/4096 bytes at offset 4096
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 4096/4096 bytes at offset 8192
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== Superset overlapping requests ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+wrote 6144/6144 bytes at offset 0
+6 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verify pattern ==
+read 1024/1024 bytes at offset 0
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 3072
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 4096/4096 bytes at offset 4096
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== Subset overlapping requests ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+wrote 6144/6144 bytes at offset 1024
+6 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verify pattern ==
+read 1024/1024 bytes at offset 0
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 3072
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 4096/4096 bytes at offset 4096
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== Head overlapping requests ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+wrote 6144/6144 bytes at offset 0
+6 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verify pattern ==
+read 2048/2048 bytes at offset 2048
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 4096/4096 bytes at offset 4096
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== Tail overlapping requests ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+wrote 6144/6144 bytes at offset 2048
+6 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verify pattern ==
+read 2048/2048 bytes at offset 0
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 4096/4096 bytes at offset 4096
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== Disjoint requests ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+wrote 8192/8192 bytes at offset 0
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verify pattern ==
+read 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 61440/61440 bytes at offset 4096
+60 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 4096/4096 bytes at offset 65536
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 4096/4096 bytes at offset 69632
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+*** done
diff --git a/tests/qemu-iotests/101 b/tests/qemu-iotests/101
new file mode 100755
index 000000000..70fbf25f6
--- /dev/null
+++ b/tests/qemu-iotests/101
@@ -0,0 +1,58 @@
+#!/bin/bash
+#
+# Test short file I/O
+#
+# Copyright (C) 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=stefanha@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt raw
+_supported_proto file
+_supported_os Linux
+
+
+echo
+echo "== creating short image file =="
+dd if=/dev/zero of="$TEST_IMG" bs=1 count=320
+
+echo
+echo "== reading bytes beyond EOF gives zeroes =="
+$QEMU_IO -c "read -P 0 0 512" "$TEST_IMG" | _filter_qemu_io
+
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/101.out b/tests/qemu-iotests/101.out
new file mode 100644
index 000000000..9a996e8fc
--- /dev/null
+++ b/tests/qemu-iotests/101.out
@@ -0,0 +1,10 @@
+QA output created by 101
+
+== creating short image file ==
+320+0 records in
+320+0 records out
+
+== reading bytes beyond EOF gives zeroes ==
+read 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+*** done
diff --git a/tests/qemu-iotests/102 b/tests/qemu-iotests/102
new file mode 100755
index 000000000..161b1974c
--- /dev/null
+++ b/tests/qemu-iotests/102
@@ -0,0 +1,81 @@
+#!/bin/bash
+#
+# Test case for qemu-io -c map and qemu-img map
+#
+# Copyright (C) 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq=$(basename $0)
+echo "QA output created by $seq"
+
+here=$PWD
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and qemu instance handling
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+IMG_SIZE=64K
+
+echo
+echo '=== Testing map command on truncated image ==='
+echo
+
+_make_test_img $IMG_SIZE
+# Create cluster
+$QEMU_IO -c 'write 0 64k' "$TEST_IMG" | _filter_qemu_io
+# Remove data cluster from image (first cluster: image header, second: reftable,
+# third: refblock, fourth: L1 table, fifth: L2 table)
+$QEMU_IMG resize -f raw "$TEST_IMG" $((5 * 64 * 1024))
+
+$QEMU_IO -c map "$TEST_IMG"
+$QEMU_IMG map "$TEST_IMG"
+
+echo
+echo '=== Testing map on an image file truncated outside of qemu ==='
+echo
+
+# Same as above, only now we concurrently truncate and map the image
+_make_test_img $IMG_SIZE
+$QEMU_IO -c 'write 0 64k' "$TEST_IMG" | _filter_qemu_io
+
+qemu_comm_method=monitor _launch_qemu -drive if=none,file="$TEST_IMG",id=drv0
+
+$QEMU_IMG resize -f raw "$TEST_IMG" $((5 * 64 * 1024))
+
+_send_qemu_cmd $QEMU_HANDLE 'qemu-io drv0 map' 'allocated' \
+    | sed -e 's/^(qemu).*qemu-io drv0 map...$/(qemu) qemu-io drv0 map/'
+_send_qemu_cmd $QEMU_HANDLE 'quit' ''
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/102.out b/tests/qemu-iotests/102.out
new file mode 100644
index 000000000..eecde16ad
--- /dev/null
+++ b/tests/qemu-iotests/102.out
@@ -0,0 +1,21 @@
+QA output created by 102
+
+=== Testing map command on truncated image ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65536
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Image resized.
+[                       0]      128/     128 sectors     allocated at offset 0 bytes (1)
+Offset          Length          Mapped to       File
+
+=== Testing map on an image file truncated outside of qemu ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65536
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Image resized.
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qemu-io drv0 map
+[                       0]      128/     128 sectors     allocated at offset 0 bytes (1)
+*** done
diff --git a/tests/qemu-iotests/103 b/tests/qemu-iotests/103
new file mode 100755
index 000000000..ccab551f6
--- /dev/null
+++ b/tests/qemu-iotests/103
@@ -0,0 +1,99 @@
+#!/bin/bash
+#
+# Test case for qcow2 metadata cache size specification
+#
+# Copyright (C) 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq=$(basename $0)
+echo "QA output created by $seq"
+
+here=$PWD
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto file nfs
+_supported_os Linux
+
+IMG_SIZE=64K
+
+_make_test_img $IMG_SIZE
+$QEMU_IO -c 'write -P 42 0 64k' "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo '=== Testing invalid option combinations ==='
+echo
+
+# all sizes set at the same time
+$QEMU_IO -c "open -o cache-size=1.25M,l2-cache-size=1M,refcount-cache-size=0.25M $TEST_IMG" \
+    2>&1 | _filter_testdir | _filter_imgfmt
+# l2-cache-size may not exceed cache-size
+$QEMU_IO -c "open -o cache-size=1M,l2-cache-size=2M $TEST_IMG" 2>&1 \
+    | _filter_testdir | _filter_imgfmt
+# refcount-cache-size may not exceed cache-size
+$QEMU_IO -c "open -o cache-size=1M,refcount-cache-size=2M $TEST_IMG" 2>&1 \
+    | _filter_testdir | _filter_imgfmt
+# 0 should be a valid size (e.g. for enforcing the minimum), so this should not
+# work
+$QEMU_IO -c "open -o cache-size=0,l2-cache-size=0,refcount-cache-size=0 $TEST_IMG" \
+    2>&1 | _filter_testdir | _filter_imgfmt
+
+echo
+echo '=== Testing valid option combinations ==='
+echo
+
+# There should be a reasonable and working minimum
+$QEMU_IO -c "open -o cache-size=0 $TEST_IMG" -c 'read -P 42 0 64k' \
+    | _filter_qemu_io
+$QEMU_IO -c "open -o l2-cache-size=0 $TEST_IMG" -c 'read -P 42 0 64k' \
+    | _filter_qemu_io
+$QEMU_IO -c "open -o refcount-cache-size=0 $TEST_IMG" -c 'read -P 42 0 64k' \
+    | _filter_qemu_io
+
+# Derive cache sizes from combined size (with a reasonable ratio, but we cannot
+# test that)
+$QEMU_IO -c "open -o cache-size=2M $TEST_IMG" -c 'read -P 42 0 64k' \
+    | _filter_qemu_io
+# Fix one cache, derive the other
+$QEMU_IO -c "open -o cache-size=2M,l2-cache-size=1M $TEST_IMG" \
+         -c 'read -P 42 0 64k' \
+    | _filter_qemu_io
+$QEMU_IO -c "open -o cache-size=2M,refcount-cache-size=1M $TEST_IMG" \
+         -c 'read -P 42 0 64k' \
+    | _filter_qemu_io
+# Directly set both caches
+$QEMU_IO -c "open -o l2-cache-size=1M,refcount-cache-size=0.25M $TEST_IMG" \
+         -c 'read -P 42 0 64k' \
+    | _filter_qemu_io
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/103.out b/tests/qemu-iotests/103.out
new file mode 100644
index 000000000..ddf6b5a07
--- /dev/null
+++ b/tests/qemu-iotests/103.out
@@ -0,0 +1,29 @@
+QA output created by 103
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65536 
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Testing invalid option combinations ===
+
+qemu-io: can't open device TEST_DIR/t.IMGFMT: cache-size, l2-cache-size and refcount-cache-size may not be set the same time
+qemu-io: can't open device TEST_DIR/t.IMGFMT: l2-cache-size may not exceed cache-size
+qemu-io: can't open device TEST_DIR/t.IMGFMT: refcount-cache-size may not exceed cache-size
+qemu-io: can't open device TEST_DIR/t.IMGFMT: cache-size, l2-cache-size and refcount-cache-size may not be set the same time
+
+=== Testing valid option combinations ===
+
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+*** done
diff --git a/tests/qemu-iotests/104 b/tests/qemu-iotests/104
new file mode 100755
index 000000000..b471aa5bb
--- /dev/null
+++ b/tests/qemu-iotests/104
@@ -0,0 +1,57 @@
+#!/bin/bash
+#
+# Test image creation with aligned and unaligned sizes
+#
+# Copyright (C) 2014 Fujitsu.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=hutao@cn.fujitsu.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt generic
+_supported_proto generic
+_supported_os Linux
+
+echo "=== Check qemu-img info output ==="
+echo
+image_sizes="1024 1234"
+
+for s in $image_sizes; do
+    _make_test_img $s | _filter_img_create
+    _img_info | _filter_img_info
+done
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/104.out b/tests/qemu-iotests/104.out
new file mode 100644
index 000000000..de27852e3
--- /dev/null
+++ b/tests/qemu-iotests/104.out
@@ -0,0 +1,12 @@
+QA output created by 104
+=== Check qemu-img info output ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1024 
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
+virtual size: 1.0K (1024 bytes)
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1234 
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
+virtual size: 1.5K (1536 bytes)
+***done
diff --git a/tests/qemu-iotests/105 b/tests/qemu-iotests/105
new file mode 100755
index 000000000..9bae49e32
--- /dev/null
+++ b/tests/qemu-iotests/105
@@ -0,0 +1,70 @@
+#!/bin/bash
+#
+# Create, read, write big image
+#
+# Copyright (C) 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=famz@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2 vmdk vhdx qed
+_supported_proto generic
+_supported_os Linux
+_unsupported_imgopts "subformat=twoGbMaxExtentFlat" \
+                     "subformat=twoGbMaxExtentSparse"
+
+echo
+echo "creating large image"
+_make_test_img 16T
+
+echo
+echo "small read"
+$QEMU_IO -c "read 1024 4096" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "small write"
+$QEMU_IO -c "write 8192 4096" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "small read at high offset"
+$QEMU_IO -c "read 14T 4096" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "small write at high offset"
+$QEMU_IO -c "write 14T 4096" "$TEST_IMG" | _filter_qemu_io
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/105.out b/tests/qemu-iotests/105.out
new file mode 100644
index 000000000..13ffcb593
--- /dev/null
+++ b/tests/qemu-iotests/105.out
@@ -0,0 +1,21 @@
+QA output created by 105
+
+creating large image
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=17592186044416
+
+small read
+read 4096/4096 bytes at offset 1024
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+small write
+wrote 4096/4096 bytes at offset 8192
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+small read at high offset
+read 4096/4096 bytes at offset 15393162788864
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+small write at high offset
+wrote 4096/4096 bytes at offset 15393162788864
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+*** done
diff --git a/tests/qemu-iotests/107 b/tests/qemu-iotests/107
new file mode 100755
index 000000000..986203046
--- /dev/null
+++ b/tests/qemu-iotests/107
@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# Tests updates of the qcow2 L1 table
+#
+# Copyright (C) 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto file nfs
+_supported_os Linux
+
+
+IMG_SIZE=64K
+
+echo
+echo '=== Updates should not write random data ==='
+echo
+
+_make_test_img $IMG_SIZE
+$QEMU_IO -c 'write 0 64k' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "open -o driver=raw $TEST_IMG" -c 'read -p -P 0 196616 65528' \
+    | _filter_qemu_io
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
+
diff --git a/tests/qemu-iotests/107.out b/tests/qemu-iotests/107.out
new file mode 100644
index 000000000..93445b731
--- /dev/null
+++ b/tests/qemu-iotests/107.out
@@ -0,0 +1,10 @@
+QA output created by 107
+
+=== Updates should not write random data ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65536 
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65528/65528 bytes at offset 196616
+63.992 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+*** done
diff --git a/tests/qemu-iotests/108 b/tests/qemu-iotests/108
new file mode 100755
index 000000000..12fc92a63
--- /dev/null
+++ b/tests/qemu-iotests/108
@@ -0,0 +1,141 @@
+#!/bin/bash
+#
+# Test case for repairing qcow2 images which cannot be repaired using
+# the on-disk refcount structures
+#
+# Copyright (C) 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+# This tests qocw2-specific low-level functionality
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+echo
+echo '=== Repairing an image without any refcount table ==='
+echo
+
+_make_test_img 64M
+# just write some data
+$QEMU_IO -c 'write -P 42 0 64k' "$TEST_IMG" | _filter_qemu_io
+
+# refcount_table_offset
+poke_file "$TEST_IMG" $((0x30)) "\x00\x00\x00\x00\x00\x00\x00\x00"
+# refcount_table_clusters
+poke_file "$TEST_IMG" $((0x38)) "\x00\x00\x00\x00"
+
+_check_test_img -r all
+
+$QEMU_IO -c 'read -P 42 0 64k' "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo '=== Repairing unreferenced data cluster in new refblock area ==='
+echo
+
+IMGOPTS='cluster_size=512' _make_test_img 64M
+# Allocate the first 128 kB in the image (first refblock)
+$QEMU_IO -c 'write 0 0x1b200' "$TEST_IMG" | _filter_qemu_io
+# should be 131072 == 0x20000
+stat -c '%s' "$TEST_IMG"
+
+# Enter a cluster at 128 kB (0x20000)
+# XXX: This should be the first free entry in the last L2 table, but we cannot
+# be certain
+poke_file "$TEST_IMG" $((0x1ccc8)) "\x80\x00\x00\x00\x00\x02\x00\x00"
+
+# Fill the cluster
+truncate -s $((0x20200)) "$TEST_IMG"
+$QEMU_IO -c "open -o driver=raw $TEST_IMG" -c 'write -P 42 128k 512' \
+    | _filter_qemu_io
+
+# The data should now appear at this guest offset
+$QEMU_IO -c 'read -P 42 0x1b200 512' "$TEST_IMG" | _filter_qemu_io
+
+# This cluster is unallocated; fix it
+_check_test_img -r all
+
+# This repair operation must have allocated a new refblock; and that refblock
+# should not overlap with the unallocated data cluster. If it does, the data
+# will be damaged, so check it.
+$QEMU_IO -c 'read -P 42 0x1b200 512' "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo '=== Repairing refblock beyond the image end ==='
+echo
+
+echo
+echo '--- Otherwise clean ---'
+echo
+
+_make_test_img 64M
+# Normally, qemu doesn't create empty refblocks, so we just have to do it by
+# hand
+# XXX: This should be the entry for the second refblock
+poke_file "$TEST_IMG" $((0x10008)) "\x00\x00\x00\x00\x00\x10\x00\x00"
+# Mark that refblock as used
+# XXX: This should be the 17th entry (cluster 16) of the first
+# refblock
+poke_file "$TEST_IMG" $((0x20020)) "\x00\x01"
+_check_test_img -r all
+
+echo
+echo '--- Refblock is unallocated ---'
+echo
+
+_make_test_img 64M
+poke_file "$TEST_IMG" $((0x10008)) "\x00\x00\x00\x00\x00\x10\x00\x00"
+_check_test_img -r all
+
+echo
+echo '--- Signed overflow after the refblock ---'
+echo
+
+_make_test_img 64M
+poke_file "$TEST_IMG" $((0x10008)) "\x7f\xff\xff\xff\xff\xff\x00\x00"
+_check_test_img -r all
+
+echo
+echo '--- Unsigned overflow after the refblock ---'
+echo
+
+_make_test_img 64M
+poke_file "$TEST_IMG" $((0x10008)) "\xff\xff\xff\xff\xff\xff\x00\x00"
+_check_test_img -r all
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/108.out b/tests/qemu-iotests/108.out
new file mode 100644
index 000000000..824d5cf13
--- /dev/null
+++ b/tests/qemu-iotests/108.out
@@ -0,0 +1,110 @@
+QA output created by 108
+
+=== Repairing an image without any refcount table ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+ERROR cluster 0 refcount=0 reference=1
+ERROR cluster 3 refcount=0 reference=1
+ERROR cluster 4 refcount=0 reference=1
+ERROR cluster 5 refcount=0 reference=1
+Rebuilding refcount structure
+The following inconsistencies were found and repaired:
+
+    0 leaked clusters
+    4 corruptions
+
+Double checking the fixed image now...
+No errors were found on the image.
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Repairing unreferenced data cluster in new refblock area ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+wrote 111104/111104 bytes at offset 0
+108.500 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+131072
+wrote 512/512 bytes at offset 131072
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 111104
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+ERROR cluster 256 refcount=0 reference=1
+Rebuilding refcount structure
+Repairing cluster 1 refcount=1 reference=0
+Repairing cluster 2 refcount=1 reference=0
+The following inconsistencies were found and repaired:
+
+    0 leaked clusters
+    1 corruptions
+
+Double checking the fixed image now...
+No errors were found on the image.
+read 512/512 bytes at offset 111104
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Repairing refblock beyond the image end ===
+
+
+--- Otherwise clean ---
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Repairing refcount block 1 is outside image
+The following inconsistencies were found and repaired:
+
+    0 leaked clusters
+    1 corruptions
+
+Double checking the fixed image now...
+No errors were found on the image.
+
+--- Refblock is unallocated ---
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Repairing refcount block 1 is outside image
+ERROR cluster 16 refcount=0 reference=1
+Rebuilding refcount structure
+Repairing cluster 1 refcount=1 reference=0
+Repairing cluster 2 refcount=1 reference=0
+Repairing cluster 16 refcount=1 reference=0
+The following inconsistencies were found and repaired:
+
+    0 leaked clusters
+    2 corruptions
+
+Double checking the fixed image now...
+No errors were found on the image.
+
+--- Signed overflow after the refblock ---
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Repairing refcount block 1 is outside image
+ERROR could not resize image: Invalid argument
+Rebuilding refcount structure
+Repairing cluster 1 refcount=1 reference=0
+Repairing cluster 2 refcount=1 reference=0
+The following inconsistencies were found and repaired:
+
+    0 leaked clusters
+    1 corruptions
+
+Double checking the fixed image now...
+No errors were found on the image.
+
+--- Unsigned overflow after the refblock ---
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Repairing refcount block 1 is outside image
+ERROR could not resize image: Invalid argument
+Rebuilding refcount structure
+Repairing cluster 1 refcount=1 reference=0
+Repairing cluster 2 refcount=1 reference=0
+The following inconsistencies were found and repaired:
+
+    0 leaked clusters
+    1 corruptions
+
+Double checking the fixed image now...
+No errors were found on the image.
+*** done
diff --git a/tests/qemu-iotests/111 b/tests/qemu-iotests/111
new file mode 100755
index 000000000..6011c94b7
--- /dev/null
+++ b/tests/qemu-iotests/111
@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+# Test case for non-existing backing file when creating a qcow2 image
+# and not specifying the size
+#
+# Copyright (C) 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qed qcow qcow2 vmdk
+_supported_proto file
+_supported_os Linux
+_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat"
+
+$QEMU_IMG create -f $IMGFMT -b "$TEST_IMG.inexistent" "$TEST_IMG" 2>&1 \
+    | _filter_testdir | _filter_imgfmt
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/111.out b/tests/qemu-iotests/111.out
new file mode 100644
index 000000000..683c01a67
--- /dev/null
+++ b/tests/qemu-iotests/111.out
@@ -0,0 +1,3 @@
+QA output created by 111
+qemu-img: TEST_DIR/t.IMGFMT: Could not open 'TEST_DIR/t.IMGFMT.inexistent': No such file or directory
+*** done
diff --git a/tests/qemu-iotests/common b/tests/qemu-iotests/common
index e4083f421..9e12bec2b 100644
--- a/tests/qemu-iotests/common
+++ b/tests/qemu-iotests/common
@@ -136,7 +136,6 @@ common options
 check options
     -raw                test raw (default)
     -bochs              test bochs
-    -cow                test cow
     -cloop              test cloop
     -parallels          test parallels
     -qcow               test qcow
@@ -152,6 +151,7 @@ check options
     -nbd                test nbd
     -ssh                test ssh
     -nfs                test nfs
+    -archipelago        test archipelago
     -xdiff              graphical mode diff
     -nocache            use O_DIRECT on backing file
     -misalign           misalign memory allocations
@@ -181,11 +181,6 @@ testlist options
             xpand=false
             ;;
 
-        -cow)
-            IMGFMT=cow
-            xpand=false
-            ;;
-
         -cloop)
             IMGFMT=cloop
             IMGFMT_GENERIC=false
@@ -263,6 +258,11 @@ testlist options
             xpand=false
             ;;
 
+        -archipelago)
+            IMGPROTO=archipelago
+            xpand=false
+            ;;
+
         -nocache)
             CACHEMODE="none"
             CACHEMODE_IS_DEFAULT=false
@@ -376,10 +376,16 @@ BEGIN        { for (t='$start'; t<='$end'; t++) printf "%03d\n",t }' \
                     echo $id >>$tmp.list
                 else
                     # oops
-                    echo "$id - unknown test, ignored"
+                    if [ "$start" == "$end" -a "$id" == "$end" ]
+                    then
+                        echo "$id - unknown test"
+                        exit 1
+                    else
+                        echo "$id - unknown test, ignored"
+                    fi
                 fi
             fi
-        done
+        done || exit 1
     fi
 
 done
diff --git a/tests/qemu-iotests/common.filter b/tests/qemu-iotests/common.filter
index a04df7f6d..3acdb307f 100644
--- a/tests/qemu-iotests/common.filter
+++ b/tests/qemu-iotests/common.filter
@@ -170,5 +170,55 @@ _filter_qmp()
         -e 's#^{"QMP":.*}$#QMP_VERSION#'
 }
 
+# replace driver-specific options in the "Formatting..." line
+_filter_img_create()
+{
+    sed -e "s#$IMGPROTO:$TEST_DIR#TEST_DIR#g" \
+        -e "s#$TEST_DIR#TEST_DIR#g" \
+        -e "s#$IMGFMT#IMGFMT#g" \
+        -e "s# encryption=off##g" \
+        -e "s# cluster_size=[0-9]\\+##g" \
+        -e "s# table_size=[0-9]\\+##g" \
+        -e "s# compat='[^']*'##g" \
+        -e "s# compat6=\\(on\\|off\\)##g" \
+        -e "s# static=\\(on\\|off\\)##g" \
+        -e "s# zeroed_grain=\\(on\\|off\\)##g" \
+        -e "s# subformat='[^']*'##g" \
+        -e "s# adapter_type='[^']*'##g" \
+        -e "s# lazy_refcounts=\\(on\\|off\\)##g" \
+        -e "s# block_size=[0-9]\\+##g" \
+        -e "s# block_state_zero=\\(on\\|off\\)##g" \
+        -e "s# log_size=[0-9]\\+##g" \
+        -e "s/archipelago:a/TEST_DIR\//g"
+}
+
+_filter_img_info()
+{
+    sed -e "s#$IMGPROTO:$TEST_DIR#TEST_DIR#g" \
+        -e "s#$TEST_DIR#TEST_DIR#g" \
+        -e "s#$IMGFMT#IMGFMT#g" \
+        -e "/encrypted: yes/d" \
+        -e "/cluster_size: [0-9]\\+/d" \
+        -e "/table_size: [0-9]\\+/d" \
+        -e "/compat: '[^']*'/d" \
+        -e "/compat6: \\(on\\|off\\)/d" \
+        -e "/static: \\(on\\|off\\)/d" \
+        -e "/zeroed_grain: \\(on\\|off\\)/d" \
+        -e "/subformat: '[^']*'/d" \
+        -e "/adapter_type: '[^']*'/d" \
+        -e "/lazy_refcounts: \\(on\\|off\\)/d" \
+        -e "/block_size: [0-9]\\+/d" \
+        -e "/block_state_zero: \\(on\\|off\\)/d" \
+        -e "/log_size: [0-9]\\+/d" \
+        -e "s/archipelago:a/TEST_DIR\//g"
+}
+
+# filter out offsets and file names from qemu-img map
+_filter_qemu_img_map()
+{
+    sed -e 's/\([0-9a-fx]* *[0-9a-fx]* *\)[0-9a-fx]* */\1/g' \
+        -e 's/Mapped to *//' | _filter_testdir | _filter_imgfmt
+}
+
 # make sure this script returns success
 /bin/true
diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc
index e0ea7e3a7..9c49deb3d 100644
--- a/tests/qemu-iotests/common.rc
+++ b/tests/qemu-iotests/common.rc
@@ -64,6 +64,8 @@ elif [ "$IMGPROTO" = "ssh" ]; then
 elif [ "$IMGPROTO" = "nfs" ]; then
     TEST_DIR="nfs://127.0.0.1/$TEST_DIR"
     TEST_IMG=$TEST_DIR/t.$IMGFMT
+elif [ "$IMGPROTO" = "archipelago" ]; then
+    TEST_IMG="archipelago:at.$IMGFMT"
 else
     TEST_IMG=$IMGPROTO:$TEST_DIR/t.$IMGFMT
 fi
@@ -147,23 +149,7 @@ _make_test_img()
      else
         $QEMU_IMG create -f $IMGFMT $extra_img_options "$img_name" $image_size 2>&1
      fi
-    ) | \
-        sed -e "s#$IMGPROTO:$TEST_DIR#TEST_DIR#g" \
-            -e "s#$TEST_DIR#TEST_DIR#g" \
-            -e "s#$IMGFMT#IMGFMT#g" \
-            -e "s# encryption=off##g" \
-            -e "s# cluster_size=[0-9]\\+##g" \
-            -e "s# table_size=[0-9]\\+##g" \
-            -e "s# compat='[^']*'##g" \
-            -e "s# compat6=\\(on\\|off\\)##g" \
-            -e "s# static=\\(on\\|off\\)##g" \
-            -e "s# zeroed_grain=\\(on\\|off\\)##g" \
-            -e "s# subformat='[^']*'##g" \
-            -e "s# adapter_type='[^']*'##g" \
-            -e "s# lazy_refcounts=\\(on\\|off\\)##g" \
-            -e "s# block_size=[0-9]\\+##g" \
-            -e "s# block_state_zero=\\(on\\|off\\)##g" \
-            -e "s# log_size=[0-9]\\+##g"
+    ) | _filter_img_create
 
     # Start an NBD server on the image file, which is what we'll be talking to
     if [ $IMGPROTO = "nbd" ]; then
@@ -206,6 +192,10 @@ _cleanup_test_img()
             rbd --no-progress rm "$TEST_DIR/t.$IMGFMT" > /dev/null
             ;;
 
+        archipelago)
+            vlmc remove "at.$IMGFMT" > /dev/null
+            ;;
+
         sheepdog)
             collie vdi delete "$TEST_DIR/t.$IMGFMT"
             ;;
diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
index 6e67f6126..7dfe46940 100644
--- a/tests/qemu-iotests/group
+++ b/tests/qemu-iotests/group
@@ -67,7 +67,7 @@
 058 rw auto quick
 059 rw auto quick
 060 rw auto quick
-061 rw auto quick
+061 rw auto
 062 rw auto quick
 063 rw auto quick
 064 rw auto quick
@@ -100,3 +100,15 @@
 091 rw auto quick
 092 rw auto quick
 095 rw auto quick
+097 rw auto backing
+098 rw auto backing quick
+099 rw auto quick
+100 rw auto quick
+101 rw auto quick
+102 rw auto quick
+103 rw auto quick
+104 rw auto
+105 rw auto quick
+107 rw auto quick
+108 rw auto quick
+111 rw auto quick
diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index 39a4cfcf4..f57f1548a 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -267,8 +267,7 @@ class QMPTestCase(unittest.TestCase):
                     self.assert_qmp(event, 'data/device', drive)
                     self.assert_qmp_absent(event, 'data/error')
                     if check_offset:
-                        self.assert_qmp(event, 'data/offset', self.image_len)
-                    self.assert_qmp(event, 'data/len', self.image_len)
+                        self.assert_qmp(event, 'data/offset', event['data']['len'])
                     completed = True
 
         self.assert_no_active_block_jobs()
diff --git a/tests/qemu-iotests/sample_images/fake.parallels.bz2 b/tests/qemu-iotests/sample_images/fake.parallels.bz2
deleted file mode 100644
index ffb5f13ba..000000000
--- a/tests/qemu-iotests/sample_images/fake.parallels.bz2
+++ /dev/null
diff --git a/tests/qemu-iotests/sample_images/iotest-version3.vmdk.bz2 b/tests/qemu-iotests/sample_images/iotest-version3.vmdk.bz2
index 30abf217e..619329a24 100644
--- a/tests/qemu-iotests/sample_images/iotest-version3.vmdk.bz2
+++ b/tests/qemu-iotests/sample_images/iotest-version3.vmdk.bz2
diff --git a/tests/qemu-iotests/sample_images/parallels-v1.bz2 b/tests/qemu-iotests/sample_images/parallels-v1.bz2
new file mode 100644
index 000000000..d1ef14205
--- /dev/null
+++ b/tests/qemu-iotests/sample_images/parallels-v1.bz2
diff --git a/tests/qemu-iotests/sample_images/parallels-v2.bz2 b/tests/qemu-iotests/sample_images/parallels-v2.bz2
new file mode 100644
index 000000000..fd8614d06
--- /dev/null
+++ b/tests/qemu-iotests/sample_images/parallels-v2.bz2
diff --git a/tests/qemu-iotests/socket_scm_helper.c b/tests/qemu-iotests/socket_scm_helper.c
index 0e2b2859a..81959835e 100644
--- a/tests/qemu-iotests/socket_scm_helper.c
+++ b/tests/qemu-iotests/socket_scm_helper.c
@@ -52,7 +52,7 @@ static int send_fd(int fd, int fd_to_send)
     cmsg->cmsg_len = CMSG_LEN(sizeof(int));
     cmsg->cmsg_level = SOL_SOCKET;
     cmsg->cmsg_type = SCM_RIGHTS;
-    memcpy(CMSG_DATA(cmsg), &fd, sizeof(int));
+    memcpy(CMSG_DATA(cmsg), &fd_to_send, sizeof(int));
 
     do {
         ret = sendmsg(fd, &msg, 0);
diff --git a/tests/tcg/xtensa/Makefile b/tests/tcg/xtensa/Makefile
index a70c92be7..522a63e36 100644
--- a/tests/tcg/xtensa/Makefile
+++ b/tests/tcg/xtensa/Makefile
@@ -13,6 +13,7 @@ SIMFLAGS = --xtensa-core=DC_B_232L --exit_with_target_code $(EXTFLAGS)
 SIMDEBUG = --gdbserve=0
 endif
 
+HOST_CC = gcc
 CC      = $(CROSS)gcc
 AS      = $(CROSS)gcc -x assembler-with-cpp
 LD      = $(CROSS)ld
@@ -21,7 +22,7 @@ XTENSA_SRC_PATH = $(SRC_PATH)/tests/tcg/xtensa
 INCLUDE_DIRS = $(XTENSA_SRC_PATH) $(SRC_PATH)/target-xtensa/core-$(CORE)
 XTENSA_INC = $(addprefix -I,$(INCLUDE_DIRS))
 
-LDFLAGS = -T$(XTENSA_SRC_PATH)/linker.ld
+LDFLAGS = -Tlinker.ld
 
 CRT        = crt.o vectors.o
 
@@ -59,13 +60,16 @@ TESTCASES += test_windowed.tst
 
 all: build
 
+linker.ld: $(XTENSA_SRC_PATH)/linker.ld.S
+	$(HOST_CC) $(XTENSA_INC) -E -P $< -o $@
+
 %.o: $(XTENSA_SRC_PATH)/%.c
 	$(CC) $(XTENSA_INC) $(CFLAGS) -c $< -o $@
 
 %.o: $(XTENSA_SRC_PATH)/%.S
 	$(CC) $(XTENSA_INC) $(ASFLAGS) -c $< -o $@
 
-%.tst: %.o $(XTENSA_SRC_PATH)/macros.inc $(CRT) Makefile
+%.tst: %.o linker.ld $(XTENSA_SRC_PATH)/macros.inc $(CRT) Makefile
 	$(LD) $(LDFLAGS) $(NOSTDFLAGS) $(CRT) $< -o $@
 
 build: $(TESTCASES)
@@ -85,4 +89,4 @@ host-debug-%.tst: %.tst
 	gdb --args $(SIM) $(SIMFLAGS) ./$<
 
 clean:
-	$(RM) -fr $(TESTCASES) $(CRT)
+	$(RM) -fr $(TESTCASES) $(CRT) linker.ld
diff --git a/tests/tcg/xtensa/linker.ld b/tests/tcg/xtensa/linker.ld
deleted file mode 100644
index 4d0b307fd..000000000
--- a/tests/tcg/xtensa/linker.ld
+++ /dev/null
@@ -1,112 +0,0 @@
-OUTPUT_FORMAT("elf32-xtensa-le")
-ENTRY(_start)
-
-__DYNAMIC = 0;
-
-MEMORY {
-	ram : ORIGIN = 0xd0000000, LENGTH = 0x08000000  /* 128M */
-	rom : ORIGIN = 0xfe000000, LENGTH = 0x00001000  /* 4k */
-}
-
-SECTIONS
-{
-    .init :
-    {
-        *(.init)
-		*(.init.*)
-    } > rom
-
-    .vector :
-    {
-    . = 0x00000000;
-        *(.vector.window_overflow_4)
-        *(.vector.window_overflow_4.*)
-    . = 0x00000040;
-        *(.vector.window_underflow_4)
-        *(.vector.window_underflow_4.*)
-    . = 0x00000080;
-        *(.vector.window_overflow_8)
-        *(.vector.window_overflow_8.*)
-    . = 0x000000c0;
-        *(.vector.window_underflow_8)
-        *(.vector.window_underflow_8.*)
-    . = 0x00000100;
-        *(.vector.window_overflow_12)
-        *(.vector.window_overflow_12.*)
-    . = 0x00000140;
-        *(.vector.window_underflow_12)
-        *(.vector.window_underflow_12.*)
-
-    . = 0x00000180;
-        *(.vector.level2)
-        *(.vector.level2.*)
-    . = 0x000001c0;
-        *(.vector.level3)
-        *(.vector.level3.*)
-    . = 0x00000200;
-        *(.vector.level4)
-        *(.vector.level4.*)
-    . = 0x00000240;
-        *(.vector.level5)
-        *(.vector.level5.*)
-    . = 0x00000280;
-        *(.vector.level6)
-        *(.vector.level6.*)
-    . = 0x000002c0;
-        *(.vector.level7)
-        *(.vector.level7.*)
-
-    . = 0x00000300;
-        *(.vector.kernel)
-        *(.vector.kernel.*)
-    . = 0x00000340;
-        *(.vector.user)
-        *(.vector.user.*)
-    . = 0x000003c0;
-        *(.vector.double)
-        *(.vector.double.*)
-    } > ram
-
-	.text :
-	{
-		_ftext = .;
-		*(.text .stub .text.* .gnu.linkonce.t.* .literal .literal.*)
-		_etext = .;
-	} > ram
-
-	.rodata :
-	{
-		. = ALIGN(4);
-		_frodata = .;
-		*(.rodata .rodata.* .gnu.linkonce.r.*)
-		*(.rodata1)
-		_erodata = .;
-	} > ram
-
-	.data :
-	{
-		. = ALIGN(4);
-		_fdata = .;
-		*(.data .data.* .gnu.linkonce.d.*)
-		*(.data1)
-		_gp = ALIGN(16);
-		*(.sdata .sdata.* .gnu.linkonce.s.*)
-		_edata = .;
-	} > ram
-
-	.bss :
-	{
-		. = ALIGN(4);
-		_fbss = .;
-		*(.dynsbss)
-		*(.sbss .sbss.* .gnu.linkonce.sb.*)
-		*(.scommon)
-		*(.dynbss)
-		*(.bss .bss.* .gnu.linkonce.b.*)
-		*(COMMON)
-		_ebss = .;
-		_end = .;
-	} > ram
-}
-
-PROVIDE(_fstack = ORIGIN(ram) + LENGTH(ram) - 4);
diff --git a/tests/tcg/xtensa/linker.ld.S b/tests/tcg/xtensa/linker.ld.S
new file mode 100644
index 000000000..f1e7fa9f8
--- /dev/null
+++ b/tests/tcg/xtensa/linker.ld.S
@@ -0,0 +1,130 @@
+#include <core-isa.h>
+
+#if XTENSA_HAVE_BE
+OUTPUT_FORMAT("elf32-xtensa-be")
+#else
+OUTPUT_FORMAT("elf32-xtensa-le")
+#endif
+ENTRY(_start)
+
+__DYNAMIC = 0;
+
+MEMORY {
+    ram : ORIGIN = XCHAL_VECBASE_RESET_VADDR, LENGTH = 0x08000000  /* 128M */
+    rom : ORIGIN = XCHAL_RESET_VECTOR_VADDR, LENGTH = 0x00001000  /* 4k */
+}
+
+SECTIONS
+{
+    .init :
+    {
+        *(.init)
+        *(.init.*)
+    } > rom
+
+    .vector :
+    {
+    . = XCHAL_WINDOW_OF4_VECOFS;
+        *(.vector.window_overflow_4)
+        *(.vector.window_overflow_4.*)
+    . = XCHAL_WINDOW_UF4_VECOFS;
+        *(.vector.window_underflow_4)
+        *(.vector.window_underflow_4.*)
+    . = XCHAL_WINDOW_OF8_VECOFS;
+        *(.vector.window_overflow_8)
+        *(.vector.window_overflow_8.*)
+    . = XCHAL_WINDOW_UF8_VECOFS;
+        *(.vector.window_underflow_8)
+        *(.vector.window_underflow_8.*)
+    . = XCHAL_WINDOW_OF12_VECOFS;
+        *(.vector.window_overflow_12)
+        *(.vector.window_overflow_12.*)
+    . = XCHAL_WINDOW_UF12_VECOFS;
+        *(.vector.window_underflow_12)
+        *(.vector.window_underflow_12.*)
+
+#if XCHAL_NUM_INTLEVELS + XCHAL_HAVE_NMI >= 2
+    . = XCHAL_INTLEVEL2_VECOFS;
+        *(.vector.level2)
+        *(.vector.level2.*)
+#endif
+#if XCHAL_NUM_INTLEVELS + XCHAL_HAVE_NMI >= 3
+    . = XCHAL_INTLEVEL3_VECOFS;
+        *(.vector.level3)
+        *(.vector.level3.*)
+#endif
+#if XCHAL_NUM_INTLEVELS + XCHAL_HAVE_NMI >= 4
+    . = XCHAL_INTLEVEL4_VECOFS;
+        *(.vector.level4)
+        *(.vector.level4.*)
+#endif
+#if XCHAL_NUM_INTLEVELS + XCHAL_HAVE_NMI >= 5
+    . = XCHAL_INTLEVEL5_VECOFS;
+        *(.vector.level5)
+        *(.vector.level5.*)
+#endif
+#if XCHAL_NUM_INTLEVELS + XCHAL_HAVE_NMI >= 6
+    . = XCHAL_INTLEVEL6_VECOFS;
+        *(.vector.level6)
+        *(.vector.level6.*)
+#endif
+#if XCHAL_NUM_INTLEVELS + XCHAL_HAVE_NMI >= 7
+    . = XCHAL_INTLEVEL7_VECOFS;
+        *(.vector.level7)
+        *(.vector.level7.*)
+#endif
+
+    . = XCHAL_KERNEL_VECOFS;
+        *(.vector.kernel)
+        *(.vector.kernel.*)
+    . = XCHAL_USER_VECOFS;
+        *(.vector.user)
+        *(.vector.user.*)
+    . = XCHAL_DOUBLEEXC_VECOFS;
+        *(.vector.double)
+        *(.vector.double.*)
+    } > ram
+
+    .text :
+    {
+        _ftext = .;
+        *(.text .stub .text.* .gnu.linkonce.t.* .literal .literal.*)
+        _etext = .;
+    } > ram
+
+    .rodata :
+    {
+        . = ALIGN(4);
+        _frodata = .;
+        *(.rodata .rodata.* .gnu.linkonce.r.*)
+        *(.rodata1)
+        _erodata = .;
+    } > ram
+
+    .data :
+    {
+        . = ALIGN(4);
+        _fdata = .;
+        *(.data .data.* .gnu.linkonce.d.*)
+        *(.data1)
+        _gp = ALIGN(16);
+        *(.sdata .sdata.* .gnu.linkonce.s.*)
+        _edata = .;
+    } > ram
+
+    .bss :
+    {
+        . = ALIGN(4);
+        _fbss = .;
+        *(.dynsbss)
+        *(.sbss .sbss.* .gnu.linkonce.sb.*)
+        *(.scommon)
+        *(.dynbss)
+        *(.bss .bss.* .gnu.linkonce.b.*)
+        *(COMMON)
+        _ebss = .;
+        _end = .;
+    } > ram
+}
+
+PROVIDE(_fstack = (ORIGIN(ram) & 0xf0000000) + LENGTH(ram) - 16);
diff --git a/tests/tcg/xtensa/test_windowed.S b/tests/tcg/xtensa/test_windowed.S
index 3de6d3763..d851e8f43 100644
--- a/tests/tcg/xtensa/test_windowed.S
+++ b/tests/tcg/xtensa/test_windowed.S
@@ -299,4 +299,55 @@ test entry
     entry_test 12
 test_end
 
+.macro entry_overflow_test window, free, next_window
+    set_vector window_overflow_4, 0
+    set_vector window_overflow_8, 0
+    set_vector window_overflow_12, 0
+    set_vector window_overflow_\next_window, 10f
+
+    movi    a2, \window
+    movi    a2, \free
+    movi    a2, \next_window
+    reset_window %(1 | ((1 | (1 << ((\next_window) / 4))) << ((\free) / 4)))
+    reset_ps
+    movi    a2, 0x4000f | ((\window) << 14)
+    wsr     a2, ps
+    isync
+    movi    a3, 0x12345678
+    j       1f
+    .align  4
+1:
+    entry   a3, 0x5678
+    test_fail
+    .align  4
+10:
+    rsr     a2, epc1
+    movi    a3, 1b
+    assert  eq, a2, a3
+    movi    a2, 2f
+    wsr     a2, epc1
+
+    rsr     a2, windowbase
+    movi    a3, (\free) / 4
+    assert  eq, a2, a3
+    rfwo
+2:
+.endm
+
+.macro all_entry_overflow_tests
+    .irp window, 4, 8, 12
+    .irp next_window, 4, 8, 12
+    .irp free, 4, 8, 12
+    .if \free <= \window
+    entry_overflow_test \window, \free, \next_window
+    .endif
+    .endr
+    .endr
+    .endr
+.endm
+
+test entry_overflow
+    all_entry_overflow_tests
+test_end
+
 test_suite_end
diff --git a/tests/test-aio.c b/tests/test-aio.c
index f12b6e0ae..a7cb5c991 100644
--- a/tests/test-aio.c
+++ b/tests/test-aio.c
@@ -14,6 +14,7 @@
 #include "block/aio.h"
 #include "qemu/timer.h"
 #include "qemu/sockets.h"
+#include "qemu/error-report.h"
 
 static AioContext *ctx;
 
@@ -57,8 +58,6 @@ static void bh_test_cb(void *opaque)
     }
 }
 
-#if !defined(_WIN32)
-
 static void timer_test_cb(void *opaque)
 {
     TimerTestData *data = opaque;
@@ -68,12 +67,10 @@ static void timer_test_cb(void *opaque)
     }
 }
 
-static void dummy_io_handler_read(void *opaque)
+static void dummy_io_handler_read(EventNotifier *e)
 {
 }
 
-#endif /* !_WIN32 */
-
 static void bh_delete_cb(void *opaque)
 {
     BHTestData *data = opaque;
@@ -428,24 +425,18 @@ static void test_wait_event_notifier_noflush(void)
     event_notifier_cleanup(&data.e);
 }
 
-#if !defined(_WIN32)
-
 static void test_timer_schedule(void)
 {
     TimerTestData data = { .n = 0, .ctx = ctx, .ns = SCALE_MS * 750LL,
                            .max = 2,
                            .clock_type = QEMU_CLOCK_VIRTUAL };
-    int pipefd[2];
+    EventNotifier e;
 
     /* aio_poll will not block to wait for timers to complete unless it has
      * an fd to wait on. Fixing this breaks other tests. So create a dummy one.
      */
-    g_assert(!qemu_pipe(pipefd));
-    qemu_set_nonblock(pipefd[0]);
-    qemu_set_nonblock(pipefd[1]);
-
-    aio_set_fd_handler(ctx, pipefd[0],
-                       dummy_io_handler_read, NULL, NULL);
+    event_notifier_init(&e, false);
+    aio_set_event_notifier(ctx, &e, dummy_io_handler_read);
     aio_poll(ctx, false);
 
     aio_timer_init(ctx, &data.timer, data.clock_type,
@@ -484,15 +475,12 @@ static void test_timer_schedule(void)
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 2);
 
-    aio_set_fd_handler(ctx, pipefd[0], NULL, NULL, NULL);
-    close(pipefd[0]);
-    close(pipefd[1]);
+    aio_set_event_notifier(ctx, &e, NULL);
+    event_notifier_cleanup(&e);
 
     timer_del(&data.timer);
 }
 
-#endif /* !_WIN32 */
-
 /* Now the same tests, using the context as a GSource.  They are
  * very similar to the ones above, with g_main_context_iteration
  * replacing aio_poll.  However:
@@ -775,25 +763,19 @@ static void test_source_wait_event_notifier_noflush(void)
     event_notifier_cleanup(&data.e);
 }
 
-#if !defined(_WIN32)
-
 static void test_source_timer_schedule(void)
 {
     TimerTestData data = { .n = 0, .ctx = ctx, .ns = SCALE_MS * 750LL,
                            .max = 2,
                            .clock_type = QEMU_CLOCK_VIRTUAL };
-    int pipefd[2];
+    EventNotifier e;
     int64_t expiry;
 
     /* aio_poll will not block to wait for timers to complete unless it has
      * an fd to wait on. Fixing this breaks other tests. So create a dummy one.
      */
-    g_assert(!qemu_pipe(pipefd));
-    qemu_set_nonblock(pipefd[0]);
-    qemu_set_nonblock(pipefd[1]);
-
-    aio_set_fd_handler(ctx, pipefd[0],
-                       dummy_io_handler_read, NULL, NULL);
+    event_notifier_init(&e, false);
+    aio_set_event_notifier(ctx, &e, dummy_io_handler_read);
     do {} while (g_main_context_iteration(NULL, false));
 
     aio_timer_init(ctx, &data.timer, data.clock_type,
@@ -818,25 +800,29 @@ static void test_source_timer_schedule(void)
     g_assert_cmpint(data.n, ==, 2);
     g_assert(qemu_clock_get_ns(data.clock_type) > expiry);
 
-    aio_set_fd_handler(ctx, pipefd[0], NULL, NULL, NULL);
-    close(pipefd[0]);
-    close(pipefd[1]);
+    aio_set_event_notifier(ctx, &e, NULL);
+    event_notifier_cleanup(&e);
 
     timer_del(&data.timer);
 }
 
-#endif /* !_WIN32 */
-
 
 /* End of tests.  */
 
 int main(int argc, char **argv)
 {
+    Error *local_error = NULL;
     GSource *src;
 
     init_clocks();
 
-    ctx = aio_context_new();
+    ctx = aio_context_new(&local_error);
+    if (!ctx) {
+        error_report("Failed to create AIO Context: '%s'",
+                     error_get_pretty(local_error));
+        error_free(local_error);
+        exit(1);
+    }
     src = aio_get_g_source(ctx);
     g_source_attach(src, NULL);
     g_source_unref(src);
@@ -857,9 +843,7 @@ int main(int argc, char **argv)
     g_test_add_func("/aio/event/wait",              test_wait_event_notifier);
     g_test_add_func("/aio/event/wait/no-flush-cb",  test_wait_event_notifier_noflush);
     g_test_add_func("/aio/event/flush",             test_flush_event_notifier);
-#if !defined(_WIN32)
     g_test_add_func("/aio/timer/schedule",          test_timer_schedule);
-#endif
 
     g_test_add_func("/aio-gsource/notify",                  test_source_notify);
     g_test_add_func("/aio-gsource/flush",                   test_source_flush);
@@ -874,8 +858,6 @@ int main(int argc, char **argv)
     g_test_add_func("/aio-gsource/event/wait",              test_source_wait_event_notifier);
     g_test_add_func("/aio-gsource/event/wait/no-flush-cb",  test_source_wait_event_notifier_noflush);
     g_test_add_func("/aio-gsource/event/flush",             test_source_flush_event_notifier);
-#if !defined(_WIN32)
     g_test_add_func("/aio-gsource/timer/schedule",          test_source_timer_schedule);
-#endif
     return g_test_run();
 }
diff --git a/tests/test-bitops.c b/tests/test-bitops.c
index 8238eb5f6..47b5d3ed9 100644
--- a/tests/test-bitops.c
+++ b/tests/test-bitops.c
@@ -8,6 +8,7 @@
 
 #include <glib.h>
 #include <stdint.h>
+#include "qemu/osdep.h"
 #include "qemu/bitops.h"
 
 typedef struct {
diff --git a/tests/test-coroutine.c b/tests/test-coroutine.c
index 760636ddc..e22fae170 100644
--- a/tests/test-coroutine.c
+++ b/tests/test-coroutine.c
@@ -288,6 +288,58 @@ static void perf_yield(void)
         maxcycles, duration);
 }
 
+static __attribute__((noinline)) void dummy(unsigned *i)
+{
+    (*i)--;
+}
+
+static void perf_baseline(void)
+{
+    unsigned int i, maxcycles;
+    double duration;
+
+    maxcycles = 100000000;
+    i = maxcycles;
+
+    g_test_timer_start();
+    while (i > 0) {
+        dummy(&i);
+    }
+    duration = g_test_timer_elapsed();
+
+    g_test_message("Function call %u iterations: %f s\n",
+        maxcycles, duration);
+}
+
+static __attribute__((noinline)) void perf_cost_func(void *opaque)
+{
+    qemu_coroutine_yield();
+}
+
+static void perf_cost(void)
+{
+    const unsigned long maxcycles = 40000000;
+    unsigned long i = 0;
+    double duration;
+    unsigned long ops;
+    Coroutine *co;
+
+    g_test_timer_start();
+    while (i++ < maxcycles) {
+        co = qemu_coroutine_create(perf_cost_func);
+        qemu_coroutine_enter(co, &i);
+        qemu_coroutine_enter(co, NULL);
+    }
+    duration = g_test_timer_elapsed();
+    ops = (long)(maxcycles / (duration * 1000));
+
+    g_test_message("Run operation %lu iterations %f s, %luK operations/s, "
+                   "%luns per coroutine",
+                   maxcycles,
+                   duration, ops,
+                   (unsigned long)(1000000000 * duration) / maxcycles);
+}
+
 int main(int argc, char **argv)
 {
     g_test_init(&argc, &argv, NULL);
@@ -301,6 +353,8 @@ int main(int argc, char **argv)
         g_test_add_func("/perf/lifecycle", perf_lifecycle);
         g_test_add_func("/perf/nesting", perf_nesting);
         g_test_add_func("/perf/yield", perf_yield);
+        g_test_add_func("/perf/function-call", perf_baseline);
+        g_test_add_func("/perf/cost", perf_cost);
     }
     return g_test_run();
 }
diff --git a/tests/test-qdev-global-props.c b/tests/test-qdev-global-props.c
index 2bef04c76..0be98355c 100644
--- a/tests/test-qdev-global-props.c
+++ b/tests/test-qdev-global-props.c
@@ -65,7 +65,7 @@ static const TypeInfo static_prop_type = {
 };
 
 /* Test simple static property setting to default value */
-static void test_static_prop(void)
+static void test_static_prop_subprocess(void)
 {
     MyType *mt;
 
@@ -75,8 +75,16 @@ static void test_static_prop(void)
     g_assert_cmpuint(mt->prop1, ==, PROP_DEFAULT);
 }
 
+static void test_static_prop(void)
+{
+    g_test_trap_subprocess("/qdev/properties/static/default/subprocess", 0, 0);
+    g_test_trap_assert_passed();
+    g_test_trap_assert_stderr("");
+    g_test_trap_assert_stdout("");
+}
+
 /* Test setting of static property using global properties */
-static void test_static_globalprop(void)
+static void test_static_globalprop_subprocess(void)
 {
     MyType *mt;
     static GlobalProperty props[] = {
@@ -93,10 +101,21 @@ static void test_static_globalprop(void)
     g_assert_cmpuint(mt->prop2, ==, PROP_DEFAULT);
 }
 
+static void test_static_globalprop(void)
+{
+    g_test_trap_subprocess("/qdev/properties/static/global/subprocess", 0, 0);
+    g_test_trap_assert_passed();
+    g_test_trap_assert_stderr("");
+    g_test_trap_assert_stdout("");
+}
+
 #define TYPE_DYNAMIC_PROPS "dynamic-prop-type"
 #define DYNAMIC_TYPE(obj) \
     OBJECT_CHECK(MyType, (obj), TYPE_DYNAMIC_PROPS)
 
+#define TYPE_UNUSED_HOTPLUG   "hotplug-type"
+#define TYPE_UNUSED_NOHOTPLUG "nohotplug-type"
+
 static void prop1_accessor(Object *obj,
                            Visitor *v,
                            void *opaque,
@@ -143,14 +162,101 @@ static const TypeInfo dynamic_prop_type = {
     .class_init = dynamic_class_init,
 };
 
-/* Test setting of static property using global properties */
+static void hotplug_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = NULL;
+    dc->hotpluggable = true;
+}
+
+static const TypeInfo hotplug_type = {
+    .name = TYPE_UNUSED_HOTPLUG,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(MyType),
+    .instance_init = dynamic_instance_init,
+    .class_init = hotplug_class_init,
+};
+
+static void nohotplug_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = NULL;
+    dc->hotpluggable = false;
+}
+
+static const TypeInfo nohotplug_type = {
+    .name = TYPE_UNUSED_NOHOTPLUG,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(MyType),
+    .instance_init = dynamic_instance_init,
+    .class_init = nohotplug_class_init,
+};
+
+#define TYPE_NONDEVICE "nondevice-type"
+
+static const TypeInfo nondevice_type = {
+    .name = TYPE_NONDEVICE,
+    .parent = TYPE_OBJECT,
+};
+
+/* Test setting of dynamic properties using global properties */
+static void test_dynamic_globalprop_subprocess(void)
+{
+    MyType *mt;
+    static GlobalProperty props[] = {
+        { TYPE_DYNAMIC_PROPS, "prop1", "101", true },
+        { TYPE_DYNAMIC_PROPS, "prop2", "102", true },
+        { TYPE_DYNAMIC_PROPS"-bad", "prop3", "103", true },
+        { TYPE_UNUSED_HOTPLUG, "prop4", "104", true },
+        { TYPE_UNUSED_NOHOTPLUG, "prop5", "105", true },
+        { TYPE_NONDEVICE, "prop6", "106", true },
+        {}
+    };
+    int all_used;
+
+    qdev_prop_register_global_list(props);
+
+    mt = DYNAMIC_TYPE(object_new(TYPE_DYNAMIC_PROPS));
+    qdev_init_nofail(DEVICE(mt));
+
+    g_assert_cmpuint(mt->prop1, ==, 101);
+    g_assert_cmpuint(mt->prop2, ==, 102);
+    all_used = qdev_prop_check_globals();
+    g_assert_cmpuint(all_used, ==, 1);
+    g_assert(props[0].used);
+    g_assert(props[1].used);
+    g_assert(!props[2].used);
+    g_assert(!props[3].used);
+    g_assert(!props[4].used);
+    g_assert(!props[5].used);
+}
+
 static void test_dynamic_globalprop(void)
 {
+    g_test_trap_subprocess("/qdev/properties/dynamic/global/subprocess", 0, 0);
+    g_test_trap_assert_passed();
+    g_test_trap_assert_stderr_unmatched("*prop1*");
+    g_test_trap_assert_stderr_unmatched("*prop2*");
+    g_test_trap_assert_stderr("*Warning: global dynamic-prop-type-bad.prop3 has invalid class name\n*");
+    g_test_trap_assert_stderr_unmatched("*prop4*");
+    g_test_trap_assert_stderr("*Warning: global nohotplug-type.prop5=105 not used\n*");
+    g_test_trap_assert_stderr("*Warning: global nondevice-type.prop6 has invalid class name\n*");
+    g_test_trap_assert_stdout("");
+}
+
+/* Test setting of dynamic properties using user_provided=false properties */
+static void test_dynamic_globalprop_nouser_subprocess(void)
+{
     MyType *mt;
     static GlobalProperty props[] = {
         { TYPE_DYNAMIC_PROPS, "prop1", "101" },
         { TYPE_DYNAMIC_PROPS, "prop2", "102" },
-        { TYPE_DYNAMIC_PROPS"-bad", "prop3", "103", true },
+        { TYPE_DYNAMIC_PROPS"-bad", "prop3", "103" },
+        { TYPE_UNUSED_HOTPLUG, "prop4", "104" },
+        { TYPE_UNUSED_NOHOTPLUG, "prop5", "105" },
+        { TYPE_NONDEVICE, "prop6", "106" },
         {}
     };
     int all_used;
@@ -162,8 +268,22 @@ static void test_dynamic_globalprop(void)
 
     g_assert_cmpuint(mt->prop1, ==, 101);
     g_assert_cmpuint(mt->prop2, ==, 102);
-    all_used = qdev_prop_check_global();
-    g_assert_cmpuint(all_used, ==, 1);
+    all_used = qdev_prop_check_globals();
+    g_assert_cmpuint(all_used, ==, 0);
+    g_assert(props[0].used);
+    g_assert(props[1].used);
+    g_assert(!props[2].used);
+    g_assert(!props[3].used);
+    g_assert(!props[4].used);
+    g_assert(!props[5].used);
+}
+
+static void test_dynamic_globalprop_nouser(void)
+{
+    g_test_trap_subprocess("/qdev/properties/dynamic/global/nouser/subprocess", 0, 0);
+    g_test_trap_assert_passed();
+    g_test_trap_assert_stderr("");
+    g_test_trap_assert_stdout("");
 }
 
 int main(int argc, char **argv)
@@ -173,10 +293,29 @@ int main(int argc, char **argv)
     module_call_init(MODULE_INIT_QOM);
     type_register_static(&static_prop_type);
     type_register_static(&dynamic_prop_type);
-
-    g_test_add_func("/qdev/properties/static/default", test_static_prop);
-    g_test_add_func("/qdev/properties/static/global", test_static_globalprop);
-    g_test_add_func("/qdev/properties/dynamic/global", test_dynamic_globalprop);
+    type_register_static(&hotplug_type);
+    type_register_static(&nohotplug_type);
+    type_register_static(&nondevice_type);
+
+    g_test_add_func("/qdev/properties/static/default/subprocess",
+                    test_static_prop_subprocess);
+    g_test_add_func("/qdev/properties/static/default",
+                    test_static_prop);
+
+    g_test_add_func("/qdev/properties/static/global/subprocess",
+                    test_static_globalprop_subprocess);
+    g_test_add_func("/qdev/properties/static/global",
+                    test_static_globalprop);
+
+    g_test_add_func("/qdev/properties/dynamic/global/subprocess",
+                    test_dynamic_globalprop_subprocess);
+    g_test_add_func("/qdev/properties/dynamic/global",
+                    test_dynamic_globalprop);
+
+    g_test_add_func("/qdev/properties/dynamic/global/nouser/subprocess",
+                    test_dynamic_globalprop_nouser_subprocess);
+    g_test_add_func("/qdev/properties/dynamic/global/nouser",
+                    test_dynamic_globalprop_nouser);
 
     g_test_run();
 
diff --git a/tests/test-qmp-input-strict.c b/tests/test-qmp-input-strict.c
index 0f770034b..d5360c6a8 100644
--- a/tests/test-qmp-input-strict.c
+++ b/tests/test-qmp-input-strict.c
@@ -260,6 +260,21 @@ static void test_validate_fail_union_flat(TestInputVisitorData *data,
     qapi_free_UserDefFlatUnion(tmp);
 }
 
+static void test_validate_fail_union_flat_no_discrim(TestInputVisitorData *data,
+                                                     const void *unused)
+{
+    UserDefFlatUnion2 *tmp = NULL;
+    Error *err = NULL;
+    Visitor *v;
+
+    /* test situation where discriminator field ('enum1' here) is missing */
+    v = validate_test_init(data, "{ 'string': 'c', 'string1': 'd', 'string2': 'e' }");
+
+    visit_type_UserDefFlatUnion2(v, &tmp, NULL, &err);
+    g_assert(err);
+    qapi_free_UserDefFlatUnion2(tmp);
+}
+
 static void test_validate_fail_union_anon(TestInputVisitorData *data,
                                           const void *unused)
 {
@@ -310,6 +325,8 @@ int main(int argc, char **argv)
                        &testdata, test_validate_fail_union);
     validate_test_add("/visitor/input-strict/fail/union-flat",
                        &testdata, test_validate_fail_union_flat);
+    validate_test_add("/visitor/input-strict/fail/union-flat-no-discriminator",
+                       &testdata, test_validate_fail_union_flat_no_discrim);
     validate_test_add("/visitor/input-strict/fail/union-anon",
                        &testdata, test_validate_fail_union_anon);
 
diff --git a/tests/test-thread-pool.c b/tests/test-thread-pool.c
index f40b7fc17..6a0b9813f 100644
--- a/tests/test-thread-pool.c
+++ b/tests/test-thread-pool.c
@@ -4,13 +4,14 @@
 #include "block/thread-pool.h"
 #include "block/block.h"
 #include "qemu/timer.h"
+#include "qemu/error-report.h"
 
 static AioContext *ctx;
 static ThreadPool *pool;
 static int active;
 
 typedef struct {
-    BlockDriverAIOCB *aiocb;
+    BlockAIOCB *aiocb;
     int n;
     int ret;
 } WorkerTestData;
@@ -33,7 +34,7 @@ static int long_cb(void *opaque)
 static void done_cb(void *opaque, int ret)
 {
     WorkerTestData *data = opaque;
-    g_assert_cmpint(data->ret, ==, -EINPROGRESS);
+    g_assert(data->ret == -EINPROGRESS || data->ret == -ECANCELED);
     data->ret = ret;
     data->aiocb = NULL;
 
@@ -132,7 +133,7 @@ static void test_submit_many(void)
     }
 }
 
-static void test_cancel(void)
+static void do_test_cancel(bool sync)
 {
     WorkerTestData data[100];
     int num_canceled;
@@ -170,18 +171,25 @@ static void test_cancel(void)
     for (i = 0; i < 100; i++) {
         if (atomic_cmpxchg(&data[i].n, 0, 3) == 0) {
             data[i].ret = -ECANCELED;
-            bdrv_aio_cancel(data[i].aiocb);
-            active--;
+            if (sync) {
+                bdrv_aio_cancel(data[i].aiocb);
+            } else {
+                bdrv_aio_cancel_async(data[i].aiocb);
+            }
             num_canceled++;
         }
     }
     g_assert_cmpint(active, >, 0);
     g_assert_cmpint(num_canceled, <, 100);
 
-    /* Canceling the others will be a blocking operation.  */
     for (i = 0; i < 100; i++) {
         if (data[i].aiocb && data[i].n != 3) {
-            bdrv_aio_cancel(data[i].aiocb);
+            if (sync) {
+                /* Canceling the others will be a blocking operation.  */
+                bdrv_aio_cancel(data[i].aiocb);
+            } else {
+                bdrv_aio_cancel_async(data[i].aiocb);
+            }
         }
     }
 
@@ -193,22 +201,39 @@ static void test_cancel(void)
     for (i = 0; i < 100; i++) {
         if (data[i].n == 3) {
             g_assert_cmpint(data[i].ret, ==, -ECANCELED);
-            g_assert(data[i].aiocb != NULL);
+            g_assert(data[i].aiocb == NULL);
         } else {
             g_assert_cmpint(data[i].n, ==, 2);
-            g_assert_cmpint(data[i].ret, ==, 0);
+            g_assert(data[i].ret == 0 || data[i].ret == -ECANCELED);
             g_assert(data[i].aiocb == NULL);
         }
     }
 }
 
+static void test_cancel(void)
+{
+    do_test_cancel(true);
+}
+
+static void test_cancel_async(void)
+{
+    do_test_cancel(false);
+}
+
 int main(int argc, char **argv)
 {
     int ret;
+    Error *local_error = NULL;
 
     init_clocks();
 
-    ctx = aio_context_new();
+    ctx = aio_context_new(&local_error);
+    if (!ctx) {
+        error_report("Failed to create AIO Context: '%s'",
+                     error_get_pretty(local_error));
+        error_free(local_error);
+        exit(1);
+    }
     pool = aio_get_thread_pool(ctx);
 
     g_test_init(&argc, &argv, NULL);
@@ -217,6 +242,7 @@ int main(int argc, char **argv)
     g_test_add_func("/thread-pool/submit-co", test_submit_co);
     g_test_add_func("/thread-pool/submit-many", test_submit_many);
     g_test_add_func("/thread-pool/cancel", test_cancel);
+    g_test_add_func("/thread-pool/cancel-async", test_cancel_async);
 
     ret = g_test_run();
 
diff --git a/tests/test-throttle.c b/tests/test-throttle.c
index 000ae31af..d8ba415e4 100644
--- a/tests/test-throttle.c
+++ b/tests/test-throttle.c
@@ -14,6 +14,7 @@
 #include <math.h>
 #include "block/aio.h"
 #include "qemu/throttle.h"
+#include "qemu/error-report.h"
 
 static AioContext     *ctx;
 static LeakyBucket    bkt;
@@ -492,10 +493,17 @@ static void test_accounting(void)
 int main(int argc, char **argv)
 {
     GSource *src;
+    Error *local_error = NULL;
 
     init_clocks();
 
-    ctx = aio_context_new();
+    ctx = aio_context_new(&local_error);
+    if (!ctx) {
+        error_report("Failed to create AIO Context: '%s'",
+                     error_get_pretty(local_error));
+        error_free(local_error);
+        exit(1);
+    }
     src = aio_get_g_source(ctx);
     g_source_attach(src, NULL);
     g_source_unref(src);
diff --git a/tests/test-vmstate.c b/tests/test-vmstate.c
index d72c64c90..5e0fd13cc 100644
--- a/tests/test-vmstate.c
+++ b/tests/test-vmstate.c
@@ -43,6 +43,12 @@ void yield_until_fd_readable(int fd)
     select(fd + 1, &fds, NULL, NULL, NULL);
 }
 
+/*
+ * Some tests use 'open_test_file' to work on a real fd, some use
+ * an in memory file (QEMUSizedBuffer+qemu_bufopen); we could pick one
+ * but this way we test both.
+ */
+
 /* Duplicate temp_fd and seek to the beginning of the file */
 static QEMUFile *open_test_file(bool write)
 {
@@ -54,6 +60,30 @@ static QEMUFile *open_test_file(bool write)
     return qemu_fdopen(fd, write ? "wb" : "rb");
 }
 
+/* Open a read-only qemu-file from an existing memory block */
+static QEMUFile *open_mem_file_read(const void *data, size_t len)
+{
+    /* The qsb gets freed by qemu_fclose */
+    QEMUSizedBuffer *qsb = qsb_create(data, len);
+    g_assert(qsb);
+
+    return qemu_bufopen("r", qsb);
+}
+
+/*
+ * Check that the contents of the memory-buffered file f match
+ * the given size/data.
+ */
+static void check_mem_file(QEMUFile *f, void *data, size_t size)
+{
+    uint8_t *result = g_malloc(size);
+    const QEMUSizedBuffer *qsb = qemu_buf_get(f);
+    g_assert_cmpint(qsb_get_length(qsb), ==, size);
+    g_assert_cmpint(qsb_get_buffer(qsb, 0, size, result), ==, size);
+    g_assert_cmpint(memcmp(result, data, size), ==, 0);
+    g_free(result);
+}
+
 #define SUCCESS(val) \
     g_assert_cmpint((val), ==, 0)
 
@@ -371,14 +401,12 @@ static const VMStateDescription vmstate_skipping = {
 
 static void test_save_noskip(void)
 {
-    QEMUFile *fsave = open_test_file(true);
+    QEMUFile *fsave = qemu_bufopen("w", NULL);
     TestStruct obj = { .a = 1, .b = 2, .c = 3, .d = 4, .e = 5, .f = 6,
                        .skip_c_e = false };
     vmstate_save_state(fsave, &vmstate_skipping, &obj);
     g_assert(!qemu_file_get_error(fsave));
-    qemu_fclose(fsave);
 
-    QEMUFile *loading = open_test_file(false);
     uint8_t expected[] = {
         0, 0, 0, 1,             /* a */
         0, 0, 0, 2,             /* b */
@@ -387,52 +415,31 @@ static void test_save_noskip(void)
         0, 0, 0, 5,             /* e */
         0, 0, 0, 0, 0, 0, 0, 6, /* f */
     };
-    uint8_t result[sizeof(expected)];
-    g_assert_cmpint(qemu_get_buffer(loading, result, sizeof(result)), ==,
-                    sizeof(result));
-    g_assert(!qemu_file_get_error(loading));
-    g_assert_cmpint(memcmp(result, expected, sizeof(result)), ==, 0);
-
-    /* Must reach EOF */
-    qemu_get_byte(loading);
-    g_assert_cmpint(qemu_file_get_error(loading), ==, -EIO);
-
-    qemu_fclose(loading);
+    check_mem_file(fsave, expected, sizeof(expected));
+    qemu_fclose(fsave);
 }
 
 static void test_save_skip(void)
 {
-    QEMUFile *fsave = open_test_file(true);
+    QEMUFile *fsave = qemu_bufopen("w", NULL);
     TestStruct obj = { .a = 1, .b = 2, .c = 3, .d = 4, .e = 5, .f = 6,
                        .skip_c_e = true };
     vmstate_save_state(fsave, &vmstate_skipping, &obj);
     g_assert(!qemu_file_get_error(fsave));
-    qemu_fclose(fsave);
 
-    QEMUFile *loading = open_test_file(false);
     uint8_t expected[] = {
         0, 0, 0, 1,             /* a */
         0, 0, 0, 2,             /* b */
         0, 0, 0, 0, 0, 0, 0, 4, /* d */
         0, 0, 0, 0, 0, 0, 0, 6, /* f */
     };
-    uint8_t result[sizeof(expected)];
-    g_assert_cmpint(qemu_get_buffer(loading, result, sizeof(result)), ==,
-                    sizeof(result));
-    g_assert(!qemu_file_get_error(loading));
-    g_assert_cmpint(memcmp(result, expected, sizeof(result)), ==, 0);
-
-
-    /* Must reach EOF */
-    qemu_get_byte(loading);
-    g_assert_cmpint(qemu_file_get_error(loading), ==, -EIO);
+    check_mem_file(fsave, expected, sizeof(expected));
 
-    qemu_fclose(loading);
+    qemu_fclose(fsave);
 }
 
 static void test_load_noskip(void)
 {
-    QEMUFile *fsave = open_test_file(true);
     uint8_t buf[] = {
         0, 0, 0, 10,             /* a */
         0, 0, 0, 20,             /* b */
@@ -442,10 +449,8 @@ static void test_load_noskip(void)
         0, 0, 0, 0, 0, 0, 0, 60, /* f */
         QEMU_VM_EOF, /* just to ensure we won't get EOF reported prematurely */
     };
-    qemu_put_buffer(fsave, buf, sizeof(buf));
-    qemu_fclose(fsave);
 
-    QEMUFile *loading = open_test_file(false);
+    QEMUFile *loading = open_mem_file_read(buf, sizeof(buf));
     TestStruct obj = { .skip_c_e = false };
     vmstate_load_state(loading, &vmstate_skipping, &obj, 2);
     g_assert(!qemu_file_get_error(loading));
@@ -460,7 +465,6 @@ static void test_load_noskip(void)
 
 static void test_load_skip(void)
 {
-    QEMUFile *fsave = open_test_file(true);
     uint8_t buf[] = {
         0, 0, 0, 10,             /* a */
         0, 0, 0, 20,             /* b */
@@ -468,10 +472,8 @@ static void test_load_skip(void)
         0, 0, 0, 0, 0, 0, 0, 60, /* f */
         QEMU_VM_EOF, /* just to ensure we won't get EOF reported prematurely */
     };
-    qemu_put_buffer(fsave, buf, sizeof(buf));
-    qemu_fclose(fsave);
 
-    QEMUFile *loading = open_test_file(false);
+    QEMUFile *loading = open_mem_file_read(buf, sizeof(buf));
     TestStruct obj = { .skip_c_e = true, .c = 300, .e = 500 };
     vmstate_load_state(loading, &vmstate_skipping, &obj, 2);
     g_assert(!qemu_file_get_error(loading));
diff --git a/tests/usb-hcd-ehci-test.c b/tests/usb-hcd-ehci-test.c
index bcdf62fc3..75073bf24 100644
--- a/tests/usb-hcd-ehci-test.c
+++ b/tests/usb-hcd-ehci-test.c
@@ -15,11 +15,7 @@
 #include "qemu/osdep.h"
 #include "hw/usb/uhci-regs.h"
 #include "hw/usb/ehci-regs.h"
-
-struct qhc {
-    QPCIDevice *dev;
-    void *base;
-};
+#include "libqos/usb.h"
 
 static QPCIBus *pcibus;
 static struct qhc uhci1;
@@ -29,15 +25,6 @@ static struct qhc ehci1;
 
 /* helpers */
 
-static void pci_init_one(struct qhc *hc, uint32_t devfn, int bar)
-{
-    hc->dev = qpci_device_find(pcibus, devfn);
-    g_assert(hc->dev != NULL);
-    qpci_device_enable(hc->dev);
-    hc->base = qpci_iomap(hc->dev, bar);
-    g_assert(hc->base != NULL);
-}
-
 #if 0
 static void uhci_port_update(struct qhc *hc, int port,
                              uint16_t set, uint16_t clear)
@@ -52,19 +39,6 @@ static void uhci_port_update(struct qhc *hc, int port,
 }
 #endif
 
-static void uhci_port_test(struct qhc *hc, int port, uint16_t expect)
-{
-    void *addr = hc->base + 0x10 + 2 * port;
-    uint16_t value = qpci_io_readw(hc->dev, addr);
-    uint16_t mask = ~(UHCI_PORT_WRITE_CLEAR | UHCI_PORT_RSVD1);
-
-#if 0
-    fprintf(stderr, "%s: %d, have 0x%04x, want 0x%04x\n",
-            __func__, port, value & mask, expect & mask);
-#endif
-    g_assert((value & mask) == (expect & mask));
-}
-
 static void ehci_port_test(struct qhc *hc, int port, uint32_t expect)
 {
     void *addr = hc->base + 0x64 + 4 * port;
@@ -88,10 +62,10 @@ static void pci_init(void)
     pcibus = qpci_init_pc();
     g_assert(pcibus != NULL);
 
-    pci_init_one(&uhci1, QPCI_DEVFN(0x1d, 0), 4);
-    pci_init_one(&uhci2, QPCI_DEVFN(0x1d, 1), 4);
-    pci_init_one(&uhci3, QPCI_DEVFN(0x1d, 2), 4);
-    pci_init_one(&ehci1, QPCI_DEVFN(0x1d, 7), 0);
+    qusb_pci_init_one(pcibus, &uhci1, QPCI_DEVFN(0x1d, 0), 4);
+    qusb_pci_init_one(pcibus, &uhci2, QPCI_DEVFN(0x1d, 1), 4);
+    qusb_pci_init_one(pcibus, &uhci3, QPCI_DEVFN(0x1d, 2), 4);
+    qusb_pci_init_one(pcibus, &ehci1, QPCI_DEVFN(0x1d, 7), 0);
 }
 
 static void pci_uhci_port_1(void)
@@ -154,6 +128,19 @@ static void pci_ehci_port_2(void)
     }
 }
 
+static void pci_ehci_port_3_hotplug(void)
+{
+    /* check for presence of hotplugged usb-tablet */
+    g_assert(pcibus != NULL);
+    ehci_port_test(&ehci1, 2, PORTSC_PPOWER | PORTSC_CONNECT);
+}
+
+static void pci_ehci_port_hotplug(void)
+{
+    usb_test_hotplug("ich9-ehci-1", 3, pci_ehci_port_3_hotplug);
+}
+
+
 int main(int argc, char **argv)
 {
     int ret;
@@ -165,6 +152,7 @@ int main(int argc, char **argv)
     qtest_add_func("/ehci/pci/ehci-config", pci_ehci_config);
     qtest_add_func("/ehci/pci/uhci-port-2", pci_uhci_port_2);
     qtest_add_func("/ehci/pci/ehci-port-2", pci_ehci_port_2);
+    qtest_add_func("/ehci/pci/ehci-port-3-hotplug", pci_ehci_port_hotplug);
 
     qtest_start("-machine q35 -device ich9-usb-ehci1,bus=pcie.0,addr=1d.7,"
                 "multifunction=on,id=ich9-ehci-1 "
diff --git a/tests/usb-hcd-ohci-test.c b/tests/usb-hcd-ohci-test.c
new file mode 100644
index 000000000..1160bde84
--- /dev/null
+++ b/tests/usb-hcd-ohci-test.c
@@ -0,0 +1,41 @@
+/*
+ * QTest testcase for USB OHCI controller
+ *
+ * Copyright (c) 2014 HUAWEI TECHNOLOGIES CO.,LTD.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include <glib.h>
+#include <string.h>
+#include "libqtest.h"
+#include "qemu/osdep.h"
+#include "libqos/usb.h"
+
+
+static void test_ohci_init(void)
+{
+
+}
+
+static void test_ohci_hotplug(void)
+{
+    usb_test_hotplug("ohci", 1, NULL);
+}
+
+int main(int argc, char **argv)
+{
+    int ret;
+
+    g_test_init(&argc, &argv, NULL);
+
+    qtest_add_func("/ohci/pci/init", test_ohci_init);
+    qtest_add_func("/ohci/pci/hotplug", test_ohci_hotplug);
+
+    qtest_start("-device pci-ohci,id=ohci");
+    ret = g_test_run();
+    qtest_end();
+
+    return ret;
+}
diff --git a/tests/usb-hcd-uhci-test.c b/tests/usb-hcd-uhci-test.c
new file mode 100644
index 000000000..8cf2c5bca
--- /dev/null
+++ b/tests/usb-hcd-uhci-test.c
@@ -0,0 +1,96 @@
+/*
+ * QTest testcase for USB UHCI controller
+ *
+ * Copyright (c) 2014 HUAWEI TECHNOLOGIES CO.,LTD.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include <glib.h>
+#include <string.h>
+#include "libqtest.h"
+#include "qemu/osdep.h"
+#include "libqos/usb.h"
+#include "hw/usb/uhci-regs.h"
+
+
+static void test_uhci_init(void)
+{
+}
+
+static void test_port(int port)
+{
+    QPCIBus *pcibus;
+    struct qhc uhci;
+
+    g_assert(port > 0);
+    pcibus = qpci_init_pc();
+    g_assert(pcibus != NULL);
+    qusb_pci_init_one(pcibus, &uhci, QPCI_DEVFN(0x1d, 0), 4);
+    uhci_port_test(&uhci, port - 1, UHCI_PORT_CCS);
+}
+
+static void test_port_1(void)
+{
+    test_port(1);
+}
+
+static void test_port_2(void)
+{
+    test_port(2);
+}
+
+static void test_uhci_hotplug(void)
+{
+    usb_test_hotplug("uhci", 2, test_port_2);
+}
+
+static void test_usb_storage_hotplug(void)
+{
+    QDict *response;
+
+    response = qmp("{'execute': 'device_add',"
+                   " 'arguments': {"
+                   "   'driver': 'usb-storage',"
+                   "   'drive': 'drive0',"
+                   "   'id': 'usbdev0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'device_del',"
+                           " 'arguments': {"
+                           "   'id': 'usbdev0'"
+                           "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("");
+    g_assert(response);
+    g_assert(qdict_haskey(response, "event"));
+    g_assert(!strcmp(qdict_get_str(response, "event"), "DEVICE_DELETED"));
+    QDECREF(response);
+}
+
+int main(int argc, char **argv)
+{
+    int ret;
+
+    g_test_init(&argc, &argv, NULL);
+
+    qtest_add_func("/uhci/pci/init", test_uhci_init);
+    qtest_add_func("/uhci/pci/port1", test_port_1);
+    qtest_add_func("/uhci/pci/hotplug", test_uhci_hotplug);
+    qtest_add_func("/uhci/pci/hotplug/usb-storage", test_usb_storage_hotplug);
+
+    qtest_start("-device piix3-usb-uhci,id=uhci,addr=1d.0"
+                " -drive id=drive0,if=none,file=/dev/null"
+                " -device usb-tablet,bus=uhci.0,port=1");
+    ret = g_test_run();
+    qtest_end();
+
+    return ret;
+}
diff --git a/tests/usb-hcd-xhci-test.c b/tests/usb-hcd-xhci-test.c
new file mode 100644
index 000000000..b1a7dec5b
--- /dev/null
+++ b/tests/usb-hcd-xhci-test.c
@@ -0,0 +1,99 @@
+/*
+ * QTest testcase for USB xHCI controller
+ *
+ * Copyright (c) 2014 HUAWEI TECHNOLOGIES CO.,LTD.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include <glib.h>
+#include <string.h>
+#include "libqtest.h"
+#include "qemu/osdep.h"
+#include "libqos/usb.h"
+
+
+static void test_xhci_init(void)
+{
+}
+
+static void test_xhci_hotplug(void)
+{
+    usb_test_hotplug("xhci", 1, NULL);
+}
+
+static void test_usb_uas_hotplug(void)
+{
+    QDict *response;
+
+    response = qmp("{'execute': 'device_add',"
+                   " 'arguments': {"
+                   "   'driver': 'usb-uas',"
+                   "   'id': 'uas'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'device_add',"
+                   " 'arguments': {"
+                   "   'driver': 'scsi-hd',"
+                   "   'drive': 'drive0',"
+                   "   'id': 'scsi-hd'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    /* TODO:
+        UAS HBA driver in libqos, to check that
+        added disk is visible after BUS rescan
+    */
+
+    response = qmp("{'execute': 'device_del',"
+                           " 'arguments': {"
+                           "   'id': 'scsi-hd'"
+                           "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("");
+    g_assert(qdict_haskey(response, "event"));
+    g_assert(!strcmp(qdict_get_str(response, "event"), "DEVICE_DELETED"));
+    QDECREF(response);
+
+
+    response = qmp("{'execute': 'device_del',"
+                           " 'arguments': {"
+                           "   'id': 'uas'"
+                           "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("");
+    g_assert(response);
+    g_assert(qdict_haskey(response, "event"));
+    g_assert(!strcmp(qdict_get_str(response, "event"), "DEVICE_DELETED"));
+    QDECREF(response);
+}
+
+int main(int argc, char **argv)
+{
+    int ret;
+
+    g_test_init(&argc, &argv, NULL);
+
+    qtest_add_func("/xhci/pci/init", test_xhci_init);
+    qtest_add_func("/xhci/pci/hotplug", test_xhci_hotplug);
+    qtest_add_func("/xhci/pci/hotplug/usb-uas", test_usb_uas_hotplug);
+
+    qtest_start("-device nec-usb-xhci,id=xhci"
+                " -drive id=drive0,if=none,file=/dev/null");
+    ret = g_test_run();
+    qtest_end();
+
+    return ret;
+}
diff --git a/tests/virtio-blk-test.c b/tests/virtio-blk-test.c
index d53f875b8..ead3911e2 100644
--- a/tests/virtio-blk-test.c
+++ b/tests/virtio-blk-test.c
@@ -2,6 +2,7 @@
  * QTest testcase for VirtIO Block Device
  *
  * Copyright (c) 2014 SUSE LINUX Products GmbH
+ * Copyright (c) 2014 Marc Marí
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
@@ -9,12 +10,657 @@
 
 #include <glib.h>
 #include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <stdio.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
+#include "libqos/virtio.h"
+#include "libqos/virtio-pci.h"
+#include "libqos/pci-pc.h"
+#include "libqos/malloc.h"
+#include "libqos/malloc-pc.h"
+#include "qemu/bswap.h"
 
-/* Tests only initialization so far. TODO: Replace with functional tests */
-static void pci_nop(void)
+#define QVIRTIO_BLK_F_BARRIER       0x00000001
+#define QVIRTIO_BLK_F_SIZE_MAX      0x00000002
+#define QVIRTIO_BLK_F_SEG_MAX       0x00000004
+#define QVIRTIO_BLK_F_GEOMETRY      0x00000010
+#define QVIRTIO_BLK_F_RO            0x00000020
+#define QVIRTIO_BLK_F_BLK_SIZE      0x00000040
+#define QVIRTIO_BLK_F_SCSI          0x00000080
+#define QVIRTIO_BLK_F_WCE           0x00000200
+#define QVIRTIO_BLK_F_TOPOLOGY      0x00000400
+#define QVIRTIO_BLK_F_CONFIG_WCE    0x00000800
+
+#define QVIRTIO_BLK_T_IN            0
+#define QVIRTIO_BLK_T_OUT           1
+#define QVIRTIO_BLK_T_SCSI_CMD      2
+#define QVIRTIO_BLK_T_SCSI_CMD_OUT  3
+#define QVIRTIO_BLK_T_FLUSH         4
+#define QVIRTIO_BLK_T_FLUSH_OUT     5
+#define QVIRTIO_BLK_T_GET_ID        8
+
+#define TEST_IMAGE_SIZE         (64 * 1024 * 1024)
+#define QVIRTIO_BLK_TIMEOUT_US  (30 * 1000 * 1000)
+#define PCI_SLOT                0x04
+#define PCI_FN                  0x00
+
+#define PCI_SLOT_HP             0x06
+
+typedef struct QVirtioBlkReq {
+    uint32_t type;
+    uint32_t ioprio;
+    uint64_t sector;
+    char *data;
+    uint8_t status;
+} QVirtioBlkReq;
+
+static QPCIBus *test_start(void)
+{
+    char *cmdline;
+    char tmp_path[] = "/tmp/qtest.XXXXXX";
+    int fd, ret;
+
+    /* Create a temporary raw image */
+    fd = mkstemp(tmp_path);
+    g_assert_cmpint(fd, >=, 0);
+    ret = ftruncate(fd, TEST_IMAGE_SIZE);
+    g_assert_cmpint(ret, ==, 0);
+    close(fd);
+
+    cmdline = g_strdup_printf("-drive if=none,id=drive0,file=%s "
+                              "-drive if=none,id=drive1,file=/dev/null "
+                              "-device virtio-blk-pci,id=drv0,drive=drive0,"
+                              "addr=%x.%x",
+                              tmp_path, PCI_SLOT, PCI_FN);
+    qtest_start(cmdline);
+    unlink(tmp_path);
+    g_free(cmdline);
+
+    return qpci_init_pc();
+}
+
+static void test_end(void)
 {
+    qtest_end();
+}
+
+static QVirtioPCIDevice *virtio_blk_init(QPCIBus *bus, int slot)
+{
+    QVirtioPCIDevice *dev;
+
+    dev = qvirtio_pci_device_find(bus, QVIRTIO_BLK_DEVICE_ID);
+    g_assert(dev != NULL);
+    g_assert_cmphex(dev->vdev.device_type, ==, QVIRTIO_BLK_DEVICE_ID);
+    g_assert_cmphex(dev->pdev->devfn, ==, ((slot << 3) | PCI_FN));
+
+    qvirtio_pci_device_enable(dev);
+    qvirtio_reset(&qvirtio_pci, &dev->vdev);
+    qvirtio_set_acknowledge(&qvirtio_pci, &dev->vdev);
+    qvirtio_set_driver(&qvirtio_pci, &dev->vdev);
+
+    return dev;
+}
+
+static inline void virtio_blk_fix_request(QVirtioBlkReq *req)
+{
+#ifdef HOST_WORDS_BIGENDIAN
+    bool host_endian = true;
+#else
+    bool host_endian = false;
+#endif
+
+    if (qtest_big_endian() != host_endian) {
+        req->type = bswap32(req->type);
+        req->ioprio = bswap32(req->ioprio);
+        req->sector = bswap64(req->sector);
+    }
+}
+
+static uint64_t virtio_blk_request(QGuestAllocator *alloc, QVirtioBlkReq *req,
+                                                            uint64_t data_size)
+{
+    uint64_t addr;
+    uint8_t status = 0xFF;
+
+    g_assert_cmpuint(data_size % 512, ==, 0);
+    addr = guest_alloc(alloc, sizeof(*req) + data_size);
+
+    virtio_blk_fix_request(req);
+
+    memwrite(addr, req, 16);
+    memwrite(addr + 16, req->data, data_size);
+    memwrite(addr + 16 + data_size, &status, sizeof(status));
+
+    return addr;
+}
+
+static void pci_basic(void)
+{
+    QVirtioPCIDevice *dev;
+    QPCIBus *bus;
+    QVirtQueuePCI *vqpci;
+    QGuestAllocator *alloc;
+    QVirtioBlkReq req;
+    void *addr;
+    uint64_t req_addr;
+    uint64_t capacity;
+    uint32_t features;
+    uint32_t free_head;
+    uint8_t status;
+    char *data;
+
+    bus = test_start();
+
+    dev = virtio_blk_init(bus, PCI_SLOT);
+
+    /* MSI-X is not enabled */
+    addr = dev->addr + QVIRTIO_DEVICE_SPECIFIC_NO_MSIX;
+
+    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev, addr);
+    g_assert_cmpint(capacity, ==, TEST_IMAGE_SIZE / 512);
+
+    features = qvirtio_get_features(&qvirtio_pci, &dev->vdev);
+    features = features & ~(QVIRTIO_F_BAD_FEATURE |
+                    QVIRTIO_F_RING_INDIRECT_DESC | QVIRTIO_F_RING_EVENT_IDX |
+                            QVIRTIO_BLK_F_SCSI);
+    qvirtio_set_features(&qvirtio_pci, &dev->vdev, features);
+
+    alloc = pc_alloc_init();
+    vqpci = (QVirtQueuePCI *)qvirtqueue_setup(&qvirtio_pci, &dev->vdev,
+                                                                    alloc, 0);
+
+    qvirtio_set_driver_ok(&qvirtio_pci, &dev->vdev);
+
+    /* Write and read with 2 descriptor layout */
+    /* Write request */
+    req.type = QVIRTIO_BLK_T_OUT;
+    req.ioprio = 1;
+    req.sector = 0;
+    req.data = g_malloc0(512);
+    strcpy(req.data, "TEST");
+
+    req_addr = virtio_blk_request(alloc, &req, 512);
+
+    g_free(req.data);
+
+    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 528, false, true);
+    qvirtqueue_add(&vqpci->vq, req_addr + 528, 1, true, false);
+    qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
+
+    qvirtio_wait_queue_isr(&qvirtio_pci, &dev->vdev, &vqpci->vq,
+                           QVIRTIO_BLK_TIMEOUT_US);
+    status = readb(req_addr + 528);
+    g_assert_cmpint(status, ==, 0);
+
+    guest_free(alloc, req_addr);
+
+    /* Read request */
+    req.type = QVIRTIO_BLK_T_IN;
+    req.ioprio = 1;
+    req.sector = 0;
+    req.data = g_malloc0(512);
+
+    req_addr = virtio_blk_request(alloc, &req, 512);
+
+    g_free(req.data);
+
+    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 16, false, true);
+    qvirtqueue_add(&vqpci->vq, req_addr + 16, 513, true, false);
+
+    qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
+
+    qvirtio_wait_queue_isr(&qvirtio_pci, &dev->vdev, &vqpci->vq,
+                           QVIRTIO_BLK_TIMEOUT_US);
+    status = readb(req_addr + 528);
+    g_assert_cmpint(status, ==, 0);
+
+    data = g_malloc0(512);
+    memread(req_addr + 16, data, 512);
+    g_assert_cmpstr(data, ==, "TEST");
+    g_free(data);
+
+    guest_free(alloc, req_addr);
+
+    /* Write and read with 3 descriptor layout */
+    /* Write request */
+    req.type = QVIRTIO_BLK_T_OUT;
+    req.ioprio = 1;
+    req.sector = 1;
+    req.data = g_malloc0(512);
+    strcpy(req.data, "TEST");
+
+    req_addr = virtio_blk_request(alloc, &req, 512);
+
+    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 16, false, true);
+    qvirtqueue_add(&vqpci->vq, req_addr + 16, 512, false, true);
+    qvirtqueue_add(&vqpci->vq, req_addr + 528, 1, true, false);
+
+    qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
+
+    qvirtio_wait_queue_isr(&qvirtio_pci, &dev->vdev, &vqpci->vq,
+                           QVIRTIO_BLK_TIMEOUT_US);
+    status = readb(req_addr + 528);
+    g_assert_cmpint(status, ==, 0);
+
+    guest_free(alloc, req_addr);
+
+    /* Read request */
+    req.type = QVIRTIO_BLK_T_IN;
+    req.ioprio = 1;
+    req.sector = 1;
+    req.data = g_malloc0(512);
+
+    req_addr = virtio_blk_request(alloc, &req, 512);
+
+    g_free(req.data);
+
+    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 16, false, true);
+    qvirtqueue_add(&vqpci->vq, req_addr + 16, 512, true, true);
+    qvirtqueue_add(&vqpci->vq, req_addr + 528, 1, true, false);
+
+    qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
+
+    qvirtio_wait_queue_isr(&qvirtio_pci, &dev->vdev, &vqpci->vq,
+                           QVIRTIO_BLK_TIMEOUT_US);
+    status = readb(req_addr + 528);
+    g_assert_cmpint(status, ==, 0);
+
+    data = g_malloc0(512);
+    memread(req_addr + 16, data, 512);
+    g_assert_cmpstr(data, ==, "TEST");
+    g_free(data);
+
+    guest_free(alloc, req_addr);
+
+    /* End test */
+    guest_free(alloc, vqpci->vq.desc);
+    qvirtio_pci_device_disable(dev);
+    g_free(dev);
+    test_end();
+}
+
+static void pci_indirect(void)
+{
+    QVirtioPCIDevice *dev;
+    QPCIBus *bus;
+    QVirtQueuePCI *vqpci;
+    QGuestAllocator *alloc;
+    QVirtioBlkReq req;
+    QVRingIndirectDesc *indirect;
+    void *addr;
+    uint64_t req_addr;
+    uint64_t capacity;
+    uint32_t features;
+    uint32_t free_head;
+    uint8_t status;
+    char *data;
+
+    bus = test_start();
+
+    dev = virtio_blk_init(bus, PCI_SLOT);
+
+    /* MSI-X is not enabled */
+    addr = dev->addr + QVIRTIO_DEVICE_SPECIFIC_NO_MSIX;
+
+    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev, addr);
+    g_assert_cmpint(capacity, ==, TEST_IMAGE_SIZE / 512);
+
+    features = qvirtio_get_features(&qvirtio_pci, &dev->vdev);
+    g_assert_cmphex(features & QVIRTIO_F_RING_INDIRECT_DESC, !=, 0);
+    features = features & ~(QVIRTIO_F_BAD_FEATURE | QVIRTIO_F_RING_EVENT_IDX |
+                                                            QVIRTIO_BLK_F_SCSI);
+    qvirtio_set_features(&qvirtio_pci, &dev->vdev, features);
+
+    alloc = pc_alloc_init();
+    vqpci = (QVirtQueuePCI *)qvirtqueue_setup(&qvirtio_pci, &dev->vdev,
+                                                                    alloc, 0);
+    qvirtio_set_driver_ok(&qvirtio_pci, &dev->vdev);
+
+    /* Write request */
+    req.type = QVIRTIO_BLK_T_OUT;
+    req.ioprio = 1;
+    req.sector = 0;
+    req.data = g_malloc0(512);
+    strcpy(req.data, "TEST");
+
+    req_addr = virtio_blk_request(alloc, &req, 512);
+
+    g_free(req.data);
+
+    indirect = qvring_indirect_desc_setup(&dev->vdev, alloc, 2);
+    qvring_indirect_desc_add(indirect, req_addr, 528, false);
+    qvring_indirect_desc_add(indirect, req_addr + 528, 1, true);
+    free_head = qvirtqueue_add_indirect(&vqpci->vq, indirect);
+    qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
+
+    qvirtio_wait_queue_isr(&qvirtio_pci, &dev->vdev, &vqpci->vq,
+                           QVIRTIO_BLK_TIMEOUT_US);
+    status = readb(req_addr + 528);
+    g_assert_cmpint(status, ==, 0);
+
+    g_free(indirect);
+    guest_free(alloc, req_addr);
+
+    /* Read request */
+    req.type = QVIRTIO_BLK_T_IN;
+    req.ioprio = 1;
+    req.sector = 0;
+    req.data = g_malloc0(512);
+    strcpy(req.data, "TEST");
+
+    req_addr = virtio_blk_request(alloc, &req, 512);
+
+    g_free(req.data);
+
+    indirect = qvring_indirect_desc_setup(&dev->vdev, alloc, 2);
+    qvring_indirect_desc_add(indirect, req_addr, 16, false);
+    qvring_indirect_desc_add(indirect, req_addr + 16, 513, true);
+    free_head = qvirtqueue_add_indirect(&vqpci->vq, indirect);
+    qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
+
+    qvirtio_wait_queue_isr(&qvirtio_pci, &dev->vdev, &vqpci->vq,
+                           QVIRTIO_BLK_TIMEOUT_US);
+    status = readb(req_addr + 528);
+    g_assert_cmpint(status, ==, 0);
+
+    data = g_malloc0(512);
+    memread(req_addr + 16, data, 512);
+    g_assert_cmpstr(data, ==, "TEST");
+    g_free(data);
+
+    g_free(indirect);
+    guest_free(alloc, req_addr);
+
+    /* End test */
+    guest_free(alloc, vqpci->vq.desc);
+    qvirtio_pci_device_disable(dev);
+    g_free(dev);
+    test_end();
+}
+
+static void pci_config(void)
+{
+    QVirtioPCIDevice *dev;
+    QPCIBus *bus;
+    int n_size = TEST_IMAGE_SIZE / 2;
+    void *addr;
+    uint64_t capacity;
+
+    bus = test_start();
+
+    dev = virtio_blk_init(bus, PCI_SLOT);
+
+    /* MSI-X is not enabled */
+    addr = dev->addr + QVIRTIO_DEVICE_SPECIFIC_NO_MSIX;
+
+    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev, addr);
+    g_assert_cmpint(capacity, ==, TEST_IMAGE_SIZE / 512);
+
+    qvirtio_set_driver_ok(&qvirtio_pci, &dev->vdev);
+
+    qmp("{ 'execute': 'block_resize', 'arguments': { 'device': 'drive0', "
+                                                    " 'size': %d } }", n_size);
+    qvirtio_wait_config_isr(&qvirtio_pci, &dev->vdev, QVIRTIO_BLK_TIMEOUT_US);
+
+    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev, addr);
+    g_assert_cmpint(capacity, ==, n_size / 512);
+
+    qvirtio_pci_device_disable(dev);
+    g_free(dev);
+    test_end();
+}
+
+static void pci_msix(void)
+{
+    QVirtioPCIDevice *dev;
+    QPCIBus *bus;
+    QVirtQueuePCI *vqpci;
+    QGuestAllocator *alloc;
+    QVirtioBlkReq req;
+    int n_size = TEST_IMAGE_SIZE / 2;
+    void *addr;
+    uint64_t req_addr;
+    uint64_t capacity;
+    uint32_t features;
+    uint32_t free_head;
+    uint8_t status;
+    char *data;
+
+    bus = test_start();
+    alloc = pc_alloc_init();
+
+    dev = virtio_blk_init(bus, PCI_SLOT);
+    qpci_msix_enable(dev->pdev);
+
+    qvirtio_pci_set_msix_configuration_vector(dev, alloc, 0);
+
+    /* MSI-X is enabled */
+    addr = dev->addr + QVIRTIO_DEVICE_SPECIFIC_MSIX;
+
+    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev, addr);
+    g_assert_cmpint(capacity, ==, TEST_IMAGE_SIZE / 512);
+
+    features = qvirtio_get_features(&qvirtio_pci, &dev->vdev);
+    features = features & ~(QVIRTIO_F_BAD_FEATURE |
+                            QVIRTIO_F_RING_INDIRECT_DESC |
+                            QVIRTIO_F_RING_EVENT_IDX | QVIRTIO_BLK_F_SCSI);
+    qvirtio_set_features(&qvirtio_pci, &dev->vdev, features);
+
+    vqpci = (QVirtQueuePCI *)qvirtqueue_setup(&qvirtio_pci, &dev->vdev,
+                                                                    alloc, 0);
+    qvirtqueue_pci_msix_setup(dev, vqpci, alloc, 1);
+
+    qvirtio_set_driver_ok(&qvirtio_pci, &dev->vdev);
+
+    qmp("{ 'execute': 'block_resize', 'arguments': { 'device': 'drive0', "
+                                                    " 'size': %d } }", n_size);
+
+    qvirtio_wait_config_isr(&qvirtio_pci, &dev->vdev, QVIRTIO_BLK_TIMEOUT_US);
+
+    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev, addr);
+    g_assert_cmpint(capacity, ==, n_size / 512);
+
+    /* Write request */
+    req.type = QVIRTIO_BLK_T_OUT;
+    req.ioprio = 1;
+    req.sector = 0;
+    req.data = g_malloc0(512);
+    strcpy(req.data, "TEST");
+
+    req_addr = virtio_blk_request(alloc, &req, 512);
+
+    g_free(req.data);
+
+    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 528, false, true);
+    qvirtqueue_add(&vqpci->vq, req_addr + 528, 1, true, false);
+    qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
+
+    qvirtio_wait_queue_isr(&qvirtio_pci, &dev->vdev, &vqpci->vq,
+                           QVIRTIO_BLK_TIMEOUT_US);
+
+    status = readb(req_addr + 528);
+    g_assert_cmpint(status, ==, 0);
+
+    guest_free(alloc, req_addr);
+
+    /* Read request */
+    req.type = QVIRTIO_BLK_T_IN;
+    req.ioprio = 1;
+    req.sector = 0;
+    req.data = g_malloc0(512);
+
+    req_addr = virtio_blk_request(alloc, &req, 512);
+
+    g_free(req.data);
+
+    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 16, false, true);
+    qvirtqueue_add(&vqpci->vq, req_addr + 16, 513, true, false);
+
+    qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
+
+
+    qvirtio_wait_queue_isr(&qvirtio_pci, &dev->vdev, &vqpci->vq,
+                           QVIRTIO_BLK_TIMEOUT_US);
+
+    status = readb(req_addr + 528);
+    g_assert_cmpint(status, ==, 0);
+
+    data = g_malloc0(512);
+    memread(req_addr + 16, data, 512);
+    g_assert_cmpstr(data, ==, "TEST");
+    g_free(data);
+
+    guest_free(alloc, req_addr);
+
+    /* End test */
+    guest_free(alloc, (uint64_t)vqpci->vq.desc);
+    qpci_msix_disable(dev->pdev);
+    qvirtio_pci_device_disable(dev);
+    g_free(dev);
+    test_end();
+}
+
+static void pci_idx(void)
+{
+    QVirtioPCIDevice *dev;
+    QPCIBus *bus;
+    QVirtQueuePCI *vqpci;
+    QGuestAllocator *alloc;
+    QVirtioBlkReq req;
+    void *addr;
+    uint64_t req_addr;
+    uint64_t capacity;
+    uint32_t features;
+    uint32_t free_head;
+    uint8_t status;
+    char *data;
+
+    bus = test_start();
+    alloc = pc_alloc_init();
+
+    dev = virtio_blk_init(bus, PCI_SLOT);
+    qpci_msix_enable(dev->pdev);
+
+    qvirtio_pci_set_msix_configuration_vector(dev, alloc, 0);
+
+    /* MSI-X is enabled */
+    addr = dev->addr + QVIRTIO_DEVICE_SPECIFIC_MSIX;
+
+    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev, addr);
+    g_assert_cmpint(capacity, ==, TEST_IMAGE_SIZE / 512);
+
+    features = qvirtio_get_features(&qvirtio_pci, &dev->vdev);
+    features = features & ~(QVIRTIO_F_BAD_FEATURE |
+                            QVIRTIO_F_RING_INDIRECT_DESC |
+                            QVIRTIO_F_NOTIFY_ON_EMPTY | QVIRTIO_BLK_F_SCSI);
+    qvirtio_set_features(&qvirtio_pci, &dev->vdev, features);
+
+    vqpci = (QVirtQueuePCI *)qvirtqueue_setup(&qvirtio_pci, &dev->vdev,
+                                                                    alloc, 0);
+    qvirtqueue_pci_msix_setup(dev, vqpci, alloc, 1);
+
+    qvirtio_set_driver_ok(&qvirtio_pci, &dev->vdev);
+
+    /* Write request */
+    req.type = QVIRTIO_BLK_T_OUT;
+    req.ioprio = 1;
+    req.sector = 0;
+    req.data = g_malloc0(512);
+    strcpy(req.data, "TEST");
+
+    req_addr = virtio_blk_request(alloc, &req, 512);
+
+    g_free(req.data);
+
+    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 528, false, true);
+    qvirtqueue_add(&vqpci->vq, req_addr + 528, 1, true, false);
+    qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
+
+    qvirtio_wait_queue_isr(&qvirtio_pci, &dev->vdev, &vqpci->vq,
+                           QVIRTIO_BLK_TIMEOUT_US);
+
+    /* Write request */
+    req.type = QVIRTIO_BLK_T_OUT;
+    req.ioprio = 1;
+    req.sector = 1;
+    req.data = g_malloc0(512);
+    strcpy(req.data, "TEST");
+
+    req_addr = virtio_blk_request(alloc, &req, 512);
+
+    g_free(req.data);
+
+    /* Notify after processing the third request */
+    qvirtqueue_set_used_event(&vqpci->vq, 2);
+    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 528, false, true);
+    qvirtqueue_add(&vqpci->vq, req_addr + 528, 1, true, false);
+    qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
+
+    /* No notification expected */
+    status = qvirtio_wait_status_byte_no_isr(&qvirtio_pci, &dev->vdev,
+                                             &vqpci->vq, req_addr + 528,
+                                             QVIRTIO_BLK_TIMEOUT_US);
+    g_assert_cmpint(status, ==, 0);
+
+    guest_free(alloc, req_addr);
+
+    /* Read request */
+    req.type = QVIRTIO_BLK_T_IN;
+    req.ioprio = 1;
+    req.sector = 1;
+    req.data = g_malloc0(512);
+
+    req_addr = virtio_blk_request(alloc, &req, 512);
+
+    g_free(req.data);
+
+    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 16, false, true);
+    qvirtqueue_add(&vqpci->vq, req_addr + 16, 513, true, false);
+
+    qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
+
+
+    qvirtio_wait_queue_isr(&qvirtio_pci, &dev->vdev, &vqpci->vq,
+                           QVIRTIO_BLK_TIMEOUT_US);
+
+    status = readb(req_addr + 528);
+    g_assert_cmpint(status, ==, 0);
+
+    data = g_malloc0(512);
+    memread(req_addr + 16, data, 512);
+    g_assert_cmpstr(data, ==, "TEST");
+    g_free(data);
+
+    guest_free(alloc, req_addr);
+
+    /* End test */
+    guest_free(alloc, vqpci->vq.desc);
+    qpci_msix_disable(dev->pdev);
+    qvirtio_pci_device_disable(dev);
+    g_free(dev);
+    test_end();
+}
+
+static void hotplug(void)
+{
+    QPCIBus *bus;
+    QVirtioPCIDevice *dev;
+
+    bus = test_start();
+
+    /* plug secondary disk */
+    qpci_plug_device_test("virtio-blk-pci", "drv1", PCI_SLOT_HP,
+                          "'drive': 'drive1'");
+
+    dev = virtio_blk_init(bus, PCI_SLOT_HP);
+    g_assert(dev);
+    qvirtio_pci_device_disable(dev);
+    g_free(dev);
+
+    /* unplug secondary disk */
+    qpci_unplug_acpi_device_test("drv1", PCI_SLOT_HP);
+    test_end();
 }
 
 int main(int argc, char **argv)
@@ -22,13 +668,15 @@ int main(int argc, char **argv)
     int ret;
 
     g_test_init(&argc, &argv, NULL);
-    qtest_add_func("/virtio/blk/pci/nop", pci_nop);
 
-    qtest_start("-drive id=drv0,if=none,file=/dev/null "
-                "-device virtio-blk-pci,drive=drv0");
-    ret = g_test_run();
+    g_test_add_func("/virtio/blk/pci/basic", pci_basic);
+    g_test_add_func("/virtio/blk/pci/indirect", pci_indirect);
+    g_test_add_func("/virtio/blk/pci/config", pci_config);
+    g_test_add_func("/virtio/blk/pci/msix", pci_msix);
+    g_test_add_func("/virtio/blk/pci/idx", pci_idx);
+    g_test_add_func("/virtio/blk/pci/hotplug", hotplug);
 
-    qtest_end();
+    ret = g_test_run();
 
     return ret;
 }
diff --git a/tests/virtio-net-test.c b/tests/virtio-net-test.c
index df9934323..ea7478c27 100644
--- a/tests/virtio-net-test.c
+++ b/tests/virtio-net-test.c
@@ -11,18 +11,28 @@
 #include <string.h>
 #include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqos/pci.h"
+
+#define PCI_SLOT_HP             0x06
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void pci_nop(void)
 {
 }
 
+static void hotplug(void)
+{
+    qpci_plug_device_test("virtio-net-pci", "net1", PCI_SLOT_HP, NULL);
+    qpci_unplug_acpi_device_test("net1", PCI_SLOT_HP);
+}
+
 int main(int argc, char **argv)
 {
     int ret;
 
     g_test_init(&argc, &argv, NULL);
     qtest_add_func("/virtio/net/pci/nop", pci_nop);
+    qtest_add_func("/virtio/net/pci/hotplug", hotplug);
 
     qtest_start("-device virtio-net-pci");
     ret = g_test_run();
diff --git a/tests/virtio-rng-test.c b/tests/virtio-rng-test.c
index 402c2060d..41c1cdb1a 100644
--- a/tests/virtio-rng-test.c
+++ b/tests/virtio-rng-test.c
@@ -11,18 +11,28 @@
 #include <string.h>
 #include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqos/pci.h"
+
+#define PCI_SLOT_HP             0x06
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void pci_nop(void)
 {
 }
 
+static void hotplug(void)
+{
+    qpci_plug_device_test("virtio-rng-pci", "rng1", PCI_SLOT_HP, NULL);
+    qpci_unplug_acpi_device_test("rng1", PCI_SLOT_HP);
+}
+
 int main(int argc, char **argv)
 {
     int ret;
 
     g_test_init(&argc, &argv, NULL);
     qtest_add_func("/virtio/rng/pci/nop", pci_nop);
+    qtest_add_func("/virtio/rng/pci/hotplug", hotplug);
 
     qtest_start("-device virtio-rng-pci");
     ret = g_test_run();
diff --git a/tests/virtio-scsi-test.c b/tests/virtio-scsi-test.c
index 3230908b9..41f9602e4 100644
--- a/tests/virtio-scsi-test.c
+++ b/tests/virtio-scsi-test.c
@@ -17,14 +17,43 @@ static void pci_nop(void)
 {
 }
 
+static void hotplug(void)
+{
+    QDict *response;
+
+    response = qmp("{\"execute\": \"device_add\","
+                   " \"arguments\": {"
+                   "   \"driver\": \"scsi-hd\","
+                   "   \"id\": \"scsi-hd\","
+                   "   \"drive\": \"drv1\""
+                   "}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{\"execute\": \"device_del\","
+                   " \"arguments\": {"
+                   "   \"id\": \"scsi-hd\""
+                   "}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    g_assert(qdict_haskey(response, "event"));
+    g_assert(!strcmp(qdict_get_str(response, "event"), "DEVICE_DELETED"));
+    QDECREF(response);
+}
+
 int main(int argc, char **argv)
 {
     int ret;
 
     g_test_init(&argc, &argv, NULL);
     qtest_add_func("/virtio/scsi/pci/nop", pci_nop);
+    qtest_add_func("/virtio/scsi/pci/hotplug", hotplug);
 
     qtest_start("-drive id=drv0,if=none,file=/dev/null "
+                "-drive id=drv1,if=none,file=/dev/null "
                 "-device virtio-scsi-pci,id=vscsi0 "
                 "-device scsi-hd,bus=vscsi0.0,drive=drv0");
     ret = g_test_run();
diff --git a/tests/virtio-serial-test.c b/tests/virtio-serial-test.c
index e7438751e..bf030a616 100644
--- a/tests/virtio-serial-test.c
+++ b/tests/virtio-serial-test.c
@@ -17,12 +17,39 @@ static void pci_nop(void)
 {
 }
 
+static void hotplug(void)
+{
+    QDict *response;
+
+    response = qmp("{\"execute\": \"device_add\","
+                   " \"arguments\": {"
+                   "   \"driver\": \"virtserialport\","
+                   "   \"id\": \"hp-port\""
+                   "}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{\"execute\": \"device_del\","
+                   " \"arguments\": {"
+                   "   \"id\": \"hp-port\""
+                   "}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    g_assert(qdict_haskey(response, "event"));
+    g_assert(!strcmp(qdict_get_str(response, "event"), "DEVICE_DELETED"));
+    QDECREF(response);
+}
+
 int main(int argc, char **argv)
 {
     int ret;
 
     g_test_init(&argc, &argv, NULL);
     qtest_add_func("/virtio/serial/pci/nop", pci_nop);
+    qtest_add_func("/virtio/serial/pci/hotplug", hotplug);
 
     qtest_start("-device virtio-serial-pci");
     ret = g_test_run();
diff --git a/thread-pool.c b/thread-pool.c
index dfb699dd9..e2cac8e4f 100644
--- a/thread-pool.c
+++ b/thread-pool.c
@@ -20,8 +20,6 @@
 #include "qemu/osdep.h"
 #include "block/coroutine.h"
 #include "trace.h"
-#include "block/block_int.h"
-#include "qemu/event_notifier.h"
 #include "block/thread-pool.h"
 #include "qemu/main-loop.h"
 
@@ -33,11 +31,10 @@ enum ThreadState {
     THREAD_QUEUED,
     THREAD_ACTIVE,
     THREAD_DONE,
-    THREAD_CANCELED,
 };
 
 struct ThreadPoolElement {
-    BlockDriverAIOCB common;
+    BlockAIOCB common;
     ThreadPool *pool;
     ThreadPoolFunc *func;
     void *arg;
@@ -57,10 +54,9 @@ struct ThreadPoolElement {
 };
 
 struct ThreadPool {
-    EventNotifier notifier;
     AioContext *ctx;
+    QEMUBH *completion_bh;
     QemuMutex lock;
-    QemuCond check_cancel;
     QemuCond worker_stopped;
     QemuSemaphore sem;
     int max_threads;
@@ -75,7 +71,6 @@ struct ThreadPool {
     int idle_threads;
     int new_threads;     /* backlog of threads we need to create */
     int pending_threads; /* threads created but not running yet */
-    int pending_cancellations; /* whether we need a cond_broadcast */
     bool stopping;
 };
 
@@ -115,11 +110,8 @@ static void *worker_thread(void *opaque)
         req->state = THREAD_DONE;
 
         qemu_mutex_lock(&pool->lock);
-        if (pool->pending_cancellations) {
-            qemu_cond_broadcast(&pool->check_cancel);
-        }
 
-        event_notifier_set(&pool->notifier);
+        qemu_bh_schedule(pool->completion_bh);
     }
 
     pool->cur_threads--;
@@ -168,15 +160,14 @@ static void spawn_thread(ThreadPool *pool)
     }
 }
 
-static void event_notifier_ready(EventNotifier *notifier)
+static void thread_pool_completion_bh(void *opaque)
 {
-    ThreadPool *pool = container_of(notifier, ThreadPool, notifier);
+    ThreadPool *pool = opaque;
     ThreadPoolElement *elem, *next;
 
-    event_notifier_test_and_clear(notifier);
 restart:
     QLIST_FOREACH_SAFE(elem, &pool->head, all, next) {
-        if (elem->state != THREAD_CANCELED && elem->state != THREAD_DONE) {
+        if (elem->state != THREAD_DONE) {
             continue;
         }
         if (elem->state == THREAD_DONE) {
@@ -187,18 +178,24 @@ restart:
             QLIST_REMOVE(elem, all);
             /* Read state before ret.  */
             smp_rmb();
+
+            /* Schedule ourselves in case elem->common.cb() calls aio_poll() to
+             * wait for another request that completed at the same time.
+             */
+            qemu_bh_schedule(pool->completion_bh);
+
             elem->common.cb(elem->common.opaque, elem->ret);
-            qemu_aio_release(elem);
+            qemu_aio_unref(elem);
             goto restart;
         } else {
             /* remove the request */
             QLIST_REMOVE(elem, all);
-            qemu_aio_release(elem);
+            qemu_aio_unref(elem);
         }
     }
 }
 
-static void thread_pool_cancel(BlockDriverAIOCB *acb)
+static void thread_pool_cancel(BlockAIOCB *acb)
 {
     ThreadPoolElement *elem = (ThreadPoolElement *)acb;
     ThreadPool *pool = elem->pool;
@@ -214,27 +211,31 @@ static void thread_pool_cancel(BlockDriverAIOCB *acb)
          */
         qemu_sem_timedwait(&pool->sem, 0) == 0) {
         QTAILQ_REMOVE(&pool->request_list, elem, reqs);
-        elem->state = THREAD_CANCELED;
-        event_notifier_set(&pool->notifier);
-    } else {
-        pool->pending_cancellations++;
-        while (elem->state != THREAD_CANCELED && elem->state != THREAD_DONE) {
-            qemu_cond_wait(&pool->check_cancel, &pool->lock);
-        }
-        pool->pending_cancellations--;
+        qemu_bh_schedule(pool->completion_bh);
+
+        elem->state = THREAD_DONE;
+        elem->ret = -ECANCELED;
     }
+
     qemu_mutex_unlock(&pool->lock);
-    event_notifier_ready(&pool->notifier);
+}
+
+static AioContext *thread_pool_get_aio_context(BlockAIOCB *acb)
+{
+    ThreadPoolElement *elem = (ThreadPoolElement *)acb;
+    ThreadPool *pool = elem->pool;
+    return pool->ctx;
 }
 
 static const AIOCBInfo thread_pool_aiocb_info = {
     .aiocb_size         = sizeof(ThreadPoolElement),
-    .cancel             = thread_pool_cancel,
+    .cancel_async       = thread_pool_cancel,
+    .get_aio_context    = thread_pool_get_aio_context,
 };
 
-BlockDriverAIOCB *thread_pool_submit_aio(ThreadPool *pool,
+BlockAIOCB *thread_pool_submit_aio(ThreadPool *pool,
         ThreadPoolFunc *func, void *arg,
-        BlockDriverCompletionFunc *cb, void *opaque)
+        BlockCompletionFunc *cb, void *opaque)
 {
     ThreadPoolElement *req;
 
@@ -293,10 +294,9 @@ static void thread_pool_init_one(ThreadPool *pool, AioContext *ctx)
     }
 
     memset(pool, 0, sizeof(*pool));
-    event_notifier_init(&pool->notifier, false);
     pool->ctx = ctx;
+    pool->completion_bh = aio_bh_new(ctx, thread_pool_completion_bh, pool);
     qemu_mutex_init(&pool->lock);
-    qemu_cond_init(&pool->check_cancel);
     qemu_cond_init(&pool->worker_stopped);
     qemu_sem_init(&pool->sem, 0);
     pool->max_threads = 64;
@@ -304,8 +304,6 @@ static void thread_pool_init_one(ThreadPool *pool, AioContext *ctx)
 
     QLIST_INIT(&pool->head);
     QTAILQ_INIT(&pool->request_list);
-
-    aio_set_event_notifier(ctx, &pool->notifier, event_notifier_ready);
 }
 
 ThreadPool *thread_pool_new(AioContext *ctx)
@@ -339,11 +337,9 @@ void thread_pool_free(ThreadPool *pool)
 
     qemu_mutex_unlock(&pool->lock);
 
-    aio_set_event_notifier(pool->ctx, &pool->notifier, NULL);
+    qemu_bh_delete(pool->completion_bh);
     qemu_sem_destroy(&pool->sem);
-    qemu_cond_destroy(&pool->check_cancel);
     qemu_cond_destroy(&pool->worker_stopped);
     qemu_mutex_destroy(&pool->lock);
-    event_notifier_cleanup(&pool->notifier);
     g_free(pool);
 }
diff --git a/trace-events b/trace-events
index 11a17a8a4..b5722ea8a 100644
--- a/trace-events
+++ b/trace-events
@@ -41,6 +41,11 @@ virtio_irq(void *vq) "vq %p"
 virtio_notify(void *vdev, void *vq) "vdev %p vq %p"
 virtio_set_status(void *vdev, uint8_t val) "vdev %p val %u"
 
+# hw/virtio/virtio-rng.c
+virtio_rng_guest_not_ready(void *rng) "rng %p: guest not ready"
+virtio_rng_pushed(void *rng, size_t len) "rng %p: %zd bytes pushed"
+virtio_rng_request(void *rng, size_t size, unsigned quota) "rng %p: %zd bytes requested, %u bytes quota left"
+
 # hw/char/virtio-serial-bus.c
 virtio_serial_send_control_event(unsigned int port, uint16_t event, uint16_t value) "port %u, event %u, value %u"
 virtio_serial_throttle_port(unsigned int port, bool throttle) "port %u, throttle %d"
@@ -116,7 +121,6 @@ virtio_blk_handle_read(void *req, uint64_t sector, size_t nsectors) "req %p sect
 virtio_blk_data_plane_start(void *s) "dataplane %p"
 virtio_blk_data_plane_stop(void *s) "dataplane %p"
 virtio_blk_data_plane_process_request(void *s, unsigned int out_num, unsigned int in_num, unsigned int head) "dataplane %p out_num %u in_num %u head %u"
-virtio_blk_data_plane_complete_request(void *s, unsigned int head, int ret) "dataplane %p head %u ret %d"
 
 # hw/virtio/dataplane/vring.c
 vring_setup(uint64_t physical, void *desc, void *avail, void *used) "vring physical %#"PRIx64" desc %p avail %p used %p"
@@ -191,8 +195,8 @@ fw_cfg_add_file_dupe(void *s, char *name) "%p %s"
 fw_cfg_add_file(void *s, int index, char *name, size_t len) "%p #%d: %s (%zd bytes)"
 
 # hw/block/hd-geometry.c
-hd_geometry_lchs_guess(void *bs, int cyls, int heads, int secs) "bs %p LCHS %d %d %d"
-hd_geometry_guess(void *bs, uint32_t cyls, uint32_t heads, uint32_t secs, int trans) "bs %p CHS %u %u %u trans %d"
+hd_geometry_lchs_guess(void *blk, int cyls, int heads, int secs) "blk %p LCHS %d %d %d"
+hd_geometry_guess(void *blk, uint32_t cyls, uint32_t heads, uint32_t secs, int trans) "blk %p CHS %u %u %u trans %d"
 
 # hw/display/jazz_led.c
 jazz_led_read(uint64_t addr, uint8_t val) "read addr=0x%"PRIx64": 0x%x"
@@ -290,8 +294,67 @@ usb_port_attach(int bus, const char *port, const char *devspeed, const char *por
 usb_port_detach(int bus, const char *port) "bus %d, port %s"
 usb_port_release(int bus, const char *port) "bus %d, port %s"
 
+# hw/usb/hcd-ohci.c
+usb_ohci_iso_td_read_failed(uint32_t addr) "ISO_TD read error at %x"
+usb_ohci_iso_td_head(uint32_t head, uint32_t tail, uint32_t flags, uint32_t bp, uint32_t next, uint32_t be, uint32_t framenum, uint32_t startframe, uint32_t framecount, int rel_frame_num) "ISO_TD ED head 0x%.8x tailp 0x%.8x\n0x%.8x 0x%.8x 0x%.8x 0x%.8x\nframe_number 0x%.8x starting_frame 0x%.8x\nframe_count  0x%.8x relative %d"
+usb_ohci_iso_td_head_offset(uint32_t o0, uint32_t o1, uint32_t o2, uint32_t o3, uint32_t o4, uint32_t o5, uint32_t o6, uint32_t o7) "0x%.8x 0x%.8x 0x%.8x 0x%.8x 0x%.8x 0x%.8x 0x%.8x 0x%.8x"
+usb_ohci_iso_td_relative_frame_number_neg(int rel) "ISO_TD R=%d < 0"
+usb_ohci_iso_td_relative_frame_number_big(int rel, int count) "ISO_TD R=%d > FC=%d"
+usb_ohci_iso_td_bad_direction(int dir) "Bad direction %d"
+usb_ohci_iso_td_bad_bp_be(uint32_t bp, uint32_t be) "ISO_TD bp 0x%.8x be 0x%.8x"
+usb_ohci_iso_td_bad_cc_not_accessed(uint32_t start, uint32_t next) "ISO_TD cc != not accessed 0x%.8x 0x%.8x"
+usb_ohci_iso_td_bad_cc_overrun(uint32_t start, uint32_t next) "ISO_TD start_offset=0x%.8x > next_offset=0x%.8x"
+usb_ohci_iso_td_so(uint32_t so, uint32_t eo, uint32_t s, uint32_t e, const char *str, ssize_t len, int ret) "0x%.8x eo 0x%.8x\nsa 0x%.8x ea 0x%.8x\ndir %s len %zu ret %d"
+usb_ohci_iso_td_data_overrun(int ret, ssize_t len) "DataOverrun %d > %zu"
+usb_ohci_iso_td_data_underrun(int ret) "DataUnderrun %d"
+usb_ohci_iso_td_nak(int ret) "got NAK/STALL %d"
+usb_ohci_iso_td_bad_response(int ret) "Bad device response %d"
+usb_ohci_port_attach(int index) "port #%d"
+usb_ohci_port_detach(int index) "port #%d"
+usb_ohci_port_wakeup(int index) "port #%d"
+usb_ohci_port_suspend(int index) "port #%d"
+usb_ohci_port_reset(int index) "port #%d"
+usb_ohci_remote_wakeup(const char *s) "%s: SUSPEND->RESUME"
+usb_ohci_reset(const char *s) "%s"
+usb_ohci_start(const char *s) "%s: USB Operational"
+usb_ohci_resume(const char *s) "%s: USB Resume"
+usb_ohci_stop(const char *s) "%s: USB Suspended"
+usb_ohci_exit(const char *s) "%s"
+usb_ohci_set_ctl(const char *s, uint32_t new_state) "%s: new state 0x%x"
+usb_ohci_td_underrun(void) ""
+usb_ohci_td_dev_error(void) ""
+usb_ohci_td_nak(void) ""
+usb_ohci_td_stall(void) ""
+usb_ohci_td_babble(void) ""
+usb_ohci_td_bad_device_response(int rc) "%d"
+usb_ohci_td_read_error(uint32_t addr) "TD read error at %x"
+usb_ohci_td_bad_direction(int dir) "Bad direction %d"
+usb_ohci_td_skip_async(void) ""
+usb_ohci_td_pkt_hdr(uint32_t addr, int64_t pktlen, int64_t len, const char *s, int flag_r, uint32_t cbp, uint32_t be) " TD @ 0x%.8x %" PRId64 " of %" PRId64 " bytes %s r=%d cbp=0x%.8x be=0x%.8x"
+usb_ohci_td_pkt_short(const char *dir, const char *buf) "%s data: %s"
+usb_ohci_td_pkt_full(const char *dir, const char *buf) "%s data: %s"
+usb_ohci_td_too_many_pending(void) ""
+usb_ohci_td_packet_status(int status) "status=%d"
+usb_ohci_ed_read_error(uint32_t addr) "ED read error at %x"
+usb_ohci_ed_pkt(uint32_t cur, int h, int c, uint32_t head, uint32_t tail, uint32_t next) "ED @ 0x%.8x h=%u c=%u\n  head=0x%.8x tailp=0x%.8x next=0x%.8x"
+usb_ohci_ed_pkt_flags(uint32_t fa, uint32_t en, uint32_t d, int s, int k, int f, uint32_t mps) "fa=%u en=%u d=%u s=%u k=%u f=%u mps=%u"
+usb_ohci_hcca_read_error(uint32_t addr) "HCCA read error at %x"
+usb_ohci_mem_read_unaligned(uint32_t addr) "at %x"
+usb_ohci_mem_read_bad_offset(uint32_t addr) "%x"
+usb_ohci_mem_write_unaligned(uint32_t addr) "at %x"
+usb_ohci_mem_write_bad_offset(uint32_t addr) "%x"
+usb_ohci_process_lists(uint32_t head, uint32_t cur) "head %x, cur %x"
+usb_ohci_bus_eof_timer_failed(const char *name) "%s: timer_new_ns failed"
+usb_ohci_set_frame_interval(const char *name, uint16_t fi_x, uint16_t fi_u) "%s: FrameInterval = 0x%x (%u)"
+usb_ohci_hub_power_up(void) "powered up all ports"
+usb_ohci_hub_power_down(void) "powered down all ports"
+usb_ohci_init_time(int64_t frametime, int64_t bittime) "usb_bit_time=%" PRId64 " usb_frame_time=%" PRId64
+usb_ohci_die(void) ""
+usb_ohci_async_complete(void) ""
+
 # hw/usb/hcd-ehci.c
 usb_ehci_reset(void) "=== RESET ==="
+usb_ehci_unrealize(void) "=== UNREALIZE ==="
 usb_ehci_opreg_read(uint32_t addr, const char *str, uint32_t val) "rd mmio %04x [%s] = %x"
 usb_ehci_opreg_write(uint32_t addr, const char *str, uint32_t val) "wr mmio %04x [%s] = %x"
 usb_ehci_opreg_change(uint32_t addr, const char *str, uint32_t new, uint32_t old) "ch mmio %04x [%s] = %x (old: %x)"
@@ -324,6 +387,7 @@ usb_ehci_dma_error(void) ""
 
 # hw/usb/hcd-uhci.c
 usb_uhci_reset(void) "=== RESET ==="
+usb_uhci_exit(void) "=== EXIT ==="
 usb_uhci_schedule_start(void) ""
 usb_uhci_schedule_stop(void) ""
 usb_uhci_frame_start(uint32_t num) "nr %d"
@@ -353,6 +417,7 @@ usb_uhci_td_complete(uint32_t qh, uint32_t td) "qh 0x%x, td 0x%x"
 
 # hw/usb/hcd-xhci.c
 usb_xhci_reset(void) "=== RESET ==="
+usb_xhci_exit(void) "=== EXIT ==="
 usb_xhci_run(void) ""
 usb_xhci_stop(void) ""
 usb_xhci_cap_read(uint32_t off, uint32_t val) "off 0x%04x, ret 0x%08x"
@@ -439,7 +504,6 @@ usb_mtp_command(int dev, uint16_t code, uint32_t trans, uint32_t arg0, uint32_t
 usb_mtp_success(int dev, uint32_t trans, uint32_t arg0, uint32_t arg1) "dev %d, trans 0x%x, args 0x%x, 0x%x"
 usb_mtp_error(int dev, uint16_t code, uint32_t trans, uint32_t arg0, uint32_t arg1) "dev %d, code 0x%x, trans 0x%x, args 0x%x, 0x%x"
 usb_mtp_data_in(int dev, uint32_t trans, uint32_t len) "dev %d, trans 0x%x, len %d"
-usb_mtp_data_out(int dev, uint32_t trans, uint32_t len) "dev %d, trans 0x%x, len %d"
 usb_mtp_xfer(int dev, uint32_t ep, uint32_t dlen, uint32_t plen) "dev %d, ep %d, %d/%d"
 usb_mtp_nak(int dev, uint32_t ep) "dev %d, ep %d"
 usb_mtp_stall(int dev, const char *reason) "dev %d, reason: %s"
@@ -633,37 +697,36 @@ lm32_uart_irq_state(int level) "irq state %d"
 megasas_init_firmware(uint64_t pa) "pa %" PRIx64 " "
 megasas_init_queue(uint64_t queue_pa, int queue_len, uint64_t head, uint64_t tail, uint32_t flags) "queue at %" PRIx64 " len %d head %" PRIx64 " tail %" PRIx64 " flags %x"
 megasas_initq_map_failed(int frame) "scmd %d: failed to map queue"
+megasas_initq_mapped(uint64_t pa) "queue already mapped at %" PRIx64 ""
 megasas_initq_mismatch(int queue_len, int fw_cmds) "queue size %d max fw cmds %d"
-megasas_qf_found(unsigned int index, uint64_t pa) "found mapped frame %x pa %" PRIx64 ""
-megasas_qf_new(unsigned int index, void *cmd) "return new frame %x cmd %p"
-megasas_qf_failed(unsigned long pa) "all frames busy for frame %lx"
-megasas_qf_enqueue(unsigned int index, unsigned int count, uint64_t context, unsigned int tail, int busy) "enqueue frame %x count %d context %" PRIx64 " tail %x busy %d"
-megasas_qf_update(unsigned int head, unsigned int busy) "update reply queue head %x busy %d"
+megasas_qf_mapped(unsigned int index) "skip mapped frame %x"
+megasas_qf_new(unsigned int index, uint64_t frame) "frame %x addr %" PRIx64 ""
+megasas_qf_busy(unsigned long pa) "all frames busy for frame %lx"
+megasas_qf_enqueue(unsigned int index, unsigned int count, uint64_t context, unsigned int head, unsigned int tail, int busy) "frame %x count %d context %" PRIx64 " head %x tail %x busy %d"
+megasas_qf_update(unsigned int head, unsigned int tail, unsigned int busy) "head %x tail %x busy %d"
 megasas_qf_map_failed(int cmd, unsigned long frame) "scmd %d: frame %lu"
 megasas_qf_complete_noirq(uint64_t context) "context %" PRIx64 " "
-megasas_qf_complete(uint64_t context, unsigned int tail, unsigned int offset, int busy, unsigned int doorbell) "context %" PRIx64 " tail %x offset %d busy %d doorbell %x"
+megasas_qf_complete(uint64_t context, unsigned int head, unsigned int tail, int busy) "context %" PRIx64 " head %x tail %x busy %d"
 megasas_frame_busy(uint64_t addr) "frame %" PRIx64 " busy"
-megasas_unhandled_frame_cmd(int cmd, uint8_t frame_cmd) "scmd %d: Unhandled MFI cmd %x"
+megasas_unhandled_frame_cmd(int cmd, uint8_t frame_cmd) "scmd %d: MFI cmd %x"
 megasas_handle_scsi(const char *frame, int bus, int dev, int lun, void *sdev, unsigned long size) "%s dev %x/%x/%x sdev %p xfer %lu"
-megasas_scsi_target_not_present(const char *frame, int bus, int dev, int lun) "%s dev %x/%x/%x target not present"
+megasas_scsi_target_not_present(const char *frame, int bus, int dev, int lun) "%s dev %x/%x/%x"
 megasas_scsi_invalid_cdb_len(const char *frame, int bus, int dev, int lun, int len) "%s dev %x/%x/%x invalid cdb len %d"
 megasas_iov_read_overflow(int cmd, int bytes, int len) "scmd %d: %d/%d bytes"
 megasas_iov_write_overflow(int cmd, int bytes, int len) "scmd %d: %d/%d bytes"
 megasas_iov_read_underflow(int cmd, int bytes, int len) "scmd %d: %d/%d bytes"
 megasas_iov_write_underflow(int cmd, int bytes, int len) "scmd %d: %d/%d bytes"
-megasas_scsi_req_alloc_failed(const char *frame, int dev, int lun) "%s dev %x/%x req allocation failed"
+megasas_scsi_req_alloc_failed(const char *frame, int dev, int lun) "%s dev %x/%x"
 megasas_scsi_read_start(int cmd, int len) "scmd %d: transfer %d bytes of data"
 megasas_scsi_write_start(int cmd, int len) "scmd %d: transfer %d bytes of data"
 megasas_scsi_nodata(int cmd) "scmd %d: no data to be transferred"
-megasas_scsi_complete(int cmd, uint32_t status, int len, int xfer) "scmd %d: finished with status %x, len %u/%u"
-megasas_command_complete(int cmd, uint32_t status, uint32_t resid) "scmd %d: command completed, status %x, residual %d"
+megasas_scsi_complete(int cmd, uint32_t status, int len, int xfer) "scmd %d: status %x, len %u/%u"
+megasas_command_complete(int cmd, uint32_t status, uint32_t resid) "scmd %d: status %x, residual %d"
 megasas_handle_io(int cmd, const char *frame, int dev, int lun, unsigned long lba, unsigned long count) "scmd %d: %s dev %x/%x lba %lx count %lu"
 megasas_io_target_not_present(int cmd, const char *frame, int dev, int lun) "scmd %d: %s dev 1/%x/%x LUN not present"
 megasas_io_read_start(int cmd, unsigned long lba, unsigned long count, unsigned long len) "scmd %d: start LBA %lx %lu blocks (%lu bytes)"
 megasas_io_write_start(int cmd, unsigned long lba, unsigned long count, unsigned long len) "scmd %d: start LBA %lx %lu blocks (%lu bytes)"
-megasas_io_complete(int cmd, uint32_t len) "scmd %d: %d bytes completed"
-megasas_io_read(int cmd, int bytes, int len, unsigned long offset) "scmd %d: %d/%d bytes, iov offset %lu"
-megasas_io_write(int cmd, int bytes, int len, unsigned long offset) "scmd %d: %d/%d bytes, iov offset %lu"
+megasas_io_complete(int cmd, uint32_t len) "scmd %d: %d bytes"
 megasas_iovec_sgl_overflow(int cmd, int index, int limit) "scmd %d: iovec count %d limit %d"
 megasas_iovec_sgl_underflow(int cmd, int index) "scmd %d: iovec count %d"
 megasas_iovec_sgl_invalid(int cmd, int index, uint64_t pa, uint32_t len) "scmd %d: element %d pa %" PRIx64 " len %u"
@@ -671,28 +734,29 @@ megasas_iovec_overflow(int cmd, int len, int limit) "scmd %d: len %d limit %d"
 megasas_iovec_underflow(int cmd, int len, int limit) "scmd %d: len %d limit %d"
 megasas_handle_dcmd(int cmd, int opcode) "scmd %d: MFI DCMD opcode %x"
 megasas_finish_dcmd(int cmd, int size) "scmd %d: MFI DCMD wrote %d bytes"
-megasas_dcmd_req_alloc_failed(int cmd, const char *desc) "scmd %d: %s alloc failed"
+megasas_dcmd_req_alloc_failed(int cmd, const char *desc) "scmd %d: %s"
 megasas_dcmd_internal_submit(int cmd, const char *desc, int dev) "scmd %d: %s to dev %d"
-megasas_dcmd_internal_finish(int cmd, int opcode, int lun) "scmd %d: DCMD finish internal cmd %x lun %d"
-megasas_dcmd_internal_invalid(int cmd, int opcode) "scmd %d: Invalid internal DCMD %x"
+megasas_dcmd_internal_finish(int cmd, int opcode, int lun) "scmd %d: cmd %x lun %d"
+megasas_dcmd_internal_invalid(int cmd, int opcode) "scmd %d: DCMD %x"
 megasas_dcmd_unhandled(int cmd, int opcode, int len) "scmd %d: opcode %x, len %d"
 megasas_dcmd_zero_sge(int cmd) "scmd %d: zero DCMD sge count"
-megasas_dcmd_invalid_sge(int cmd, int count) "scmd %d: invalid DCMD sge count %d"
-megasas_dcmd_invalid_xfer_len(int cmd, unsigned long size, unsigned long max) "scmd %d: invalid xfer len %ld, max %ld"
+megasas_dcmd_invalid_sge(int cmd, int count) "scmd %d: DCMD sge count %d"
+megasas_dcmd_invalid_xfer_len(int cmd, unsigned long size, unsigned long max) "scmd %d: xfer len %ld, max %ld"
 megasas_dcmd_enter(int cmd, const char *dcmd, int len) "scmd %d: DCMD %s len %d"
-megasas_dcmd_dummy(int cmd, unsigned long size) "scmd %d: DCMD dummy xfer len %ld"
+megasas_dcmd_dummy(int cmd, unsigned long size) "scmd %d: xfer len %ld"
 megasas_dcmd_set_fw_time(int cmd, unsigned long time) "scmd %d: Set FW time %lx"
 megasas_dcmd_pd_get_list(int cmd, int num, int max, int offset) "scmd %d: DCMD PD get list: %d / %d PDs, size %d"
 megasas_dcmd_ld_get_list(int cmd, int num, int max) "scmd %d: DCMD LD get list: found %d / %d LDs"
-megasas_dcmd_ld_get_info(int cmd, int ld_id) "scmd %d: DCMD LD get info for dev %d"
-megasas_dcmd_pd_get_info(int cmd, int pd_id) "scmd %d: DCMD PD get info for dev %d"
-megasas_dcmd_pd_list_query(int cmd, int flags) "scmd %d: DCMD PD list query flags %x"
-megasas_dcmd_ld_list_query(int cmd, int flags) "scmd %d: DCMD LD list query flags %x"
+megasas_dcmd_ld_get_info(int cmd, int ld_id) "scmd %d: dev %d"
+megasas_dcmd_ld_list_query(int cmd, int flags) "scmd %d: query flags %x"
+megasas_dcmd_pd_get_info(int cmd, int pd_id) "scmd %d: dev %d"
+megasas_dcmd_pd_list_query(int cmd, int flags) "scmd %d: query flags %x"
+megasas_dcmd_reset_ld(int cmd, int target_id) "scmd %d: dev %d"
 megasas_dcmd_unsupported(int cmd, unsigned long size) "scmd %d: set properties len %ld"
-megasas_abort_frame(int cmd, int abort_cmd) "scmd %d: aborting frame %x"
+megasas_abort_frame(int cmd, int abort_cmd) "scmd %d: frame %x"
 megasas_abort_no_cmd(int cmd, uint64_t context) "scmd %d: no active command for frame context %" PRIx64 ""
 megasas_abort_invalid_context(int cmd, uint64_t context, int abort_cmd) "scmd %d: invalid frame context %" PRIx64 " for abort frame %x"
-megasas_reset(void) "Reset"
+megasas_reset(int fw_state) "firmware state %x"
 megasas_init(int sges, int cmds, const char *mode) "Using %d sges, %d cmds, %s mode"
 megasas_msix_raise(int vector) "vector %d"
 megasas_msi_raise(int vector) "vector %d"
@@ -702,9 +766,9 @@ megasas_intr_enabled(void) "Interrupts enabled"
 megasas_intr_disabled(void) "Interrupts disabled"
 megasas_msix_enabled(int vector) "vector %d"
 megasas_msi_enabled(int vector) "vector %d"
-megasas_mmio_readl(unsigned long addr, uint32_t val) "addr 0x%lx: 0x%x"
+megasas_mmio_readl(const char *reg, uint32_t val) "reg %s: 0x%x"
 megasas_mmio_invalid_readl(unsigned long addr) "addr 0x%lx"
-megasas_mmio_writel(uint32_t addr, uint32_t val) "addr 0x%x: 0x%x"
+megasas_mmio_writel(const char *reg, uint32_t val) "reg %s: 0x%x"
 megasas_mmio_invalid_writel(uint32_t addr, uint32_t val) "addr 0x%x: 0x%x"
 
 # hw/audio/milkymist-ac97.c
@@ -830,7 +894,7 @@ pvscsi_state(const char* state) "starting %s ..."
 pvscsi_tx_rings_ppn(const char* label, uint64_t ppn) "%s page: %"PRIx64""
 pvscsi_tx_rings_num_pages(const char* label, uint32_t num) "Number of %s pages: %u"
 
-# xen-all.c
+# xen-hvm.c
 xen_ram_alloc(unsigned long ram_addr, unsigned long size) "requested: %#lx, size %#lx"
 xen_client_set_memory(uint64_t start_addr, unsigned long size, bool log_dirty) "%#"PRIx64" size %#lx, log_dirty %i"
 
@@ -839,7 +903,7 @@ xen_map_cache(uint64_t phys_addr) "want %#"PRIx64
 xen_remap_bucket(uint64_t index) "index %#"PRIx64
 xen_map_cache_return(void* ptr) "%p"
 
-# hw/xen/xen_platform.c
+# hw/i386/xen/xen_platform.c
 xen_platform_log(char *s) "xen platform: %s"
 
 # qemu-coroutine.c
@@ -870,12 +934,6 @@ escc_sunkbd_event_out(int ch) "Translated keycode 0x%2.2x"
 escc_kbd_command(int val) "Command %d"
 escc_sunmouse_event(int dx, int dy, int buttons_state) "dx=%d dy=%d buttons=%01x"
 
-# block/iscsi.c
-iscsi_aio_write16_cb(void *iscsi, int status, void *acb, int canceled) "iscsi %p status %d acb %p canceled %d"
-iscsi_aio_writev(void *iscsi, int64_t sector_num, int nb_sectors, void *opaque, void *acb) "iscsi %p sector_num %"PRId64" nb_sectors %d opaque %p acb %p"
-iscsi_aio_read16_cb(void *iscsi, int status, void *acb, int canceled) "iscsi %p status %d acb %p canceled %d"
-iscsi_aio_readv(void *iscsi, int64_t sector_num, int nb_sectors, void *opaque, void *acb) "iscsi %p sector_num %"PRId64" nb_sectors %d opaque %p acb %p"
-
 # hw/scsi/esp.c
 esp_error_fifo_overrun(void) "FIFO overrun"
 esp_error_unhandled_command(uint32_t val) "unhandled command (%2.2x)"
@@ -929,7 +987,6 @@ esp_pci_sbac_write(uint32_t reg, uint32_t val) "sbac: 0x%8.8x -> 0x%8.8x"
 # monitor.c
 handle_qmp_command(void *mon, const char *cmd_name) "mon %p cmd_name \"%s\""
 monitor_protocol_emitter(void *mon) "mon %p"
-monitor_protocol_event(uint32_t event, const char *evname, void *data) "event=%d name \"%s\" data %p"
 monitor_protocol_event_handler(uint32_t event, void *data, uint64_t last, uint64_t now) "event=%d data=%p last=%" PRId64 " now=%" PRId64
 monitor_protocol_event_emit(uint32_t event, void *data) "event=%d data=%p"
 monitor_protocol_event_queue(uint32_t event, void *data, uint64_t rate, uint64_t last, uint64_t now) "event=%d data=%p rate=%" PRId64 " last=%" PRId64 " now=%" PRId64
@@ -1023,10 +1080,10 @@ win_helper_done(uint32_t tl) "tl=%d"
 win_helper_retry(uint32_t tl) "tl=%d"
 
 # dma-helpers.c
-dma_bdrv_io(void *dbs, void *bs, int64_t sector_num, bool to_dev) "dbs=%p bs=%p sector_num=%" PRId64 " to_dev=%d"
+dma_blk_io(void *dbs, void *bs, int64_t sector_num, bool to_dev) "dbs=%p bs=%p sector_num=%" PRId64 " to_dev=%d"
 dma_aio_cancel(void *dbs) "dbs=%p"
 dma_complete(void *dbs, int ret, void *cb) "dbs=%p ret=%d cb=%p"
-dma_bdrv_cb(void *dbs, int ret) "dbs=%p ret=%d"
+dma_blk_cb(void *dbs, int ret) "dbs=%p ret=%d"
 dma_map_wait(void *dbs) "dbs=%p"
 
 # ui/console.c
@@ -1037,7 +1094,7 @@ console_txt_new(int w, int h) "%dx%d"
 console_select(int nr) "%d"
 console_refresh(int interval) "interval %d ms"
 displaysurface_create(void *display_surface, int w, int h) "surface=%p, %dx%d"
-displaysurface_create_from(void *display_surface, int w, int h, int bpp, int swap) "surface=%p, %dx%d, bpp %d, bswap %d"
+displaysurface_create_from(void *display_surface, int w, int h, uint32_t format) "surface=%p, %dx%d, format 0x%x"
 displaysurface_free(void *display_surface) "surface=%p"
 displaychangelistener_register(void *dcl, const char *name) "%p [ %s ]"
 displaychangelistener_unregister(void *dcl, const char *name) "%p [ %s ]"
@@ -1084,9 +1141,11 @@ savevm_state_complete(void) ""
 savevm_state_cancel(void) ""
 vmstate_save(const char *idstr, const char *vmsd_name) "%s, %s"
 vmstate_load(const char *idstr, const char *vmsd_name) "%s, %s"
-vmstate_load_field_error(const char *field, int ret) "field \"%s\" load failed, ret = %d"
 qemu_announce_self_iter(const char *mac) "%s"
 
+# vmstate.c
+vmstate_load_field_error(const char *field, int ret) "field \"%s\" load failed, ret = %d"
+
 # qemu-file.c
 qemu_file_fclose(void) ""
 
@@ -1241,7 +1300,7 @@ css_adapter_interrupt(uint8_t isc) "CSS: adapter I/O interrupt (isc %x)"
 virtio_ccw_interpret_ccw(int cssid, int ssid, int schid, int cmd_code) "VIRTIO-CCW: %x.%x.%04x: interpret command %x"
 virtio_ccw_new_device(int cssid, int ssid, int schid, int devno, const char *devno_mode) "VIRTIO-CCW: add subchannel %x.%x.%04x, devno %04x (%s)"
 
-# hw/intc/s390_flic.c
+# hw/intc/s390_flic_kvm.c
 flic_create_device(int err) "flic: create device failed %d"
 flic_no_device_api(int err) "flic: no Device Contral API support %d"
 flic_reset_failed(int err) "flic: reset failed %d"
@@ -1260,11 +1319,22 @@ kvm_vm_ioctl(int type, void *arg) "type 0x%x, arg %p"
 kvm_vcpu_ioctl(int cpu_index, int type, void *arg) "cpu_index %d, type 0x%x, arg %p"
 kvm_run_exit(int cpu_index, uint32_t reason) "cpu_index %d, reason %d"
 kvm_device_ioctl(int fd, int type, void *arg) "dev fd %d, type 0x%x, arg %p"
-kvm_failed_spr_set(int str, const char *msg) "Warning: Unable to set SPR %d to KVM: %s"
-kvm_failed_spr_get(int str, const char *msg) "Warning: Unable to retrieve SPR %d from KVM: %s"
 kvm_failed_reg_get(uint64_t id, const char *msg) "Warning: Unable to retrieve ONEREG %" PRIu64 " from KVM: %s"
 kvm_failed_reg_set(uint64_t id, const char *msg) "Warning: Unable to set ONEREG %" PRIu64 " to KVM: %s"
 
+# target-ppc/kvm.c
+kvm_failed_spr_set(int str, const char *msg) "Warning: Unable to set SPR %d to KVM: %s"
+kvm_failed_spr_get(int str, const char *msg) "Warning: Unable to retrieve SPR %d from KVM: %s"
+
+# TCG related tracing (mostly disabled by default)
+# cpu-exec.c
+disable exec_tb(void *tb, uintptr_t pc) "tb:%p pc=0x%"PRIxPTR
+disable exec_tb_nocache(void *tb, uintptr_t pc) "tb:%p pc=0x%"PRIxPTR
+disable exec_tb_exit(void *next_tb, unsigned int flags) "tb:%p flags=%x"
+
+# translate-all.c
+translate_block(void *tb, uintptr_t pc, uint8_t *tb_code) "tb:%p, pc:0x%"PRIxPTR", tb_code:%p"
+
 # memory.c
 memory_region_ops_read(void *mr, uint64_t addr, uint64_t value, unsigned size) "mr %p addr %#"PRIx64" value %#"PRIx64" size %u"
 memory_region_ops_write(void *mr, uint64_t addr, uint64_t value, unsigned size) "mr %p addr %#"PRIx64" value %#"PRIx64" size %u"
@@ -1273,7 +1343,7 @@ memory_region_ops_write(void *mr, uint64_t addr, uint64_t value, unsigned size)
 object_dynamic_cast_assert(const char *type, const char *target, const char *file, int line, const char *func) "%s->%s (%s:%d:%s)"
 object_class_dynamic_cast_assert(const char *type, const char *target, const char *file, int line, const char *func) "%s->%s (%s:%d:%s)"
 
-# hw/xen/xen_pvdevice.c
+# hw/i386/xen/xen_pvdevice.c
 xen_pv_mmio_read(uint64_t addr) "WARNING: read from Xen PV Device MMIO space (address %"PRIx64")"
 xen_pv_mmio_write(uint64_t addr) "WARNING: write to Xen PV Device MMIO space (address %"PRIx64")"
 
@@ -1281,7 +1351,7 @@ xen_pv_mmio_write(uint64_t addr) "WARNING: write to Xen PV Device MMIO space (ad
 pci_cfg_read(const char *dev, unsigned devid, unsigned fnid, unsigned offs, unsigned val) "%s %02u:%u @0x%x -> 0x%x"
 pci_cfg_write(const char *dev, unsigned devid, unsigned fnid, unsigned offs, unsigned val) "%s %02u:%u @0x%x <- 0x%x"
 
-#hw/acpi/memory_hotplug.c
+# hw/acpi/memory_hotplug.c
 mhp_acpi_invalid_slot_selected(uint32_t slot) "0x%"PRIx32
 mhp_acpi_read_addr_lo(uint32_t slot, uint32_t addr) "slot[0x%"PRIx32"] addr lo: 0x%"PRIx32
 mhp_acpi_read_addr_hi(uint32_t slot, uint32_t addr) "slot[0x%"PRIx32"] addr hi: 0x%"PRIx32
@@ -1294,10 +1364,19 @@ mhp_acpi_write_ost_ev(uint32_t slot, uint32_t ev) "slot[0x%"PRIx32"] OST EVENT:
 mhp_acpi_write_ost_status(uint32_t slot, uint32_t st) "slot[0x%"PRIx32"] OST STATUS: 0x%"PRIx32
 mhp_acpi_clear_insert_evt(uint32_t slot) "slot[0x%"PRIx32"] clear insert event"
 
-#hw/i386/pc.c
+# hw/i386/pc.c
 mhp_pc_dimm_assigned_slot(int slot) "0x%d"
 mhp_pc_dimm_assigned_address(uint64_t addr) "0x%"PRIx64
 
 # target-s390x/kvm.c
 kvm_enable_cmma(int rc) "CMMA: enabling with result code %d"
 kvm_clear_cmma(int rc) "CMMA: clearing with result code %d"
+kvm_failed_cpu_state_set(int cpu_index, uint8_t state, const char *msg) "Warning: Unable to set cpu %d state %" PRIu8 " to KVM: %s"
+
+# hw/dma/i8257.c
+i8257_unregistered_dma(int nchan, int dma_pos, int dma_len) "unregistered DMA channel used nchan=%d dma_pos=%d dma_len=%d"
+
+# target-s390x/cpu.c
+cpu_set_state(int cpu_index, uint8_t state) "setting cpu %d state to %" PRIu8
+cpu_halt(int cpu_index) "halting cpu %d"
+cpu_unhalt(int cpu_index) "unhalting cpu %d"
diff --git a/trace/Makefile.objs b/trace/Makefile.objs
index d7a86969a..32f7a32ce 100644
--- a/trace/Makefile.objs
+++ b/trace/Makefile.objs
@@ -49,6 +49,9 @@ util-obj-y += generated-events.o
 ######################################################################
 # Auto-generated tracing routines
 
+##################################################
+# Execution level
+
 $(obj)/generated-tracers.h: $(obj)/generated-tracers.h-timestamp
 	@cmp -s $< $@ || cp $< $@
 $(obj)/generated-tracers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
@@ -57,8 +60,8 @@ $(obj)/generated-tracers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/conf
 		--backends=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
 
-######################################################################
-# Auto-generated tracing routines (non-DTrace)
+##############################
+# non-DTrace
 
 $(obj)/generated-tracers.c: $(obj)/generated-tracers.c-timestamp
 	@cmp -s $< $@ || cp $< $@
@@ -70,9 +73,8 @@ $(obj)/generated-tracers.c-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/conf
 
 $(obj)/generated-tracers.o: $(obj)/generated-tracers.c $(obj)/generated-tracers.h
 
-
-######################################################################
-# Auto-generated DTrace code
+##############################
+# DTrace
 
 # Normal practice is to name DTrace probe file with a '.d' extension
 # but that gets picked up by QEMU's Makefile as an external dependency
@@ -94,11 +96,52 @@ $(obj)/generated-tracers-dtrace.o: $(obj)/generated-tracers-dtrace.dtrace
 util-obj-y += generated-tracers-dtrace.o
 endif
 
+##################################################
+# Translation level
+
+$(obj)/generated-helpers-wrappers.h: $(obj)/generated-helpers-wrappers.h-timestamp
+$(obj)/generated-helpers-wrappers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	$(call quiet-command,$(TRACETOOL) \
+		--format=tcg-helper-wrapper-h \
+		--backend=$(TRACE_BACKENDS) \
+		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
+	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
+
+$(obj)/generated-helpers.h: $(obj)/generated-helpers.h-timestamp
+$(obj)/generated-helpers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	$(call quiet-command,$(TRACETOOL) \
+		--format=tcg-helper-h \
+		--backend=$(TRACE_BACKENDS) \
+		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
+	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
+
+$(obj)/generated-helpers.c: $(obj)/generated-helpers.c-timestamp
+$(obj)/generated-helpers.c-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	$(call quiet-command,$(TRACETOOL) \
+		--format=tcg-helper-c \
+		--backend=$(TRACE_BACKENDS) \
+		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
+	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
+
+$(obj)/generated-helpers.o: $(obj)/generated-helpers.c
+
+target-obj-y += generated-helpers.o
+
+
+$(obj)/generated-tcg-tracers.h: $(obj)/generated-tcg-tracers.h-timestamp
+$(obj)/generated-tcg-tracers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	$(call quiet-command,$(TRACETOOL) \
+		--format=tcg-h \
+		--backend=$(TRACE_BACKENDS) \
+		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
+	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
+
+
 ######################################################################
 # Backend code
 
-util-obj-$(CONFIG_TRACE_SIMPLE) += simple.o
+util-obj-$(CONFIG_TRACE_SIMPLE) += simple.o generated-tracers.o
 util-obj-$(CONFIG_TRACE_FTRACE) += ftrace.o
 util-obj-$(CONFIG_TRACE_UST) += generated-ust.o
 util-obj-y += control.o
-util-obj-y += generated-tracers.o
+util-obj-y += qmp.o
diff --git a/trace/control.c b/trace/control.c
index 9631a40ef..0d308011a 100644
--- a/trace/control.c
+++ b/trace/control.c
@@ -85,19 +85,6 @@ TraceEvent *trace_event_pattern(const char *pat, TraceEvent *ev)
     return NULL;
 }
 
-void trace_print_events(FILE *stream, fprintf_function stream_printf)
-{
-    TraceEventID i;
-
-    for (i = 0; i < trace_event_count(); i++) {
-        TraceEvent *ev = trace_event_id(i);
-        stream_printf(stream, "%s [Event ID %u] : state %u\n",
-                      trace_event_get_name(ev), i,
-                      trace_event_get_state_static(ev) &&
-                      trace_event_get_state_dynamic(ev));
-    }
-}
-
 static void trace_init_events(const char *fname)
 {
     Location loc;
diff --git a/trace/control.h b/trace/control.h
index e1ec03370..da9bb6b77 100644
--- a/trace/control.h
+++ b/trace/control.h
@@ -149,13 +149,6 @@ static void trace_event_set_state_dynamic(TraceEvent *ev, bool state);
 
 
 /**
- * trace_print_events:
- *
- * Print the state of all events.
- */
-void trace_print_events(FILE *stream, fprintf_function stream_printf);
-
-/**
  * trace_init_backends:
  * @events: Name of file with events to be enabled at startup; may be NULL.
  *          Corresponds to commandline option "-trace events=...".
diff --git a/trace/qmp.c b/trace/qmp.c
new file mode 100644
index 000000000..0b1948952
--- /dev/null
+++ b/trace/qmp.c
@@ -0,0 +1,75 @@
+/*
+ * QMP commands for tracing events.
+ *
+ * Copyright (C) 2014 Lluís Vilanova <vilanova@ac.upc.edu>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/typedefs.h"
+#include "qmp-commands.h"
+#include "trace/control.h"
+
+
+TraceEventInfoList *qmp_trace_event_get_state(const char *name, Error **errp)
+{
+    TraceEventInfoList *events = NULL;
+    bool found = false;
+    TraceEvent *ev;
+
+    ev = NULL;
+    while ((ev = trace_event_pattern(name, ev)) != NULL) {
+        TraceEventInfoList *elem = g_new(TraceEventInfoList, 1);
+        elem->value = g_new(TraceEventInfo, 1);
+        elem->value->name = g_strdup(trace_event_get_name(ev));
+        if (!trace_event_get_state_static(ev)) {
+            elem->value->state = TRACE_EVENT_STATE_UNAVAILABLE;
+        } else if (!trace_event_get_state_dynamic(ev)) {
+            elem->value->state = TRACE_EVENT_STATE_DISABLED;
+        } else {
+            elem->value->state = TRACE_EVENT_STATE_ENABLED;
+        }
+        elem->next = events;
+        events = elem;
+        found = true;
+    }
+
+    if (!found && !trace_event_is_pattern(name)) {
+        error_setg(errp, "unknown event \"%s\"", name);
+    }
+
+    return events;
+}
+
+void qmp_trace_event_set_state(const char *name, bool enable,
+                               bool has_ignore_unavailable,
+                               bool ignore_unavailable, Error **errp)
+{
+    bool found = false;
+    TraceEvent *ev;
+
+    /* Check all selected events are dynamic */
+    ev = NULL;
+    while ((ev = trace_event_pattern(name, ev)) != NULL) {
+        found = true;
+        if (!(has_ignore_unavailable && ignore_unavailable) &&
+            !trace_event_get_state_static(ev)) {
+            error_setg(errp, "cannot set dynamic tracing state for \"%s\"",
+                       trace_event_get_name(ev));
+            return;
+        }
+    }
+    if (!found && !trace_event_is_pattern(name)) {
+        error_setg(errp, "unknown event \"%s\"", name);
+        return;
+    }
+
+    /* Apply changes */
+    ev = NULL;
+    while ((ev = trace_event_pattern(name, ev)) != NULL) {
+        if (trace_event_get_state_static(ev)) {
+            trace_event_set_state_dynamic(ev, enable);
+        }
+    }
+}
diff --git a/translate-all.c b/translate-all.c
index 8f7e11b0a..ba5c8403d 100644
--- a/translate-all.c
+++ b/translate-all.c
@@ -33,6 +33,7 @@
 #include "qemu-common.h"
 #define NO_CPU_IO_DEFS
 #include "cpu.h"
+#include "trace.h"
 #include "disas/disas.h"
 #include "tcg.h"
 #if defined(CONFIG_USER_ONLY)
@@ -158,6 +159,8 @@ int cpu_gen_code(CPUArchState *env, TranslationBlock *tb, int *gen_code_size_ptr
 
     gen_intermediate_code(env, tb);
 
+    trace_translate_block(tb, tb->pc, tb->tc_ptr);
+
     /* generate machine code */
     gen_code_buf = tb->tc_ptr;
     tb->tb_next_offset[0] = 0xffff;
@@ -1657,30 +1660,30 @@ void cpu_interrupt(CPUState *cpu, int mask)
 struct walk_memory_regions_data {
     walk_memory_regions_fn fn;
     void *priv;
-    uintptr_t start;
+    target_ulong start;
     int prot;
 };
 
 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
-                                   abi_ulong end, int new_prot)
+                                   target_ulong end, int new_prot)
 {
-    if (data->start != -1ul) {
+    if (data->start != -1u) {
         int rc = data->fn(data->priv, data->start, end, data->prot);
         if (rc != 0) {
             return rc;
         }
     }
 
-    data->start = (new_prot ? end : -1ul);
+    data->start = (new_prot ? end : -1u);
     data->prot = new_prot;
 
     return 0;
 }
 
 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
-                                 abi_ulong base, int level, void **lp)
+                                 target_ulong base, int level, void **lp)
 {
-    abi_ulong pa;
+    target_ulong pa;
     int i, rc;
 
     if (*lp == NULL) {
@@ -1705,7 +1708,7 @@ static int walk_memory_regions_1(struct walk_memory_regions_data *data,
         void **pp = *lp;
 
         for (i = 0; i < V_L2_SIZE; ++i) {
-            pa = base | ((abi_ulong)i <<
+            pa = base | ((target_ulong)i <<
                 (TARGET_PAGE_BITS + V_L2_BITS * level));
             rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
             if (rc != 0) {
@@ -1724,13 +1727,12 @@ int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
 
     data.fn = fn;
     data.priv = priv;
-    data.start = -1ul;
+    data.start = -1u;
     data.prot = 0;
 
     for (i = 0; i < V_L1_SIZE; i++) {
-        int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
+        int rc = walk_memory_regions_1(&data, (target_ulong)i << (V_L1_SHIFT + TARGET_PAGE_BITS),
                                        V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
-
         if (rc != 0) {
             return rc;
         }
@@ -1739,13 +1741,13 @@ int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
     return walk_memory_regions_end(&data, 0, 0);
 }
 
-static int dump_region(void *priv, abi_ulong start,
-    abi_ulong end, unsigned long prot)
+static int dump_region(void *priv, target_ulong start,
+    target_ulong end, unsigned long prot)
 {
     FILE *f = (FILE *)priv;
 
-    (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
-        " "TARGET_ABI_FMT_lx" %c%c%c\n",
+    (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx
+        " "TARGET_FMT_lx" %c%c%c\n",
         start, end, end - start,
         ((prot & PAGE_READ) ? 'r' : '-'),
         ((prot & PAGE_WRITE) ? 'w' : '-'),
@@ -1757,7 +1759,7 @@ static int dump_region(void *priv, abi_ulong start,
 /* dump memory mappings */
 void page_dump(FILE *f)
 {
-    const int length = sizeof(abi_ulong) * 2;
+    const int length = sizeof(target_ulong) * 2;
     (void) fprintf(f, "%-*s %-*s %-*s %s\n",
             length, "start", length, "end", length, "size", "prot");
     walk_memory_regions(f, dump_region);
@@ -1785,7 +1787,7 @@ void page_set_flags(target_ulong start, target_ulong end, int flags)
        guest address space.  If this assert fires, it probably indicates
        a missing call to h2g_valid.  */
 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
-    assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
+    assert(end < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
 #endif
     assert(start < end);
 
@@ -1822,7 +1824,7 @@ int page_check_range(target_ulong start, target_ulong len, int flags)
        guest address space.  If this assert fires, it probably indicates
        a missing call to h2g_valid.  */
 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
-    assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
+    assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
 #endif
 
     if (len == 0) {
diff --git a/ui/Makefile.objs b/ui/Makefile.objs
index 6afb52a93..801cba267 100644
--- a/ui/Makefile.objs
+++ b/ui/Makefile.objs
@@ -10,12 +10,13 @@ vnc-obj-y += vnc-jobs.o
 common-obj-y += keymaps.o console.o cursor.o qemu-pixman.o
 common-obj-y += input.o input-keymap.o input-legacy.o
 common-obj-$(CONFIG_SPICE) += spice-core.o spice-input.o spice-display.o
-common-obj-$(CONFIG_SDL) += sdl.o sdl_zoom.o x_keymap.o sdl2.o
+common-obj-$(CONFIG_SDL) += sdl.mo x_keymap.o
 common-obj-$(CONFIG_COCOA) += cocoa.o
 common-obj-$(CONFIG_CURSES) += curses.o
 common-obj-$(CONFIG_VNC) += $(vnc-obj-y)
 common-obj-$(CONFIG_GTK) += gtk.o x_keymap.o
 
-$(obj)/sdl.o $(obj)/sdl_zoom.o $(obj)/sdl2.o: QEMU_CFLAGS += $(SDL_CFLAGS)
+sdl.mo-objs := sdl.o sdl_zoom.o sdl2.o
+sdl.mo-cflags := $(SDL_CFLAGS)
 
 gtk.o-cflags := $(GTK_CFLAGS) $(VTE_CFLAGS)
diff --git a/ui/cocoa.m b/ui/cocoa.m
index 314717811..d37c29b4a 100644
--- a/ui/cocoa.m
+++ b/ui/cocoa.m
@@ -857,7 +857,7 @@ QemuCocoaView *cocoaView;
         [op setPrompt:@"Boot image"];
         [op setMessage:@"Select the disk image you want to boot.\n\nHit the \"Cancel\" button to quit"];
         NSArray *filetypes = [NSArray arrayWithObjects:@"img", @"iso", @"dmg",
-                                 @"qcow", @"qcow2", @"cow", @"cloop", @"vmdk", nil];
+                                 @"qcow", @"qcow2", @"cloop", @"vmdk", nil];
 #if (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)
         [op setAllowedFileTypes:filetypes];
         [op beginSheetModalForWindow:normalWindow
diff --git a/ui/console.c b/ui/console.c
index ab8454903..258af5dff 100644
--- a/ui/console.c
+++ b/ui/console.c
@@ -28,6 +28,7 @@
 #include "qmp-commands.h"
 #include "sysemu/char.h"
 #include "trace.h"
+#include "exec/memory.h"
 
 #define DEFAULT_BACKSCROLL 512
 #define CONSOLE_CURSOR_PERIOD 500
@@ -1224,61 +1225,77 @@ static QemuConsole *new_console(DisplayState *ds, console_type_t console_type,
     return s;
 }
 
-static void qemu_alloc_display(DisplaySurface *surface, int width, int height,
-                               int linesize, PixelFormat pf, int newflags)
+static void qemu_alloc_display(DisplaySurface *surface, int width, int height)
 {
-    surface->pf = pf;
-
     qemu_pixman_image_unref(surface->image);
     surface->image = NULL;
 
-    surface->format = qemu_pixman_get_format(&pf);
-    assert(surface->format != 0);
+    surface->format = PIXMAN_x8r8g8b8;
     surface->image = pixman_image_create_bits(surface->format,
                                               width, height,
-                                              NULL, linesize);
+                                              NULL, width * 4);
     assert(surface->image != NULL);
 
-    surface->flags = newflags | QEMU_ALLOCATED_FLAG;
-#ifdef HOST_WORDS_BIGENDIAN
-    surface->flags |= QEMU_BIG_ENDIAN_FLAG;
-#endif
+    surface->flags = QEMU_ALLOCATED_FLAG;
 }
 
 DisplaySurface *qemu_create_displaysurface(int width, int height)
 {
     DisplaySurface *surface = g_new0(DisplaySurface, 1);
-    int linesize = width * 4;
 
     trace_displaysurface_create(surface, width, height);
-    qemu_alloc_display(surface, width, height, linesize,
-                       qemu_default_pixelformat(32), 0);
+    qemu_alloc_display(surface, width, height);
     return surface;
 }
 
-DisplaySurface *qemu_create_displaysurface_from(int width, int height, int bpp,
-                                                int linesize, uint8_t *data,
-                                                bool byteswap)
+DisplaySurface *qemu_create_displaysurface_from(int width, int height,
+                                                pixman_format_code_t format,
+                                                int linesize, uint8_t *data)
 {
     DisplaySurface *surface = g_new0(DisplaySurface, 1);
 
-    trace_displaysurface_create_from(surface, width, height, bpp, byteswap);
-    if (byteswap) {
-        surface->pf = qemu_different_endianness_pixelformat(bpp);
-    } else {
-        surface->pf = qemu_default_pixelformat(bpp);
-    }
-
-    surface->format = qemu_pixman_get_format(&surface->pf);
-    assert(surface->format != 0);
+    trace_displaysurface_create_from(surface, width, height, format);
+    surface->format = format;
     surface->image = pixman_image_create_bits(surface->format,
                                               width, height,
                                               (void *)data, linesize);
     assert(surface->image != NULL);
 
-#ifdef HOST_WORDS_BIGENDIAN
-    surface->flags = QEMU_BIG_ENDIAN_FLAG;
-#endif
+    return surface;
+}
+
+static void qemu_unmap_displaysurface_guestmem(pixman_image_t *image,
+                                               void *unused)
+{
+    void *data = pixman_image_get_data(image);
+    uint32_t size = pixman_image_get_stride(image) *
+        pixman_image_get_height(image);
+    cpu_physical_memory_unmap(data, size, 0, 0);
+}
+
+DisplaySurface *qemu_create_displaysurface_guestmem(int width, int height,
+                                                    pixman_format_code_t format,
+                                                    int linesize, uint64_t addr)
+{
+    DisplaySurface *surface;
+    hwaddr size;
+    void *data;
+
+    if (linesize == 0) {
+        linesize = width * PIXMAN_FORMAT_BPP(format) / 8;
+    }
+
+    size = linesize * height;
+    data = cpu_physical_memory_map(addr, &size, 0);
+    if (size != linesize * height) {
+        cpu_physical_memory_unmap(data, size, 0, 0);
+        return NULL;
+    }
+
+    surface = qemu_create_displaysurface_from
+        (width, height, format, linesize, data);
+    pixman_image_set_destroy_function
+        (surface->image, qemu_unmap_displaysurface_guestmem, NULL);
 
     return surface;
 }
@@ -1557,6 +1574,67 @@ bool dpy_cursor_define_supported(QemuConsole *con)
     return false;
 }
 
+/*
+ * Call dpy_gfx_update for all dirity scanlines.  Works for
+ * DisplaySurfaces backed by guest memory (i.e. the ones created
+ * using qemu_create_displaysurface_guestmem).
+ */
+void dpy_gfx_update_dirty(QemuConsole *con,
+                          MemoryRegion *address_space,
+                          hwaddr base,
+                          bool invalidate)
+{
+    DisplaySurface *ds = qemu_console_surface(con);
+    int width = surface_stride(ds);
+    int height = surface_height(ds);
+    hwaddr size = width * height;
+    MemoryRegionSection mem_section;
+    MemoryRegion *mem;
+    ram_addr_t addr;
+    int first, last, i;
+    bool dirty;
+
+    mem_section = memory_region_find(address_space, base, size);
+    mem = mem_section.mr;
+    if (int128_get64(mem_section.size) != size ||
+        !memory_region_is_ram(mem_section.mr)) {
+        goto out;
+    }
+    assert(mem);
+
+    memory_region_sync_dirty_bitmap(mem);
+    addr = mem_section.offset_within_region;
+
+    first = -1;
+    last = -1;
+    for (i = 0; i < height; i++, addr += width) {
+        dirty = invalidate ||
+            memory_region_get_dirty(mem, addr, width, DIRTY_MEMORY_VGA);
+        if (dirty) {
+            if (first == -1) {
+                first = i;
+            }
+            last = i;
+        }
+        if (first != -1 && !dirty) {
+            assert(last != -1 && last >= first);
+            dpy_gfx_update(con, 0, first, surface_width(ds),
+                           last - first + 1);
+            first = -1;
+        }
+    }
+    if (first != -1) {
+        assert(last != -1 && last >= first);
+        dpy_gfx_update(con, 0, first, surface_width(ds),
+                       last - first + 1);
+    }
+
+    memory_region_reset_dirty(mem, mem_section.offset_within_region, size,
+                              DIRTY_MEMORY_VGA);
+out:
+    memory_region_unref(mem);
+}
+
 /***********************************************************/
 /* register display */
 
@@ -1599,6 +1677,14 @@ DisplayState *init_displaystate(void)
     return display_state;
 }
 
+void graphic_console_set_hwops(QemuConsole *con,
+                               const GraphicHwOps *hw_ops,
+                               void *opaque)
+{
+    con->hw_ops = hw_ops;
+    con->hw = opaque;
+}
+
 QemuConsole *graphic_console_init(DeviceState *dev, uint32_t head,
                                   const GraphicHwOps *hw_ops,
                                   void *opaque)
@@ -1613,8 +1699,7 @@ QemuConsole *graphic_console_init(DeviceState *dev, uint32_t head,
     ds = get_alloc_displaystate();
     trace_console_gfx_new();
     s = new_console(ds, GRAPHIC_CONSOLE, head);
-    s->hw_ops = hw_ops;
-    s->hw = opaque;
+    graphic_console_set_hwops(s, hw_ops, opaque);
     if (dev) {
         object_property_set_link(OBJECT(s), OBJECT(dev), "device",
                                  &error_abort);
@@ -1902,124 +1987,15 @@ DisplayState *qemu_console_displaystate(QemuConsole *console)
 
 PixelFormat qemu_different_endianness_pixelformat(int bpp)
 {
-    PixelFormat pf;
-
-    memset(&pf, 0x00, sizeof(PixelFormat));
-
-    pf.bits_per_pixel = bpp;
-    pf.bytes_per_pixel = DIV_ROUND_UP(bpp, 8);
-    pf.depth = bpp == 32 ? 24 : bpp;
-
-    switch (bpp) {
-        case 24:
-            pf.rmask = 0x000000FF;
-            pf.gmask = 0x0000FF00;
-            pf.bmask = 0x00FF0000;
-            pf.rmax = 255;
-            pf.gmax = 255;
-            pf.bmax = 255;
-            pf.rshift = 0;
-            pf.gshift = 8;
-            pf.bshift = 16;
-            pf.rbits = 8;
-            pf.gbits = 8;
-            pf.bbits = 8;
-            break;
-        case 32:
-            pf.rmask = 0x0000FF00;
-            pf.gmask = 0x00FF0000;
-            pf.bmask = 0xFF000000;
-            pf.amask = 0x00000000;
-            pf.amax = 255;
-            pf.rmax = 255;
-            pf.gmax = 255;
-            pf.bmax = 255;
-            pf.ashift = 0;
-            pf.rshift = 8;
-            pf.gshift = 16;
-            pf.bshift = 24;
-            pf.rbits = 8;
-            pf.gbits = 8;
-            pf.bbits = 8;
-            pf.abits = 8;
-            break;
-        default:
-            break;
-    }
+    pixman_format_code_t fmt = qemu_default_pixman_format(bpp, false);
+    PixelFormat pf = qemu_pixelformat_from_pixman(fmt);
     return pf;
 }
 
 PixelFormat qemu_default_pixelformat(int bpp)
 {
-    PixelFormat pf;
-
-    memset(&pf, 0x00, sizeof(PixelFormat));
-
-    pf.bits_per_pixel = bpp;
-    pf.bytes_per_pixel = DIV_ROUND_UP(bpp, 8);
-    pf.depth = bpp == 32 ? 24 : bpp;
-
-    switch (bpp) {
-        case 15:
-            pf.bits_per_pixel = 16;
-            pf.rmask = 0x00007c00;
-            pf.gmask = 0x000003E0;
-            pf.bmask = 0x0000001F;
-            pf.rmax = 31;
-            pf.gmax = 31;
-            pf.bmax = 31;
-            pf.rshift = 10;
-            pf.gshift = 5;
-            pf.bshift = 0;
-            pf.rbits = 5;
-            pf.gbits = 5;
-            pf.bbits = 5;
-            break;
-        case 16:
-            pf.rmask = 0x0000F800;
-            pf.gmask = 0x000007E0;
-            pf.bmask = 0x0000001F;
-            pf.rmax = 31;
-            pf.gmax = 63;
-            pf.bmax = 31;
-            pf.rshift = 11;
-            pf.gshift = 5;
-            pf.bshift = 0;
-            pf.rbits = 5;
-            pf.gbits = 6;
-            pf.bbits = 5;
-            break;
-        case 24:
-            pf.rmask = 0x00FF0000;
-            pf.gmask = 0x0000FF00;
-            pf.bmask = 0x000000FF;
-            pf.rmax = 255;
-            pf.gmax = 255;
-            pf.bmax = 255;
-            pf.rshift = 16;
-            pf.gshift = 8;
-            pf.bshift = 0;
-            pf.rbits = 8;
-            pf.gbits = 8;
-            pf.bbits = 8;
-            break;
-        case 32:
-            pf.rmask = 0x00FF0000;
-            pf.gmask = 0x0000FF00;
-            pf.bmask = 0x000000FF;
-            pf.rmax = 255;
-            pf.gmax = 255;
-            pf.bmax = 255;
-            pf.rshift = 16;
-            pf.gshift = 8;
-            pf.bshift = 0;
-            pf.rbits = 8;
-            pf.gbits = 8;
-            pf.bbits = 8;
-            break;
-        default:
-            break;
-    }
+    pixman_format_code_t fmt = qemu_default_pixman_format(bpp, true);
+    PixelFormat pf = qemu_pixelformat_from_pixman(fmt);
     return pf;
 }
 
@@ -2066,8 +2042,7 @@ static const TypeInfo qemu_console_info = {
 static void register_types(void)
 {
     type_register_static(&qemu_console_info);
-    register_char_driver_qapi("vc", CHARDEV_BACKEND_KIND_VC,
-                              qemu_chr_parse_vc);
+    register_char_driver("vc", CHARDEV_BACKEND_KIND_VC, qemu_chr_parse_vc);
 }
 
 type_init(register_types);
diff --git a/ui/gtk.c b/ui/gtk.c
index 2345d7e3a..0385757bf 100644
--- a/ui/gtk.c
+++ b/ui/gtk.c
@@ -128,6 +128,7 @@ static inline void gdk_drawable_get_size(GdkWindow *w, gint *ww, gint *wh)
 #define GDK_KEY_q GDK_q
 #define GDK_KEY_plus GDK_plus
 #define GDK_KEY_minus GDK_minus
+#define GDK_KEY_Pause GDK_Pause
 #endif
 
 #define HOTKEY_MODIFIERS        (GDK_CONTROL_MASK | GDK_MOD1_MASK)
@@ -435,6 +436,15 @@ static void gtk_release_modifiers(GtkDisplayState *s)
     }
 }
 
+static void gd_widget_reparent(GtkWidget *from, GtkWidget *to,
+                               GtkWidget *widget)
+{
+    g_object_ref(G_OBJECT(widget));
+    gtk_container_remove(GTK_CONTAINER(from), widget);
+    gtk_container_add(GTK_CONTAINER(to), widget);
+    g_object_unref(G_OBJECT(widget));
+}
+
 /** DisplayState Callbacks **/
 
 static void gd_update(DisplayChangeListener *dcl,
@@ -931,6 +941,12 @@ static gboolean gd_key_event(GtkWidget *widget, GdkEventKey *key, void *opaque)
     int qemu_keycode;
     int i;
 
+    if (key->keyval == GDK_KEY_Pause) {
+        qemu_input_event_send_key_qcode(vc->gfx.dcl.con, Q_KEY_CODE_PAUSE,
+                                        key->type == GDK_KEY_PRESS);
+        return TRUE;
+    }
+
     qemu_keycode = gd_map_keycode(s, gtk_widget_get_display(widget),
                                   gdk_keycode);
 
@@ -1005,6 +1021,12 @@ static void gd_menu_switch_vc(GtkMenuItem *item, void *opaque)
     }
 }
 
+static void gd_accel_switch_vc(void *opaque)
+{
+    VirtualConsole *vc = opaque;
+    gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(vc->menu_item), TRUE);
+}
+
 static void gd_menu_show_tabs(GtkMenuItem *item, void *opaque)
 {
     GtkDisplayState *s = opaque;
@@ -1025,7 +1047,7 @@ static gboolean gd_tab_window_close(GtkWidget *widget, GdkEvent *event,
     GtkDisplayState *s = vc->s;
 
     gtk_widget_set_sensitive(vc->menu_item, true);
-    gtk_widget_reparent(vc->tab_item, s->notebook);
+    gd_widget_reparent(vc->window, s->notebook, vc->tab_item);
     gtk_notebook_set_tab_label_text(GTK_NOTEBOOK(s->notebook),
                                     vc->tab_item, vc->label);
     gtk_widget_destroy(vc->window);
@@ -1059,7 +1081,7 @@ static void gd_menu_untabify(GtkMenuItem *item, void *opaque)
     if (!vc->window) {
         gtk_widget_set_sensitive(vc->menu_item, false);
         vc->window = gtk_window_new(GTK_WINDOW_TOPLEVEL);
-        gtk_widget_reparent(vc->tab_item, vc->window);
+        gd_widget_reparent(s->notebook, vc->window, vc->tab_item);
 
         g_signal_connect(vc->window, "delete-event",
                          G_CALLBACK(gd_tab_window_close), vc);
@@ -1083,7 +1105,7 @@ static void gd_menu_full_screen(GtkMenuItem *item, void *opaque)
 
     if (!s->full_screen) {
         gtk_notebook_set_show_tabs(GTK_NOTEBOOK(s->notebook), FALSE);
-        gtk_widget_set_size_request(s->menu_bar, 0, 0);
+        gtk_widget_hide(s->menu_bar);
         if (vc->type == GD_VC_GFX) {
             gtk_widget_set_size_request(vc->gfx.drawing_area, -1, -1);
             gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item),
@@ -1094,7 +1116,7 @@ static void gd_menu_full_screen(GtkMenuItem *item, void *opaque)
     } else {
         gtk_window_unfullscreen(GTK_WINDOW(s->window));
         gd_menu_show_tabs(GTK_MENU_ITEM(s->show_tabs_item), s);
-        gtk_widget_set_size_request(s->menu_bar, -1, -1);
+        gtk_widget_show(s->menu_bar);
         s->full_screen = FALSE;
         if (vc->type == GD_VC_GFX) {
             vc->gfx.scale_x = 1.0;
@@ -1108,6 +1130,12 @@ static void gd_menu_full_screen(GtkMenuItem *item, void *opaque)
     gd_update_cursor(vc);
 }
 
+static void gd_accel_full_screen(void *opaque)
+{
+    GtkDisplayState *s = opaque;
+    gtk_menu_item_activate(GTK_MENU_ITEM(s->full_screen_item));
+}
+
 static void gd_menu_zoom_in(GtkMenuItem *item, void *opaque)
 {
     GtkDisplayState *s = opaque;
@@ -1386,19 +1414,21 @@ static gboolean gd_focus_out_event(GtkWidget *widget,
 static GSList *gd_vc_menu_init(GtkDisplayState *s, VirtualConsole *vc,
                                int idx, GSList *group, GtkWidget *view_menu)
 {
-    char path[32];
-
-    snprintf(path, sizeof(path), "<QEMU>/View/VC%d", idx);
-
     vc->menu_item = gtk_radio_menu_item_new_with_mnemonic(group, vc->label);
-    group = gtk_radio_menu_item_get_group(GTK_RADIO_MENU_ITEM(vc->menu_item));
-    gtk_menu_item_set_accel_path(GTK_MENU_ITEM(vc->menu_item), path);
-    gtk_accel_map_add_entry(path, GDK_KEY_1 + idx, HOTKEY_MODIFIERS);
+    gtk_accel_group_connect(s->accel_group, GDK_KEY_1 + idx,
+            HOTKEY_MODIFIERS, 0,
+            g_cclosure_new_swap(G_CALLBACK(gd_accel_switch_vc), vc, NULL));
+#if GTK_CHECK_VERSION(3, 8, 0)
+    gtk_accel_label_set_accel(
+            GTK_ACCEL_LABEL(gtk_bin_get_child(GTK_BIN(vc->menu_item))),
+            GDK_KEY_1 + idx, HOTKEY_MODIFIERS);
+#endif
 
     g_signal_connect(vc->menu_item, "activate",
                      G_CALLBACK(gd_menu_switch_vc), s);
     gtk_menu_shell_append(GTK_MENU_SHELL(view_menu), vc->menu_item);
 
+    group = gtk_radio_menu_item_get_group(GTK_RADIO_MENU_ITEM(vc->menu_item));
     return group;
 }
 
@@ -1590,13 +1620,13 @@ static void gd_connect_signals(GtkDisplayState *s)
                      G_CALLBACK(gd_change_page), s);
 }
 
-static GtkWidget *gd_create_menu_machine(GtkDisplayState *s, GtkAccelGroup *accel_group)
+static GtkWidget *gd_create_menu_machine(GtkDisplayState *s)
 {
     GtkWidget *machine_menu;
     GtkWidget *separator;
 
     machine_menu = gtk_menu_new();
-    gtk_menu_set_accel_group(GTK_MENU(machine_menu), accel_group);
+    gtk_menu_set_accel_group(GTK_MENU(machine_menu), s->accel_group);
 
     s->pause_item = gtk_check_menu_item_new_with_mnemonic(_("_Pause"));
     gtk_menu_shell_append(GTK_MENU_SHELL(machine_menu), s->pause_item);
@@ -1636,10 +1666,9 @@ static GSList *gd_vc_gfx_init(GtkDisplayState *s, VirtualConsole *vc,
                               QemuConsole *con, int idx,
                               GSList *group, GtkWidget *view_menu)
 {
-    Error *local_err = NULL;
     Object *obj;
 
-    obj = object_property_get_link(OBJECT(con), "device", &local_err);
+    obj = object_property_get_link(OBJECT(con), "device", NULL);
     if (obj) {
         vc->label = g_strdup_printf("%s", object_get_typename(obj));
     } else {
@@ -1660,7 +1689,6 @@ static GSList *gd_vc_gfx_init(GtkDisplayState *s, VirtualConsole *vc,
                           GDK_LEAVE_NOTIFY_MASK |
                           GDK_SCROLL_MASK |
                           GDK_KEY_PRESS_MASK);
-    gtk_widget_set_double_buffered(vc->gfx.drawing_area, FALSE);
     gtk_widget_set_can_focus(vc->gfx.drawing_area, TRUE);
 
     vc->type = GD_VC_GFX;
@@ -1678,7 +1706,7 @@ static GSList *gd_vc_gfx_init(GtkDisplayState *s, VirtualConsole *vc,
     return group;
 }
 
-static GtkWidget *gd_create_menu_view(GtkDisplayState *s, GtkAccelGroup *accel_group)
+static GtkWidget *gd_create_menu_view(GtkDisplayState *s)
 {
     GSList *group = NULL;
     GtkWidget *view_menu;
@@ -1687,13 +1715,17 @@ static GtkWidget *gd_create_menu_view(GtkDisplayState *s, GtkAccelGroup *accel_g
     int vc;
 
     view_menu = gtk_menu_new();
-    gtk_menu_set_accel_group(GTK_MENU(view_menu), accel_group);
+    gtk_menu_set_accel_group(GTK_MENU(view_menu), s->accel_group);
 
     s->full_screen_item = gtk_menu_item_new_with_mnemonic(_("_Fullscreen"));
-    gtk_menu_item_set_accel_path(GTK_MENU_ITEM(s->full_screen_item),
-                                 "<QEMU>/View/Full Screen");
-    gtk_accel_map_add_entry("<QEMU>/View/Full Screen", GDK_KEY_f,
-                            HOTKEY_MODIFIERS);
+
+    gtk_accel_group_connect(s->accel_group, GDK_KEY_f, HOTKEY_MODIFIERS, 0,
+            g_cclosure_new_swap(G_CALLBACK(gd_accel_full_screen), s, NULL));
+#if GTK_CHECK_VERSION(3, 8, 0)
+    gtk_accel_label_set_accel(
+            GTK_ACCEL_LABEL(gtk_bin_get_child(GTK_BIN(s->full_screen_item))),
+            GDK_KEY_f, HOTKEY_MODIFIERS);
+#endif
     gtk_menu_shell_append(GTK_MENU_SHELL(view_menu), s->full_screen_item);
 
     separator = gtk_separator_menu_item_new();
@@ -1769,11 +1801,9 @@ static GtkWidget *gd_create_menu_view(GtkDisplayState *s, GtkAccelGroup *accel_g
 
 static void gd_create_menus(GtkDisplayState *s)
 {
-    GtkAccelGroup *accel_group;
-
-    accel_group = gtk_accel_group_new();
-    s->machine_menu = gd_create_menu_machine(s, accel_group);
-    s->view_menu = gd_create_menu_view(s, accel_group);
+    s->accel_group = gtk_accel_group_new();
+    s->machine_menu = gd_create_menu_machine(s);
+    s->view_menu = gd_create_menu_view(s);
 
     s->machine_menu_item = gtk_menu_item_new_with_mnemonic(_("_Machine"));
     gtk_menu_item_set_submenu(GTK_MENU_ITEM(s->machine_menu_item),
@@ -1784,9 +1814,8 @@ static void gd_create_menus(GtkDisplayState *s)
     gtk_menu_item_set_submenu(GTK_MENU_ITEM(s->view_menu_item), s->view_menu);
     gtk_menu_shell_append(GTK_MENU_SHELL(s->menu_bar), s->view_menu_item);
 
-    g_object_set_data(G_OBJECT(s->window), "accel_group", accel_group);
-    gtk_window_add_accel_group(GTK_WINDOW(s->window), accel_group);
-    s->accel_group = accel_group;
+    g_object_set_data(G_OBJECT(s->window), "accel_group", s->accel_group);
+    gtk_window_add_accel_group(GTK_WINDOW(s->window), s->accel_group);
 }
 
 static void gd_set_keycode_type(GtkDisplayState *s)
@@ -1810,6 +1839,13 @@ static void gd_set_keycode_type(GtkDisplayState *s)
             fprintf(stderr, "unknown keycodes `%s', please report to "
                     "qemu-devel@nongnu.org\n", keycodes);
         }
+
+        if (desc) {
+            XkbFreeKeyboard(desc, XkbGBN_AllComponentsMask, True);
+        }
+        if (keycodes) {
+            XFree(keycodes);
+        }
     }
 #endif
 }
@@ -1873,15 +1909,17 @@ void gtk_display_init(DisplayState *ds, bool full_screen, bool grab_on_hover)
 #ifdef VTE_RESIZE_HACK
     {
         VirtualConsole *cur = gd_vc_find_current(s);
-        int i;
-
-        for (i = 0; i < s->nb_vcs; i++) {
-            VirtualConsole *vc = &s->vc[i];
-            if (vc && vc->type == GD_VC_VTE && vc != cur) {
-                gtk_widget_hide(vc->vte.terminal);
+        if (cur) {
+            int i;
+
+            for (i = 0; i < s->nb_vcs; i++) {
+                VirtualConsole *vc = &s->vc[i];
+                if (vc && vc->type == GD_VC_VTE && vc != cur) {
+                    gtk_widget_hide(vc->vte.terminal);
+                }
             }
+            gd_update_windowsize(cur);
         }
-        gd_update_windowsize(cur);
     }
 #endif
 
diff --git a/ui/input-legacy.c b/ui/input-legacy.c
index 3025f50f4..a698a342b 100644
--- a/ui/input-legacy.c
+++ b/ui/input-legacy.c
@@ -85,6 +85,8 @@ void qmp_send_key(KeyValueList *keys, bool has_hold_time, int64_t hold_time,
                   Error **errp)
 {
     KeyValueList *p;
+    KeyValue **up = NULL;
+    int count = 0;
 
     if (!has_hold_time) {
         hold_time = 0; /* use default */
@@ -93,11 +95,16 @@ void qmp_send_key(KeyValueList *keys, bool has_hold_time, int64_t hold_time,
     for (p = keys; p != NULL; p = p->next) {
         qemu_input_event_send_key(NULL, copy_key_value(p->value), true);
         qemu_input_event_send_key_delay(hold_time);
+        up = g_realloc(up, sizeof(*up) * (count+1));
+        up[count] = copy_key_value(p->value);
+        count++;
     }
-    for (p = keys; p != NULL; p = p->next) {
-        qemu_input_event_send_key(NULL, copy_key_value(p->value), false);
+    while (count) {
+        count--;
+        qemu_input_event_send_key(NULL, up[count], false);
         qemu_input_event_send_key_delay(hold_time);
     }
+    g_free(up);
 }
 
 static void legacy_kbd_event(DeviceState *dev, QemuConsole *src,
diff --git a/ui/input.c b/ui/input.c
index 89d9db78c..7ba99e56b 100644
--- a/ui/input.c
+++ b/ui/input.c
@@ -122,6 +122,46 @@ qemu_input_find_handler(uint32_t mask, QemuConsole *con)
     return NULL;
 }
 
+void qmp_x_input_send_event(bool has_console, int64_t console,
+                            InputEventList *events, Error **errp)
+{
+    InputEventList *e;
+    QemuConsole *con;
+
+    con = NULL;
+    if (has_console) {
+        con = qemu_console_lookup_by_index(console);
+        if (!con) {
+            error_setg(errp, "console %" PRId64 " not found", console);
+            return;
+        }
+    }
+
+    if (!runstate_is_running() && !runstate_check(RUN_STATE_SUSPENDED)) {
+        error_setg(errp, "VM not running");
+        return;
+    }
+
+    for (e = events; e != NULL; e = e->next) {
+        InputEvent *event = e->value;
+
+        if (!qemu_input_find_handler(1 << event->kind, con)) {
+            error_setg(errp, "Input handler not found for "
+                             "event type %s",
+                            InputEventKind_lookup[event->kind]);
+            return;
+        }
+    }
+
+    for (e = events; e != NULL; e = e->next) {
+        InputEvent *event = e->value;
+
+        qemu_input_event_send(con, event);
+    }
+
+    qemu_input_event_sync();
+}
+
 static void qemu_input_transform_abs_rotate(InputEvent *evt)
 {
     switch (graphic_rotate) {
diff --git a/ui/qemu-pixman.c b/ui/qemu-pixman.c
index 254bd8ce1..1f6fea535 100644
--- a/ui/qemu-pixman.c
+++ b/ui/qemu-pixman.c
@@ -6,6 +6,87 @@
 #include "qemu-common.h"
 #include "ui/console.h"
 
+PixelFormat qemu_pixelformat_from_pixman(pixman_format_code_t format)
+{
+    PixelFormat pf;
+    uint8_t bpp;
+
+    bpp = pf.bits_per_pixel = PIXMAN_FORMAT_BPP(format);
+    pf.bytes_per_pixel = PIXMAN_FORMAT_BPP(format) / 8;
+    pf.depth = PIXMAN_FORMAT_DEPTH(format);
+
+    pf.abits = PIXMAN_FORMAT_A(format);
+    pf.rbits = PIXMAN_FORMAT_R(format);
+    pf.gbits = PIXMAN_FORMAT_G(format);
+    pf.bbits = PIXMAN_FORMAT_B(format);
+
+    switch (PIXMAN_FORMAT_TYPE(format)) {
+    case PIXMAN_TYPE_ARGB:
+        pf.ashift = pf.bbits + pf.gbits + pf.rbits;
+        pf.rshift = pf.bbits + pf.gbits;
+        pf.gshift = pf.bbits;
+        pf.bshift = 0;
+        break;
+    case PIXMAN_TYPE_ABGR:
+        pf.ashift = pf.rbits + pf.gbits + pf.bbits;
+        pf.bshift = pf.rbits + pf.gbits;
+        pf.gshift = pf.rbits;
+        pf.rshift = 0;
+        break;
+    case PIXMAN_TYPE_BGRA:
+	pf.bshift = bpp - pf.bbits;
+        pf.gshift = bpp - (pf.bbits + pf.gbits);
+        pf.rshift = bpp - (pf.bbits + pf.gbits + pf.rbits);
+        pf.ashift = 0;
+        break;
+    case PIXMAN_TYPE_RGBA:
+        pf.rshift = bpp - pf.rbits;
+        pf.gshift = bpp - (pf.rbits + pf.gbits);
+        pf.bshift = bpp - (pf.rbits + pf.gbits + pf.bbits);
+        pf.ashift = 0;
+        break;
+    default:
+        g_assert_not_reached();
+        break;
+    }
+
+    pf.amax = (1 << pf.abits) - 1;
+    pf.rmax = (1 << pf.rbits) - 1;
+    pf.gmax = (1 << pf.gbits) - 1;
+    pf.bmax = (1 << pf.bbits) - 1;
+    pf.amask = pf.amax << pf.ashift;
+    pf.rmask = pf.rmax << pf.rshift;
+    pf.gmask = pf.gmax << pf.gshift;
+    pf.bmask = pf.bmax << pf.bshift;
+
+    return pf;
+}
+
+pixman_format_code_t qemu_default_pixman_format(int bpp, bool native_endian)
+{
+    if (native_endian) {
+        switch (bpp) {
+        case 15:
+            return PIXMAN_x1r5g5b5;
+        case 16:
+            return PIXMAN_r5g6b5;
+        case 24:
+            return PIXMAN_r8g8b8;
+        case 32:
+            return PIXMAN_x8r8g8b8;
+        }
+    } else {
+        switch (bpp) {
+        case 24:
+            return PIXMAN_b8g8r8;
+        case 32:
+            return PIXMAN_b8g8r8x8;
+        break;
+        }
+    }
+    g_assert_not_reached();
+}
+
 int qemu_pixman_get_type(int rshift, int gshift, int bshift)
 {
     int type = PIXMAN_TYPE_OTHER;
@@ -52,6 +133,7 @@ pixman_image_t *qemu_pixman_linebuf_create(pixman_format_code_t format,
     return image;
 }
 
+/* fill linebuf from framebuffer */
 void qemu_pixman_linebuf_fill(pixman_image_t *linebuf, pixman_image_t *fb,
                               int width, int x, int y)
 {
@@ -59,6 +141,14 @@ void qemu_pixman_linebuf_fill(pixman_image_t *linebuf, pixman_image_t *fb,
                            x, y, 0, 0, 0, 0, width, 1);
 }
 
+/* copy linebuf to framebuffer */
+void qemu_pixman_linebuf_copy(pixman_image_t *fb, int width, int x, int y,
+                              pixman_image_t *linebuf)
+{
+    pixman_image_composite(PIXMAN_OP_SRC, linebuf, NULL, fb,
+                           0, 0, 0, 0, x, y, width, 1);
+}
+
 pixman_image_t *qemu_pixman_mirror_create(pixman_format_code_t format,
                                           pixman_image_t *image)
 {
diff --git a/ui/sdl.c b/ui/sdl.c
index 4e7f920e3..94c1d9dc3 100644
--- a/ui/sdl.c
+++ b/ui/sdl.c
@@ -127,6 +127,7 @@ static void do_sdl_resize(int width, int height, int bpp)
 static void sdl_switch(DisplayChangeListener *dcl,
                        DisplaySurface *new_surface)
 {
+    PixelFormat pf = qemu_pixelformat_from_pixman(new_surface->format);
 
     /* temporary hack: allows to call sdl_switch to handle scaling changes */
     if (new_surface) {
@@ -148,8 +149,8 @@ static void sdl_switch(DisplayChangeListener *dcl,
         (surface_data(surface),
          surface_width(surface), surface_height(surface),
          surface_bits_per_pixel(surface), surface_stride(surface),
-         surface->pf.rmask, surface->pf.gmask,
-         surface->pf.bmask, surface->pf.amask);
+         pf.rmask, pf.gmask,
+         pf.bmask, pf.amask);
 }
 
 /* generic keyboard conversion */
diff --git a/ui/sdl2-keymap.h b/ui/sdl2-keymap.h
index 5a12f4543..cbedaa477 100644
--- a/ui/sdl2-keymap.h
+++ b/ui/sdl2-keymap.h
@@ -105,9 +105,10 @@ static const int sdl2_scancode_to_qcode[SDL_NUM_SCANCODES] = {
     [SDL_SCANCODE_KP_9]              = Q_KEY_CODE_KP_9,
     [SDL_SCANCODE_KP_0]              = Q_KEY_CODE_KP_0,
     [SDL_SCANCODE_KP_PERIOD]         = Q_KEY_CODE_KP_DECIMAL,
+
+    [SDL_SCANCODE_NONUSBACKSLASH]    = Q_KEY_CODE_LESS,
+    [SDL_SCANCODE_APPLICATION]       = Q_KEY_CODE_MENU,
 #if 0
-    [SDL_SCANCODE_NONUSBACKSLASH]    = Q_KEY_CODE_NONUSBACKSLASH,
-    [SDL_SCANCODE_APPLICATION]       = Q_KEY_CODE_APPLICATION,
     [SDL_SCANCODE_POWER]             = Q_KEY_CODE_POWER,
     [SDL_SCANCODE_KP_EQUALS]         = Q_KEY_CODE_KP_EQUALS,
 
@@ -231,7 +232,7 @@ static const int sdl2_scancode_to_qcode[SDL_NUM_SCANCODES] = {
     [SDL_SCANCODE_LGUI]              = Q_KEY_CODE_META_L,
     [SDL_SCANCODE_RCTRL]             = Q_KEY_CODE_CTRL_R,
     [SDL_SCANCODE_RSHIFT]            = Q_KEY_CODE_SHIFT_R,
-    [SDL_SCANCODE_RALT]              = Q_KEY_CODE_ALTGR,
+    [SDL_SCANCODE_RALT]              = Q_KEY_CODE_ALT_R,
     [SDL_SCANCODE_RGUI]              = Q_KEY_CODE_META_R,
 #if 0
     [SDL_SCANCODE_MODE]              = Q_KEY_CODE_MODE,
diff --git a/ui/sdl2.c b/ui/sdl2.c
index fcac87b4b..1ad74baaf 100644
--- a/ui/sdl2.c
+++ b/ui/sdl2.c
@@ -35,7 +35,6 @@
 #include "ui/console.h"
 #include "ui/input.h"
 #include "sysemu/sysemu.h"
-#include "sdl_zoom.h"
 
 #include "sdl2-keymap.h"
 
diff --git a/ui/spice-core.c b/ui/spice-core.c
index 7bb91e6ba..6467fa477 100644
--- a/ui/spice-core.c
+++ b/ui/spice-core.c
@@ -677,7 +677,7 @@ void qemu_spice_init(void)
 
     if (tls_port) {
         x509_dir = qemu_opt_get(opts, "x509-dir");
-        if (NULL == x509_dir) {
+        if (!x509_dir) {
             x509_dir = ".";
         }
 
@@ -733,7 +733,7 @@ void qemu_spice_init(void)
                              tls_ciphers);
     }
     if (password) {
-        spice_server_set_ticket(spice_server, password, 0, 0, 0);
+        qemu_spice_set_passwd(password, false, false);
     }
     if (qemu_opt_get_bool(opts, "sasl", 0)) {
         if (spice_server_set_sasl_appname(spice_server, "qemu") == -1 ||
@@ -803,7 +803,7 @@ void qemu_spice_init(void)
 
     seamless_migration = qemu_opt_get_bool(opts, "seamless-migration", 0);
     spice_server_set_seamless_migration(spice_server, seamless_migration);
-    if (0 != spice_server_init(spice_server, &core_interface)) {
+    if (spice_server_init(spice_server, &core_interface) != 0) {
         error_report("failed to initialize spice server");
         exit(1);
     };
@@ -853,7 +853,6 @@ int qemu_spice_add_interface(SpiceBaseInstance *sin)
 }
 
 static GSList *spice_consoles;
-static int display_id;
 
 bool qemu_spice_have_display_interface(QemuConsole *con)
 {
@@ -868,7 +867,7 @@ int qemu_spice_add_display_interface(QXLInstance *qxlin, QemuConsole *con)
     if (g_slist_find(spice_consoles, con)) {
         return -1;
     }
-    qxlin->id = display_id++;
+    qxlin->id = qemu_console_get_index(con);
     spice_consoles = g_slist_append(spice_consoles, con);
     return qemu_spice_add_interface(&qxlin->base);
 }
diff --git a/ui/spice-display.c b/ui/spice-display.c
index 66e25788c..def7b52e9 100644
--- a/ui/spice-display.c
+++ b/ui/spice-display.c
@@ -334,11 +334,23 @@ void qemu_spice_create_host_memslot(SimpleSpiceDisplay *ssd)
 void qemu_spice_create_host_primary(SimpleSpiceDisplay *ssd)
 {
     QXLDevSurfaceCreate surface;
+    uint64_t surface_size;
 
     memset(&surface, 0, sizeof(surface));
 
-    dprint(1, "%s/%d: %dx%d\n", __func__, ssd->qxl.id,
-           surface_width(ssd->ds), surface_height(ssd->ds));
+    surface_size = (uint64_t) surface_width(ssd->ds) *
+        surface_height(ssd->ds) * 4;
+    assert(surface_size > 0);
+    assert(surface_size < INT_MAX);
+    if (ssd->bufsize < surface_size) {
+        ssd->bufsize = surface_size;
+        g_free(ssd->buf);
+        ssd->buf = g_malloc(ssd->bufsize);
+    }
+
+    dprint(1, "%s/%d: %ux%u (size %" PRIu64 "/%d)\n", __func__, ssd->qxl.id,
+           surface_width(ssd->ds), surface_height(ssd->ds),
+           surface_size, ssd->bufsize);
 
     surface.format     = SPICE_SURFACE_FMT_32_xRGB;
     surface.width      = surface_width(ssd->ds);
@@ -369,8 +381,6 @@ void qemu_spice_display_init_common(SimpleSpiceDisplay *ssd)
     if (ssd->num_surfaces == 0) {
         ssd->num_surfaces = 1024;
     }
-    ssd->bufsize = (16 * 1024 * 1024);
-    ssd->buf = g_malloc(ssd->bufsize);
 }
 
 /* display listener callbacks */
@@ -495,7 +505,7 @@ static void interface_get_init_info(QXLInstance *sin, QXLDevInitInfo *info)
     info->num_memslots = NUM_MEMSLOTS;
     info->num_memslots_groups = NUM_MEMSLOTS_GROUPS;
     info->internal_groupslot_id = 0;
-    info->qxl_ram_size = ssd->bufsize;
+    info->qxl_ram_size = 16 * 1024 * 1024;
     info->n_surfaces = ssd->num_surfaces;
 }
 
diff --git a/ui/vnc-enc-tight.c b/ui/vnc-enc-tight.c
index f02352cc4..3d1b5cd06 100644
--- a/ui/vnc-enc-tight.c
+++ b/ui/vnc-enc-tight.c
@@ -220,8 +220,7 @@ tight_detect_smooth_image24(VncState *vs, int w, int h)
         unsigned int errors;                                            \
         unsigned char *buf = vs->tight.tight.buffer;                    \
                                                                         \
-        endian = 0; /* FIXME: ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) != \
-                      (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG)); */ \
+        endian = 0; /* FIXME */                                         \
                                                                         \
                                                                         \
         max[0] = vs->client_pf.rmax;                                  \
@@ -563,8 +562,7 @@ tight_filter_gradient24(VncState *vs, uint8_t *buf, int w, int h)
     buf32 = (uint32_t *)buf;
     memset(vs->tight.gradient.buffer, 0, w * 3 * sizeof(int));
 
-    if (1 /* FIXME: (vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) ==
-             (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG) */) {
+    if (1 /* FIXME */) {
         shift[0] = vs->client_pf.rshift;
         shift[1] = vs->client_pf.gshift;
         shift[2] = vs->client_pf.bshift;
@@ -621,8 +619,7 @@ tight_filter_gradient24(VncState *vs, uint8_t *buf, int w, int h)
                                                                         \
         memset (vs->tight.gradient.buffer, 0, w * 3 * sizeof(int));     \
                                                                         \
-        endian = 0; /* FIXME: ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) != \
-                       (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG)); */ \
+        endian = 0; /* FIXME */                                         \
                                                                         \
         max[0] = vs->client_pf.rmax;                                    \
         max[1] = vs->client_pf.gmax;                                    \
@@ -898,8 +895,7 @@ static void tight_pack24(VncState *vs, uint8_t *buf, size_t count, size_t *ret)
 
     buf32 = (uint32_t *)buf;
 
-    if (1 /* FIXME: (vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) ==
-             (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG) */) {
+    if (1 /* FIXME */) {
         rshift = vs->client_pf.rshift;
         gshift = vs->client_pf.gshift;
         bshift = vs->client_pf.bshift;
diff --git a/ui/vnc-tls.c b/ui/vnc-tls.c
index 63923265f..0f59f9b28 100644
--- a/ui/vnc-tls.c
+++ b/ui/vnc-tls.c
@@ -444,8 +444,6 @@ static int vnc_set_x509_credential(VncDisplay *vd,
     struct stat sb;
 
     g_free(*cred);
-    *cred = NULL;
-
     *cred = g_malloc(strlen(certdir) + strlen(filename) + 2);
 
     strcpy(*cred, certdir);
diff --git a/ui/vnc.c b/ui/vnc.c
index f8d9b7db9..57070150d 100644
--- a/ui/vnc.c
+++ b/ui/vnc.c
@@ -2026,6 +2026,16 @@ static void set_pixel_format(VncState *vs,
         return;
     }
 
+    switch (bits_per_pixel) {
+    case 8:
+    case 16:
+    case 32:
+        break;
+    default:
+        vnc_client_error(vs);
+        return;
+    }
+
     vs->client_pf.rmax = red_max;
     vs->client_pf.rbits = hweight_long(red_max);
     vs->client_pf.rshift = red_shift;
@@ -2768,6 +2778,11 @@ static void vnc_refresh(DisplayChangeListener *dcl)
     VncState *vs, *vn;
     int has_dirty, rects = 0;
 
+    if (QTAILQ_EMPTY(&vd->clients)) {
+        update_displaychangelistener(&vd->dcl, VNC_REFRESH_INTERVAL_MAX);
+        return;
+    }
+
     graphic_hw_update(NULL);
 
     if (vnc_trylock_display(vd)) {
@@ -2783,11 +2798,6 @@ static void vnc_refresh(DisplayChangeListener *dcl)
         /* vs might be free()ed here */
     }
 
-    if (QTAILQ_EMPTY(&vd->clients)) {
-        update_displaychangelistener(&vd->dcl, VNC_REFRESH_INTERVAL_MAX);
-        return;
-    }
-
     if (has_dirty && rects) {
         vd->dcl.update_interval /= 2;
         if (vd->dcl.update_interval < VNC_REFRESH_INTERVAL_BASE) {
@@ -2914,6 +2924,7 @@ static void vnc_listen_read(void *opaque, bool websocket)
     }
 
     if (csock != -1) {
+        socket_set_nodelay(csock);
         vnc_connect(vs, csock, false, websocket);
     }
 }
diff --git a/util/Makefile.objs b/util/Makefile.objs
index 6b3c83b0e..93007e2f5 100644
--- a/util/Makefile.objs
+++ b/util/Makefile.objs
@@ -1,12 +1,14 @@
 util-obj-y = osdep.o cutils.o unicode.o qemu-timer-common.o
 util-obj-$(CONFIG_WIN32) += oslib-win32.o qemu-thread-win32.o event_notifier-win32.o
 util-obj-$(CONFIG_POSIX) += oslib-posix.o qemu-thread-posix.o event_notifier-posix.o qemu-openpty.o
-util-obj-y += envlist.o path.o host-utils.o module.o
+util-obj-y += envlist.o path.o module.o
+util-obj-$(call lnot,$(CONFIG_INT128)) += host-utils.o
 util-obj-y += bitmap.o bitops.o hbitmap.o
 util-obj-y += fifo8.o
 util-obj-y += acl.o
 util-obj-y += error.o qemu-error.o
 util-obj-$(CONFIG_POSIX) += compatfd.o
+util-obj-y += id.o
 util-obj-y += iov.o aes.o qemu-config.o qemu-sockets.o uri.o notify.o
 util-obj-y += qemu-option.o qemu-progress.o
 util-obj-y += hexdump.o
diff --git a/util/acl.c b/util/acl.c
index 938b7ae2d..571d68615 100644
--- a/util/acl.c
+++ b/util/acl.c
@@ -132,7 +132,6 @@ int qemu_acl_insert(qemu_acl *acl,
                     const char *match,
                     int index)
 {
-    qemu_acl_entry *entry;
     qemu_acl_entry *tmp;
     int i = 0;
 
@@ -142,13 +141,14 @@ int qemu_acl_insert(qemu_acl *acl,
         return qemu_acl_append(acl, deny, match);
     }
 
-    entry = g_malloc(sizeof(*entry));
-    entry->match = g_strdup(match);
-    entry->deny = deny;
-
     QTAILQ_FOREACH(tmp, &acl->entries, next) {
         i++;
         if (i == index) {
+            qemu_acl_entry *entry;
+            entry = g_malloc(sizeof(*entry));
+            entry->match = g_strdup(match);
+            entry->deny = deny;
+
             QTAILQ_INSERT_BEFORE(tmp, entry, next);
             acl->nentries++;
             break;
diff --git a/util/getauxval.c b/util/getauxval.c
index 25f48e545..1732ace2b 100644
--- a/util/getauxval.c
+++ b/util/getauxval.c
@@ -98,4 +98,12 @@ unsigned long qemu_getauxval(unsigned long type)
 
     return 0;
 }
+
+#else
+
+unsigned long qemu_getauxval(unsigned long type)
+{
+    return 0;
+}
+
 #endif
diff --git a/util/host-utils.c b/util/host-utils.c
index ee57ef55f..102e5bf30 100644
--- a/util/host-utils.c
+++ b/util/host-utils.c
@@ -28,7 +28,6 @@
 #include "qemu/host-utils.h"
 
 /* Long integer helpers */
-#ifndef CONFIG_INT128
 static inline void mul64(uint64_t *plow, uint64_t *phigh,
                          uint64_t a, uint64_t b)
 {
@@ -161,4 +160,3 @@ int divs128(int64_t *plow, int64_t *phigh, int64_t divisor)
     return overflow;
 }
 
-#endif /* !CONFIG_INT128 */
diff --git a/util/id.c b/util/id.c
new file mode 100644
index 000000000..09b22fb8f
--- /dev/null
+++ b/util/id.c
@@ -0,0 +1,28 @@
+/*
+ * Dealing with identifiers
+ *
+ * Copyright (C) 2014 Red Hat, Inc.
+ *
+ * Authors:
+ *  Markus Armbruster <armbru@redhat.com>,
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1
+ * or later.  See the COPYING.LIB file in the top-level directory.
+ */
+
+#include "qemu-common.h"
+
+bool id_wellformed(const char *id)
+{
+    int i;
+
+    if (!qemu_isalpha(id[0])) {
+        return false;
+    }
+    for (i = 1; id[i]; i++) {
+        if (!qemu_isalnum(id[i]) && !strchr("-._", id[i])) {
+            return false;
+        }
+    }
+    return true;
+}
diff --git a/util/oslib-posix.c b/util/oslib-posix.c
index cdbfb2e27..16fcec2f3 100644
--- a/util/oslib-posix.c
+++ b/util/oslib-posix.c
@@ -94,7 +94,7 @@ void *qemu_oom_check(void *ptr)
     return ptr;
 }
 
-void *qemu_memalign(size_t alignment, size_t size)
+void *qemu_try_memalign(size_t alignment, size_t size)
 {
     void *ptr;
 
@@ -106,21 +106,25 @@ void *qemu_memalign(size_t alignment, size_t size)
     int ret;
     ret = posix_memalign(&ptr, alignment, size);
     if (ret != 0) {
-        fprintf(stderr, "Failed to allocate %zu B: %s\n",
-                size, strerror(ret));
-        abort();
+        errno = ret;
+        ptr = NULL;
     }
 #elif defined(CONFIG_BSD)
-    ptr = qemu_oom_check(valloc(size));
+    ptr = valloc(size);
 #else
-    ptr = qemu_oom_check(memalign(alignment, size));
+    ptr = memalign(alignment, size);
 #endif
     trace_qemu_memalign(alignment, size, ptr);
     return ptr;
 }
 
+void *qemu_memalign(size_t alignment, size_t size)
+{
+    return qemu_oom_check(qemu_try_memalign(alignment, size));
+}
+
 /* alloc shared memory pages */
-void *qemu_anon_ram_alloc(size_t size)
+void *qemu_anon_ram_alloc(size_t size, uint64_t *alignment)
 {
     size_t align = QEMU_VMALLOC_ALIGN;
     size_t total = size + align - getpagesize();
@@ -132,6 +136,9 @@ void *qemu_anon_ram_alloc(size_t size)
         return NULL;
     }
 
+    if (alignment) {
+        *alignment = align;
+    }
     ptr += offset;
     total -= offset;
 
@@ -386,7 +393,8 @@ void os_mem_prealloc(int fd, char *area, size_t memory)
     pthread_sigmask(SIG_UNBLOCK, &set, &oldset);
 
     if (sigsetjmp(sigjump, 1)) {
-        fprintf(stderr, "os_mem_prealloc: failed to preallocate pages\n");
+        fprintf(stderr, "os_mem_prealloc: Insufficient free host memory "
+                        "pages available to allocate guest RAM\n");
         exit(1);
     } else {
         int i;
diff --git a/util/oslib-win32.c b/util/oslib-win32.c
index 507cedd84..87cfbe083 100644
--- a/util/oslib-win32.c
+++ b/util/oslib-win32.c
@@ -50,19 +50,24 @@ void *qemu_oom_check(void *ptr)
     return ptr;
 }
 
-void *qemu_memalign(size_t alignment, size_t size)
+void *qemu_try_memalign(size_t alignment, size_t size)
 {
     void *ptr;
 
     if (!size) {
         abort();
     }
-    ptr = qemu_oom_check(VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE));
+    ptr = VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE);
     trace_qemu_memalign(alignment, size, ptr);
     return ptr;
 }
 
-void *qemu_anon_ram_alloc(size_t size)
+void *qemu_memalign(size_t alignment, size_t size)
+{
+    return qemu_oom_check(qemu_try_memalign(alignment, size));
+}
+
+void *qemu_anon_ram_alloc(size_t size, uint64_t *align)
 {
     void *ptr;
 
diff --git a/util/path.c b/util/path.c
index 5c59d9f1d..4e4877e82 100644
--- a/util/path.c
+++ b/util/path.c
@@ -45,8 +45,8 @@ static struct pathelem *new_entry(const char *root,
                                   struct pathelem *parent,
                                   const char *name)
 {
-    struct pathelem *new = malloc(sizeof(*new));
-    new->name = strdup(name);
+    struct pathelem *new = g_malloc(sizeof(*new));
+    new->name = g_strdup(name);
     new->pathname = g_strdup_printf("%s/%s", root, name);
     new->num_entries = 0;
     return new;
@@ -88,7 +88,7 @@ static struct pathelem *add_entry(struct pathelem *root, const char *name,
 
     root->num_entries++;
 
-    root = realloc(root, sizeof(*root)
+    root = g_realloc(root, sizeof(*root)
                    + sizeof(root->entries[0])*root->num_entries);
     e = &root->entries[root->num_entries-1];
 
@@ -161,8 +161,8 @@ void init_paths(const char *prefix)
     base = add_dir_maybe(base);
     if (base->num_entries == 0) {
         g_free(base->pathname);
-        free(base->name);
-        free(base);
+        g_free(base->name);
+        g_free(base);
         base = NULL;
     } else {
         set_parents(base, base);
diff --git a/util/qemu-error.c b/util/qemu-error.c
index 7b167fd06..9bba5f53d 100644
--- a/util/qemu-error.c
+++ b/util/qemu-error.c
@@ -199,14 +199,13 @@ static void error_print_loc(void)
 bool enable_timestamp_msg;
 /*
  * Print an error message to current monitor if we have one, else to stderr.
- * Format arguments like sprintf().  The result should not contain
+ * Format arguments like vsprintf().  The result should not contain
  * newlines.
  * Prepend the current location and append a newline.
  * It's wrong to call this in a QMP monitor.  Use qerror_report() there.
  */
-void error_report(const char *fmt, ...)
+void error_vreport(const char *fmt, va_list ap)
 {
-    va_list ap;
     GTimeVal tv;
     gchar *timestr;
 
@@ -218,8 +217,22 @@ void error_report(const char *fmt, ...)
     }
 
     error_print_loc();
-    va_start(ap, fmt);
     error_vprintf(fmt, ap);
-    va_end(ap);
     error_printf("\n");
 }
+
+/*
+ * Print an error message to current monitor if we have one, else to stderr.
+ * Format arguments like sprintf().  The result should not contain
+ * newlines.
+ * Prepend the current location and append a newline.
+ * It's wrong to call this in a QMP monitor.  Use qerror_report() there.
+ */
+void error_report(const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    error_vreport(fmt, ap);
+    va_end(ap);
+}
diff --git a/util/qemu-option.c b/util/qemu-option.c
index 6dc27ce04..5d106959c 100644
--- a/util/qemu-option.c
+++ b/util/qemu-option.c
@@ -641,21 +641,6 @@ QemuOpts *qemu_opts_find(QemuOptsList *list, const char *id)
     return NULL;
 }
 
-static int id_wellformed(const char *id)
-{
-    int i;
-
-    if (!qemu_isalpha(id[0])) {
-        return 0;
-    }
-    for (i = 1; id[i]; i++) {
-        if (!qemu_isalnum(id[i]) && !strchr("-._", id[i])) {
-            return 0;
-        }
-    }
-    return 1;
-}
-
 QemuOpts *qemu_opts_create(QemuOptsList *list, const char *id,
                            int fail_if_exists, Error **errp)
 {
diff --git a/util/qemu-sockets.c b/util/qemu-sockets.c
index 74cf0786e..a76bb3c91 100644
--- a/util/qemu-sockets.c
+++ b/util/qemu-sockets.c
@@ -159,7 +159,7 @@ int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp)
         slisten = qemu_socket(e->ai_family, e->ai_socktype, e->ai_protocol);
         if (slisten < 0) {
             if (!e->ai_next) {
-                error_set_errno(errp, errno, QERR_SOCKET_CREATE_FAILED);
+                error_setg_errno(errp, errno, "Failed to create socket");
             }
             continue;
         }
@@ -183,7 +183,7 @@ int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp)
             }
             if (p == port_max) {
                 if (!e->ai_next) {
-                    error_set_errno(errp, errno, QERR_SOCKET_BIND_FAILED);
+                    error_setg_errno(errp, errno, "Failed to bind socket");
                 }
             }
         }
@@ -194,7 +194,7 @@ int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp)
 
 listen:
     if (listen(slisten,1) != 0) {
-        error_set_errno(errp, errno, QERR_SOCKET_LISTEN_FAILED);
+        error_setg_errno(errp, errno, "Failed to listen on socket");
         closesocket(slisten);
         freeaddrinfo(res);
         return -1;
@@ -234,6 +234,7 @@ static void wait_for_connect(void *opaque)
     int val = 0, rc = 0;
     socklen_t valsize = sizeof(val);
     bool in_progress;
+    Error *err = NULL;
 
     qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
 
@@ -244,10 +245,12 @@ static void wait_for_connect(void *opaque)
     /* update rc to contain error */
     if (!rc && val) {
         rc = -1;
+        errno = val;
     }
 
     /* connect error */
     if (rc < 0) {
+        error_setg_errno(&err, errno, "Error connecting to socket");
         closesocket(s->fd);
         s->fd = rc;
     }
@@ -257,9 +260,14 @@ static void wait_for_connect(void *opaque)
         while (s->current_addr->ai_next != NULL && s->fd < 0) {
             s->current_addr = s->current_addr->ai_next;
             s->fd = inet_connect_addr(s->current_addr, &in_progress, s, NULL);
+            if (s->fd < 0) {
+                error_free(err);
+                err = NULL;
+                error_setg_errno(&err, errno, "Unable to start socket connect");
+            }
             /* connect in progress */
             if (in_progress) {
-                return;
+                goto out;
             }
         }
 
@@ -267,9 +275,11 @@ static void wait_for_connect(void *opaque)
     }
 
     if (s->callback) {
-        s->callback(s->fd, s->opaque);
+        s->callback(s->fd, err, s->opaque);
     }
     g_free(s);
+out:
+    error_free(err);
 }
 
 static int inet_connect_addr(struct addrinfo *addr, bool *in_progress,
@@ -281,7 +291,7 @@ static int inet_connect_addr(struct addrinfo *addr, bool *in_progress,
 
     sock = qemu_socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
     if (sock < 0) {
-        error_set_errno(errp, errno, QERR_SOCKET_CREATE_FAILED);
+        error_setg_errno(errp, errno, "Failed to create socket");
         return -1;
     }
     socket_set_fast_reuse(sock);
@@ -302,7 +312,7 @@ static int inet_connect_addr(struct addrinfo *addr, bool *in_progress,
                              connect_state);
         *in_progress = true;
     } else if (rc < 0) {
-        error_set_errno(errp, errno, QERR_SOCKET_CONNECT_FAILED);
+        error_setg_errno(errp, errno, "Failed to connect socket");
         closesocket(sock);
         return -1;
     }
@@ -401,7 +411,7 @@ int inet_connect_opts(QemuOpts *opts, Error **errp,
         return sock;
     } else {
         if (callback) {
-            callback(sock, opaque);
+            callback(sock, NULL, opaque);
         }
     }
     g_free(connect_state);
@@ -466,20 +476,20 @@ int inet_dgram_opts(QemuOpts *opts, Error **errp)
     /* create socket */
     sock = qemu_socket(peer->ai_family, peer->ai_socktype, peer->ai_protocol);
     if (sock < 0) {
-        error_set_errno(errp, errno, QERR_SOCKET_CREATE_FAILED);
+        error_setg_errno(errp, errno, "Failed to create socket");
         goto err;
     }
     socket_set_fast_reuse(sock);
 
     /* bind socket */
     if (bind(sock, local->ai_addr, local->ai_addrlen) < 0) {
-        error_set_errno(errp, errno, QERR_SOCKET_BIND_FAILED);
+        error_setg_errno(errp, errno, "Failed to bind socket");
         goto err;
     }
 
     /* connect to peer */
     if (connect(sock,peer->ai_addr,peer->ai_addrlen) < 0) {
-        error_set_errno(errp, errno, QERR_SOCKET_CONNECT_FAILED);
+        error_setg_errno(errp, errno, "Failed to connect socket");
         goto err;
     }
 
@@ -684,7 +694,7 @@ int unix_listen_opts(QemuOpts *opts, Error **errp)
 
     sock = qemu_socket(PF_UNIX, SOCK_STREAM, 0);
     if (sock < 0) {
-        error_set_errno(errp, errno, QERR_SOCKET_CREATE_FAILED);
+        error_setg_errno(errp, errno, "Failed to create socket");
         return -1;
     }
 
@@ -709,11 +719,11 @@ int unix_listen_opts(QemuOpts *opts, Error **errp)
 
     unlink(un.sun_path);
     if (bind(sock, (struct sockaddr*) &un, sizeof(un)) < 0) {
-        error_set_errno(errp, errno, QERR_SOCKET_BIND_FAILED);
+        error_setg_errno(errp, errno, "Failed to bind socket");
         goto err;
     }
     if (listen(sock, 1) < 0) {
-        error_set_errno(errp, errno, QERR_SOCKET_LISTEN_FAILED);
+        error_setg_errno(errp, errno, "Failed to listen on socket");
         goto err;
     }
 
@@ -732,14 +742,14 @@ int unix_connect_opts(QemuOpts *opts, Error **errp,
     ConnectState *connect_state = NULL;
     int sock, rc;
 
-    if (NULL == path) {
+    if (path == NULL) {
         error_setg(errp, "unix connect: no path specified");
         return -1;
     }
 
     sock = qemu_socket(PF_UNIX, SOCK_STREAM, 0);
     if (sock < 0) {
-        error_set_errno(errp, errno, QERR_SOCKET_CREATE_FAILED);
+        error_setg_errno(errp, errno, "Failed to create socket");
         return -1;
     }
     if (callback != NULL) {
@@ -769,12 +779,12 @@ int unix_connect_opts(QemuOpts *opts, Error **errp,
     } else if (rc >= 0) {
         /* non blocking socket immediate success, call callback */
         if (callback != NULL) {
-            callback(sock, opaque);
+            callback(sock, NULL, opaque);
         }
     }
 
     if (rc < 0) {
-        error_set_errno(errp, -rc, QERR_SOCKET_CONNECT_FAILED);
+        error_setg_errno(errp, -rc, "Failed to connect socket");
         close(sock);
         sock = -1;
     }
@@ -919,7 +929,7 @@ int socket_connect(SocketAddress *addr, Error **errp,
         fd = monitor_get_fd(cur_mon, addr->fd->str, errp);
         if (fd >= 0 && callback) {
             qemu_set_nonblock(fd);
-            callback(fd, opaque);
+            callback(fd, NULL, opaque);
         }
         break;
 
@@ -966,8 +976,7 @@ int socket_dgram(SocketAddress *remote, SocketAddress *local, Error **errp)
     opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
     switch (remote->kind) {
     case SOCKET_ADDRESS_KIND_INET:
-        qemu_opt_set(opts, "host", remote->inet->host);
-        qemu_opt_set(opts, "port", remote->inet->port);
+        inet_addr_to_opts(opts, remote->inet);
         if (local) {
             qemu_opt_set(opts, "localaddr", local->inet->host);
             qemu_opt_set(opts, "localport", local->inet->port);
diff --git a/vl.c b/vl.c
index fe451aaf1..eb89d6290 100644
--- a/vl.c
+++ b/vl.c
@@ -61,8 +61,8 @@ int main(int argc, char **argv)
 #include "qemu/sockets.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
+#include "sysemu/accel.h"
 #include "hw/usb.h"
-#include "hw/pcmcia.h"
 #include "hw/i386/pc.h"
 #include "hw/isa/isa.h"
 #include "hw/bt.h"
@@ -134,6 +134,7 @@ const char* keyboard_layout = NULL;
 ram_addr_t ram_size;
 const char *mem_path = NULL;
 int mem_prealloc = 0; /* force preallocation of physical target memory */
+bool enable_mlock = false;
 int nb_nics;
 NICInfo nd_table[MAX_NICS];
 int autostart;
@@ -178,27 +179,20 @@ int ctrl_grab = 0;
 unsigned int nb_prom_envs = 0;
 const char *prom_envs[MAX_PROM_ENVS];
 int boot_menu;
-static bool boot_strict;
+bool boot_strict;
 uint8_t *boot_splash_filedata;
 size_t boot_splash_filedata_size;
 uint8_t qemu_extra_params_fw[2];
 
-typedef struct FWBootEntry FWBootEntry;
-
-struct FWBootEntry {
-    QTAILQ_ENTRY(FWBootEntry) link;
-    int32_t bootindex;
-    DeviceState *dev;
-    char *suffix;
-};
-
-static QTAILQ_HEAD(, FWBootEntry) fw_boot_order =
-    QTAILQ_HEAD_INITIALIZER(fw_boot_order);
+int icount_align_option;
 
 int nb_numa_nodes;
 int max_numa_nodeid;
 NodeInfo numa_info[MAX_NODES];
 
+/* The bytes in qemu_uuid[] are in the order specified by RFC4122, _not_ in the
+ * little-endian "wire format" described in the SMBIOS 2.6 specification.
+ */
 uint8_t qemu_uuid[16];
 bool qemu_uuid_set;
 
@@ -211,11 +205,9 @@ static NotifierList exit_notifiers =
 static NotifierList machine_init_done_notifiers =
     NOTIFIER_LIST_INITIALIZER(machine_init_done_notifiers);
 
-static bool tcg_allowed = true;
 bool xen_allowed;
 uint32_t xen_domid;
 enum xen_mode xen_mode = XEN_EMULATE;
-static int tcg_tb_size;
 
 static int has_defaults = 1;
 static int default_serial = 1;
@@ -387,6 +379,14 @@ static QemuOptsList qemu_machine_opts = {
             .name = PC_MACHINE_MAX_RAM_BELOW_4G,
             .type = QEMU_OPT_SIZE,
             .help = "maximum ram below the 4G boundary (32bit boundary)",
+        }, {
+            .name = PC_MACHINE_VMPORT,
+            .type = QEMU_OPT_STRING,
+            .help = "Enable vmport (pc & q35)",
+        },{
+            .name = "iommu",
+            .type = QEMU_OPT_BOOL,
+            .help = "Set on/off to enable/disable Intel IOMMU (VT-d)",
         },
         { /* End of list */ }
     },
@@ -537,6 +537,23 @@ static QemuOptsList qemu_mem_opts = {
     },
 };
 
+static QemuOptsList qemu_icount_opts = {
+    .name = "icount",
+    .implied_opt_name = "shift",
+    .merge_lists = true,
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_icount_opts.head),
+    .desc = {
+        {
+            .name = "shift",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "align",
+            .type = QEMU_OPT_BOOL,
+        },
+        { /* end of list */ }
+    },
+};
+
 /**
  * Get machine options
  *
@@ -1136,7 +1153,7 @@ static int drive_init_func(QemuOpts *opts, void *opaque)
 
 static int drive_enable_snapshot(QemuOpts *opts, void *opaque)
 {
-    if (NULL == qemu_opt_get(opts, "snapshot")) {
+    if (qemu_opt_get(opts, "snapshot") == NULL) {
         qemu_opt_set(opts, "snapshot", "on");
     }
     return 0;
@@ -1146,6 +1163,7 @@ static void default_drive(int enable, int snapshot, BlockInterfaceType type,
                           int index, const char *optstr)
 {
     QemuOpts *opts;
+    DriveInfo *dinfo;
 
     if (!enable || drive_get_by_index(type, index)) {
         return;
@@ -1155,9 +1173,13 @@ static void default_drive(int enable, int snapshot, BlockInterfaceType type,
     if (snapshot) {
         drive_enable_snapshot(opts, NULL);
     }
-    if (!drive_new(opts, type)) {
+
+    dinfo = drive_new(opts, type);
+    if (!dinfo) {
         exit(1);
     }
+    dinfo->is_default = true;
+
 }
 
 void qemu_register_boot_set(QEMUBootSetHandler *func, void *opaque)
@@ -1219,111 +1241,6 @@ static void restore_boot_order(void *opaque)
     g_free(normal_boot_order);
 }
 
-void add_boot_device_path(int32_t bootindex, DeviceState *dev,
-                          const char *suffix)
-{
-    FWBootEntry *node, *i;
-
-    if (bootindex < 0) {
-        return;
-    }
-
-    assert(dev != NULL || suffix != NULL);
-
-    node = g_malloc0(sizeof(FWBootEntry));
-    node->bootindex = bootindex;
-    node->suffix = g_strdup(suffix);
-    node->dev = dev;
-
-    QTAILQ_FOREACH(i, &fw_boot_order, link) {
-        if (i->bootindex == bootindex) {
-            fprintf(stderr, "Two devices with same boot index %d\n", bootindex);
-            exit(1);
-        } else if (i->bootindex < bootindex) {
-            continue;
-        }
-        QTAILQ_INSERT_BEFORE(i, node, link);
-        return;
-    }
-    QTAILQ_INSERT_TAIL(&fw_boot_order, node, link);
-}
-
-DeviceState *get_boot_device(uint32_t position)
-{
-    uint32_t counter = 0;
-    FWBootEntry *i = NULL;
-    DeviceState *res = NULL;
-
-    if (!QTAILQ_EMPTY(&fw_boot_order)) {
-        QTAILQ_FOREACH(i, &fw_boot_order, link) {
-            if (counter == position) {
-                res = i->dev;
-                break;
-            }
-            counter++;
-        }
-    }
-    return res;
-}
-
-/*
- * This function returns null terminated string that consist of new line
- * separated device paths.
- *
- * memory pointed by "size" is assigned total length of the array in bytes
- *
- */
-char *get_boot_devices_list(size_t *size, bool ignore_suffixes)
-{
-    FWBootEntry *i;
-    size_t total = 0;
-    char *list = NULL;
-
-    QTAILQ_FOREACH(i, &fw_boot_order, link) {
-        char *devpath = NULL, *bootpath;
-        size_t len;
-
-        if (i->dev) {
-            devpath = qdev_get_fw_dev_path(i->dev);
-            assert(devpath);
-        }
-
-        if (i->suffix && !ignore_suffixes && devpath) {
-            size_t bootpathlen = strlen(devpath) + strlen(i->suffix) + 1;
-
-            bootpath = g_malloc(bootpathlen);
-            snprintf(bootpath, bootpathlen, "%s%s", devpath, i->suffix);
-            g_free(devpath);
-        } else if (devpath) {
-            bootpath = devpath;
-        } else if (!ignore_suffixes) {
-            assert(i->suffix);
-            bootpath = g_strdup(i->suffix);
-        } else {
-            bootpath = g_strdup("");
-        }
-
-        if (total) {
-            list[total-1] = '\n';
-        }
-        len = strlen(bootpath) + 1;
-        list = g_realloc(list, total + len);
-        memcpy(&list[total], bootpath, len);
-        total += len;
-        g_free(bootpath);
-    }
-
-    *size = total;
-
-    if (boot_strict && *size > 0) {
-        list[total-1] = '\n';
-        list = g_realloc(list, total + 5);
-        memcpy(&list[total], "HALT", 5);
-        *size = total + 5;
-    }
-    return list;
-}
-
 static QemuOptsList qemu_smp_opts = {
     .name = "smp-opts",
     .implied_opt_name = "cpus",
@@ -1399,12 +1316,8 @@ static void smp_parse(QemuOpts *opts)
 
 }
 
-static void configure_realtime(QemuOpts *opts)
+static void realtime_init(void)
 {
-    bool enable_mlock;
-
-    enable_mlock = qemu_opt_get_bool(opts, "mlock", true);
-
     if (enable_mlock) {
         if (os_mlock() < 0) {
             fprintf(stderr, "qemu: locking memory failed\n");
@@ -1501,49 +1414,6 @@ void do_usb_del(Monitor *mon, const QDict *qdict)
 }
 
 /***********************************************************/
-/* PCMCIA/Cardbus */
-
-static struct pcmcia_socket_entry_s {
-    PCMCIASocket *socket;
-    struct pcmcia_socket_entry_s *next;
-} *pcmcia_sockets = 0;
-
-void pcmcia_socket_register(PCMCIASocket *socket)
-{
-    struct pcmcia_socket_entry_s *entry;
-
-    entry = g_malloc(sizeof(struct pcmcia_socket_entry_s));
-    entry->socket = socket;
-    entry->next = pcmcia_sockets;
-    pcmcia_sockets = entry;
-}
-
-void pcmcia_socket_unregister(PCMCIASocket *socket)
-{
-    struct pcmcia_socket_entry_s *entry, **ptr;
-
-    ptr = &pcmcia_sockets;
-    for (entry = *ptr; entry; ptr = &entry->next, entry = *ptr)
-        if (entry->socket == socket) {
-            *ptr = entry->next;
-            g_free(entry);
-        }
-}
-
-void pcmcia_info(Monitor *mon, const QDict *qdict)
-{
-    struct pcmcia_socket_entry_s *iter;
-
-    if (!pcmcia_sockets)
-        monitor_printf(mon, "No PCMCIA sockets\n");
-
-    for (iter = pcmcia_sockets; iter; iter = iter->next)
-        monitor_printf(mon, "%s: %s\n", iter->socket->slot_string,
-                       iter->socket->attached ? iter->socket->card_string :
-                       "Empty");
-}
-
-/***********************************************************/
 /* machine registration */
 
 MachineState *current_machine;
@@ -1553,6 +1423,7 @@ static void machine_class_init(ObjectClass *oc, void *data)
     MachineClass *mc = MACHINE_CLASS(oc);
     QEMUMachine *qm = data;
 
+    mc->family = qm->family;
     mc->name = qm->name;
     mc->alias = qm->alias;
     mc->desc = qm->desc;
@@ -1561,6 +1432,7 @@ static void machine_class_init(ObjectClass *oc, void *data)
     mc->hot_add_cpu = qm->hot_add_cpu;
     mc->kvm_type = qm->kvm_type;
     mc->block_default_type = qm->block_default_type;
+    mc->units_per_default_bus = qm->units_per_default_bus;
     mc->max_cpus = qm->max_cpus;
     mc->no_serial = qm->no_serial;
     mc->no_parallel = qm->no_parallel;
@@ -1569,9 +1441,11 @@ static void machine_class_init(ObjectClass *oc, void *data)
     mc->no_floppy = qm->no_floppy;
     mc->no_cdrom = qm->no_cdrom;
     mc->no_sdcard = qm->no_sdcard;
+    mc->has_dynamic_sysbus = qm->has_dynamic_sysbus;
     mc->is_default = qm->is_default;
     mc->default_machine_opts = qm->default_machine_opts;
     mc->default_boot_order = qm->default_boot_order;
+    mc->default_display = qm->default_display;
     mc->compat_props = qm->compat_props;
     mc->hw_version = qm->hw_version;
 }
@@ -1699,11 +1573,11 @@ void qemu_del_vm_change_state_handler(VMChangeStateEntry *e)
 
 void vm_state_notify(int running, RunState state)
 {
-    VMChangeStateEntry *e;
+    VMChangeStateEntry *e, *next;
 
     trace_vm_state_notify(running, state);
 
-    for (e = vm_change_state_head.lh_first; e; e = e->entries.le_next) {
+    QLIST_FOREACH_SAFE(e, &vm_change_state_head, entries, next) {
         e->cb(e->opaque, running, state);
     }
 }
@@ -1745,9 +1619,7 @@ int qemu_reset_requested_get(void)
 
 static int qemu_shutdown_requested(void)
 {
-    int r = shutdown_requested;
-    shutdown_requested = 0;
-    return r;
+    return atomic_xchg(&shutdown_requested, 0);
 }
 
 static void qemu_kill_report(void)
@@ -2488,8 +2360,9 @@ static int foreach_device_config(int type, int (*func)(const char *cmdline))
         loc_push_restore(&conf->loc);
         rc = func(conf->cmdline);
         loc_pop(&conf->loc);
-        if (0 != rc)
+        if (rc) {
             return rc;
+        }
     }
     return 0;
 }
@@ -2625,7 +2498,41 @@ static int debugcon_parse(const char *devname)
     return 0;
 }
 
-static MachineClass *machine_parse(const char *name)
+static gint machine_class_cmp(gconstpointer a, gconstpointer b)
+{
+    const MachineClass *mc1 = a, *mc2 = b;
+    int res;
+
+    if (mc1->family == NULL) {
+        if (mc2->family == NULL) {
+            /* Compare standalone machine types against each other; they sort
+             * in increasing order.
+             */
+            return strcmp(object_class_get_name(OBJECT_CLASS(mc1)),
+                          object_class_get_name(OBJECT_CLASS(mc2)));
+        }
+
+        /* Standalone machine types sort after families. */
+        return 1;
+    }
+
+    if (mc2->family == NULL) {
+        /* Families sort before standalone machine types. */
+        return -1;
+    }
+
+    /* Families sort between each other alphabetically increasingly. */
+    res = strcmp(mc1->family, mc2->family);
+    if (res != 0) {
+        return res;
+    }
+
+    /* Within the same family, machine types sort in decreasing order. */
+    return strcmp(object_class_get_name(OBJECT_CLASS(mc2)),
+                  object_class_get_name(OBJECT_CLASS(mc1)));
+}
+
+ static MachineClass *machine_parse(const char *name)
 {
     MachineClass *mc = NULL;
     GSList *el, *machines = object_class_get_list(TYPE_MACHINE, false);
@@ -2641,6 +2548,7 @@ static MachineClass *machine_parse(const char *name)
         error_printf("Use -machine help to list supported machines!\n");
     } else {
         printf("Supported machines are:\n");
+        machines = g_slist_sort(machines, machine_class_cmp);
         for (el = machines; el; el = el->next) {
             MachineClass *mc = el->data;
             if (mc->alias) {
@@ -2655,84 +2563,6 @@ static MachineClass *machine_parse(const char *name)
     exit(!name || !is_help_option(name));
 }
 
-static int tcg_init(MachineClass *mc)
-{
-    tcg_exec_init(tcg_tb_size * 1024 * 1024);
-    return 0;
-}
-
-static struct {
-    const char *opt_name;
-    const char *name;
-    int (*available)(void);
-    int (*init)(MachineClass *mc);
-    bool *allowed;
-} accel_list[] = {
-    { "tcg", "tcg", tcg_available, tcg_init, &tcg_allowed },
-    { "xen", "Xen", xen_available, xen_init, &xen_allowed },
-    { "kvm", "KVM", kvm_available, kvm_init, &kvm_allowed },
-    { "qtest", "QTest", qtest_available, qtest_init_accel, &qtest_allowed },
-};
-
-static int configure_accelerator(MachineClass *mc)
-{
-    const char *p;
-    char buf[10];
-    int i, ret;
-    bool accel_initialised = false;
-    bool init_failed = false;
-
-    p = qemu_opt_get(qemu_get_machine_opts(), "accel");
-    if (p == NULL) {
-        /* Use the default "accelerator", tcg */
-        p = "tcg";
-    }
-
-    while (!accel_initialised && *p != '\0') {
-        if (*p == ':') {
-            p++;
-        }
-        p = get_opt_name(buf, sizeof (buf), p, ':');
-        for (i = 0; i < ARRAY_SIZE(accel_list); i++) {
-            if (strcmp(accel_list[i].opt_name, buf) == 0) {
-                if (!accel_list[i].available()) {
-                    printf("%s not supported for this target\n",
-                           accel_list[i].name);
-                    break;
-                }
-                *(accel_list[i].allowed) = true;
-                ret = accel_list[i].init(mc);
-                if (ret < 0) {
-                    init_failed = true;
-                    fprintf(stderr, "failed to initialize %s: %s\n",
-                            accel_list[i].name,
-                            strerror(-ret));
-                    *(accel_list[i].allowed) = false;
-                } else {
-                    accel_initialised = true;
-                }
-                break;
-            }
-        }
-        if (i == ARRAY_SIZE(accel_list)) {
-            fprintf(stderr, "\"%s\" accelerator does not exist.\n", buf);
-        }
-    }
-
-    if (!accel_initialised) {
-        if (!init_failed) {
-            fprintf(stderr, "No accelerator found!\n");
-        }
-        exit(1);
-    }
-
-    if (init_failed) {
-        fprintf(stderr, "Back to %s accelerator.\n", accel_list[i].name);
-    }
-
-    return !accel_initialised;
-}
-
 void qemu_add_exit_notifier(Notifier *notify)
 {
     notifier_list_add(&exit_notifiers, notify);
@@ -2818,15 +2648,15 @@ static void free_and_trace(gpointer mem)
     free(mem);
 }
 
-static int object_set_property(const char *name, const char *value, void *opaque)
+static int machine_set_property(const char *name, const char *value,
+                                void *opaque)
 {
     Object *obj = OBJECT(opaque);
     StringInputVisitor *siv;
     Error *local_err = NULL;
     char *c, *qom_name;
 
-    if (strcmp(name, "qom-type") == 0 || strcmp(name, "id") == 0 ||
-        strcmp(name, "type") == 0) {
+    if (strcmp(name, "type") == 0) {
         return 0;
     }
 
@@ -2899,6 +2729,7 @@ out:
     g_free(dummy);
     if (err) {
         qerror_report_err(err);
+        error_free(err);
         return -1;
     }
     return 0;
@@ -2908,13 +2739,12 @@ int main(int argc, char **argv, char **envp)
 {
     int i;
     int snapshot, linux_boot;
-    const char *icount_option = NULL;
     const char *initrd_filename;
     const char *kernel_filename, *kernel_cmdline;
     const char *boot_order;
     DisplayState *ds;
     int cyls, heads, secs, translation;
-    QemuOpts *hda_opts = NULL, *opts, *machine_opts;
+    QemuOpts *hda_opts = NULL, *opts, *machine_opts, *icount_opts = NULL;
     QemuOptsList *olist;
     int optind;
     const char *optarg;
@@ -2945,6 +2775,7 @@ int main(int argc, char **argv, char **envp)
     ram_addr_t maxram_size = default_ram_size;
     uint64_t ram_slots = 0;
     FILE *vmstate_dump_file = NULL;
+    Error *main_loop_err = NULL;
 
     atexit(qemu_run_exit_notifiers);
     error_set_progname(argv[0]);
@@ -2979,6 +2810,7 @@ int main(int argc, char **argv, char **envp)
     qemu_add_opts(&qemu_msg_opts);
     qemu_add_opts(&qemu_name_opts);
     qemu_add_opts(&qemu_numa_opts);
+    qemu_add_opts(&qemu_icount_opts);
 
     runstate_init();
 
@@ -3334,34 +3166,34 @@ int main(int argc, char **argv, char **envp)
 
                     sz = qemu_opt_get_size(opts, "maxmem", 0);
                     if (sz < ram_size) {
-                        fprintf(stderr, "qemu: invalid -m option value: maxmem "
-                                "(%" PRIu64 ") <= initial memory ("
-                                RAM_ADDR_FMT ")\n", sz, ram_size);
+                        error_report("invalid -m option value: maxmem "
+                                "(0x%" PRIx64 ") <= initial memory (0x"
+                                RAM_ADDR_FMT ")", sz, ram_size);
                         exit(EXIT_FAILURE);
                     }
 
                     slots = qemu_opt_get_number(opts, "slots", 0);
                     if ((sz > ram_size) && !slots) {
-                        fprintf(stderr, "qemu: invalid -m option value: maxmem "
-                                "(%" PRIu64 ") more than initial memory ("
+                        error_report("invalid -m option value: maxmem "
+                                "(0x%" PRIx64 ") more than initial memory (0x"
                                 RAM_ADDR_FMT ") but no hotplug slots where "
-                                "specified\n", sz, ram_size);
+                                "specified", sz, ram_size);
                         exit(EXIT_FAILURE);
                     }
 
                     if ((sz <= ram_size) && slots) {
-                        fprintf(stderr, "qemu: invalid -m option value:  %"
+                        error_report("invalid -m option value:  %"
                                 PRIu64 " hotplug slots where specified but "
-                                "maxmem (%" PRIu64 ") <= initial memory ("
-                                RAM_ADDR_FMT ")\n", slots, sz, ram_size);
+                                "maxmem (0x%" PRIx64 ") <= initial memory (0x"
+                                RAM_ADDR_FMT ")", slots, sz, ram_size);
                         exit(EXIT_FAILURE);
                     }
                     maxram_size = sz;
                     ram_slots = slots;
                 } else if ((!maxmem_str && slots_str) ||
                            (maxmem_str && !slots_str)) {
-                    fprintf(stderr, "qemu: invalid -m option value: missing "
-                            "'%s' option\n", slots_str ? "maxmem" : "slots");
+                    error_report("invalid -m option value: missing "
+                            "'%s' option", slots_str ? "maxmem" : "slots");
                     exit(EXIT_FAILURE);
                 }
                 break;
@@ -3830,7 +3662,11 @@ int main(int argc, char **argv, char **envp)
                 }
                 break;
             case QEMU_OPTION_icount:
-                icount_option = optarg;
+                icount_opts = qemu_opts_parse(qemu_find_opts("icount"),
+                                              optarg, 1);
+                if (!icount_opts) {
+                    exit(1);
+                }
                 break;
             case QEMU_OPTION_incoming:
                 incoming = optarg;
@@ -3945,7 +3781,7 @@ int main(int argc, char **argv, char **envp)
                 if (!opts) {
                     exit(1);
                 }
-                configure_realtime(opts);
+                enable_mlock = qemu_opt_get_bool(opts, "mlock", true);
                 break;
             case QEMU_OPTION_msg:
                 opts = qemu_opts_parse(qemu_find_opts("msg"), optarg, 0);
@@ -3955,6 +3791,11 @@ int main(int argc, char **argv, char **envp)
                 configure_msg(opts);
                 break;
             case QEMU_OPTION_dump_vmstate:
+                if (vmstate_dump_file) {
+                    fprintf(stderr, "qemu: only one '-dump-vmstate' "
+                            "option may be given\n");
+                    exit(1);
+                }
                 vmstate_dump_file = fopen(optarg, "w");
                 if (vmstate_dump_file == NULL) {
                     fprintf(stderr, "open %s: %s\n", optarg, strerror(errno));
@@ -3970,8 +3811,8 @@ int main(int argc, char **argv, char **envp)
 
     os_daemonize();
 
-    if (qemu_init_main_loop()) {
-        fprintf(stderr, "qemu_init_main_loop failed\n");
+    if (qemu_init_main_loop(&main_loop_err)) {
+        error_report("%s", error_get_pretty(main_loop_err));
         exit(1);
     }
 
@@ -4009,11 +3850,6 @@ int main(int argc, char **argv, char **envp)
         qemu_set_version(machine_class->hw_version);
     }
 
-    if (qemu_opts_foreach(qemu_find_opts("object"),
-                          object_create, NULL, 0) != 0) {
-        exit(1);
-    }
-
     /* Init CPU def lists, based on config
      * - Must be called after all the qemu_read_config_file() calls
      * - Must be called before list_cpus()
@@ -4213,7 +4049,7 @@ int main(int argc, char **argv, char **envp)
 #endif
 
     if (pid_file && qemu_create_pidfile(pid_file) != 0) {
-        os_pidfile_error();
+        fprintf(stderr, "Could not acquire pid file: %s\n", strerror(errno));
         exit(1);
     }
 
@@ -4225,14 +4061,19 @@ int main(int argc, char **argv, char **envp)
         exit(0);
     }
 
+    if (qemu_opts_foreach(qemu_find_opts("object"),
+                          object_create, NULL, 0) != 0) {
+        exit(1);
+    }
+
     machine_opts = qemu_get_machine_opts();
-    if (qemu_opt_foreach(machine_opts, object_set_property, current_machine,
+    if (qemu_opt_foreach(machine_opts, machine_set_property, current_machine,
                          1) < 0) {
         object_unref(OBJECT(current_machine));
         exit(1);
     }
 
-    configure_accelerator(machine_class);
+    configure_accelerator(current_machine);
 
     if (qtest_chrdev) {
         Error *local_err = NULL;
@@ -4306,11 +4147,15 @@ int main(int argc, char **argv, char **envp)
     qemu_spice_init();
 #endif
 
-    if (icount_option && (kvm_enabled() || xen_enabled())) {
-        fprintf(stderr, "-icount is not allowed with kvm or xen\n");
-        exit(1);
+    cpu_ticks_init();
+    if (icount_opts) {
+        if (kvm_enabled() || xen_enabled()) {
+            fprintf(stderr, "-icount is not allowed with kvm or xen\n");
+            exit(1);
+        }
+        configure_icount(icount_opts, &error_abort);
+        qemu_opts_del(icount_opts);
     }
-    configure_icount(icount_option);
 
     /* clean up network at qemu process termination */
     atexit(&net_cleanup);
@@ -4340,6 +4185,13 @@ int main(int argc, char **argv, char **envp)
     blk_mig_init();
     ram_mig_init();
 
+    /* If the currently selected machine wishes to override the units-per-bus
+     * property of its default HBA interface type, do so now. */
+    if (machine_class->units_per_default_bus) {
+        override_max_devs(machine_class->block_default_type,
+                          machine_class->units_per_default_bus);
+    }
+
     /* open the virtual block devices */
     if (snapshot)
         qemu_opts_foreach(qemu_find_opts("drive"), drive_enable_snapshot, NULL, 0);
@@ -4378,7 +4230,9 @@ int main(int argc, char **argv, char **envp)
 
     /* If no default VGA is requested, the default is "none".  */
     if (default_vga) {
-        if (cirrus_vga_available()) {
+        if (machine_class->default_display) {
+            vga_model = machine_class->default_display;
+        } else if (cirrus_vga_available()) {
             vga_model = "cirrus";
         } else if (vga_available()) {
             vga_model = "std";
@@ -4409,6 +4263,8 @@ int main(int argc, char **argv, char **envp)
 
     machine_class->init(current_machine);
 
+    realtime_init();
+
     audio_init();
 
     cpu_synchronize_all_post_init();
@@ -4425,6 +4281,9 @@ int main(int argc, char **argv, char **envp)
     if (qemu_opts_foreach(qemu_find_opts("device"), device_init_func, NULL, 1) != 0)
         exit(1);
 
+    /* Did we create any drives that we failed to create a device for? */
+    drive_check_orphaned();
+
     net_check_clients();
 
     ds = init_displaystate();
@@ -4510,7 +4369,7 @@ int main(int argc, char **argv, char **envp)
         }
     }
 
-    qdev_prop_check_global();
+    qdev_prop_check_globals();
     if (vmstate_dump_file) {
         /* dump and exit */
         dump_vmstate_json_to_file(vmstate_dump_file);
diff --git a/vmstate.c b/vmstate.c
index ef2f87bda..3dde574c0 100644
--- a/vmstate.c
+++ b/vmstate.c
@@ -49,9 +49,16 @@ static void *vmstate_base_addr(void *opaque, VMStateField *field, bool alloc)
 
     if (field->flags & VMS_POINTER) {
         if (alloc && (field->flags & VMS_ALLOC)) {
-            int n_elems = vmstate_n_elems(opaque, field);
-            if (n_elems) {
-                gsize size = n_elems * field->size;
+            gsize size = 0;
+            if (field->flags & VMS_VBUFFER) {
+                size = vmstate_size(opaque, field);
+            } else {
+                int n_elems = vmstate_n_elems(opaque, field);
+                if (n_elems) {
+                    size = n_elems * field->size;
+                }
+            }
+            if (size) {
                 *((void **)base_addr + field->start) = g_malloc(size);
             }
         }
diff --git a/xen-common-stub.c b/xen-common-stub.c
index bd56ca2ce..906f991f1 100644
--- a/xen-common-stub.c
+++ b/xen-common-stub.c
@@ -11,9 +11,3 @@
 void xenstore_store_pv_console_info(int i, CharDriverState *chr)
 {
 }
-
-int xen_init(MachineClass *mc)
-{
-    return -ENOSYS;
-}
-
diff --git a/xen-common.c b/xen-common.c
index f07b35e47..56359ca72 100644
--- a/xen-common.c
+++ b/xen-common.c
@@ -11,6 +11,7 @@
 #include "hw/xen/xen_backend.h"
 #include "qmp-commands.h"
 #include "sysemu/char.h"
+#include "sysemu/accel.h"
 
 //#define DEBUG_XEN
 
@@ -109,7 +110,7 @@ static void xen_change_state_handler(void *opaque, int running,
     }
 }
 
-int xen_init(MachineClass *mc)
+static int xen_init(MachineState *ms)
 {
     xen_xc = xen_xc_interface_open(0, 0, 0);
     if (xen_xc == XC_HANDLER_INITIAL_VALUE) {
@@ -121,3 +122,25 @@ int xen_init(MachineClass *mc)
     return 0;
 }
 
+static void xen_accel_class_init(ObjectClass *oc, void *data)
+{
+    AccelClass *ac = ACCEL_CLASS(oc);
+    ac->name = "Xen";
+    ac->init_machine = xen_init;
+    ac->allowed = &xen_allowed;
+}
+
+#define TYPE_XEN_ACCEL ACCEL_CLASS_NAME("xen")
+
+static const TypeInfo xen_accel_type = {
+    .name = TYPE_XEN_ACCEL,
+    .parent = TYPE_ACCEL,
+    .class_init = xen_accel_class_init,
+};
+
+static void xen_type_init(void)
+{
+    type_register_static(&xen_accel_type);
+}
+
+type_init(xen_type_init);
diff --git a/xen-hvm.c b/xen-hvm.c
index a2486cfd7..754879481 100644
--- a/xen-hvm.c
+++ b/xen-hvm.c
@@ -41,6 +41,29 @@ static MemoryRegion *framebuffer;
 static bool xen_in_migration;
 
 /* Compatibility with older version */
+
+/* This allows QEMU to build on a system that has Xen 4.5 or earlier
+ * installed.  This here (not in hw/xen/xen_common.h) because xen/hvm/ioreq.h
+ * needs to be included before this block and hw/xen/xen_common.h needs to
+ * be included before xen/hvm/ioreq.h
+ */
+#ifndef IOREQ_TYPE_VMWARE_PORT
+#define IOREQ_TYPE_VMWARE_PORT  3
+struct vmware_regs {
+    uint32_t esi;
+    uint32_t edi;
+    uint32_t ebx;
+    uint32_t ecx;
+    uint32_t edx;
+};
+typedef struct vmware_regs vmware_regs_t;
+
+struct shared_vmport_iopage {
+    struct vmware_regs vcpu_vmport_regs[1];
+};
+typedef struct shared_vmport_iopage shared_vmport_iopage_t;
+#endif
+
 #if __XEN_LATEST_INTERFACE_VERSION__ < 0x0003020a
 static inline uint32_t xen_vcpu_eport(shared_iopage_t *shared_page, int i)
 {
@@ -71,7 +94,7 @@ static inline ioreq_t *xen_vcpu_ioreq(shared_iopage_t *shared_page, int vcpu)
 typedef struct XenPhysmap {
     hwaddr start_addr;
     ram_addr_t size;
-    char *name;
+    const char *name;
     hwaddr phys_offset;
 
     QLIST_ENTRY(XenPhysmap) list;
@@ -79,8 +102,10 @@ typedef struct XenPhysmap {
 
 typedef struct XenIOState {
     shared_iopage_t *shared_page;
+    shared_vmport_iopage_t *shared_vmport_page;
     buffered_iopage_t *buffered_io_page;
     QEMUTimer *buffered_io_timer;
+    CPUState **cpu_by_vcpu_id;
     /* the evtchn port for polling the notification, */
     evtchn_port_t *ioreq_local_port;
     /* evtchn local port for buffered io */
@@ -188,7 +213,8 @@ static void xen_ram_init(ram_addr_t *below_4g_mem_size,
          */
         block_len = (1ULL << 32) + *above_4g_mem_size;
     }
-    memory_region_init_ram(&ram_memory, NULL, "xen.ram", block_len);
+    memory_region_init_ram(&ram_memory, NULL, "xen.ram", block_len,
+                           &error_abort);
     *ram_memory_p = &ram_memory;
     vmstate_register_ram_global(&ram_memory);
 
@@ -291,6 +317,7 @@ static int xen_add_to_physmap(XenIOState *state,
     hwaddr pfn, start_gpfn;
     hwaddr phys_offset = memory_region_get_ram_addr(mr);
     char path[80], value[17];
+    const char *mr_name;
 
     if (get_physmapping(state, start_addr, size)) {
         return 0;
@@ -326,11 +353,13 @@ go_physmap:
         }
     }
 
+    mr_name = memory_region_name(mr);
+
     physmap = g_malloc(sizeof (XenPhysmap));
 
     physmap->start_addr = start_addr;
     physmap->size = size;
-    physmap->name = (char *)mr->name;
+    physmap->name = mr_name;
     physmap->phys_offset = phys_offset;
 
     QLIST_INSERT_HEAD(&state->physmap, physmap, list);
@@ -354,11 +383,11 @@ go_physmap:
     if (!xs_write(state->xenstore, 0, path, value, strlen(value))) {
         return -1;
     }
-    if (mr->name) {
+    if (mr_name) {
         snprintf(path, sizeof(path),
                 "/local/domain/0/device-model/%d/physmap/%"PRIx64"/name",
                 xen_domid, (uint64_t)phys_offset);
-        if (!xs_write(state->xenstore, 0, path, mr->name, strlen(mr->name))) {
+        if (!xs_write(state->xenstore, 0, path, mr_name, strlen(mr_name))) {
             return -1;
         }
     }
@@ -513,11 +542,14 @@ static void xen_sync_dirty_bitmap(XenIOState *state,
                                  start_addr >> TARGET_PAGE_BITS, npages,
                                  bitmap);
     if (rc < 0) {
-        if (rc != -ENODATA) {
+#ifndef ENODATA
+#define ENODATA  ENOENT
+#endif
+        if (errno == ENODATA) {
             memory_region_set_dirty(framebuffer, 0, size);
             DPRINTF("xen: track_dirty_vram failed (0x" TARGET_FMT_plx
                     ", 0x" TARGET_FMT_plx "): %s\n",
-                    start_addr, start_addr + size, strerror(-rc));
+                    start_addr, start_addr + size, strerror(errno));
         }
         return;
     }
@@ -766,7 +798,50 @@ static void cpu_ioreq_move(ioreq_t *req)
     }
 }
 
-static void handle_ioreq(ioreq_t *req)
+static void regs_to_cpu(vmware_regs_t *vmport_regs, ioreq_t *req)
+{
+    X86CPU *cpu;
+    CPUX86State *env;
+
+    cpu = X86_CPU(current_cpu);
+    env = &cpu->env;
+    env->regs[R_EAX] = req->data;
+    env->regs[R_EBX] = vmport_regs->ebx;
+    env->regs[R_ECX] = vmport_regs->ecx;
+    env->regs[R_EDX] = vmport_regs->edx;
+    env->regs[R_ESI] = vmport_regs->esi;
+    env->regs[R_EDI] = vmport_regs->edi;
+}
+
+static void regs_from_cpu(vmware_regs_t *vmport_regs)
+{
+    X86CPU *cpu = X86_CPU(current_cpu);
+    CPUX86State *env = &cpu->env;
+
+    vmport_regs->ebx = env->regs[R_EBX];
+    vmport_regs->ecx = env->regs[R_ECX];
+    vmport_regs->edx = env->regs[R_EDX];
+    vmport_regs->esi = env->regs[R_ESI];
+    vmport_regs->edi = env->regs[R_EDI];
+}
+
+static void handle_vmport_ioreq(XenIOState *state, ioreq_t *req)
+{
+    vmware_regs_t *vmport_regs;
+
+    assert(state->shared_vmport_page);
+    vmport_regs =
+        &state->shared_vmport_page->vcpu_vmport_regs[state->send_vcpu];
+    QEMU_BUILD_BUG_ON(sizeof(*req) < sizeof(*vmport_regs));
+
+    current_cpu = state->cpu_by_vcpu_id[state->send_vcpu];
+    regs_to_cpu(vmport_regs, req);
+    cpu_ioreq_pio(req);
+    regs_from_cpu(vmport_regs);
+    current_cpu = NULL;
+}
+
+static void handle_ioreq(XenIOState *state, ioreq_t *req)
 {
     if (!req->data_is_ptr && (req->dir == IOREQ_WRITE) &&
             (req->size < sizeof (target_ulong))) {
@@ -780,6 +855,9 @@ static void handle_ioreq(ioreq_t *req)
         case IOREQ_TYPE_COPY:
             cpu_ioreq_move(req);
             break;
+        case IOREQ_TYPE_VMWARE_PORT:
+            handle_vmport_ioreq(state, req);
+            break;
         case IOREQ_TYPE_TIMEOFFSET:
             break;
         case IOREQ_TYPE_INVALIDATE:
@@ -821,7 +899,7 @@ static int handle_buffered_iopage(XenIOState *state)
             req.data |= ((uint64_t)buf_req->data) << 32;
         }
 
-        handle_ioreq(&req);
+        handle_ioreq(state, &req);
 
         xen_mb();
         state->buffered_io_page->read_pointer += qw ? 2 : 1;
@@ -850,14 +928,16 @@ static void cpu_handle_ioreq(void *opaque)
 
     handle_buffered_iopage(state);
     if (req) {
-        handle_ioreq(req);
+        handle_ioreq(state, req);
 
         if (req->state != STATE_IOREQ_INPROCESS) {
             fprintf(stderr, "Badness in I/O request ... not in service?!: "
                     "%x, ptr: %x, port: %"PRIx64", "
-                    "data: %"PRIx64", count: %" FMT_ioreq_size ", size: %" FMT_ioreq_size "\n",
+                    "data: %"PRIx64", count: %" FMT_ioreq_size
+                    ", size: %" FMT_ioreq_size
+                    ", type: %"FMT_ioreq_size"\n",
                     req->state, req->data_is_ptr, req->addr,
-                    req->data, req->count, req->size);
+                    req->data, req->count, req->size, req->type);
             destroy_hvm_domain(false);
             return;
         }
@@ -897,6 +977,14 @@ static void xen_main_loop_prepare(XenIOState *state)
                                                  state);
 
     if (evtchn_fd != -1) {
+        CPUState *cpu_state;
+
+        DPRINTF("%s: Init cpu_by_vcpu_id\n", __func__);
+        CPU_FOREACH(cpu_state) {
+            DPRINTF("%s: cpu_by_vcpu_id[%d]=%p\n",
+                    __func__, cpu_state->cpu_index, cpu_state);
+            state->cpu_by_vcpu_id[cpu_state->cpu_index] = cpu_state;
+        }
         qemu_set_fd_handler(evtchn_fd, cpu_handle_ioreq, NULL, state);
     }
 }
@@ -905,9 +993,8 @@ static void xen_main_loop_prepare(XenIOState *state)
 static void xen_hvm_change_state_handler(void *opaque, int running,
                                          RunState rstate)
 {
-    XenIOState *xstate = opaque;
     if (running) {
-        xen_main_loop_prepare(xstate);
+        xen_main_loop_prepare((XenIOState *)opaque);
     }
 }
 
@@ -972,6 +1059,7 @@ static void xen_wakeup_notifier(Notifier *notifier, void *data)
     xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 0);
 }
 
+/* return 0 means OK, or -1 means critical issue -- will exit(1) */
 int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
                  MemoryRegion **ram_memory)
 {
@@ -985,15 +1073,13 @@ int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
     state->xce_handle = xen_xc_evtchn_open(NULL, 0);
     if (state->xce_handle == XC_HANDLER_INITIAL_VALUE) {
         perror("xen: event channel open");
-        g_free(state);
-        return -errno;
+        return -1;
     }
 
     state->xenstore = xs_daemon_open();
     if (state->xenstore == NULL) {
         perror("xen: xenstore open");
-        g_free(state);
-        return -errno;
+        return -1;
     }
 
     state->exit.notify = xen_exit_notifier;
@@ -1014,6 +1100,20 @@ int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
                  errno, xen_xc);
     }
 
+    rc = xen_get_vmport_regs_pfn(xen_xc, xen_domid, &ioreq_pfn);
+    if (!rc) {
+        DPRINTF("shared vmport page at pfn %lx\n", ioreq_pfn);
+        state->shared_vmport_page =
+            xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
+                                 PROT_READ|PROT_WRITE, ioreq_pfn);
+        if (state->shared_vmport_page == NULL) {
+            hw_error("map shared vmport IO page returned error %d handle="
+                     XC_INTERFACE_FMT, errno, xen_xc);
+        }
+    } else if (rc != -ENOSYS) {
+        hw_error("get vmport regs pfn returned error %d, rc=%d", errno, rc);
+    }
+
     xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_BUFIOREQ_PFN, &ioreq_pfn);
     DPRINTF("buffered io page at pfn %lx\n", ioreq_pfn);
     state->buffered_io_page = xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
@@ -1022,6 +1122,9 @@ int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
         hw_error("map buffered IO page returned error %d", errno);
     }
 
+    /* Note: cpus is empty at this point in init */
+    state->cpu_by_vcpu_id = g_malloc0(max_cpus * sizeof(CPUState *));
+
     state->ioreq_local_port = g_malloc0(max_cpus * sizeof (evtchn_port_t));
 
     /* FIXME: how about if we overflow the page here? */
@@ -1063,7 +1166,7 @@ int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
     /* Initialize backend core & drivers */
     if (xen_be_init() != 0) {
         fprintf(stderr, "%s: xen backend core setup failed\n", __FUNCTION__);
-        exit(1);
+        return -1;
     }
     xen_be_register("console", &xen_console_ops);
     xen_be_register("vkbd", &xen_kbdmouse_ops);